thread_job 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7bd76842b19742485270bbe794a33f9f852f0133
+  data.tar.gz: 4a5926023d47505c75b56155090d8ddb38ea97c8
+SHA512:
+  metadata.gz: 441dd1924c0b8b2b008e3127c9329d5b029bf9b38b29e94f06a90e68765e838a27616c077210d7500dd9ac4af16ee98a0b33eff59ded44d4324d245e13faf806
+  data.tar.gz: 7b1136aaa31fc69684e807b8db0e4e521d07276f27ef405b1abbf466299a752c6e7be2ce5f8a96b6a9ce902981d7d7b19ede76fd42c27d08e4b0994d7dac88ac

data/lib/thread_job.rb

@@ -0,0 +1,6 @@
+require 'thread'
+require 'thread_job/job'
+require 'thread_job/job_store'
+require 'thread_job/backends/memory/store'
+require 'thread_job/scheduler'
+require 'thread_job/thread_pool'

data/lib/thread_job/backends/memory/store.rb

@@ -0,0 +1,105 @@
+require 'logger'
+
+module ThreadJob
+  module Memory
+    AVAILABLE = 1
+    WORKING = 2
+    COMPLETE = 3
+    FAILED = 4
+
+    class Record
+      attr_accessor :id, :attempts, :job, :job_name, :queue_name, :status
+    end
+
+    class Store < JobStore
+      def initialize(max_retries=10, logger=Logger.new(STDOUT))
+        @jobs = {}
+        @failed_jobs = {}
+        @mutex = Mutex.new
+        @logger = logger
+        @max_retries = max_retries
+      end
+
+      def save_job(queue_name, job_name, job)
+        @mutex.synchronize {
+          queued_jobs = @jobs[queue_name] ||= []
+          failed_queue_jobs = @failed_jobs[queue_name] ||= []
+
+          rec = Memory::Record.new
+          rec.attempts = 0
+          rec.id = queued_jobs.count + 1
+          rec.job_name = job_name
+          rec.job = job
+          rec.status = AVAILABLE
+          rec.queue_name = queue_name
+
+          queued_jobs.push(rec)
+        }
+
+        @logger.info("[MemoryStore] Saved #{job_name}")
+      end
+
+      def poll_for_job(queue_name)
+        @jobs[queue_name] ||= []
+        @logger.debug("[MemoryStore] Polling for jobs, #{@jobs[queue_name].length} in the queue")
+
+        @mutex.synchronize {
+          @jobs[queue_name].each do |record|
+            if record.status == AVAILABLE || (record.status == FAILED && record.attempts < @max_retries)
+              record.status = WORKING
+              @logger.debug("[MemoryStore] Sending job '#{record.job_name}' to the thread pool for work")
+              return {id: record.id, job: record.job, job_name: record.job_name}
+            end
+          end
+        }
+
+        return nil
+      end
+
+      def get_job(queue_name, job_id)
+        found_job = false
+        if @jobs[queue_name] != nil
+          @jobs[queue_name].each do |job|
+            if job.id == job_id
+              found_job = true
+              return job
+            end
+          end
+        end
+
+        @logger.warn("[MemoryStore] unable to get job: #{job_id} from queue: #{queue_name}")
+
+        return nil
+      end
+
+      def complete_job(queue_name, job_id)
+        @mutex.synchronize {
+          job = get_job(queue_name, job_id)
+          if job
+            @jobs[queue_name].delete(job)
+            @logger.info("[MemoryStore] job: '#{job.job_name}' has been completed and removed from the queue")
+          end
+        }
+      end
+
+      def fail_job(queue_name, job_id)
+        @mutex.synchronize {
+          job = get_job(queue_name, job_id)
+          if job
+            job.status = FAILED
+            job.attempts += 1
+
+            if job.attempts == @max_retries
+              @failed_jobs[queue_name].push(job)
+              @jobs[queue_name].delete(job)
+              @logger.warn("[MemoryStore] job: '#{job.job_name}' has failed the reached the maximum amount of retries (#{@max_retries}) and is being removed from the queue.")
+            else
+              @logger.info("[MemoryStore] failed job: '#{job.job_name}' has been requeued and attempted #{job.attempts} times")
+            end
+          end
+        }
+      end
+
+    end
+  end
+end

data/lib/thread_job/job.rb

@@ -0,0 +1,7 @@
+module ThreadJob
+  class Job
+    def run
+      raise Exception.new 'not implemented'
+    end
+  end
+end

data/lib/thread_job/job_store.rb

@@ -0,0 +1,15 @@
+module ThreadJob
+  class JobStore
+    def save_job(queue_name, job_name, job)
+    end
+
+    def poll_for_job(queue_name)
+    end
+
+    def complete_job(queue_name, job_name)
+    end
+
+    def fail_job(queue_name, job_name)
+    end
+  end
+end

data/lib/thread_job/scheduler.rb

@@ -0,0 +1,59 @@
+require 'logger'
+
+module ThreadJob
+  class Scheduler
+
+    def initialize(queue_name, job_store=ThreadJob::Memory::Store.new, poll_delay_seconds=5, thread_pool_size=5, logger=Logger.new(STDOUT))
+      @job_store = job_store
+      @logger = logger
+      @queue_name = queue_name
+      @poll_delay = poll_delay_seconds
+      @scheduler_thread = nil
+      @thread_pool = ThreadPool.new(thread_pool_size, logger)
+    end
+
+    def start
+      return @scheduler_thread = Thread.new do
+        do_start
+      end
+    end
+
+    def kill
+      @logger.info("[Scheduler] scheduler stopping...")
+      @scheduler_thread.kill
+    end
+
+    def kill_workers
+      @logger.info("[Scheduler] Stopping all worker threads")
+      @thread_pool.kill
+    end
+
+    def add_job(job_name, job)
+      @logger.info("[Scheduler] Added job: '#{job_name}' to the '#{@queue_name}' queue")
+      @job_store.save_job(@queue_name, job_name, job)
+    end
+
+    def add_workers(num_workers)
+      @logger.info("[Scheduler] Adding #{num_workers} to the worker pool")
+      @thread_pool.add_workers(num_workers)
+    end
+
+    private
+    def do_start
+      @logger.info("[Scheduler] starting...")
+      while true
+        if @thread_pool.has_available_thread?
+          job_hash = @job_store.poll_for_job(@queue_name)
+          if job_hash
+            job_hash[:queue_name] = @queue_name
+            job_hash[:job_store] = @job_store
+            @logger.info("[Scheduler] scheudled job '#{job_hash[:job_name]}', sending to thread pool")
+            @thread_pool.run(job_hash)
+          end
+        end
+        sleep(@poll_delay)
+      end
+    end
+
+  end
+end

data/lib/thread_job/thread_pool.rb

@@ -0,0 +1,83 @@
+require 'logger'
+
+module ThreadJob
+  class ThreadPool
+
+    def initialize(max_size=5, logger=Logger.new(STDOUT))
+      @queue = Queue.new
+      @logger = logger
+      @avail_pool = max_size.times.map do
+        Thread.new do
+          @logger.debug("[ThreadPool] started thread #{Thread.current}")
+          while true
+            monitor_queue
+          end
+        end
+      end
+      @use_pool = []
+      @mutex = Mutex.new
+    end
+
+    def has_available_thread?
+      @mutex.synchronize {
+        @logger.debug("[ThreadPool] #{@avail_pool.length} threads available, #{@use_pool.length} threads in use")
+        return @avail_pool.length > 0
+      }
+    end
+
+    def add_workers(num_workers)
+      num_workers.times do
+        thread = Thread.new do
+          @logger.debug("[ThreadPool] started thread #{Thread.current}")
+          while true
+            monitor_queue
+          end
+        end
+        @avail_pool.push(thread)
+      end
+    end
+
+    def kill
+      @avail_pool.each do |avail_thread|
+        avail_thread.kill
+      end
+
+      @use_pool.each do |used_thread|
+        used_thread.kill
+      end
+    end
+
+    def monitor_queue
+      work = @queue.pop
+      if work
+        @mutex.synchronize {
+          @use_pool.push(Thread.current)
+          @avail_pool.delete(Thread.current)
+        }
+
+        @logger.debug("[ThreadPool] Running job '#{work[:job_name]}' on thread #{Thread.current}")
+        begin
+          work[:job].run
+        rescue => e
+          @logger.error("[ThreadPool] Worker thread #{Thread.current} encountered an error #{e} while processing job '#{work[:job_name]}'")
+          work[:job_store].fail_job(work[:queue_name], work[:id])
+          @mutex.synchronize {
+            @avail_pool.push(Thread.current)
+            @use_pool.delete(Thread.current)
+          }
+          return
+        end
+        work[:job_store].complete_job(work[:queue_name], work[:id])
+
+        @mutex.synchronize {
+          @avail_pool.push(Thread.current)
+          @use_pool.delete(Thread.current)
+        }
+      end
+    end
+
+    def run(job_hash)
+      @queue.push(job_hash)
+    end
+  end
+end

data/thread_job.gemspec

@@ -0,0 +1,17 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |spec|
+  spec.authors        = ['Jeff Kuchta']
+  spec.description    = 'A simple framework to asynchronously execute longer running tasks in the background using threads.'
+  spec.email          = ['jeff@flywiremedia.com']
+  spec.files          = %w[thread_job.gemspec]
+  spec.files          += Dir.glob('{lib}/**/*')
+  spec.homepage       = 'https://github.com/jkuchta/thread_job'
+  spec.licenses       = ['MIT']
+  spec.name           = 'thread_job'
+  spec.require_paths  = ['lib']
+  spec.summary        = 'Lightweight asynchronous scheduler and workers'
+  #spec.test_files     = Dir.glob('spec/**/*')
+  spec.add_development_dependency 'rspec'
+  spec.version        = '0.1.0'
+end

metadata

@@ -0,0 +1,66 @@
+--- !ruby/object:Gem::Specification
+name: thread_job
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jeff Kuchta
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-03-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A simple framework to asynchronously execute longer running tasks in
+  the background using threads.
+email:
+- jeff@flywiremedia.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/thread_job.rb
+- lib/thread_job/backends/memory/store.rb
+- lib/thread_job/job.rb
+- lib/thread_job/job_store.rb
+- lib/thread_job/scheduler.rb
+- lib/thread_job/thread_pool.rb
+- thread_job.gemspec
+homepage: https://github.com/jkuchta/thread_job
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Lightweight asynchronous scheduler and workers
+test_files: []