litestack 0.1.7 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2662941f303da99554039370e6e53e4c13956c52070e93ad6862742881ca8063
-  data.tar.gz: ccd5583a9e0e7f5c559f8dafdde913bba256b7b5a679fe927cd31805c1289022
+  metadata.gz: d08724a0b9293f55ebed24ba6d738b103795b69563a210c4454322fc382e174e
+  data.tar.gz: 01401bfca727b4ef9452a9efafba846c886845dd43c1d8f5f5ce84cee22ab61f
 SHA512:
-  metadata.gz: 1a139b3c42cd3f9a327fc7d8f1487dab2e95f2c88d07e063eecc2e438c91236f301a8e3928c9a3b41030fd8c08f4c118c449e56ea4be35b73b00fe2aa240faf8
-  data.tar.gz: 409221a087df5707bd6da477bb9091daf2ee0fc5f8114d71a7666324676bc62e5d77b6edfd1e3e7d29d601985f7936b2e3545e618b1c888b6c7a748544bbafde
+  metadata.gz: 729320670e62261596eabbfd8f8d931117507317127e9fd9e5a796928e7031418455ca351dd2de815d4860edc2e688c0bbdb79560d3907cb3545deb75a1b4fae
+  data.tar.gz: d75cc4f23694c726a361b0bc474f561b8ffc4db074ef275839917d4c82461aca7a1c4a0b0e939cd749a7e7afbb58de86f3de254a6adadfc7f6d91ea9021218f3

data/BENCHMARKS.md

@@ -67,7 +67,7 @@ For testing the cache we attempted to try writing and reading different payload
 
 ### Write
 
-|Payload Size (bytes)|Redis|litecahce|
+|Payload Size (bytes)|Redis|litecache|
 |-:|-:|-:|
 |10|4.2K q/s|11.0K q/s|
 |100|4.7K q/s|11.6K q/s|

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 ## [Unreleased]
 
+## [0.1.8] - 2022-03-08
+
+- More code cleanups, more test coverage
+- Retry support for jobs in Litejob
+- Job storage and garbage collection for failed jobs
+- Initial graceful shutdown support for Litejob (incomplete)
+- More configuration options for Litejob
+
 ## [0.1.7] - 2022-03-05
 
 - Code cleanup, removal of references to older name

data/README.md

@@ -120,7 +120,7 @@ litejob is a fast and very efficient job queue processor for Ruby applications.
 require 'litestack'
 # define your job class
 class MyJob
-  include ::litejob
+  include ::Litejob
       
   queue = :default
       

data/bench/bench_cache_raw.rb

@@ -16,15 +16,18 @@ redis = Redis.new # default settings
 
 values = []
 keys = []
-count = 1000
+count = 5
 count.times { keys << random_str(10) }
 
-[10, 100, 1000, 10000].each do |size|
+[10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000].each do |size|
   count.times do
     values << random_str(size) 
   end
   
   random_keys = keys.shuffle
+  
+  GC.compact
+  
   puts "Benchmarks for values of size #{size} bytes"
   puts "=========================================================="
   puts "== Writes =="
@@ -33,7 +36,7 @@ count.times { keys << random_str(10) }
   end
 
   bench("Redis writes", count) do |i|
-    redis.set(keys[i], values[i])
+    #redis.set(keys[i], values[i])
   end
 
   puts "== Reads =="
@@ -42,11 +45,13 @@ count.times { keys << random_str(10) }
   end
 
   bench("Redis reads", count) do |i|
-    redis.get(random_keys[i])
+    #redis.get(random_keys[i])
   end
   puts "=========================================================="
 
   values = []
+  
+  
 end
 
 
@@ -64,5 +69,5 @@ end
 cache.clear
 redis.flushdb
 
-sleep
+#sleep
 

data/bench/bench_jobs_rails.rb

@@ -1,22 +1,14 @@
 require './bench'
-require 'async/scheduler'
-
-#ActiveJob::Base.logger = Logger.new(IO::NULL)
 
+count = ARGV[0].to_i rescue 1000
+env = ARGV[1] || "t"
+delay = ARGV[2].to_f rescue 0
 
-Fiber.set_scheduler Async::Scheduler.new
 
-require_relative '../lib/active_job/queue_adapters/litejob_adapter'
-
-ActiveSupport::IsolatedExecutionState.isolation_level = :fiber
+#ActiveJob::Base.logger = Logger.new(IO::NULL)
 
 require './rails_job.rb'
 
-
-puts Litesupport.environment
-
-count = 1000
-
 RailsJob.queue_adapter = :sidekiq
 t = Time.now.to_f
 puts "Make sure sidekiq is started with -c ./rails_job.rb"     
@@ -26,13 +18,27 @@ end
 
 puts "Don't forget to check the sidekiq log for processing time conclusion"
 
+
+# Litejob bench
+###############
+
+if env == "a" # threaded
+  require 'async/scheduler'
+  Fiber.set_scheduler Async::Scheduler.new
+  ActiveSupport::IsolatedExecutionState.isolation_level = :fiber
+end
+
+require_relative '../lib/active_job/queue_adapters/litejob_adapter'
+puts Litesupport.environment
+
 RailsJob.queue_adapter = :litejob
 t = Time.now.to_f
 bench("enqueuing litejobs", count) do 
   RailsJob.perform_later(count, t)
 end
 
-Fiber.scheduler.run
-
+if env == "a" # threaded
+  Fiber.scheduler.run
+end
 
 sleep

data/bench/bench_jobs_raw.rb

@@ -24,8 +24,6 @@ if env == "t" # threaded
 elsif env == "a" # async
   require 'async/scheduler'
   Fiber.set_scheduler Async::Scheduler.new
-elsif env == "p" # polyphony
-  require 'polyphony'
 end
 
 require './uljob.rb'

data/lib/active_job/queue_adapters/litejob_adapter.rb

@@ -16,12 +16,16 @@ module ActiveJob
       DEFAULT_OPTIONS = {
         config_path: "./config/litejob.yml",
         path: "../db/queue.db",
-        queues: [["default", 1, "spawn"]],
-        workers: 1
+        queues: [["default", 1]],
+        logger: nil, # Rails performs its logging already
+        retries: 5, # It is recommended to stop retries at the Rails level
+        workers: 5
       }      
     
       def initialize(options={})
-        Job.options = DEFAULT_OPTIONS.merge(options)
+        # we currently don't honour individual options per job class
+        # possible in the future?
+        # Job.options = DEFAULT_OPTIONS.merge(options)
       end
     
       def enqueue(job) # :nodoc:

data/lib/litestack/litejob.rb

@@ -43,7 +43,7 @@ module Litejob
   private
   def self.included(klass)
     klass.extend(ClassMethods)
-    klass.get_jobqueue
+    #klass.get_jobqueue
   end
   
   module ClassMethods
@@ -52,32 +52,37 @@ module Litejob
     end
     
     def perform_at(time, *params)
-      delay = time - Time.now.to_i
+      delay = time.to_i - Time.now.to_i
       get_jobqueue.push(self.name, params, delay, queue)           
     end
     
-    def perfrom_in(delay, *params)
+    def perform_in(delay, *params)
       get_jobqueue.push(self.name, params, delay, queue)           
     end
-        
-    def options
-      @@options ||= {}
+
+    def perform_after(delay, *params)
+      perform_in(delay, *params)
     end
-    
-    def options=(options)
-      @@options = options
+
+    def process_jobs
+      get_jobqueue
     end
-    
+       
+    def delete(id, queue_name=nil) 
+      queue_name ||= queue
+      get_jobqueue.delete(id, queue)
+    end  
+         
     def queue
-      @@queue_name ||= "default"
+      @@queue ||= "default"
     end
     
     def queue=(queue_name)
-      @@queue_name = queue_name.to_s
+      @@queue = queue_name.to_s
     end
           
     def get_jobqueue
-      Litejobqueue.jobqueue(options)
+      Litejobqueue.jobqueue
     end
   end
     

data/lib/litestack/litejobqueue.rb

@@ -34,12 +34,19 @@ class Litejobqueue
     path: "./queue.db",
     queues: [["default", 1]],
     workers: 5,
-    logger: STDOUT,
+    retries: 5, 
+    retry_delay: 60,
+    retry_delay_multiplier: 10,
+    dead_job_retention: 10 * 24 * 3600,
+    gc_sleep_interval: 7200, 
+    logger: 'STDOUT',
     sleep_intervals: [0.001, 0.005, 0.025, 0.125, 0.625, 1.0, 2.0]
   }
   
   @@queue = nil
   
+  attr_reader :running
+  
   # a method that returns a single instance of the job queue
   # for use by Litejob
   def self.jobqueue(options = {})
@@ -64,11 +71,23 @@ class Litejobqueue
       config.delete k
     end
     @options.merge!(config)
+    @options.merge!(options) # make sure options passed to initialize trump everything else
+
     @queue = Litequeue.new(@options) # create a new queue object
+    
+    # create logger
     if @options[:logger].respond_to? :info
       @logger = @options[:logger] 
-    else
+    elsif @options[:logger] == 'STDOUT'
+      @logger = Logger.new(STDOUT)      
+    elsif @options[:logger] == 'STDERR'
+      @logger = Logger.new(STDERR)      
+    elsif @options[:logger].nil?
+      @logger = Logger.new(IO::NULL)      
+    elsif @options[:logger].is_a? String 
       @logger = Logger.new(@options[:logger])
+    else
+      @logger = Logger.new(IO::NULL)      
     end
     # group and order queues according to their priority
     pgroups = {}
@@ -77,7 +96,25 @@ class Litejobqueue
       pgroups[q[1]] << [q[0], q[2] == "spawn"]
     end
     @queues = pgroups.keys.sort.reverse.collect{|p| [p, pgroups[p]]}
-    @workers = @options[:workers].times.collect{ create_worker } 
+    @running = true
+    @workers = @options[:workers].times.collect{ create_worker }
+    
+    @gc = create_garbage_collector
+    @jobs_in_flight = 0
+    @mutex = Litesupport::Mutex.new
+    
+    at_exit do
+      @running = false
+      puts "--- Litejob detected an exit attempt, cleaning up"
+      index = 0
+      while @jobs_in_flight > 0 and index < 5
+        puts "--- Waiting for #{@jobs_in_flight} jobs to finish"
+        sleep 1
+        index += 1
+      end
+      puts " --- Exiting with #{@jobs_in_flight} jobs in flight"
+    end
+  
   end
   
   # push a job to the queue
@@ -89,10 +126,9 @@ class Litejobqueue
   #   jobqueue = Litejobqueue.new
   #   jobqueue.push(EasyJob, params) # the job will be performed asynchronously
   def push(jobclass, params, delay=0, queue=nil)
-    payload = Oj.dump([jobclass, params])
-    #res = 
+    payload = Oj.dump({klass: jobclass, params: params, retries: @options[:retries], queue: queue})
     res = @queue.push(payload, delay, queue)
-    @logger.info("[litejob]:[ENQ] id: #{res} class: #{jobclass}")
+    @logger.info("[litejob]:[ENQ] id: #{res} job: #{jobclass}")
     res
   end
   
@@ -104,16 +140,42 @@ class Litejobqueue
   #   end
   #   jobqueue = Litejobqueue.new
   #   id = jobqueue.push(EasyJob, params, 10) # queue for processing in 10 seconds
-  #   jobqueue.delete(id)    
-  def delete(id)
-    job = @queue.delete(id)
+  #   jobqueue.delete(id, 'default')    
+  def delete(id, queue=nil)
+    job = @queue.delete(id, queue)
     @logger.info("[litejob]:[DEL] job: #{job}")
-    Oj.load(job) if job
+    job = Oj.load(job[0]) if job
     job
   end
   
+  # delete all jobs in a certain named queue
+  # or delete all jobs if the queue name is nil
+  def clear(queue=nil)
+    @queue.clear(queue)
+  end
+  
+  # stop the queue object (does not delete the jobs in the queue)
+  # specifically useful for testing
+  def stop
+    @running = false
+    @@queue = nil
+  end
+  
+  
+  def count(queue=nil)
+    @queue.count(queue)
+  end
+  
   private
   
+  def job_started
+    Litesupport.synchronize(@mutex){@jobs_in_flight += 1}
+  end
+  
+  def job_finished
+    Litesupport.synchronize(@mutex){@jobs_in_flight -= 1}
+  end
+  
   # optionally run a job in its own context
   def schedule(spawn = false, &block)
     if spawn
@@ -126,14 +188,8 @@ class Litejobqueue
   # create a worker according to environment
   def create_worker
     Litesupport.spawn do
-      if @options[:logger].respond_to? :info
-        logger = @options[:logger] 
-      else
-        logger = Logger.new(@options[:logger])
-      end
       worker_sleep_index = 0
-      i = 0
-      loop do
+      while @running do
         processed = 0
         @queues.each do |level| # iterate through the levels
           level[1].each do |q| # iterate through the queues in the level
@@ -145,22 +201,34 @@ class Litejobqueue
               begin
                 id, job = payload[0], payload[1]
                 job = Oj.load(job)
-                logger.info "[litejob]:[DEQ] id: #{id} class: #{job[0]}" 
-                klass = eval(job[0])
+                # first capture the original job id                
+                job[:id] = id if job[:retries].to_i == @options[:retries].to_i
+                @logger.info "[litejob]:[DEQ] job:#{job}" 
+                klass = eval(job[:klass])
                 schedule(q[1]) do # run the job in a new context
+                  job_started #(Litesupport.current_context)
                   begin
-                    klass.new.perform(*job[1])
-                    logger.info "[litejob]:[END] id: #{id} class: #{job[0]}" 
+                    klass.new.perform(*job[:params])
+                    @logger.info "[litejob]:[END] job:#{job}" 
                   rescue Exception => e
-                    puts e
-                    puts e.message
-                    puts e.backtrace
+                    # we can retry the failed job now
+                    if job[:retries] == 0
+                      @logger.error "[litejob]:[ERR] job: #{job} failed with #{e}:#{e.message}, retries exhausted, moved to _dead queue"
+                      @queue.push(Oj.dump(job), @options[:dead_job_retention], '_dead')
+                    else
+                      retry_delay = @options[:retry_delay_multiplier].pow(@options[:retries] - job[:retries]) * @options[:retry_delay] 
+                      job[:retries] -=  1
+                      @logger.error "[litejob]:[ERR] job: #{job} failed with #{e}:#{e.message}, retrying in #{retry_delay}"
+                      @queue.push(Oj.dump(job), retry_delay, q[0])
+                      @logger.info "[litejob]:[ENQ] job: #{job} enqueued"
+                    end
                   end
+                  job_finished #(Litesupport.current_context)
                 end
               rescue Exception => e
-                puts e
-                puts e.message
-                puts e.backtrace
+                # this is an error in the extraction of job info
+                # retrying here will not be useful
+                @logger.error "[litejob]:[ERR] failed to extract job info for: #{payload} with #{e}:#{e.message}"
               end
               Litesupport.switch #give other contexts a chance to run here
             end
@@ -176,4 +244,20 @@ class Litejobqueue
     end
   end  
   
+  # create a gc for dead jobs
+  def create_garbage_collector
+    Litesupport.spawn do
+      while @running do
+        while jobs = @queue.pop('_dead', 100)
+          if jobs[0].is_a? Array
+            @logger.info "[litejob]:[DEL] garbage collector deleted #{jobs.length} dead jobs"
+          else
+            @logger.info "[litejob]:[DEL] garbage collector deleted 1 dead job"
+          end
+        end
+        sleep @options[:gc_sleep_interval]      
+      end
+    end
+  end
+  
 end

data/lib/litestack/litequeue.rb

@@ -53,9 +53,10 @@ class Litequeue
   
   # pop an item from the queue, optionally with a specific queue name (default queue name is 'default')
   def pop(queue='default', limit = 1)
-   res = @queue.acquire {|q| res = q.stmts[:pop].execute!(queue, limit)[0] }
-   #return res[0] if res.length == 1
-   #res
+   res = @queue.acquire {|q| res = q.stmts[:pop].execute!(queue, limit) }
+   return res[0] if res.length == 1
+   return nil if res.empty?
+   res
   end
   
   # delete an item from the queue
@@ -64,7 +65,7 @@ class Litequeue
   #   queue.delete(id) # => "somevalue"
   #   queue.pop # => nil
   def delete(id, queue='default')
-    fire_at, id = id.split("_")
+    fire_at, id = id.split("-")
     result = @queue.acquire{|q| q.stmts[:delete].execute!(queue, fire_at.to_i, id)[0] } 
   end
   
@@ -82,6 +83,13 @@ class Litequeue
   def size
     @queue.acquire{|q| q.get_first_value("SELECT size.page_size * count.page_count FROM pragma_page_size() AS size, pragma_page_count() AS count") }
   end
+  
+  def close
+    @queue.acquire do |q| 
+      q.stmts.each_pair {|k, v| q.stmts[k].close }
+      q.close
+    end
+  end
 
   private  
   

data/lib/litestack/litesupport.rb

@@ -1,5 +1,4 @@
 require 'sqlite3'
-require 'hiredis'
 
 module Litesupport
 
@@ -38,8 +37,8 @@ module Litesupport
     end
     # we should never reach here
   end
-  
-  def self.detect_context
+    
+  def self.context
     if environment == :fiber || environment == :poylphony
       Fiber.current.storage
     else
@@ -47,8 +46,12 @@ module Litesupport
     end
   end
   
-  def self.context
-    @ctx ||= detect_context
+  def self.current_context
+    if environment == :fiber || environment == :poylphony
+      Fiber.current
+    else
+      Thread.current
+    end
   end
   
   # switch the execution context to allow others to run
@@ -111,6 +114,21 @@ module Litesupport
     end
   
   end
+   
+  module Forkable
+    
+    def _fork(*args)
+      ppid = Process.pid
+      result = super      
+      if Process.pid != ppid
+        # trigger a restart of all connections owned by Litesupport::Pool
+      end
+      result
+    end
+    
+  end
+
+  #::Process.singleton_class.prepend(::Litesupport::Forkable)
   
   class Pool
   
@@ -126,6 +144,7 @@ module Litesupport
     end
     
     def acquire
+      # check for pid changes
       acquired = false
       result = nil
       while !acquired do

data/lib/litestack/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Litestack
-  VERSION = "0.1.7"
+  VERSION = "0.1.8"
 end       

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: litestack
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.8
 platform: ruby
 authors:
 - Mohamed Hassan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-03-05 00:00:00.000000000 Z
+date: 2023-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sqlite3