litestack 0.1.6 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 01c10017cec21bcabea357aecb8746779060718bb2e6b6e36f250c37d80656f1
-  data.tar.gz: 863b319c7a658f2a19410c614035068f8a2dec72b189698a5a081f595ecf5853
+  metadata.gz: d08724a0b9293f55ebed24ba6d738b103795b69563a210c4454322fc382e174e
+  data.tar.gz: 01401bfca727b4ef9452a9efafba846c886845dd43c1d8f5f5ce84cee22ab61f
 SHA512:
-  metadata.gz: 7bb0fd84ee582eddfe32e48b0373a7545cad8581217ba56d7ac6c984b60b137b86d5196e1bc37486b88d21c28bc5ff62da573a1ba6ce6b49d66ba744ddbcabc6
-  data.tar.gz: 38da99eae6a37222a8689577900504ea1b131f0ec56f67531dff00f5ae4f4c62a29baca21ba2a102435cc334773f8ef4709a361dff6f28de988471eac0937e08
+  metadata.gz: 729320670e62261596eabbfd8f8d931117507317127e9fd9e5a796928e7031418455ca351dd2de815d4860edc2e688c0bbdb79560d3907cb3545deb75a1b4fae
+  data.tar.gz: d75cc4f23694c726a361b0bc474f561b8ffc4db074ef275839917d4c82461aca7a1c4a0b0e939cd749a7e7afbb58de86f3de254a6adadfc7f6d91ea9021218f3

data/BENCHMARKS.md

@@ -67,7 +67,7 @@ For testing the cache we attempted to try writing and reading different payload
 
 ### Write
 
-|Payload Size (bytes)|Redis|litecahce|
+|Payload Size (bytes)|Redis|litecache|
 |-:|-:|-:|
 |10|4.2K q/s|11.0K q/s|
 |100|4.7K q/s|11.6K q/s|

data/CHANGELOG.md

@@ -1,5 +1,20 @@
 ## [Unreleased]
 
+## [0.1.8] - 2022-03-08
+
+- More code cleanups, more test coverage
+- Retry support for jobs in Litejob
+- Job storage and garbage collection for failed jobs
+- Initial graceful shutdown support for Litejob (incomplete)
+- More configuration options for Litejob
+
+## [0.1.7] - 2022-03-05
+
+- Code cleanup, removal of references to older name
+- Fix for the litedb rake tasks (thanks: netmute)
+- More fixes for the new concurrency model
+- Introduced a logger for the Litejobqueue (doesn't work with Polyphony, fix should come soon)
+
 ## [0.1.6] - 2022-03-03
 
 - Revamped the locking model, more robust, minimal performance hit

data/README.md

@@ -1,9 +1,9 @@
 ![litestack](https://github.com/oldmoe/litestack/blob/master/assets/litestack_logo_teal_large.png?raw=true)
 
 
-litestack is a revolutionary gem for Ruby and Ruby on Rails that provides an all-in-one solution for web application development. It exploits the power and embeddedness of SQLite to include a full-fledged SQL database, a fast cache, a robust job queue, and a simple yet performant full-text search all in a single package.
+litestack is a revolutionary gem for Ruby and Ruby on Rails that provides an all-in-one solution for web application development. It exploits the power and embeddedness of SQLite to include a full-fledged SQL database, a fast cache and a robust job queue all in a single package.
 
-Compared to conventional approaches that require separate servers and databases, LiteStack offers superior performance, efficiency, ease of use, and cost savings. Its embedded database and cache reduce memory and CPU usage, while its simple interface streamlines the development process. Overall, LiteStack sets a new standard for web application development and is an excellent choice for those who demand speed, efficiency, and simplicity.
+Compared to conventional approaches that require separate servers and databases, Litestack offers superior performance, efficiency, ease of use, and cost savings. Its embedded database and cache reduce memory and CPU usage, while its simple interface streamlines the development process. Overall, LiteStack sets a new standard for web application development and is an excellent choice for those who demand speed, efficiency, and simplicity.
 
 You can read more about why litestack can be a good choice for your next web application **[here](WHYLITESTACK.md)**, you might also be interested in litestack **[benchmarks](BENCHMARKS.md)**.
 
@@ -83,7 +83,7 @@ adapter: litedb
 litedb offers integration with the Sequel database toolkit and can be configured as follows
 
 ```ruby
-DB = Sequel.conncet("litedb://path_to_db_file")    
+DB = Sequel.connect("litedb://path_to_db_file")
 ```
 
 
@@ -120,7 +120,7 @@ litejob is a fast and very efficient job queue processor for Ruby applications.
 require 'litestack'
 # define your job class
 class MyJob
-  include ::litejob
+  include ::Litejob
       
   queue = :default
       

data/bench/bench.rb

@@ -2,8 +2,8 @@ require 'sqlite3'
 
 def bench(msg, iterations=1000)
   GC.start
-  GC.compact
-  print "Starting #{iterations} iterations of #{msg} ... "
+  #GC.compact
+  STDERR.puts "Starting #{iterations} iterations of #{msg} ... "
   t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
   iterations.times do |i|
     yield i
@@ -12,7 +12,7 @@ def bench(msg, iterations=1000)
   time = ((t2 - t1)*1000).to_i.to_f / 1000 rescue 0
   ips = ((iterations/(t2-t1))*100).to_i.to_f / 100 rescue "infinity?"
   #{m: msg, t: time, ips: iteratinos/time, i: iterations}
-  puts "finished in #{time} seconds (#{ips} ips)"
+ STDERR.puts " .. finished in #{time} seconds (#{ips} ips)"
 end
 
 @db = SQLite3::Database.new(":memory:") # sqlite database for fast random string generation

data/bench/bench_cache_raw.rb

@@ -16,15 +16,18 @@ redis = Redis.new # default settings
 
 values = []
 keys = []
-count = 1000
+count = 5
 count.times { keys << random_str(10) }
 
-[10, 100, 1000, 10000].each do |size|
+[10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000].each do |size|
   count.times do
     values << random_str(size) 
   end
   
   random_keys = keys.shuffle
+  
+  GC.compact
+  
   puts "Benchmarks for values of size #{size} bytes"
   puts "=========================================================="
   puts "== Writes =="
@@ -33,7 +36,7 @@ count.times { keys << random_str(10) }
   end
 
   bench("Redis writes", count) do |i|
-    redis.set(keys[i], values[i])
+    #redis.set(keys[i], values[i])
   end
 
   puts "== Reads =="
@@ -42,11 +45,13 @@ count.times { keys << random_str(10) }
   end
 
   bench("Redis reads", count) do |i|
-    redis.get(random_keys[i])
+    #redis.get(random_keys[i])
   end
   puts "=========================================================="
 
   values = []
+  
+  
 end
 
 
@@ -64,5 +69,5 @@ end
 cache.clear
 redis.flushdb
 
-sleep
+#sleep
 

data/bench/bench_jobs_rails.rb

@@ -1,22 +1,14 @@
 require './bench'
-require 'async/scheduler'
-
-#ActiveJob::Base.logger = Logger.new(IO::NULL)
 
+count = ARGV[0].to_i rescue 1000
+env = ARGV[1] || "t"
+delay = ARGV[2].to_f rescue 0
 
-Fiber.set_scheduler Async::Scheduler.new
 
-require_relative '../lib/active_job/queue_adapters/litejob_adapter'
-
-ActiveSupport::IsolatedExecutionState.isolation_level = :fiber
+#ActiveJob::Base.logger = Logger.new(IO::NULL)
 
 require './rails_job.rb'
 
-
-puts Litesupport.environment
-
-count = 1000
-
 RailsJob.queue_adapter = :sidekiq
 t = Time.now.to_f
 puts "Make sure sidekiq is started with -c ./rails_job.rb"     
@@ -26,13 +18,27 @@ end
 
 puts "Don't forget to check the sidekiq log for processing time conclusion"
 
+
+# Litejob bench
+###############
+
+if env == "a" # threaded
+  require 'async/scheduler'
+  Fiber.set_scheduler Async::Scheduler.new
+  ActiveSupport::IsolatedExecutionState.isolation_level = :fiber
+end
+
+require_relative '../lib/active_job/queue_adapters/litejob_adapter'
+puts Litesupport.environment
+
 RailsJob.queue_adapter = :litejob
 t = Time.now.to_f
 bench("enqueuing litejobs", count) do 
   RailsJob.perform_later(count, t)
 end
 
-Fiber.scheduler.run
-
+if env == "a" # threaded
+  Fiber.scheduler.run
+end
 
 sleep

data/bench/bench_jobs_raw.rb

@@ -1,29 +1,43 @@
-#require 'polyphony'
-require 'async/scheduler'
 require './bench'
 
-Fiber.set_scheduler Async::Scheduler.new
+count = ARGV[0].to_i rescue 1000
+env = ARGV[1] || "t"
+delay = ARGV[2].to_f rescue 0
 
-count = 10000
+# Sidekiq bench
+###############
 require './skjob.rb'
-require './uljob.rb'
-
-puts Litesupport.environment
 
-t = Time.now.to_f
+t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 puts "make sure sidekiq is started with skjob.rb as the job"     
 bench("enqueuing sidekiq jobs", count) do |i|
-  SidekiqJob.perform_async(count, t)
+  SidekiqJob.perform_async(count, t, delay)
 end
 
 puts "Don't forget to check the sidekiq log for processing time conclusion"
 
-t = Time.now.to_f
+# Litejob bench
+###############
+
+if env == "t" # threaded
+  # do nothing 
+elsif env == "a" # async
+  require 'async/scheduler'
+  Fiber.set_scheduler Async::Scheduler.new
+end
+
+require './uljob.rb'
+
+STDERR.puts "litejob started in #{Litesupport.environment} environmnet"
+
+t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 bench("enqueuing litejobs", count) do |i|
-  MyJob.perform_async(count, t)
+  MyJob.perform_async(count, t, delay)
 end
 
-Fiber.scheduler.run
+puts "Please wait for the benchmark to finish .."
+
+Fiber.scheduler.run if env == "a"
 
 sleep
 

data/bench/skjob.rb

@@ -3,11 +3,13 @@ require 'sidekiq'
 class SidekiqJob
   include Sidekiq::Job
   @@count = 0
-  def perform(count, time)
-    sleep 0.1  
+  def perform(count, time, sleep_interval = nil)
+    sleep sleep_interval if sleep_interval  
     @@count += 1
     if @@count == count  
-      puts "finished in #{Time.now.to_f - time} seconds (#{count / (Time.now.to_f - time)} jps)"
+      now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      STDERR.puts "Sidekiq finished in #{now - time} seconds (#{count / (now - time)} jps)"
+      @@count = 0
     end
   end
 end

data/bench/uljob.rb

@@ -4,12 +4,13 @@ require '../lib/litestack'
 class MyJob
   include Litejob
   @@count = 0
-#  self.queue = :normal
-  def perform(count, time)
-    #sleep 0.1
+  #self.queue = :default
+  def perform(count, time, sleep_interval = nil)
+    sleep sleep_interval if sleep_interval  
     @@count += 1
     if @@count == count  
-      puts "UL finished in #{Time.now.to_f - time} seconds (#{count / (Time.now.to_f - time)} jps)"
+      now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      STDERR.puts "Litejob finished in #{now - time} seconds (#{count / (now - time)} jps)"
     end
   end
 end

data/lib/active_job/queue_adapters/litejob_adapter.rb

@@ -16,12 +16,16 @@ module ActiveJob
       DEFAULT_OPTIONS = {
         config_path: "./config/litejob.yml",
         path: "../db/queue.db",
-        queues: [["default", 1, "spawn"]],
-        workers: 1
+        queues: [["default", 1]],
+        logger: nil, # Rails performs its logging already
+        retries: 5, # It is recommended to stop retries at the Rails level
+        workers: 5
       }      
     
       def initialize(options={})
-        Job.options = DEFAULT_OPTIONS.merge(options)
+        # we currently don't honour individual options per job class
+        # possible in the future?
+        # Job.options = DEFAULT_OPTIONS.merge(options)
       end
     
       def enqueue(job) # :nodoc:

data/lib/active_record/connection_adapters/litedb_adapter.rb

@@ -95,7 +95,7 @@ module ActiveRecord
     end
     
     module DatabaseTasks
-      register_task(/ultralite/,       "ActiveRecord::Tasks::LitedbDatabaseTasks")
+      register_task(/litedb/, "ActiveRecord::Tasks::LitedbDatabaseTasks")
 
     end
 	end

data/lib/litestack/litejob.rb

@@ -43,7 +43,7 @@ module Litejob
   private
   def self.included(klass)
     klass.extend(ClassMethods)
-    klass.get_jobqueue
+    #klass.get_jobqueue
   end
   
   module ClassMethods
@@ -52,32 +52,37 @@ module Litejob
     end
     
     def perform_at(time, *params)
-      delay = time - Time.now.to_i
+      delay = time.to_i - Time.now.to_i
       get_jobqueue.push(self.name, params, delay, queue)           
     end
     
-    def perfrom_in(delay, *params)
+    def perform_in(delay, *params)
       get_jobqueue.push(self.name, params, delay, queue)           
     end
-        
-    def options
-      @@options ||= {}
+
+    def perform_after(delay, *params)
+      perform_in(delay, *params)
     end
-    
-    def options=(options)
-      @@options = options
+
+    def process_jobs
+      get_jobqueue
     end
-    
+       
+    def delete(id, queue_name=nil) 
+      queue_name ||= queue
+      get_jobqueue.delete(id, queue)
+    end  
+         
     def queue
-      @@queue_name ||= "default"
+      @@queue ||= "default"
     end
     
     def queue=(queue_name)
-      @@queue_name = queue_name.to_s
+      @@queue = queue_name.to_s
     end
           
     def get_jobqueue
-      Litejobqueue.jobqueue(options)
+      Litejobqueue.jobqueue
     end
   end
     

data/lib/litestack/litejobqueue.rb

@@ -1,5 +1,5 @@
 # frozen_stringe_literal: true
-
+require 'logger'
 require 'oj'
 require 'yaml'
 require_relative './litequeue'
@@ -32,13 +32,21 @@ class Litejobqueue
   DEFAULT_OPTIONS = {
     config_path: "./litejob.yml",
     path: "./queue.db",
-    queues: [["default", 5]],
-    workers: 1,
-    sleep_intervals: [0.001, 0.005, 0.025, 0.125, 0.625, 3.125]
+    queues: [["default", 1]],
+    workers: 5,
+    retries: 5, 
+    retry_delay: 60,
+    retry_delay_multiplier: 10,
+    dead_job_retention: 10 * 24 * 3600,
+    gc_sleep_interval: 7200, 
+    logger: 'STDOUT',
+    sleep_intervals: [0.001, 0.005, 0.025, 0.125, 0.625, 1.0, 2.0]
   }
   
   @@queue = nil
   
+  attr_reader :running
+  
   # a method that returns a single instance of the job queue
   # for use by Litejob
   def self.jobqueue(options = {})
@@ -57,14 +65,30 @@ class Litejobqueue
   #   
   def initialize(options = {})
     @options = DEFAULT_OPTIONS.merge(options)
-    @worker_sleep_index = 0
     config = YAML.load_file(@options[:config_path]) rescue {} # an empty hash won't hurt
     config.keys.each do |k| # symbolize keys
       config[k.to_sym] = config[k]
       config.delete k
     end
     @options.merge!(config)
+    @options.merge!(options) # make sure options passed to initialize trump everything else
+
     @queue = Litequeue.new(@options) # create a new queue object
+    
+    # create logger
+    if @options[:logger].respond_to? :info
+      @logger = @options[:logger] 
+    elsif @options[:logger] == 'STDOUT'
+      @logger = Logger.new(STDOUT)      
+    elsif @options[:logger] == 'STDERR'
+      @logger = Logger.new(STDERR)      
+    elsif @options[:logger].nil?
+      @logger = Logger.new(IO::NULL)      
+    elsif @options[:logger].is_a? String 
+      @logger = Logger.new(@options[:logger])
+    else
+      @logger = Logger.new(IO::NULL)      
+    end
     # group and order queues according to their priority
     pgroups = {}
     @options[:queues].each do |q|
@@ -72,7 +96,25 @@ class Litejobqueue
       pgroups[q[1]] << [q[0], q[2] == "spawn"]
     end
     @queues = pgroups.keys.sort.reverse.collect{|p| [p, pgroups[p]]}
-    @workers = @options[:workers].times.collect{create_worker} 
+    @running = true
+    @workers = @options[:workers].times.collect{ create_worker }
+    
+    @gc = create_garbage_collector
+    @jobs_in_flight = 0
+    @mutex = Litesupport::Mutex.new
+    
+    at_exit do
+      @running = false
+      puts "--- Litejob detected an exit attempt, cleaning up"
+      index = 0
+      while @jobs_in_flight > 0 and index < 5
+        puts "--- Waiting for #{@jobs_in_flight} jobs to finish"
+        sleep 1
+        index += 1
+      end
+      puts " --- Exiting with #{@jobs_in_flight} jobs in flight"
+    end
+  
   end
   
   # push a job to the queue
@@ -84,8 +126,10 @@ class Litejobqueue
   #   jobqueue = Litejobqueue.new
   #   jobqueue.push(EasyJob, params) # the job will be performed asynchronously
   def push(jobclass, params, delay=0, queue=nil)
-    payload = Oj.dump([jobclass, params])
-    @queue.push(payload, delay, queue)
+    payload = Oj.dump({klass: jobclass, params: params, retries: @options[:retries], queue: queue})
+    res = @queue.push(payload, delay, queue)
+    @logger.info("[litejob]:[ENQ] id: #{res} job: #{jobclass}")
+    res
   end
   
   # delete a job from the job queue
@@ -96,14 +140,42 @@ class Litejobqueue
   #   end
   #   jobqueue = Litejobqueue.new
   #   id = jobqueue.push(EasyJob, params, 10) # queue for processing in 10 seconds
-  #   jobqueue.delete(id)    
-  def delete(id)
-    job = @queue.delete(id)
-    Oj.load(job) if job
+  #   jobqueue.delete(id, 'default')    
+  def delete(id, queue=nil)
+    job = @queue.delete(id, queue)
+    @logger.info("[litejob]:[DEL] job: #{job}")
+    job = Oj.load(job[0]) if job
+    job
+  end
+  
+  # delete all jobs in a certain named queue
+  # or delete all jobs if the queue name is nil
+  def clear(queue=nil)
+    @queue.clear(queue)
+  end
+  
+  # stop the queue object (does not delete the jobs in the queue)
+  # specifically useful for testing
+  def stop
+    @running = false
+    @@queue = nil
+  end
+  
+  
+  def count(queue=nil)
+    @queue.count(queue)
   end
   
   private
   
+  def job_started
+    Litesupport.synchronize(@mutex){@jobs_in_flight += 1}
+  end
+  
+  def job_finished
+    Litesupport.synchronize(@mutex){@jobs_in_flight -= 1}
+  end
+  
   # optionally run a job in its own context
   def schedule(spawn = false, &block)
     if spawn
@@ -111,50 +183,81 @@ class Litejobqueue
     else
       yield
     end
-  end
-  
+  end  
+    
   # create a worker according to environment
   def create_worker
     Litesupport.spawn do
-      loop do
+      worker_sleep_index = 0
+      while @running do
         processed = 0
         @queues.each do |level| # iterate through the levels
           level[1].each do |q| # iterate through the queues in the level
             index = 0
             max = level[0]
-            while index < max && payload = @queue.pop(q[0]) # fearlessly use the same queue object 
+            while index < max && payload = @queue.pop(q[0], 1) # fearlessly use the same queue object 
               processed += 1
               index += 1
               begin
                 id, job = payload[0], payload[1]
                 job = Oj.load(job)
-                klass = eval(job[0])
+                # first capture the original job id                
+                job[:id] = id if job[:retries].to_i == @options[:retries].to_i
+                @logger.info "[litejob]:[DEQ] job:#{job}" 
+                klass = eval(job[:klass])
                 schedule(q[1]) do # run the job in a new context
+                  job_started #(Litesupport.current_context)
                   begin
-                    klass.new.perform(*job[1])
+                    klass.new.perform(*job[:params])
+                    @logger.info "[litejob]:[END] job:#{job}" 
                   rescue Exception => e
-                    puts e
-                    puts e.message
-                    puts e.backtrace
+                    # we can retry the failed job now
+                    if job[:retries] == 0
+                      @logger.error "[litejob]:[ERR] job: #{job} failed with #{e}:#{e.message}, retries exhausted, moved to _dead queue"
+                      @queue.push(Oj.dump(job), @options[:dead_job_retention], '_dead')
+                    else
+                      retry_delay = @options[:retry_delay_multiplier].pow(@options[:retries] - job[:retries]) * @options[:retry_delay] 
+                      job[:retries] -=  1
+                      @logger.error "[litejob]:[ERR] job: #{job} failed with #{e}:#{e.message}, retrying in #{retry_delay}"
+                      @queue.push(Oj.dump(job), retry_delay, q[0])
+                      @logger.info "[litejob]:[ENQ] job: #{job} enqueued"
+                    end
                   end
+                  job_finished #(Litesupport.current_context)
                 end
               rescue Exception => e
-                puts e
-                puts e.message
-                puts e.backtrace
+                # this is an error in the extraction of job info
+                # retrying here will not be useful
+                @logger.error "[litejob]:[ERR] failed to extract job info for: #{payload} with #{e}:#{e.message}"
               end
-              Litesupport.switch #give other context a chance to run here
+              Litesupport.switch #give other contexts a chance to run here
             end
           end
         end
         if processed == 0 
-          sleep @options[:sleep_intervals][@worker_sleep_index]      
-          @worker_sleep_index += 1 if @worker_sleep_index < @options[:sleep_intervals].length - 1          
+          sleep @options[:sleep_intervals][worker_sleep_index]      
+          worker_sleep_index += 1 if worker_sleep_index < @options[:sleep_intervals].length - 1          
         else
-          @worker_sleep_index = 0 # reset the index
+          worker_sleep_index = 0 # reset the index
         end
       end
     end
-  end     
+  end  
+  
+  # create a gc for dead jobs
+  def create_garbage_collector
+    Litesupport.spawn do
+      while @running do
+        while jobs = @queue.pop('_dead', 100)
+          if jobs[0].is_a? Array
+            @logger.info "[litejob]:[DEL] garbage collector deleted #{jobs.length} dead jobs"
+          else
+            @logger.info "[litejob]:[DEL] garbage collector deleted 1 dead job"
+          end
+        end
+        sleep @options[:gc_sleep_interval]      
+      end
+    end
+  end
   
 end