litestack 0.4.1 → 0.4.3

data/lib/litestack/litedb.rb

@@ -1,23 +1,22 @@
 # all components should require the support module
-require_relative 'litesupport'
+require_relative "litesupport"
 
 # all measurable components should require the litemetric class
-require_relative 'litemetric'
+require_relative "litemetric"
 
 # litedb in particular gets access to litesearch
-require_relative 'litesearch'
+require_relative "litesearch"
 
 # Litedb inherits from the SQLite3::Database class and adds a few initialization options
 class Litedb < ::SQLite3::Database
-
   # add litemetric support
   include Litemetric::Measurable
-  
+
   # add litesearch support
   include Litesearch
-  
+
   # overrride the original initilaizer to allow for connection configuration
-  def initialize(file, options = {}, zfs = nil )
+  def initialize(file, options = {}, zfs = nil)
     if block_given?
       super(file, options, zfs) do |db|
         init unless options[:noinit] == true
@@ -41,15 +40,15 @@ class Litedb < ::SQLite3::Database
     super(mode)
   end
 
-  # return the size of the database file  
+  # return the size of the database file
   def size
     execute("SELECT s.page_size * c.page_count FROM pragma_page_size() AS s, pragma_page_count() AS c")[0][0]
   end
-  
+
   def schema_object_count(type = nil)
     execute("SELECT count(*) FROM SQLITE_MASTER WHERE iif(?1 IS NOT NULL, type = ?1, TRUE)", type)[0][0]
   end
-  
+
   # collect snapshot information
   def snapshot
     {
@@ -58,8 +57,8 @@ class Litedb < ::SQLite3::Database
         journal_mode: journal_mode,
         synchronous: synchronous,
         size: size.to_f / (1024 * 1024),
-        tables: schema_object_count('table'), 
-        indexes: schema_object_count('index')
+        tables: schema_object_count("table"),
+        indexes: schema_object_count("index")
       }
     }
   end
@@ -74,42 +73,41 @@ class Litedb < ::SQLite3::Database
     ensure
       stmt.close unless stmt.closed?
     end
-  end  
+  end
 
-  # override execute to capture metrics  
+  # override execute to capture metrics
   def execute(sql, bind_vars = [], *args, &block)
     if bind_vars.nil? || !args.empty?
-      if args.empty?
-        bind_vars = []
+      bind_vars = if args.empty?
+        []
       else
-        bind_vars = [bind_vars] + args
+        [bind_vars] + args
       end
     end
-    
+
     prepare(sql) do |stmt|
       measure(stmt.stmt_type, stmt.sql) do
         stmt.bind_params(bind_vars)
         stmt = SQLite3::ResultSet.new self, stmt
       end
-      if block_given?
+      if block
         stmt.each do |row|
           yield row
         end
       else
         stmt.to_a
       end
-    end  
-   
+    end
   end
 
   private
 
-  # default connection configuration values 
+  # default connection configuration values
   def init
     # version 3.37 is required for strict typing support and the newest json operators
     raise Litesupport::Error if SQLite3::SQLITE_VERSION_NUMBER < 3037000
     # time to wait to obtain a write lock before raising an exception
-    self.busy_handler{|i| sleep 0.001}
+    busy_handler { |i| sleep 0.001 }
     # level of database durability, 2 = "FULL" (sync on every write), other values include 1 = "NORMAL" (sync every 1000 written pages) and 0 = "NONE"
     self.synchronous = 1
     # Journal mode WAL allows for greater concurrency (many readers + one writer)
@@ -117,16 +115,14 @@ class Litedb < ::SQLite3::Database
     # impose a limit on the WAL file to prevent unlimited growth (with a negative impact on read performance as well)
     self.journal_size_limit = 64 * 1024 * 1024
     # set the global memory map so all processes can share data
-    self.mmap_size = 128 * 1024 * 1024 
+    self.mmap_size = 128 * 1024 * 1024
     # increase the local connection cache to 2000 pages
     self.cache_size = 2000
   end
-  
 end
 
 # the Litedb::Statement also inherits from SQLite3::Statement
-class Litedb::Statement < SQLite3::Statement      
-    
+class Litedb::Statement < SQLite3::Statement
   include Litemetric::Measurable
 
   attr_accessor :sql
@@ -139,32 +135,32 @@ class Litedb::Statement < SQLite3::Statement
   def metrics_identifier
     "Litedb" # overridden to match the parent class
   end
-  
-  # return the type of the statement 
+
+  # return the type of the statement
   def stmt_type
     @stmt_type ||= detect_stmt_type
   end
-  
+
   def detect_stmt_type
-    if @sql.start_with?("SEL") || @sql.start_with?("WITH")
+    if @sql.start_with?("SEL", "WITH")
       "Read"
-    elsif @sql.start_with?("CRE") || @sql.start_with?("ALT") || @sql.start_with?("DRO")
+    elsif @sql.start_with?("CRE", "ALT", "DRO")
       "Schema change"
-    elsif @sql.start_with?("PRA") 
+    elsif @sql.start_with?("PRA")
       "Pragma"
     else
       "Write"
-    end        
+    end
   end
-  
-  # overriding each to measure the query time (plus the processing time as well, sadly) 
+
+  # overriding each to measure the query time (plus the processing time as well, sadly)
   def each
     measure(stmt_type, @sql) do
       super
-    end 
+    end
   end
-  
-  # overriding execute to measure the query time  
+
+  # overriding execute to measure the query time
   def execute(*bind_vars)
     res = nil
     measure(stmt_type, @sql) do
@@ -172,5 +168,4 @@ class Litedb::Statement < SQLite3::Statement
     end
     res
   end
-
 end

data/lib/litestack/litejob.rb

@@ -1,36 +1,36 @@
 # frozen_stringe_literal: true
 
-require_relative './litejobqueue'
+require_relative "./litejobqueue"
 
 ##
-#Litejob is a Ruby module that enables seamless integration of the Litejobqueue job queueing system into Ruby applications. By including the Litejob module in a class and implementing the #perform method, developers can easily enqueue and process jobs asynchronously.
+# Litejob is a Ruby module that enables seamless integration of the Litejobqueue job queueing system into Ruby applications. By including the Litejob module in a class and implementing the #perform method, developers can easily enqueue and process jobs asynchronously.
 #
-#When a job is enqueued, Litejob creates a new instance of the class and passes it any necessary arguments. The class's #perform method is then called asynchronously to process the job. This allows the application to continue running without waiting for the job to finish, improving overall performance and responsiveness.
+# When a job is enqueued, Litejob creates a new instance of the class and passes it any necessary arguments. The class's #perform method is then called asynchronously to process the job. This allows the application to continue running without waiting for the job to finish, improving overall performance and responsiveness.
 #
-#One of the main benefits of using Litejob is its simplicity. Because it integrates directly with Litejobqueue, developers do not need to worry about managing job queues or processing logic themselves. Instead, they can focus on implementing the #perform method to handle the specific job tasks.
+# One of the main benefits of using Litejob is its simplicity. Because it integrates directly with Litejobqueue, developers do not need to worry about managing job queues or processing logic themselves. Instead, they can focus on implementing the #perform method to handle the specific job tasks.
 #
-#Litejob also provides a number of useful features, including the ability to set job priorities, retry failed jobs, and limit the number of retries. These features can be configured using simple configuration options in the class that includes the Litejob module.
+# Litejob also provides a number of useful features, including the ability to set job priorities, retry failed jobs, and limit the number of retries. These features can be configured using simple configuration options in the class that includes the Litejob module.
 #
-#Overall, Litejob is a powerful and flexible module that allows developers to easily integrate Litejobqueue job queueing into their Ruby applications. By enabling asynchronous job processing, Litejob can help improve application performance and scalability, while simplifying the development and management of background job processing logic.
+# Overall, Litejob is a powerful and flexible module that allows developers to easily integrate Litejobqueue job queueing into their Ruby applications. By enabling asynchronous job processing, Litejob can help improve application performance and scalability, while simplifying the development and management of background job processing logic.
 #  class EasyJob
 #    include ::Litejob
-#   
+#
 #    def perform(params)
 #      # do stuff
 #    end
 #  end
 #
-#Then later you can perform a job asynchronously:
+# Then later you can perform a job asynchronously:
 #
 #  EasyJob.perform_async(params) # perform a job synchronously
-#Or perform it at a specific time:
+# Or perform it at a specific time:
 #  EasyJob.perform_at(time, params) # perform a job at a specific time
-#Or perform it after a certain delay:
+# Or perform it after a certain delay:
 #  EasyJob.perform_in(delay, params) # perform a job after a certain delay
-#You can also specify a specific queue to be used
+# You can also specify a specific queue to be used
 #  class EasyJob
 #    include ::Litejob
-#    
+#
 #    self.queue = :urgent
 #
 #    def perform(params)
@@ -39,25 +39,23 @@ require_relative './litejobqueue'
 #  end
 #
 module Litejob
-
-  private
   def self.included(klass)
     klass.extend(ClassMethods)
     klass.get_jobqueue
   end
-  
+
   module ClassMethods
     def perform_async(*params)
-      get_jobqueue.push(self.name, params, 0, queue)      
+      get_jobqueue.push(name, params, 0, queue)
     end
-    
+
     def perform_at(time, *params)
       delay = time.to_i - Time.now.to_i
-      get_jobqueue.push(self.name, params, delay, queue)           
+      get_jobqueue.push(name, params, delay, queue)
     end
-    
+
     def perform_in(delay, *params)
-      get_jobqueue.push(self.name, params, delay, queue)           
+      get_jobqueue.push(name, params, delay, queue)
     end
 
     def perform_after(delay, *params)
@@ -67,26 +65,29 @@ module Litejob
     def process_jobs
       get_jobqueue
     end
-       
-    def delete(id) 
+
+    def delete(id)
       get_jobqueue.delete(id)
-    end  
-         
+    end
+
     def queue
       @queue_name ||= "default"
     end
-    
+
     def queue=(queue_name)
       @queue_name = queue_name.to_s
     end
 
     def options
-      @options ||= self::DEFAULT_OPTIONS rescue {}
+      @options ||= begin
+        self::DEFAULT_OPTIONS
+      rescue
+        {}
+      end
     end
-          
+
     def get_jobqueue
       Litejobqueue.jobqueue(options)
     end
   end
-    
-end  
+end

data/lib/litestack/litejobqueue.rb

@@ -1,152 +1,150 @@
 # frozen_stringe_literal: true
 
-require_relative './litequeue'
-require_relative './litemetric'
+require_relative "./litequeue"
+require_relative "./litemetric"
 
 ##
-#Litejobqueue is a job queueing and processing system designed for Ruby applications. It is built on top of SQLite, which is an embedded relational database management system that is #lightweight and fast.
+# Litejobqueue is a job queueing and processing system designed for Ruby applications. It is built on top of SQLite, which is an embedded relational database management system that is #lightweight and fast.
 #
-#One of the main benefits of Litejobqueue is that it is very low on resources, making it an ideal choice for applications that need to manage a large number of jobs without incurring #high resource costs. In addition, because it is built on SQLite, it is easy to use and does not require any additional configuration or setup.
+# One of the main benefits of Litejobqueue is that it is very low on resources, making it an ideal choice for applications that need to manage a large number of jobs without incurring #high resource costs. In addition, because it is built on SQLite, it is easy to use and does not require any additional configuration or setup.
 #
-#Litejobqueue also integrates well with various I/O frameworks like Async and Polyphony, making it a great choice for Ruby applications that use these frameworks. It provides a #simple and easy-to-use API for adding jobs to the queue and for processing them.
+# Litejobqueue also integrates well with various I/O frameworks like Async and Polyphony, making it a great choice for Ruby applications that use these frameworks. It provides a #simple and easy-to-use API for adding jobs to the queue and for processing them.
 #
-#Overall, LiteJobQueue is an excellent choice for Ruby applications that require a lightweight, embedded job queueing and processing system that is fast, efficient, and easy to use.
+# Overall, LiteJobQueue is an excellent choice for Ruby applications that require a lightweight, embedded job queueing and processing system that is fast, efficient, and easy to use.
 class Litejobqueue < Litequeue
-
   include Litemetric::Measurable
 
   # the default options for the job queue
-  # can be overriden by passing new options in a hash 
+  # can be overriden by passing new options in a hash
   # to Litejobqueue.new, it will also be then passed to the underlying Litequeue object
   #   config_path: "./litejob.yml" -> were to find the configuration file (if any)
   #   path: "./db/queue.db"
   #   mmap_size: 128 * 1024 * 1024 -> 128MB to be held in memory
   #   sync: 1 -> sync only when checkpointing
-  #   queues: [["default", 1, "spawn"]] -> an array of queues to process 
+  #   queues: [["default", 1, "spawn"]] -> an array of queues to process
   #   workers: 1 -> number of job processing workers
   #   sleep_intervals: [0.001, 0.005, 0.025, 0.125, 0.625, 3.125] -> sleep intervals for workers
   # queues will be processed according to priority, such that if the queues are as such
   #   queues: [["default", 1, "spawn"], ["urgent", 10]]
   # it means that roughly, if the queues are full, for each 10 urgent jobs, 1 default job will be processed
-  # the priority value is mandatory. The optional "spawn" parameter tells the job workers to spawn a separate execution context (thread or fiber, based on environment) for each job. 
-  # This can be particularly useful for long running, IO bound jobs. It is not recommended though for threaded environments, as it can result in creating many threads that may consudme a lot of memory. 
+  # the priority value is mandatory. The optional "spawn" parameter tells the job workers to spawn a separate execution context (thread or fiber, based on environment) for each job.
+  # This can be particularly useful for long running, IO bound jobs. It is not recommended though for threaded environments, as it can result in creating many threads that may consudme a lot of memory.
   DEFAULT_OPTIONS = {
     config_path: "./litejob.yml",
     path: Litesupport.root.join("queue.sqlite3"),
     queues: [["default", 1]],
     workers: 5,
-    retries: 5, 
+    retries: 5,
     retry_delay: 60,
     retry_delay_multiplier: 10,
     dead_job_retention: 10 * 24 * 3600,
-    gc_sleep_interval: 7200, 
-    logger: 'STDOUT',
+    gc_sleep_interval: 7200,
+    logger: "STDOUT",
     sleep_intervals: [0.001, 0.005, 0.025, 0.125, 0.625, 1.0, 2.0],
-    metrics: false 
+    metrics: false
   }
-  
+
   @@queue = nil
-  
+
   attr_reader :running
-  
+
   alias_method :_push, :push
-  
+
   # a method that returns a single instance of the job queue
   # for use by Litejob
   def self.jobqueue(options = {})
-    @@queue ||= Litescheduler.synchronize{self.new(options)}
+    @@queue ||= Litescheduler.synchronize { new(options) }
   end
 
   def self.new(options = {})
     return @@queue if @@queue
     @@queue = allocate
     @@queue.send(:initialize, options)
-    @@queue 
+    @@queue
   end
 
   # create new queue instance (only once instance will be created in the process)
   #   jobqueue = Litejobqueue.new
-  #   
+  #
   def initialize(options = {})
-
     @queues = [] # a place holder to allow workers to process
     super(options)
-    
+
     # group and order queues according to their priority
     pgroups = {}
     @options[:queues].each do |q|
       pgroups[q[1]] = [] unless pgroups[q[1]]
       pgroups[q[1]] << [q[0], q[2] == "spawn"]
     end
-    @queues = pgroups.keys.sort.reverse.collect{|p| [p, pgroups[p]]}
+    @queues = pgroups.keys.sort.reverse.collect { |p| [p, pgroups[p]] }
     collect_metrics if @options[:metrics]
   end
 
   def metrics_identifier
     "Litejob" # overrides default identifier
   end
-  
+
   # push a job to the queue
   #   class EasyJob
   #      def perform(any, number, of_params)
   #         # do anything
-  #      end 
+  #      end
   #   end
   #   jobqueue = Litejobqueue.new
   #   jobqueue.push(EasyJob, params) # the job will be performed asynchronously
-  def push(jobclass, params, delay=0, queue=nil)
+  def push(jobclass, params, delay = 0, queue = nil)
     payload = Oj.dump({klass: jobclass, params: params, retries: @options[:retries], queue: queue}, mode: :strict)
     res = super(payload, delay, queue)
     capture(:enqueue, queue)
     @logger.info("[litejob]:[ENQ] queue:#{res[1]} class:#{jobclass} job:#{res[0]}")
     res
   end
-  
-  def repush(id, job, delay=0, queue=nil)
+
+  def repush(id, job, delay = 0, queue = nil)
     res = super(id, Oj.dump(job, mode: :strict), delay, queue)
     capture(:enqueue, queue)
     @logger.info("[litejob]:[ENQ] queue:#{res[0]} class:#{job[:klass]} job:#{id}")
     res
   end
-  
+
   # delete a job from the job queue
   #   class EasyJob
   #      def perform(any, number, of_params)
   #         # do anything
-  #      end 
+  #      end
   #   end
   #   jobqueue = Litejobqueue.new
   #   id = jobqueue.push(EasyJob, params, 10) # queue for processing in 10 seconds
-  #   jobqueue.delete(id)    
+  #   jobqueue.delete(id)
   def delete(id)
     job = super(id)
     @logger.info("[litejob]:[DEL] job: #{job}")
     job = Oj.load(job[0], symbol_keys: true) if job
     job
   end
-  
+
   # delete all jobs in a certain named queue
   # or delete all jobs if the queue name is nil
-  #def clear(queue=nil)
-    #@queue.clear(queue)
-  #end
-  
+  # def clear(queue=nil)
+  # @queue.clear(queue)
+  # end
+
   # stop the queue object (does not delete the jobs in the queue)
   # specifically useful for testing
   def stop
     @running = false
-    #@@queue = nil
+    # @@queue = nil
     close
   end
-  
-  
+
   private
 
   def exit_callback
     @running = false # stop all workers
+    return unless @jobs_in_flight > 0
     puts "--- Litejob detected an exit, cleaning up"
     index = 0
-    while @jobs_in_flight > 0 and index < 30 # 3 seconds grace period for jobs to finish
+    while @jobs_in_flight > 0 && index < 30 # 3 seconds grace period for jobs to finish
       puts "--- Waiting for #{@jobs_in_flight} jobs to finish"
       sleep 0.1
       index += 1
@@ -157,76 +155,76 @@ class Litejobqueue < Litequeue
   def setup
     super
     @jobs_in_flight = 0
-    @workers = @options[:workers].times.collect{ create_worker }    
+    @workers = @options[:workers].times.collect { create_worker }
     @gc = create_garbage_collector
-    @mutex = Litesupport::Mutex.new 
+    @mutex = Litesupport::Mutex.new
   end
-  
+
   def job_started
-    Litescheduler.synchronize(@mutex){@jobs_in_flight += 1}
+    Litescheduler.synchronize(@mutex) { @jobs_in_flight += 1 }
   end
-  
+
   def job_finished
-    Litescheduler.synchronize(@mutex){@jobs_in_flight -= 1}
+    Litescheduler.synchronize(@mutex) { @jobs_in_flight -= 1 }
   end
-    
+
   # optionally run a job in its own context
   def schedule(spawn = false, &block)
     if spawn
-      Litescheduler.spawn &block
+      Litescheduler.spawn(&block)
     else
       yield
     end
-  end  
-    
+  end
+
   # create a worker according to environment
   def create_worker
     Litescheduler.spawn do
       worker_sleep_index = 0
-      while @running do
+      while @running
         processed = 0
         @queues.each do |priority, queues| # iterate through the levels
           queues.each do |queue, spawns| # iterate through the queues in the level
             batched = 0
-            
+
             while (batched < priority) && (payload = pop(queue, 1)) # fearlessly use the same queue object
               capture(:dequeue, queue)
               processed += 1
               batched += 1
-              
+
               id, serialized_job = payload
               process_job(queue, id, serialized_job, spawns)
-              
+
               Litescheduler.switch # give other contexts a chance to run here
             end
           end
         end
-        if processed == 0 
-          sleep @options[:sleep_intervals][worker_sleep_index]      
-          worker_sleep_index += 1 if worker_sleep_index < @options[:sleep_intervals].length - 1          
+        if processed == 0
+          sleep @options[:sleep_intervals][worker_sleep_index]
+          worker_sleep_index += 1 if worker_sleep_index < @options[:sleep_intervals].length - 1
         else
           worker_sleep_index = 0 # reset the index
         end
       end
     end
-  end  
-  
+  end
+
   # create a gc for dead jobs
   def create_garbage_collector
     Litescheduler.spawn do
-      while @running do
-        while jobs = pop('_dead', 100)
+      while @running
+        while (jobs = pop("_dead", 100))
           if jobs[0].is_a? Array
             @logger.info "[litejob]:[DEL] garbage collector deleted #{jobs.length} dead jobs"
           else
             @logger.info "[litejob]:[DEL] garbage collector deleted 1 dead job"
           end
         end
-        sleep @options[:gc_sleep_interval]      
+        sleep @options[:gc_sleep_interval]
       end
     end
   end
-  
+
   def process_job(queue, id, serialized_job, spawns)
     job = Oj.load(serialized_job)
     @logger.info "[litejob]:[DEQ] queue:#{queue} class:#{job["klass"]} job:#{id}"