litestack 0.1.7 → 0.2.0

data/lib/litestack/litecache.rb

@@ -2,6 +2,7 @@
 
 # all components should require the support module
 require_relative 'litesupport'
+require_relative 'litemetric'
 
 ##
 #Litecache is a caching library for Ruby applications that is built on top of SQLite. It is designed to be simple to use, very fast, and feature-rich, providing developers with a reliable and efficient way to cache data.
@@ -16,6 +17,9 @@ require_relative 'litesupport'
 
 class Litecache
 
+  include Litesupport::Liteconnection
+  include Litemetric::Measurable
+
   # the default options for the cache
   # can be overriden by passing new options in a hash 
   # to Litecache.new
@@ -29,12 +33,15 @@ class Litecache
 
   DEFAULT_OPTIONS = {
     path: "./cache.db",
+    config_path: "./litecache.yml",
+    sync: 0,
     expiry: 60 * 60 * 24 * 30, # one month 
     size: 128 * 1024 * 1024, #128MB 
     mmap_size: 128 * 1024 * 1024, #128MB
-    min_size: 32 * 1024, #32MB
+    min_size: 8  * 1024 * 1024, #16MB
     return_full_record: false, #only return the payload
-    sleep_interval: 1 # 1 second
+    sleep_interval: 1, # 1 second
+    metrics: false
   }
   
   # creates a new instance of Litecache
@@ -56,36 +63,20 @@ class Litecache
   #   litecache.close # optional, you can safely kill the process
   
   def initialize(options = {})
-    @options = DEFAULT_OPTIONS.merge(options)
-    @options[:size] = @options[:min_size] if @options[:size] < @options[:min_size]
-    @sql = {
-      :pruner => "DELETE FROM data WHERE expires_in <= $1",
-      :extra_pruner => "DELETE FROM data WHERE id IN (SELECT id FROM data ORDER BY last_used ASC LIMIT (SELECT CAST((count(*) * $1) AS int) FROM data))",
-      :limited_pruner => "DELETE FROM data WHERE id IN (SELECT id FROM data ORDER BY last_used asc limit $1)",
-      :toucher => "UPDATE data SET  last_used = unixepoch('now') WHERE id = $1",
-      :setter => "INSERT into data (id, value, expires_in, last_used) VALUES   ($1, $2, unixepoch('now') + $3, unixepoch('now')) on conflict(id) do UPDATE SET value = excluded.value, last_used = excluded.last_used, expires_in = excluded.expires_in",
-      :inserter => "INSERT into data (id, value, expires_in, last_used) VALUES   ($1, $2, unixepoch('now') + $3, unixepoch('now')) on conflict(id) do UPDATE SET value = excluded.value, last_used = excluded.last_used, expires_in = excluded.expires_in WHERE id = $1 and expires_in <= unixepoch('now')",
-      :finder => "SELECT id FROM data WHERE id = $1",
-      :getter => "SELECT id, value, expires_in FROM data WHERE id = $1",
-      :deleter => "delete FROM data WHERE id = $1 returning value",
-      :incrementer => "INSERT into data (id, value, expires_in, last_used) VALUES   ($1, $2, unixepoch('now') + $3, unixepoch('now')) on conflict(id) do UPDATE SET value = cast(value AS int) + cast(excluded.value as int), last_used = excluded.last_used, expires_in = excluded.expires_in",
-      :counter => "SELECT count(*) FROM data",
-      :sizer => "SELECT size.page_size * count.page_count FROM pragma_page_size() AS size, pragma_page_count() AS count" 
-    }
-    @cache = Litesupport::Pool.new(1){create_db}
-    @stats = {hit: 0, miss: 0}
+    options[:size] = DEFAULT_OPTIONS[:min_size] if options[:size] && options[:size] < DEFAULT_OPTIONS[:min_size]        
+    init(options)    
     @last_visited = {}
-    @running = true
-    @bgthread = spawn_worker 
+    collect_metrics if @options[:metrics]
   end
   
   # add a key, value pair to the cache, with an optional expiry value (number of seconds)      
   def set(key, value, expires_in = nil)
     key = key.to_s
     expires_in = @options[:expires_in] if expires_in.nil? or expires_in.zero?
-    @cache.acquire do |cache|
+    @conn.acquire do |cache|
       begin
         cache.stmts[:setter].execute!(key, value, expires_in)
+        capture(:write, key)
       rescue SQLite3::FullException
         cache.stmts[:extra_pruner].execute!(0.2)
         cache.execute("vacuum")
@@ -100,12 +91,13 @@ class Litecache
     key = key.to_s
     expires_in = @options[:expires_in] if expires_in.nil? or expires_in.zero?
     changes = 0
-    @cache.acquire do |cache|
+    @conn.acquire do |cache|
       begin
-        transaction(:immediate) do
+        cache.transaction(:immediate) do
           cache.stmts[:inserter].execute!(key, value, expires_in)
-          changes = @cache.changes
+          changes = cache.changes
         end
+        capture(:write, key)
       rescue SQLite3::FullException
         cache.stmts[:extra_pruner].execute!(0.2)
         cache.execute("vacuum")
@@ -119,19 +111,19 @@ class Litecache
   # if the key doesn't exist or it is expired then null will be returned
   def get(key)
     key = key.to_s
-    if record = @cache.acquire{|cache| cache.stmts[:getter].execute!(key)[0] }
+    if record = @conn.acquire{|cache| cache.stmts[:getter].execute!(key)[0] }
       @last_visited[key] = true
-      @stats[:hit] +=1
+      capture(:hit, key)
       return record[1]
     end
-    @stats[:miss] += 1
+    capture(:miss, key)
     nil
   end
   
   # delete a key, value pair from the cache
   def delete(key)
     changes = 0
-    @cache.aquire do |cache|
+    @conn.acquire do |cache|
       cache.stmts[:deleter].execute!(key)
       changes = cache.changes
     end
@@ -141,7 +133,7 @@ class Litecache
   # increment an integer value by amount, optionally add an expiry value (in seconds)
   def increment(key, amount, expires_in = nil)
     expires_in = @expires_in unless expires_in
-    @cache.acquire{|cache| cache.stmts[:incrementer].execute!(key.to_s, amount, expires_in) }
+    @conn.acquire{|cache| cache.stmts[:incrementer].execute!(key.to_s, amount, expires_in) }
   end
   
   # decrement an integer value by amount, optionally add an expiry value (in seconds)
@@ -151,7 +143,7 @@ class Litecache
   
   # delete all entries in the cache up limit (ordered by LRU), if no limit is provided approximately 20% of the entries will be deleted
   def prune(limit=nil)
-    @cache.acquire do |cache|
+    @conn.acquire do |cache|
       if limit and limit.is_a? Integer
         cache.stmts[:limited_pruner].execute!(limit)
       elsif limit and limit.is_a? Float
@@ -164,42 +156,34 @@ class Litecache
   
   # return the number of key, value pairs in the cache
   def count
-    @cache.acquire{|cache| cache.stmts[:counter].execute!.to_a[0][0] }
+    run_stmt(:counter)[0][0]
   end
   
   # return the actual size of the cache file
   def size
-    @cache.acquire{|cache| cache.stmts[:sizer].execute!.to_a[0][0] }
+    run_stmt(:sizer)[0][0]
   end
   
   # delete all key, value pairs in the cache
   def clear
-    @cache.acquire{|cache| cache.execute("delete FROM data") }
+    run_sql("delete FROM data")
   end
   
   # close the connection to the cache file
   def close
     @running = false
-    #Litesupport.synchronize do
-    @cache.acquire{|cache| cache.close }
-    #end
+    super
   end
   
   # return the maximum size of the cache
   def max_size
-    @cache.acquire{|cache| cache.get_first_value("SELECT s.page_size * c.max_page_count FROM pragma_page_size() as s, pragma_max_page_count() as c") }
+    run_sql("SELECT s.page_size * c.max_page_count FROM pragma_page_size() as s, pragma_max_page_count() as c")[0][0]
   end
   
-  # hits and misses for get operations performed over this particular connection (not cache wide)
-  # 
-  #   litecache.stats # => {hit: 543, miss: 31} 
-  def stats
-    @stats
-  end
-
   # low level access to SQLite transactions, use with caution
-  def transaction(mode)
-    @cache.acquire do |cache|
+  def transaction(mode, acquire=true)
+    return cache.transaction(mode){yield} unless acquire
+    @conn.acquire do |cache|
       cache.transaction(mode) do
         yield
       end
@@ -208,10 +192,15 @@ class Litecache
 
   private 
   
+  def setup
+    super # create connection
+    @bgthread = spawn_worker # create backgroud pruner thread
+  end
+    
   def spawn_worker
     Litesupport.spawn do
       while @running
-        @cache.acquire do |cache|
+        @conn.acquire do |cache|
           begin
             cache.transaction(:immediate) do
               @last_visited.delete_if do |k| # there is a race condition here, but not a serious one
@@ -232,19 +221,24 @@ class Litecache
     end
   end
   
-  def create_db
-    db = Litesupport.create_db(@options[:path])
-    db.synchronous = 0
-    db.cache_size = 2000
-    db.journal_size_limit = [(@options[:size]/2).to_i, @options[:min_size]].min
-    db.mmap_size = @options[:mmap_size]
-    db.max_page_count = (@options[:size] / db.page_size).to_i
-    db.case_sensitive_like = true
-    db.execute("CREATE table if not exists data(id text primary key, value text, expires_in integer, last_used integer)")
-    db.execute("CREATE index if not exists expiry_index on data (expires_in)")
-    db.execute("CREATE index if not exists last_used_index on data (last_used)")
-    @sql.each_pair{|k, v| db.stmts[k] = db.prepare(v)}
-    db
+  def create_connection
+    conn = super 
+    conn.cache_size = 2000
+    conn.journal_size_limit = [(@options[:size]/2).to_i, @options[:min_size]].min
+    conn.max_page_count = (@options[:size] / conn.page_size).to_i
+    conn.case_sensitive_like = true
+    sql = YAML.load_file("#{__dir__}/litecache.sql.yml")
+    version = conn.get_first_value("PRAGMA user_version")
+    sql["schema"].each_pair do |v, obj| 
+      if v > version
+        conn.transaction do 
+          obj.each{|k, s| conn.execute(s)}
+          conn.user_version = v
+        end
+      end
+    end 
+    sql["stmts"].each { |k, v| conn.stmts[k.to_sym] = conn.prepare(v) }
+    conn
   end
   
 end

data/lib/litestack/litecache.sql.yml

@@ -0,0 +1,28 @@
+schema:
+  1:    
+    create_table_data: >  
+      CREATE table if not exists data(id text primary key, value text, expires_in integer, last_used integer)      
+    create_expiry_index: >
+      CREATE index if not exists expiry_index on data (expires_in)
+    create_last_used_index: >
+      CREATE index if not exists last_used_index on data (last_used)
+
+stmts:
+  pruner: DELETE FROM data WHERE expires_in <= $1
+  extra_pruner: DELETE FROM data WHERE id IN (SELECT id FROM data ORDER BY last_used ASC LIMIT (SELECT CAST((count(*) * $1) AS int) FROM data))
+  limited_pruner: DELETE FROM data WHERE id IN (SELECT id FROM data ORDER BY last_used asc limit $1)
+  toucher: UPDATE data SET  last_used = unixepoch('now') WHERE id = $1
+  setter: >
+    INSERT into data (id, value, expires_in, last_used) VALUES ($1, $2, unixepoch('now') + $3, unixepoch('now')) on conflict(id) do 
+    UPDATE SET value = excluded.value, last_used = excluded.last_used, expires_in = excluded.expires_in
+  inserter: > 
+    INSERT into data (id, value, expires_in, last_used) VALUES ($1, $2, unixepoch('now') + $3, unixepoch('now')) on conflict(id) do 
+    UPDATE SET value = excluded.value, last_used = excluded.last_used, expires_in = excluded.expires_in WHERE id = $1 and expires_in <= unixepoch('now')
+  finder: SELECT id FROM data WHERE id = $1
+  getter: SELECT id, value, expires_in FROM data WHERE id = $1
+  deleter: delete FROM data WHERE id = $1 returning value
+  incrementer: > 
+    INSERT into data (id, value, expires_in, last_used) VALUES ($1, $2, unixepoch('now') + $3, unixepoch('now')) on conflict(id) do 
+    UPDATE SET value = cast(value AS int) + cast(excluded.value as int), last_used = excluded.last_used, expires_in = excluded.expires_in
+  counter: SELECT count(*) FROM data
+  sizer: SELECT size.page_size * count.page_count FROM pragma_page_size() AS size, pragma_page_count() AS count

data/lib/litestack/litecache.yml

@@ -0,0 +1,7 @@
+data: 123
+
+production: 
+  data: abc
+
+developments:
+  data: xyz

data/lib/litestack/litejob.rb

@@ -52,28 +52,37 @@ module Litejob
     end
     
     def perform_at(time, *params)
-      delay = time - Time.now.to_i
+      delay = time.to_i - Time.now.to_i
       get_jobqueue.push(self.name, params, delay, queue)           
     end
     
-    def perfrom_in(delay, *params)
+    def perform_in(delay, *params)
       get_jobqueue.push(self.name, params, delay, queue)           
     end
-        
-    def options
-      @@options ||= {}
+
+    def perform_after(delay, *params)
+      perform_in(delay, *params)
     end
-    
-    def options=(options)
-      @@options = options
+
+    def process_jobs
+      get_jobqueue
     end
-    
+       
+    def delete(id, queue_name=nil) 
+      queue_name ||= queue
+      get_jobqueue.delete(id, queue)
+    end  
+         
     def queue
-      @@queue_name ||= "default"
+      @@queue ||= "default"
     end
     
     def queue=(queue_name)
-      @@queue_name = queue_name.to_s
+      @@queue = queue_name.to_s
+    end
+
+    def options
+      @options ||= self::DEFAULT_OPTIONS rescue {}
     end
           
     def get_jobqueue

data/lib/litestack/litejobqueue.rb

@@ -1,8 +1,7 @@
 # frozen_stringe_literal: true
-require 'logger'
-require 'oj'
-require 'yaml'
+
 require_relative './litequeue'
+require_relative './litemetric'
 
 ##
 #Litejobqueue is a job queueing and processing system designed for Ruby applications. It is built on top of SQLite, which is an embedded relational database management system that is #lightweight and fast.
@@ -12,7 +11,9 @@ require_relative './litequeue'
 #Litejobqueue also integrates well with various I/O frameworks like Async and Polyphony, making it a great choice for Ruby applications that use these frameworks. It provides a #simple and easy-to-use API for adding jobs to the queue and for processing them.
 #
 #Overall, LiteJobQueue is an excellent choice for Ruby applications that require a lightweight, embedded job queueing and processing system that is fast, efficient, and easy to use.
-class Litejobqueue
+class Litejobqueue < Litequeue
+
+  include Litemetric::Measurable
 
   # the default options for the job queue
   # can be overriden by passing new options in a hash 
@@ -34,12 +35,22 @@ class Litejobqueue
     path: "./queue.db",
     queues: [["default", 1]],
     workers: 5,
-    logger: STDOUT,
-    sleep_intervals: [0.001, 0.005, 0.025, 0.125, 0.625, 1.0, 2.0]
+    retries: 5, 
+    retry_delay: 60,
+    retry_delay_multiplier: 10,
+    dead_job_retention: 10 * 24 * 3600,
+    gc_sleep_interval: 7200, 
+    logger: 'STDOUT',
+    sleep_intervals: [0.001, 0.005, 0.025, 0.125, 0.625, 1.0, 2.0],
+    metrics: false 
   }
   
   @@queue = nil
   
+  attr_reader :running
+  
+  alias_method :_push, :push
+  
   # a method that returns a single instance of the job queue
   # for use by Litejob
   def self.jobqueue(options = {})
@@ -57,19 +68,10 @@ class Litejobqueue
   #   jobqueue = Litejobqueue.new
   #   
   def initialize(options = {})
-    @options = DEFAULT_OPTIONS.merge(options)
-    config = YAML.load_file(@options[:config_path]) rescue {} # an empty hash won't hurt
-    config.keys.each do |k| # symbolize keys
-      config[k.to_sym] = config[k]
-      config.delete k
-    end
-    @options.merge!(config)
-    @queue = Litequeue.new(@options) # create a new queue object
-    if @options[:logger].respond_to? :info
-      @logger = @options[:logger] 
-    else
-      @logger = Logger.new(@options[:logger])
-    end
+
+    @queues = [] # a place holder to allow workers to process
+    super(options)
+    
     # group and order queues according to their priority
     pgroups = {}
     @options[:queues].each do |q|
@@ -77,7 +79,6 @@ class Litejobqueue
       pgroups[q[1]] << [q[0], q[2] == "spawn"]
     end
     @queues = pgroups.keys.sort.reverse.collect{|p| [p, pgroups[p]]}
-    @workers = @options[:workers].times.collect{ create_worker } 
   end
   
   # push a job to the queue
@@ -89,10 +90,17 @@ class Litejobqueue
   #   jobqueue = Litejobqueue.new
   #   jobqueue.push(EasyJob, params) # the job will be performed asynchronously
   def push(jobclass, params, delay=0, queue=nil)
-    payload = Oj.dump([jobclass, params])
-    #res = 
-    res = @queue.push(payload, delay, queue)
-    @logger.info("[litejob]:[ENQ] id: #{res} class: #{jobclass}")
+    payload = Oj.dump({klass: jobclass, params: params, retries: @options[:retries], queue: queue})
+    res = super(payload, delay, queue)
+    capture(:enqueue, queue)
+    @logger.info("[litejob]:[ENQ] queue:#{res[1]} class:#{jobclass} job:#{res[0]}")
+    res
+  end
+  
+  def repush(id, job, delay=0, queue=nil)
+    res = super(id, Oj.dump(job), delay, queue)
+    capture(:enqueue, queue)
+    @logger.info("[litejob]:[ENQ] queue:#{res[0]} class:#{job[:klass]} job:#{id}")
     res
   end
   
@@ -106,13 +114,61 @@ class Litejobqueue
   #   id = jobqueue.push(EasyJob, params, 10) # queue for processing in 10 seconds
   #   jobqueue.delete(id)    
   def delete(id)
-    job = @queue.delete(id)
+    job = super(id)
     @logger.info("[litejob]:[DEL] job: #{job}")
-    Oj.load(job) if job
+    job = Oj.load(job[0]) if job
     job
   end
   
+  # delete all jobs in a certain named queue
+  # or delete all jobs if the queue name is nil
+  #def clear(queue=nil)
+    #@queue.clear(queue)
+  #end
+  
+  # stop the queue object (does not delete the jobs in the queue)
+  # specifically useful for testing
+  def stop
+    @running = false
+    #@@queue = nil
+    close
+  end
+  
+  
   private
+
+  def exit_callback
+    @running = false # stop all workers
+    puts "--- Litejob detected an exit, cleaning up"
+    index = 0
+    while @jobs_in_flight > 0 and index < 30 # 3 seconds grace period for jobs to finish
+      puts "--- Waiting for #{@jobs_in_flight} jobs to finish"
+      sleep 0.1
+      index += 1
+    end
+    puts " --- Exiting with #{@jobs_in_flight} jobs in flight"
+  end
+
+  def setup
+    super
+    @jobs_in_flight = 0
+    @workers = @options[:workers].times.collect{ create_worker }    
+    @gc = create_garbage_collector
+    @mutex = Litesupport::Mutex.new 
+  end
+  
+  def job_started
+    Litesupport.synchronize(@mutex){@jobs_in_flight += 1}
+  end
+  
+  def job_finished
+    Litesupport.synchronize(@mutex){@jobs_in_flight -= 1}
+  end
+  
+  # return a hash encapsulating the info about the current jobqueue
+  def snapshot
+    info
+  end
   
   # optionally run a job in its own context
   def schedule(spawn = false, &block)
@@ -126,41 +182,47 @@ class Litejobqueue
   # create a worker according to environment
   def create_worker
     Litesupport.spawn do
-      if @options[:logger].respond_to? :info
-        logger = @options[:logger] 
-      else
-        logger = Logger.new(@options[:logger])
-      end
       worker_sleep_index = 0
-      i = 0
-      loop do
+      while @running do
         processed = 0
         @queues.each do |level| # iterate through the levels
           level[1].each do |q| # iterate through the queues in the level
             index = 0
             max = level[0]
-            while index < max && payload = @queue.pop(q[0], 1) # fearlessly use the same queue object 
+            while index < max && payload = pop(q[0], 1) # fearlessly use the same queue object 
+              capture(:dequeue, q[0])
               processed += 1
               index += 1
               begin
                 id, job = payload[0], payload[1]
                 job = Oj.load(job)
-                logger.info "[litejob]:[DEQ] id: #{id} class: #{job[0]}" 
-                klass = eval(job[0])
+                @logger.info "[litejob]:[DEQ] queue:#{q[0]} class:#{job[:klass]} job:#{id}" 
+                klass = eval(job[:klass])
                 schedule(q[1]) do # run the job in a new context
+                  job_started #(Litesupport.current_context)
                   begin
-                    klass.new.perform(*job[1])
-                    logger.info "[litejob]:[END] id: #{id} class: #{job[0]}" 
+                    measure(:perform, q[0]){ klass.new.perform(*job[:params]) }
+                    @logger.info "[litejob]:[END] queue:#{q[0]} class:#{job[:klass]} job:#{id}" 
                   rescue Exception => e
-                    puts e
-                    puts e.message
-                    puts e.backtrace
+                    # we can retry the failed job now
+                    capture(:fail, q[0])
+                    if job[:retries] == 0
+                      @logger.error "[litejob]:[ERR] queue:#{q[0]} class:#{job[:klass]} job:#{id} failed with #{e}:#{e.message}, retries exhausted, moved to _dead queue"
+                      repush(id, job, @options[:dead_job_retention], '_dead')
+                    else
+                      capture(:retry, q[0])
+                      retry_delay = @options[:retry_delay_multiplier].pow(@options[:retries] - job[:retries]) * @options[:retry_delay] 
+                      job[:retries] -=  1
+                      @logger.error "[litejob]:[ERR] queue:#{q[0]} class:#{job[:klass]} job:#{id} failed with #{e}:#{e.message}, retrying in #{retry_delay} seconds"
+                      repush(id, job, retry_delay, q[0])                      
+                    end
                   end
+                  job_finished #(Litesupport.current_context)
                 end
               rescue Exception => e
-                puts e
-                puts e.message
-                puts e.backtrace
+                # this is an error in the extraction of job info, retrying here will not be useful
+                @logger.error "[litejob]:[ERR] failed to extract job info for: #{payload} with #{e}:#{e.message}"
+                job_finished #(Litesupport.current_context)
               end
               Litesupport.switch #give other contexts a chance to run here
             end
@@ -176,4 +238,20 @@ class Litejobqueue
     end
   end  
   
+  # create a gc for dead jobs
+  def create_garbage_collector
+    Litesupport.spawn do
+      while @running do
+        while jobs = pop('_dead', 100)
+          if jobs[0].is_a? Array
+            @logger.info "[litejob]:[DEL] garbage collector deleted #{jobs.length} dead jobs"
+          else
+            @logger.info "[litejob]:[DEL] garbage collector deleted 1 dead job"
+          end
+        end
+        sleep @options[:gc_sleep_interval]      
+      end
+    end
+  end
+  
 end