litestack 0.1.8 → 0.2.1

data/lib/litestack/litecache.yml

@@ -0,0 +1,7 @@
+data: 123
+
+production: 
+  data: abc
+
+developments:
+  data: xyz

data/lib/litestack/litejob.rb

@@ -43,7 +43,7 @@ module Litejob
   private
   def self.included(klass)
     klass.extend(ClassMethods)
-    #klass.get_jobqueue
+    klass.get_jobqueue
   end
   
   module ClassMethods
@@ -80,9 +80,13 @@ module Litejob
     def queue=(queue_name)
       @@queue = queue_name.to_s
     end
+
+    def options
+      @options ||= self::DEFAULT_OPTIONS rescue {}
+    end
           
     def get_jobqueue
-      Litejobqueue.jobqueue
+      Litejobqueue.jobqueue(options)
     end
   end
     

data/lib/litestack/litejobqueue.rb

@@ -1,8 +1,7 @@
 # frozen_stringe_literal: true
-require 'logger'
-require 'oj'
-require 'yaml'
+
 require_relative './litequeue'
+require_relative './litemetric'
 
 ##
 #Litejobqueue is a job queueing and processing system designed for Ruby applications. It is built on top of SQLite, which is an embedded relational database management system that is #lightweight and fast.
@@ -12,7 +11,9 @@ require_relative './litequeue'
 #Litejobqueue also integrates well with various I/O frameworks like Async and Polyphony, making it a great choice for Ruby applications that use these frameworks. It provides a #simple and easy-to-use API for adding jobs to the queue and for processing them.
 #
 #Overall, LiteJobQueue is an excellent choice for Ruby applications that require a lightweight, embedded job queueing and processing system that is fast, efficient, and easy to use.
-class Litejobqueue
+class Litejobqueue < Litequeue
+
+  include Litemetric::Measurable
 
   # the default options for the job queue
   # can be overriden by passing new options in a hash 
@@ -40,13 +41,16 @@ class Litejobqueue
     dead_job_retention: 10 * 24 * 3600,
     gc_sleep_interval: 7200, 
     logger: 'STDOUT',
-    sleep_intervals: [0.001, 0.005, 0.025, 0.125, 0.625, 1.0, 2.0]
+    sleep_intervals: [0.001, 0.005, 0.025, 0.125, 0.625, 1.0, 2.0],
+    metrics: false 
   }
   
   @@queue = nil
   
   attr_reader :running
   
+  alias_method :_push, :push
+  
   # a method that returns a single instance of the job queue
   # for use by Litejob
   def self.jobqueue(options = {})
@@ -64,31 +68,10 @@ class Litejobqueue
   #   jobqueue = Litejobqueue.new
   #   
   def initialize(options = {})
-    @options = DEFAULT_OPTIONS.merge(options)
-    config = YAML.load_file(@options[:config_path]) rescue {} # an empty hash won't hurt
-    config.keys.each do |k| # symbolize keys
-      config[k.to_sym] = config[k]
-      config.delete k
-    end
-    @options.merge!(config)
-    @options.merge!(options) # make sure options passed to initialize trump everything else
 
-    @queue = Litequeue.new(@options) # create a new queue object
+    @queues = [] # a place holder to allow workers to process
+    super(options)
     
-    # create logger
-    if @options[:logger].respond_to? :info
-      @logger = @options[:logger] 
-    elsif @options[:logger] == 'STDOUT'
-      @logger = Logger.new(STDOUT)      
-    elsif @options[:logger] == 'STDERR'
-      @logger = Logger.new(STDERR)      
-    elsif @options[:logger].nil?
-      @logger = Logger.new(IO::NULL)      
-    elsif @options[:logger].is_a? String 
-      @logger = Logger.new(@options[:logger])
-    else
-      @logger = Logger.new(IO::NULL)      
-    end
     # group and order queues according to their priority
     pgroups = {}
     @options[:queues].each do |q|
@@ -96,25 +79,6 @@ class Litejobqueue
       pgroups[q[1]] << [q[0], q[2] == "spawn"]
     end
     @queues = pgroups.keys.sort.reverse.collect{|p| [p, pgroups[p]]}
-    @running = true
-    @workers = @options[:workers].times.collect{ create_worker }
-    
-    @gc = create_garbage_collector
-    @jobs_in_flight = 0
-    @mutex = Litesupport::Mutex.new
-    
-    at_exit do
-      @running = false
-      puts "--- Litejob detected an exit attempt, cleaning up"
-      index = 0
-      while @jobs_in_flight > 0 and index < 5
-        puts "--- Waiting for #{@jobs_in_flight} jobs to finish"
-        sleep 1
-        index += 1
-      end
-      puts " --- Exiting with #{@jobs_in_flight} jobs in flight"
-    end
-  
   end
   
   # push a job to the queue
@@ -127,8 +91,16 @@ class Litejobqueue
   #   jobqueue.push(EasyJob, params) # the job will be performed asynchronously
   def push(jobclass, params, delay=0, queue=nil)
     payload = Oj.dump({klass: jobclass, params: params, retries: @options[:retries], queue: queue})
-    res = @queue.push(payload, delay, queue)
-    @logger.info("[litejob]:[ENQ] id: #{res} job: #{jobclass}")
+    res = super(payload, delay, queue)
+    capture(:enqueue, queue)
+    @logger.info("[litejob]:[ENQ] queue:#{res[1]} class:#{jobclass} job:#{res[0]}")
+    res
+  end
+  
+  def repush(id, job, delay=0, queue=nil)
+    res = super(id, Oj.dump(job), delay, queue)
+    capture(:enqueue, queue)
+    @logger.info("[litejob]:[ENQ] queue:#{res[0]} class:#{job[:klass]} job:#{id}")
     res
   end
   
@@ -140,9 +112,9 @@ class Litejobqueue
   #   end
   #   jobqueue = Litejobqueue.new
   #   id = jobqueue.push(EasyJob, params, 10) # queue for processing in 10 seconds
-  #   jobqueue.delete(id, 'default')    
-  def delete(id, queue=nil)
-    job = @queue.delete(id, queue)
+  #   jobqueue.delete(id)    
+  def delete(id)
+    job = super(id)
     @logger.info("[litejob]:[DEL] job: #{job}")
     job = Oj.load(job[0]) if job
     job
@@ -150,23 +122,40 @@ class Litejobqueue
   
   # delete all jobs in a certain named queue
   # or delete all jobs if the queue name is nil
-  def clear(queue=nil)
-    @queue.clear(queue)
-  end
+  #def clear(queue=nil)
+    #@queue.clear(queue)
+  #end
   
   # stop the queue object (does not delete the jobs in the queue)
   # specifically useful for testing
   def stop
     @running = false
-    @@queue = nil
+    #@@queue = nil
+    close
   end
   
   
-  def count(queue=nil)
-    @queue.count(queue)
-  end
-  
   private
+
+  def exit_callback
+    @running = false # stop all workers
+    puts "--- Litejob detected an exit, cleaning up"
+    index = 0
+    while @jobs_in_flight > 0 and index < 30 # 3 seconds grace period for jobs to finish
+      puts "--- Waiting for #{@jobs_in_flight} jobs to finish"
+      sleep 0.1
+      index += 1
+    end
+    puts " --- Exiting with #{@jobs_in_flight} jobs in flight"
+  end
+
+  def setup
+    super
+    @jobs_in_flight = 0
+    @workers = @options[:workers].times.collect{ create_worker }    
+    @gc = create_garbage_collector
+    @mutex = Litesupport::Mutex.new 
+  end
   
   def job_started
     Litesupport.synchronize(@mutex){@jobs_in_flight += 1}
@@ -176,6 +165,11 @@ class Litejobqueue
     Litesupport.synchronize(@mutex){@jobs_in_flight -= 1}
   end
   
+  # return a hash encapsulating the info about the current jobqueue
+  def snapshot
+    info
+  end
+  
   # optionally run a job in its own context
   def schedule(spawn = false, &block)
     if spawn
@@ -195,40 +189,40 @@ class Litejobqueue
           level[1].each do |q| # iterate through the queues in the level
             index = 0
             max = level[0]
-            while index < max && payload = @queue.pop(q[0], 1) # fearlessly use the same queue object 
+            while index < max && payload = pop(q[0], 1) # fearlessly use the same queue object 
+              capture(:dequeue, q[0])
               processed += 1
               index += 1
               begin
                 id, job = payload[0], payload[1]
                 job = Oj.load(job)
-                # first capture the original job id                
-                job[:id] = id if job[:retries].to_i == @options[:retries].to_i
-                @logger.info "[litejob]:[DEQ] job:#{job}" 
+                @logger.info "[litejob]:[DEQ] queue:#{q[0]} class:#{job[:klass]} job:#{id}" 
                 klass = eval(job[:klass])
                 schedule(q[1]) do # run the job in a new context
                   job_started #(Litesupport.current_context)
                   begin
-                    klass.new.perform(*job[:params])
-                    @logger.info "[litejob]:[END] job:#{job}" 
+                    measure(:perform, q[0]){ klass.new.perform(*job[:params]) }
+                    @logger.info "[litejob]:[END] queue:#{q[0]} class:#{job[:klass]} job:#{id}" 
                   rescue Exception => e
                     # we can retry the failed job now
+                    capture(:fail, q[0])
                     if job[:retries] == 0
-                      @logger.error "[litejob]:[ERR] job: #{job} failed with #{e}:#{e.message}, retries exhausted, moved to _dead queue"
-                      @queue.push(Oj.dump(job), @options[:dead_job_retention], '_dead')
+                      @logger.error "[litejob]:[ERR] queue:#{q[0]} class:#{job[:klass]} job:#{id} failed with #{e}:#{e.message}, retries exhausted, moved to _dead queue"
+                      repush(id, job, @options[:dead_job_retention], '_dead')
                     else
+                      capture(:retry, q[0])
                       retry_delay = @options[:retry_delay_multiplier].pow(@options[:retries] - job[:retries]) * @options[:retry_delay] 
                       job[:retries] -=  1
-                      @logger.error "[litejob]:[ERR] job: #{job} failed with #{e}:#{e.message}, retrying in #{retry_delay}"
-                      @queue.push(Oj.dump(job), retry_delay, q[0])
-                      @logger.info "[litejob]:[ENQ] job: #{job} enqueued"
+                      @logger.error "[litejob]:[ERR] queue:#{q[0]} class:#{job[:klass]} job:#{id} failed with #{e}:#{e.message}, retrying in #{retry_delay} seconds"
+                      repush(id, job, retry_delay, q[0])                      
                     end
                   end
                   job_finished #(Litesupport.current_context)
                 end
               rescue Exception => e
-                # this is an error in the extraction of job info
-                # retrying here will not be useful
+                # this is an error in the extraction of job info, retrying here will not be useful
                 @logger.error "[litejob]:[ERR] failed to extract job info for: #{payload} with #{e}:#{e.message}"
+                job_finished #(Litesupport.current_context)
               end
               Litesupport.switch #give other contexts a chance to run here
             end
@@ -248,7 +242,7 @@ class Litejobqueue
   def create_garbage_collector
     Litesupport.spawn do
       while @running do
-        while jobs = @queue.pop('_dead', 100)
+        while jobs = pop('_dead', 100)
           if jobs[0].is_a? Array
             @logger.info "[litejob]:[DEL] garbage collector deleted #{jobs.length} dead jobs"
           else

data/lib/litestack/litemetric.rb

@@ -0,0 +1,228 @@
+# frozen_stringe_literal: true
+
+require 'singleton'
+
+require_relative './litesupport'
+
+# this class is a singleton
+# and should remain so
+class Litemetric
+
+  include Singleton
+  include Litesupport::Liteconnection
+  
+  DEFAULT_OPTIONS = {
+    config_path: "./litemetric.yml",
+    path: "./metrics.db",
+    sync: 1,
+    mmap_size: 16 * 1024 * 1024, # 16MB of memory to easily process 1 year worth of data
+    flush_interval: 10, # flush data every 1 minute
+    summarize_interval: 10 # summarize data every 1 minute
+  }
+
+  RESOLUTIONS = {
+    minute: 300, # 5 minutes (highest resolution)
+    hour: 3600, # 1 hour
+    day: 24*3600, # 1 day
+    week: 7*24*3600 # 1 week (lowest resolution)
+  }
+
+  # :nodoc: 
+  def initialize(options = {})
+    init(options)
+  end
+  
+  # registers a class for metrics to be collected
+  def register(identifier)
+    @registered[identifier] = true   
+    @metrics[identifier] = {} unless @metrics[identifier]
+    run_stmt(:register_topic, identifier) # it is safe to call register topic multiple times with the same identifier
+  end
+  
+  ## event capturing
+  ##################
+    
+  def capture(topic, event, key=event, value=nil)
+    if key.is_a? Array
+      key.each{|k| capture_single_key(topic, event, k, value)}
+    else
+      capture_single_key(topic, event, key, value)
+    end
+  end
+  
+  def capture_single_key(topic, event, key=event, value=nil)
+    @mutex.synchronize do
+      time_slot = current_time_slot # should that be 5 minutes?
+      topic_slot = @metrics[topic]
+      if event_slot = topic_slot[event]
+        if key_slot = event_slot[key]
+          if key_slot[time_slot]
+            key_slot[time_slot][:count] += 1
+            key_slot[time_slot][:value] += value unless value.nil?
+          else # new time slot
+            key_slot[time_slot] = {count: 1, value: value}
+          end
+        else
+          event_slot[key] = {time_slot => {count: 1, value: value}}
+        end
+      else # new event
+        topic_slot[event] = {key => {time_slot => {count: 1, value: value}}}
+      end
+    end
+  end
+  
+
+  ## event reporting
+  ##################
+  
+  def topics
+    run_stmt(:list_topics).to_a
+  end
+
+  def event_names(resolution, topic)
+    run_stmt(:list_event_names, resolution, topic).to_a
+  end
+
+  def keys(resolution, topic, event_name)
+    run_stmt(:list_event_keys, resolution, topic, event_name).to_a
+  end
+
+  def event_data(resolution, topic, event_name, key)
+    run_stmt(:list_events_by_key, resolution, topic, event_name, key).to_a
+  end
+  
+  ## summarize data
+  #################  
+
+  def summarize
+    run_stmt(:summarize_events, RESOLUTIONS[:hour], "hour", "minute") 
+    run_stmt(:summarize_events, RESOLUTIONS[:day], "day", "hour") 
+    run_stmt(:summarize_events, RESOLUTIONS[:week], "week", "day")
+    run_stmt(:delete_events, "minute", RESOLUTIONS[:hour]*1) 
+    run_stmt(:delete_events, "hour", RESOLUTIONS[:day]*1) 
+    run_stmt(:delete_events, "day", RESOLUTIONS[:week]*1) 
+  end
+
+  ## background stuff
+  ###################
+
+  private
+
+  def exit_callback
+    puts "--- Litemetric detected an exit, flushing metrics"
+    @running = false
+    flush
+  end
+
+  def setup
+    super
+    @metrics = {}
+    @registered = {}
+    @flusher = create_flusher
+    @summarizer = create_summarizer
+    @mutex = Litesupport::Mutex.new
+  end
+
+  def current_time_slot
+    (Time.now.to_i / 300) * 300 # every 5 minutes 
+  end
+  
+  def flush
+    to_delete = []
+    @conn.acquire do |conn|
+      conn.transaction(:immediate) do
+        @metrics.each_pair do |topic, event_hash|
+          event_hash.each_pair do |event, key_hash|
+            key_hash.each_pair do |key, time_hash|
+              time_hash.each_pair do |time, data|
+                conn.stmts[:capture_event].execute!(topic, event.to_s, key, time, data[:count], data[:value]) if data 
+                time_hash[time] = nil 
+                to_delete << [topic, event, key, time]
+              end
+            end      
+          end      
+        end
+      end
+    end
+    to_delete.each do |r| 
+      @metrics[r[0]][r[1]][r[2]].delete(r[3])
+      @metrics[r[0]][r[1]].delete(r[2]) if @metrics[r[0]][r[1]][r[2]].empty? 
+      @metrics[r[0]].delete(r[1]) if @metrics[r[0]][r[1]].empty? 
+    end
+  end  
+     
+  def create_connection
+    conn = super
+    conn.wal_autocheckpoint = 10000
+    sql = YAML.load_file("#{__dir__}/litemetric.sql.yml")
+    version = conn.get_first_value("PRAGMA user_version")
+    sql["schema"].each_pair do |v, obj| 
+      if v > version
+        conn.transaction do 
+          obj.each{|k, s| conn.execute(s)}
+          conn.user_version = v
+        end
+      end
+    end 
+    sql["stmts"].each { |k, v| conn.stmts[k.to_sym] = conn.prepare(v) }
+    conn
+  end
+  
+  def create_flusher
+    Litesupport.spawn do
+      while @running do
+        sleep @options[:flush_interval]
+        @mutex.synchronize do
+          flush
+        end
+      end      
+    end 
+  end
+  
+  def create_summarizer
+    Litesupport.spawn do
+      while @running do
+        sleep @options[:summarize_interval]
+        summarize
+      end      
+    end 
+  end
+       
+end
+
+## Measurable Module
+####################
+
+class Litemetric
+  module Measurable
+            
+    def collect_metrics
+      @litemetric = Litemetric.instance
+      @litemetric.register(metrics_identifier)
+    end
+
+    def metrics_identifier
+      self.class.name # override in included classes
+    end
+
+    def capture(event, key=event, value=nil)
+      return unless @litemetric
+      @litemetric.capture(metrics_identifier, event, key, value)
+    end
+    
+    def measure(event, key=event)
+      return yield unless @litemetric
+      t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC) 
+      res = yield
+      t2 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      value = (( t2 - t1 ) * 1000).round # capture time in milliseconds
+      capture(event, key, value)
+      res  
+    end    
+    
+    def snapshot
+      raise Litestack::NotImplementedError
+    end
+    
+  end
+end

data/lib/litestack/litemetric.sql.yml

@@ -0,0 +1,69 @@
+schema:
+  1:
+    create_topics: >
+      CREATE TABLE IF NOT EXISTS topics(
+        name text PRIMARY KEY NOT NULL
+      ) WITHOUT ROWID; 
+    create_events: >
+      CREATE TABLE IF NOT EXISTS events(
+        topic text NOT NULL references topics(name) ON DELETE CASCADE, 
+        name TEXT NOT NULL, 
+        key TEXT NOT NULL, 
+        count INTEGER DEFAULT(0) NOT NULL ON CONFLICT REPLACE, 
+        value INTEGER, 
+        minimum INTEGER, 
+        maximum INTEGER, 
+        created_at INTEGER DEFAULT((unixepoch()/300*300)) NOT NULL, 
+        resolution TEXT DEFAULT('minute') NOT NULL, 
+        PRIMARY KEY(resolution, topic, name, key, created_at)
+      ) WITHOUT ROWID;    
+    create_index_on_event: CREATE INDEX IF NOT EXISTS events_by_resolution ON events(resolution, created_at);
+
+stmts:
+  # register topic
+  register_topic: INSERT INTO topics VALUES (?) ON CONFLICT DO NOTHING;
+
+  capture_event: >
+    INSERT INTO events(topic, name, key, created_at, count, value, minimum, maximum) VALUES ($1, $2, $3, $4, $5, $6, $6, $6) 
+    ON CONFLICT DO 
+    UPDATE SET count = count + EXCLUDED.count, value = value + EXCLUDED.value, minimum = min(minimum, EXCLUDED.minimum), maximum = max(maximum, EXCLUDED.maximum) 
+  
+  # requires an index on (resolution, created_at)
+  summarize_events: >
+    INSERT INTO events (topic, name, key, count, value, minimum, maximum, created_at, resolution ) SELECT 
+      topic, 
+      name, 
+      key, 
+      sum(count) as count, 
+      sum(value) as value, 
+      min(minimum) as minimum, 
+      max(maximum) as maximum, 
+      (created_at/$1)*$1 as created,  
+      $2 
+    FROM events WHERE resolution = $3 AND created_at < (unixepoch()/$1)*$1 GROUP BY topic, name, key, created ON CONFLICT DO UPDATE 
+    SET count = count + EXCLUDED.count, value = value + EXCLUDED.value, minimum = min(minimum, EXCLUDED.minimum), maximum = max(maximum, EXCLUDED.maximum); 
+
+  # requires an index on (resolution, created_at)
+  delete_events: DELETE FROM events WHERE resolution = $3 AND created_at < (unixepoch() - $4);
+
+  # select topics from the topics table
+  list_topics: SELECT name FROM topics;
+
+  # requires an index on (resolution, topic, name)
+  list_event_names: >
+    SELECT name, sum(count) as count, count(distinct name) as name, sum(value) as value, min(minimum), max(maximum) 
+    FROM events WHERE resolution = ? AND topic = ? GROUP BY name ORDER BY count;
+
+  # requires an index on (resolution, topic, name, key)
+  list_event_keys: >
+    SELECT key, sum(count) as count, sum(value) as value, min(minimum), max(maximum)
+    FROM events WHERE resolution = ? AND topic = ? AND name = ? GROUP BY key ORDER BY count;
+  
+  # requires an index on (resolution, topic, name, key, created_at)
+  list_events_by_key: > 
+    SELECT * FROM events WHERE resolution = $1 AND topic = $2 AND name = $3 AND key = $4 ORDER BY created_at ASC;    
+    
+  # requires an index on (resolution, topic, name, key, created_at)
+  list_all_events: >
+    SELECT * FROM events WHERE resolution = ? AND topic = ? ORDER BY name, key, created_at ASC;
+