litestack 0.2.6 → 0.4.1

data/lib/litestack/litejobqueue.rb

@@ -54,7 +54,7 @@ class Litejobqueue < Litequeue
   # a method that returns a single instance of the job queue
   # for use by Litejob
   def self.jobqueue(options = {})
-    @@queue ||= Litesupport.synchronize{self.new(options)}
+    @@queue ||= Litescheduler.synchronize{self.new(options)}
   end
 
   def self.new(options = {})
@@ -95,7 +95,7 @@ class Litejobqueue < Litequeue
   #   jobqueue = Litejobqueue.new
   #   jobqueue.push(EasyJob, params) # the job will be performed asynchronously
   def push(jobclass, params, delay=0, queue=nil)
-    payload = Oj.dump({klass: jobclass, params: params, retries: @options[:retries], queue: queue})
+    payload = Oj.dump({klass: jobclass, params: params, retries: @options[:retries], queue: queue}, mode: :strict)
     res = super(payload, delay, queue)
     capture(:enqueue, queue)
     @logger.info("[litejob]:[ENQ] queue:#{res[1]} class:#{jobclass} job:#{res[0]}")
@@ -103,7 +103,7 @@ class Litejobqueue < Litequeue
   end
   
   def repush(id, job, delay=0, queue=nil)
-    res = super(id, Oj.dump(job), delay, queue)
+    res = super(id, Oj.dump(job, mode: :strict), delay, queue)
     capture(:enqueue, queue)
     @logger.info("[litejob]:[ENQ] queue:#{res[0]} class:#{job[:klass]} job:#{id}")
     res
@@ -121,7 +121,7 @@ class Litejobqueue < Litequeue
   def delete(id)
     job = super(id)
     @logger.info("[litejob]:[DEL] job: #{job}")
-    job = Oj.load(job[0]) if job
+    job = Oj.load(job[0], symbol_keys: true) if job
     job
   end
   
@@ -163,17 +163,17 @@ class Litejobqueue < Litequeue
   end
   
   def job_started
-    Litesupport.synchronize(@mutex){@jobs_in_flight += 1}
+    Litescheduler.synchronize(@mutex){@jobs_in_flight += 1}
   end
   
   def job_finished
-    Litesupport.synchronize(@mutex){@jobs_in_flight -= 1}
+    Litescheduler.synchronize(@mutex){@jobs_in_flight -= 1}
   end
     
   # optionally run a job in its own context
   def schedule(spawn = false, &block)
     if spawn
-      Litesupport.spawn &block
+      Litescheduler.spawn &block
     else
       yield
     end
@@ -181,50 +181,23 @@ class Litejobqueue < Litequeue
     
   # create a worker according to environment
   def create_worker
-    Litesupport.spawn do
+    Litescheduler.spawn do
       worker_sleep_index = 0
       while @running do
         processed = 0
-        @queues.each do |level| # iterate through the levels
-          level[1].each do |q| # iterate through the queues in the level
-            index = 0
-            max = level[0]
-            while index < max && payload = pop(q[0], 1) # fearlessly use the same queue object 
-              capture(:dequeue, q[0])
+        @queues.each do |priority, queues| # iterate through the levels
+          queues.each do |queue, spawns| # iterate through the queues in the level
+            batched = 0
+            
+            while (batched < priority) && (payload = pop(queue, 1)) # fearlessly use the same queue object
+              capture(:dequeue, queue)
               processed += 1
-              index += 1
-              begin
-                id, job = payload[0], payload[1]
-                job = Oj.load(job)
-                @logger.info "[litejob]:[DEQ] queue:#{q[0]} class:#{job[:klass]} job:#{id}" 
-                klass = eval(job[:klass])
-                schedule(q[1]) do # run the job in a new context
-                  job_started #(Litesupport.current_context)
-                  begin
-                    measure(:perform, q[0]){ klass.new.perform(*job[:params]) }
-                    @logger.info "[litejob]:[END] queue:#{q[0]} class:#{job[:klass]} job:#{id}" 
-                  rescue Exception => e
-                    # we can retry the failed job now
-                    capture(:fail, q[0])
-                    if job[:retries] == 0
-                      @logger.error "[litejob]:[ERR] queue:#{q[0]} class:#{job[:klass]} job:#{id} failed with #{e}:#{e.message}, retries exhausted, moved to _dead queue"
-                      repush(id, job, @options[:dead_job_retention], '_dead')
-                    else
-                      capture(:retry, q[0])
-                      retry_delay = @options[:retry_delay_multiplier].pow(@options[:retries] - job[:retries]) * @options[:retry_delay] 
-                      job[:retries] -=  1
-                      @logger.error "[litejob]:[ERR] queue:#{q[0]} class:#{job[:klass]} job:#{id} failed with #{e}:#{e.message}, retrying in #{retry_delay} seconds"
-                      repush(id, job, retry_delay, q[0])                      
-                    end
-                  end
-                  job_finished #(Litesupport.current_context)
-                end
-              rescue Exception => e
-                # this is an error in the extraction of job info, retrying here will not be useful
-                @logger.error "[litejob]:[ERR] failed to extract job info for: #{payload} with #{e}:#{e.message}"
-                job_finished #(Litesupport.current_context)
-              end
-              Litesupport.switch #give other contexts a chance to run here
+              batched += 1
+              
+              id, serialized_job = payload
+              process_job(queue, id, serialized_job, spawns)
+              
+              Litescheduler.switch # give other contexts a chance to run here
             end
           end
         end
@@ -240,7 +213,7 @@ class Litejobqueue < Litequeue
   
   # create a gc for dead jobs
   def create_garbage_collector
-    Litesupport.spawn do
+    Litescheduler.spawn do
       while @running do
         while jobs = pop('_dead', 100)
           if jobs[0].is_a? Array
@@ -254,4 +227,34 @@ class Litejobqueue < Litequeue
     end
   end
   
+  def process_job(queue, id, serialized_job, spawns)
+    job = Oj.load(serialized_job)
+    @logger.info "[litejob]:[DEQ] queue:#{queue} class:#{job["klass"]} job:#{id}"
+    klass = Object.const_get(job["klass"])
+    schedule(spawns) do # run the job in a new context
+      job_started # (Litesupport.current_context)
+      begin
+        measure(:perform, queue) { klass.new.perform(*job["params"]) }
+        @logger.info "[litejob]:[END] queue:#{queue} class:#{job["klass"]} job:#{id}"
+      rescue Exception => e # standard:disable Lint/RescueException
+        # we can retry the failed job now
+        capture(:fail, queue)
+        if job["retries"] == 0
+          @logger.error "[litejob]:[ERR] queue:#{queue} class:#{job["klass"]} job:#{id} failed with #{e}:#{e.message}, retries exhausted, moved to _dead queue"
+          repush(id, job, @options[:dead_job_retention], "_dead")
+        else
+          capture(:retry, queue)
+          retry_delay = @options[:retry_delay_multiplier].pow(@options[:retries] - job["retries"]) * @options[:retry_delay]
+          job["retries"] -= 1
+          @logger.error "[litejob]:[ERR] queue:#{queue} class:#{job["klass"]} job:#{id} failed with #{e}:#{e.message}, retrying in #{retry_delay} seconds"
+          repush(id, job, retry_delay, queue)
+        end
+      end
+      job_finished # (Litesupport.current_context)
+    end
+  rescue Exception => e # standard:disable Lint/RescueException
+    # this is an error in the extraction of job info, retrying here will not be useful
+    @logger.error "[litejob]:[ERR] failed to extract job info for: #{serialized_job} with #{e}:#{e.message}"
+    job_finished # (Litesupport.current_context)
+  end
 end

data/lib/litestack/litemetric.rb

@@ -55,9 +55,12 @@ class Litemetric
   ## event capturing
   ##################
   
+  def current_time_slot
+    (Time.now.to_i / 300) * 300
+  end
   
   def capture(topic, event, key=event, value=nil)
-    @collector.capture(topic, event, key, value)
+    @collector.capture(topic, event, key, value, current_time_slot)
   end
       
   def capture_snapshot(topic, state)
@@ -83,18 +86,18 @@ class Litemetric
   def events_summaries(topic, resolution, order, dir, search, count)
     search = "%#{search}%" if search
     if dir.downcase == "desc" 
-      run_stmt(:events_summaries, topic, resolution, order, search, count).to_a
+      run_stmt_hash(:events_summaries, topic, resolution, order, search, count)
     else
-      run_stmt(:events_summaries_asc, topic, resolution, order, search, count).to_a
+      run_stmt_hash(:events_summaries_asc, topic, resolution, order, search, count)
     end
   end
   
   def keys_summaries(topic, event, resolution, order, dir, search, count)
     search = "%#{search}%" if search
     if dir.downcase == "desc" 
-      run_stmt(:keys_summaries, topic, event, resolution, order, search, count).to_a
+      run_stmt_hash(:keys_summaries, topic, event, resolution, order, search, count).to_a
     else
-      run_stmt(:keys_summaries_asc, topic, event, resolution, order, search, count).to_a
+      run_stmt_hash(:keys_summaries_asc, topic, event, resolution, order, search, count).to_a
     end 
   end
 
@@ -103,11 +106,11 @@ class Litemetric
   end 
 
   def event_data_points(step, count, resolution, topic, event)
-    run_stmt(:event_data_points, step, count, resolution, topic, event).to_a
+    run_stmt_hash(:event_data_points, step, count, resolution, topic, event).to_a
   end 
 
   def key_data_points(step, count, resolution, topic, event, key)
-    run_stmt(:key_data_points, step, count, resolution, topic, event, key).to_a
+    run_stmt_hash(:key_data_points, step, count, resolution, topic, event, key).to_a
   end 
 
   def snapshot(topic)
@@ -130,7 +133,21 @@ class Litemetric
   ###################
 
   private
-
+  
+  def run_stmt_hash(stmt, *args)
+    res = run_stmt(stmt, *args)
+    cols = run_stmt_method(stmt, :columns)
+    hashes = []
+    res.each do | row |
+      hash = {}
+      row.each_with_index do |field, i|
+        hash[cols[i]] = field
+      end
+      hashes << hash
+    end
+    hashes
+  end
+  
   def exit_callback
     STDERR.puts "--- Litemetric detected an exit, flushing metrics"
     @running = false
@@ -156,29 +173,13 @@ class Litemetric
   end
      
   def create_connection
-    conn = super
-    conn.wal_autocheckpoint = 10000 # checkpoint after 10000 pages are written
-    sql = YAML.load_file("#{__dir__}/litemetric.sql.yml")
-    version = conn.get_first_value("PRAGMA user_version")
-    sql["schema"].each_pair do |v, obj| 
-      if v > version
-        begin
-          conn.transaction do 
-            obj.each{|k, s| conn.execute(s)}
-            conn.user_version = v
-          end
-        rescue Exception => e
-          STDERR.puts e.message
-          raise e
-        end
-      end
-    end 
-    sql["stmts"].each { |k, v| conn.stmts[k.to_sym] = conn.prepare(v) }
-    conn
+    super("#{__dir__}/litemetric.sql.yml") do |conn|
+      conn.wal_autocheckpoint = 10000 # checkpoint after 10000 pages are written
+    end
   end
   
   def create_flusher
-    Litesupport.spawn do
+    Litescheduler.spawn do
       while @running do
         sleep @options[:flush_interval]
         flush
@@ -187,7 +188,7 @@ class Litemetric
   end
   
   def create_summarizer
-    Litesupport.spawn do
+    Litescheduler.spawn do
       while @running do
         sleep @options[:summarize_interval]
         summarize
@@ -210,7 +211,7 @@ class Litemetric
     end
 
     def create_snapshotter
-      Litesupport.spawn do
+      Litescheduler.spawn do
         while @running do
           sleep @litemetric.options[:snapshot_interval]
           capture_snapshot
@@ -228,13 +229,16 @@ class Litemetric
     end
     
     def measure(event, key=event)
-      return yield unless @litemetric
+      unless @litemetric
+        yield
+        return 0 
+      end
       t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC) 
-      res = yield
+      yield
       t2 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      value = (( t2 - t1 ) * 1000).round # capture time in milliseconds
+      value =  t2 - t1 
       capture(event, key, value)
-      res  
+      value # return value so other events can reuse it  
     end    
     
     def capture_snapshot
@@ -273,16 +277,16 @@ class Litemetric
       init(options)
     end
     
-    def capture(topic, event, key, value=nil)
+    def capture(topic, event, key, value=nil, time=nil)
       if key.is_a? Array
-        key.each{|k| capture_single_key(topic, event, k, value)}
+        key.each{|k| capture_single_key(topic, event, k, value, time)}
       else
-        capture_single_key(topic, event, key, value)
+        capture_single_key(topic, event, key, value, time)
       end
     end
         
-    def capture_single_key(topic, event, key, value)
-      run_stmt(:capture_event, topic.to_s, event.to_s, key.to_s, nil ,1, value)
+    def capture_single_key(topic, event, key, value, time=nil)
+      run_stmt(:capture_event, topic.to_s, event.to_s, key.to_s, time ,1, value)
     end
     
     def flush
@@ -302,37 +306,10 @@ class Litemetric
     end
 
     def create_connection
-      conn = super
-      conn.execute("ATTACH ? as m", @options[:dbpath].to_s)
-      conn.wal_autocheckpoint = 10000
-      sql = YAML.load_file("#{__dir__}/litemetric_collector.sql.yml")
-      version = conn.get_first_value("PRAGMA user_version")
-      sql["schema"].each_pair do |v, obj| 
-        if v > version
-          conn.transaction do 
-            obj.each do |k, s| 
-              begin
-                conn.execute(s)
-              rescue Exception => e
-                STDERR.puts "Error parsing #{k}"
-                STDERR.puts s
-                raise e
-              end
-            end
-            conn.user_version = v
-          end
-        end
-      end 
-      sql["stmts"].each do |k, v| 
-        begin
-          conn.stmts[k.to_sym] = conn.prepare(v) 
-        rescue Exception => e
-          STDERR.puts "Error parsing #{k}"
-          STDERR.puts v
-          raise e
-        end
+      super("#{__dir__}/litemetric_collector.sql.yml") do |conn|
+        conn.execute("ATTACH ? as m", @options[:dbpath].to_s)      
+        conn.wal_autocheckpoint = 10000
       end
-      conn
     end
     
   end

data/lib/litestack/litemetric.sql.yml

@@ -13,13 +13,13 @@ schema:
         name TEXT DEFAULT('___') NOT NULL ON CONFLICT REPLACE, 
         key TEXT DEFAULT('___') NOT NULL ON CONFLICT REPLACE, 
         count INTEGER DEFAULT(0) NOT NULL ON CONFLICT REPLACE, 
-        value INTEGER, 
-        minimum INTEGER, 
-        maximum INTEGER, 
+        value REAL, 
+        minimum REAL, 
+        maximum REAL, 
         created_at INTEGER DEFAULT((unixepoch()/300*300)) NOT NULL, 
         resolution TEXT DEFAULT('minute') NOT NULL, 
         PRIMARY KEY(resolution, created_at, topic, name, key)
-      );        
+      ) STRICT;        
 
     create_topic_index_on_events: CREATE INDEX events_topic_index ON events (resolution, created_at, topic) WHERE name = '___';
     create_event_index_on_events: CREATE INDEX events_event_index ON events (resolution, created_at , topic, name) WHERE key = '___';
@@ -226,7 +226,7 @@ stmts:
     FROM events 
     WHERE topic = ?1 AND name = ?2 AND resolution = ?3 
     AND created_at <= unixepoch()
-    AND created_at >= unixepoch() -?6 
+    AND created_at >= unixepoch() - ?6 
     AND key != '___'
     AND iif(length(?5) > 0, rowid IN (SELECT rowid FROM events_search WHERE key LIKE ?5), TRUE)
     GROUP BY key
@@ -251,7 +251,7 @@ stmts:
     FROM events 
     WHERE topic = ?1 AND name = ?2 AND resolution = ?3 
     AND created_at <= unixepoch()
-    AND created_at >= unixepoch() -?6 
+    AND created_at >= unixepoch() - ?6 
     AND key != '___'
     AND iif(length(?5) > 0, rowid IN (SELECT rowid FROM events_search WHERE key LIKE ?5), TRUE)
     GROUP BY key
@@ -272,8 +272,8 @@ stmts:
     SELECT 
       datetime(ts.slot, 'unixepoch') AS rtime, 
       ts.slot AS unixtime, 
-      format('%.2f', sum(events.count)) AS rcount, 
-      format('%.2f', avg(events.value/events.count)) AS ravg 
+      sum(events.count) AS rcount, 
+      avg(events.value/events.count) AS ravg 
     FROM (
       WITH RECURSIVE timeslot(x) AS ( 
         SELECT (unixepoch()/(?1))*(?1) 
@@ -294,8 +294,9 @@ stmts:
     SELECT 
       datetime(ts.slot, 'unixepoch') AS rtime, 
       ts.slot AS unixtime, 
-      format('%.2f', sum(events.count)) AS rcount, 
-      format('%.2f', avg(events.value/events.count)) AS ravg 
+      events.count AS rcount, 
+      events.value / events.count AS ravg, 
+      events.value AS rtotal 
     FROM (
       WITH RECURSIVE timeslot(x) AS ( 
         SELECT (unixepoch()/(?1))*(?1) 
@@ -317,8 +318,9 @@ stmts:
     SELECT 
       datetime(ts.slot, 'unixepoch') AS rtime, 
       ts.slot AS unixtime, 
-      format('%.2f', sum(events.count)) AS rcount, 
-      format('%.2f', avg(events.value/events.count)) AS ravg 
+      events.count AS rcount, 
+      events.value / events.count AS ravg, 
+      events.value AS rtotal 
     FROM (
       WITH RECURSIVE timeslot(x) AS ( 
         SELECT (unixepoch()/(?1))*(?1) 

data/lib/litestack/litemetric_collector.sql.yml

@@ -7,13 +7,13 @@ schema:
         name TEXT DEFAULT('___') NOT NULL ON CONFLICT REPLACE, 
         key TEXT DEFAULT('___') NOT NULL ON CONFLICT REPLACE, 
         count INTEGER DEFAULT(0) NOT NULL ON CONFLICT REPLACE, 
-        value INTEGER, 
-        minimum INTEGER, 
-        maximum INTEGER, 
+        value REAL, 
+        minimum REAL, 
+        maximum REAL, 
         created_at INTEGER DEFAULT((unixepoch()/300*300)) NOT NULL ON CONFLICT REPLACE, 
         resolution TEXT DEFAULT('minute') NOT NULL, 
         PRIMARY KEY(resolution, created_at, topic, name, key)
-      );        
+      ) STRICT;        
 
 stmts:
       

data/lib/litestack/litequeue.rb

@@ -78,12 +78,12 @@ class Litequeue
   
   # deletes all the entries in all queues, or if a queue name is given, deletes all entries in that specific queue
   def clear(queue=nil)
-    run_sql("DELETE FROM queue WHERE iif(?, name = ?,  1)", queue)
+    run_sql("DELETE FROM queue WHERE iif(?1 IS NOT NULL, name = ?1,  TRUE)", queue)
   end
 
   # returns a count of entries in all queues, or if a queue name is given, reutrns the count of entries in that queue
   def count(queue=nil)
-    run_sql("SELECT count(*) FROM queue WHERE iif(?, name = ?, 1)", queue)[0][0]
+    run_sql("SELECT count(*) FROM queue WHERE iif(?1 IS NOT NULL, name = ?1, TRUE)", queue)[0][0]
   end
   
   # return the size of the queue file on disk
@@ -116,27 +116,16 @@ class Litequeue
   private  
       
   def create_connection
-    conn = super
-    conn.wal_autocheckpoint = 10000 
-    sql = YAML.load_file("#{__dir__}/litequeue.sql.yml")
-    version = conn.get_first_value("PRAGMA user_version")
-    sql["schema"].each_pair do |v, obj|
-      if v > version
-        conn.transaction(:immediate) do 
-          obj.each{|k, s| conn.execute(s)}
-          conn.user_version = v
+    super("#{__dir__}/litequeue.sql.yml") do |conn|
+      conn.wal_autocheckpoint = 10000 
+      # check if there is an old database and convert entries to the new format
+      if conn.get_first_value("select count(*) from sqlite_master where name = '_ul_queue_'") == 1
+        conn.transaction(:immediate) do
+          conn.execute("INSERT INTO queue(fire_at, name, value, created_at) SELECT fire_at, queue, value, created_at FROM _ul_queue_")
+          conn.execute("DROP TABLE _ul_queue_")
         end
       end
-    end 
-    sql["stmts"].each { |k, v| conn.stmts[k.to_sym] = conn.prepare(v) }
-    # check if there is an old database and convert entries to the new format
-    if conn.get_first_value("select count(*) from sqlite_master where name = '_ul_queue_'") == 1
-      conn.transaction(:immediate) do
-        conn.execute("INSERT INTO queue(fire_at, name, value, created_at) SELECT fire_at, queue, value, created_at FROM _ul_queue_")
-        conn.execute("DROP TABLE _ul_queue_")
-      end
     end
-    conn
   end
 
 end  

data/lib/litestack/litescheduler.rb

@@ -0,0 +1,84 @@
+# frozen_stringe_literal: true
+
+module Litescheduler
+  # cache the scheduler we are running in
+  # it is an error to change the scheduler for a process
+  # or for a child forked from that process
+  def self.backend
+    @backend ||= if Fiber.scheduler
+      :fiber
+    elsif defined? Polyphony
+      :polyphony
+    elsif defined? Iodine
+      :iodine 
+    else
+      :threaded
+    end
+  end
+  
+  # spawn a new execution context
+  def self.spawn(&block)
+    if backend == :fiber
+      Fiber.schedule(&block)
+    elsif backend == :polyphony
+      spin(&block)
+    elsif backend == :threaded or backend == :iodine
+      Thread.new(&block)
+    end
+    # we should never reach here
+  end
+  
+  def self.storage
+    if backend == :fiber || backend == :poylphony
+      Fiber.current.storage
+    else
+      Thread.current
+    end
+  end
+  
+  def self.current
+    if backend == :fiber || backend == :poylphony
+      Fiber.current
+    else
+      Thread.current
+    end
+  end
+  
+  # switch the execution context to allow others to run
+  def self.switch
+    if backend == :fiber
+      Fiber.scheduler.yield
+      true
+    elsif backend == :polyphony
+      Fiber.current.schedule
+      Thread.current.switch_fiber
+      true
+    else
+      #Thread.pass
+      false
+    end   
+  end
+  
+  # bold assumption, we will only synchronize threaded code!
+  # If some code explicitly wants to synchronize a fiber
+  # they must send (true) as a parameter to this method
+  # else it is a no-op for fibers
+  def self.synchronize(fiber_sync = false, &block)
+    if backend == :fiber or backend == :polyphony
+      yield # do nothing, just run the block as is
+    else
+      self.mutex.synchronize(&block)
+    end
+  end
+  
+  def self.max_contexts
+    return 50 if backend == :fiber || backend == :polyphony
+    5    
+  end
+  
+  # mutex initialization
+  def self.mutex
+    # a single mutex per process (is that ok?)
+    @@mutex ||= Mutex.new
+  end
+end