litestack 0.2.3 → 0.3.0

data/lib/litestack/litedb.rb

@@ -1,9 +1,15 @@
 # all components should require the support module
 require_relative 'litesupport'
 
+# all measurable components should require the litemetric class
+require_relative 'litemetric'
+
 # Litedb inherits from the SQLite3::Database class and adds a few initialization options
 class Litedb < ::SQLite3::Database
 
+  # add litemetric support
+  include Litemetric::Measurable
+  
   # overrride the original initilaizer to allow for connection configuration
   def initialize(file, options = {}, zfs = nil )
     if block_given?
@@ -15,6 +21,13 @@ class Litedb < ::SQLite3::Database
       super(file, options, zfs)
       init unless options[:noinit] == true
     end
+    @running = true
+    @collecting_metrics = options[:metrics]
+    collect_metrics if @collecting_metrics
+  end
+
+  def collecting_metrics?
+    @collecting_metrics
   end
 
   # enforce immediate mode to avoid deadlocks for a small performance penalty
@@ -22,6 +35,67 @@ class Litedb < ::SQLite3::Database
     super(mode)
   end
 
+  # return the size of the database file  
+  def size
+    execute("SELECT s.page_size * c.page_count FROM pragma_page_size() AS s, pragma_page_count() AS c")[0][0]
+  end
+  
+  def schema_object_count(type = nil)
+    execute("SELECT count(*) FROM SQLITE_MASTER WHERE iif(?1 IS NOT NULL, type = ?1, TRUE)", type)[0][0]
+  end
+  
+  # collect snapshot information
+  def snapshot
+    {
+      summary: {
+        path: filename,
+        journal_mode: journal_mode,
+        synchronous: synchronous,
+        size: size.to_f / (1024 * 1024),
+        tables: schema_object_count('table'), 
+        indexes: schema_object_count('index')
+      }
+    }
+  end
+
+  # override prepare to return Litedb::Statement (and pass the sql to it)
+  def prepare(sql)
+    stmt = Litedb::Statement.new(self, sql)
+    stmt.sql = sql.strip.upcase
+    return stmt unless block_given?
+    begin
+      yield stmt
+    ensure
+      stmt.close unless stmt.closed?
+    end
+  end  
+
+  # override execute to capture metrics  
+  def execute(sql, bind_vars = [], *args, &block)
+    if bind_vars.nil? || !args.empty?
+      if args.empty?
+        bind_vars = []
+      else
+        bind_vars = [bind_vars] + args
+      end
+    end
+    
+    prepare(sql) do |stmt|
+      measure(stmt.stmt_type, stmt.sql) do
+        stmt.bind_params(bind_vars)
+        stmt = SQLite3::ResultSet.new self, stmt
+      end
+      if block_given?
+        stmt.each do |row|
+          yield row
+        end
+      else
+        stmt.to_a
+      end
+    end  
+   
+  end
+
   private
 
   # default connection configuration values 
@@ -41,7 +115,56 @@ class Litedb < ::SQLite3::Database
     # increase the local connection cache to 2000 pages
     self.cache_size = 2000
   end
-
+  
 end
 
+# the Litedb::Statement also inherits from SQLite3::Statement
+class Litedb::Statement < SQLite3::Statement      
+    
+  include Litemetric::Measurable
+
+  attr_accessor :sql
+
+  def initialize(db, sql)
+    super(db, sql)
+    collect_metrics if db.collecting_metrics?
+  end
+
+  def metrics_identifier
+    "Litedb" # overridden to match the parent class
+  end
+  
+  # return the type of the statement 
+  def stmt_type
+    @stmt_type ||= detect_stmt_type
+  end
+  
+  def detect_stmt_type
+    if @sql.start_with?("SEL") || @sql.start_with?("WITH")
+      "Read"
+    elsif @sql.start_with?("CRE") || @sql.start_with?("ALT") || @sql.start_with?("DRO")
+      "Schema change"
+    elsif @sql.start_with?("PRA") 
+      "Pragma"
+    else
+      "Write"
+    end        
+  end
+  
+  # overriding each to measure the query time (plus the processing time as well, sadly) 
+  def each
+    measure(stmt_type, @sql) do
+      super
+    end 
+  end
+  
+  # overriding execute to measure the query time  
+  def execute(*bind_vars)
+    res = nil
+    measure(stmt_type, @sql) do
+      res = super(*bind_vars)
+    end
+    res
+  end
 
+end

data/lib/litestack/litejob.rb

@@ -74,11 +74,11 @@ module Litejob
     end  
          
     def queue
-      @@queue ||= "default"
+      @queue_name ||= "default"
     end
     
     def queue=(queue_name)
-      @@queue = queue_name.to_s
+      @queue_name = queue_name.to_s
     end
 
     def options

data/lib/litestack/litejobqueue.rb

@@ -19,7 +19,7 @@ class Litejobqueue < Litequeue
   # can be overriden by passing new options in a hash 
   # to Litejobqueue.new, it will also be then passed to the underlying Litequeue object
   #   config_path: "./litejob.yml" -> were to find the configuration file (if any)
-  #   path: "./queue.db"
+  #   path: "./db/queue.db"
   #   mmap_size: 128 * 1024 * 1024 -> 128MB to be held in memory
   #   sync: 1 -> sync only when checkpointing
   #   queues: [["default", 1, "spawn"]] -> an array of queues to process 
@@ -32,7 +32,7 @@ class Litejobqueue < Litequeue
   # This can be particularly useful for long running, IO bound jobs. It is not recommended though for threaded environments, as it can result in creating many threads that may consudme a lot of memory. 
   DEFAULT_OPTIONS = {
     config_path: "./litejob.yml",
-    path: "./queue.db",
+    path: Litesupport.root.join("queue.sqlite3"),
     queues: [["default", 1]],
     workers: 5,
     retries: 5, 
@@ -79,6 +79,11 @@ class Litejobqueue < Litequeue
       pgroups[q[1]] << [q[0], q[2] == "spawn"]
     end
     @queues = pgroups.keys.sort.reverse.collect{|p| [p, pgroups[p]]}
+    collect_metrics if @options[:metrics]
+  end
+
+  def metrics_identifier
+    "Litejob" # overrides default identifier
   end
   
   # push a job to the queue
@@ -164,12 +169,7 @@ class Litejobqueue < Litequeue
   def job_finished
     Litesupport.synchronize(@mutex){@jobs_in_flight -= 1}
   end
-  
-  # return a hash encapsulating the info about the current jobqueue
-  def snapshot
-    info
-  end
-  
+    
   # optionally run a job in its own context
   def schedule(spawn = false, &block)
     if spawn

data/lib/litestack/litemetric.rb

@@ -13,11 +13,12 @@ class Litemetric
   
   DEFAULT_OPTIONS = {
     config_path: "./litemetric.yml",
-    path: "./metrics.db",
+    path: Litesupport.root.join("metrics.sqlite3"),
     sync: 1,
-    mmap_size: 16 * 1024 * 1024, # 16MB of memory to easily process 1 year worth of data
-    flush_interval: 10, # flush data every 1 minute
-    summarize_interval: 10 # summarize data every 1 minute
+    mmap_size: 128 * 1024 * 1024, # 16MB of memory to easily process 1 year worth of data
+    flush_interval:  10, # flush data every 10 seconds
+    summarize_interval: 30, # summarize data every 1/2 minute
+    snapshot_interval: 10*60 # snapshot every 10 minutes
   }
 
   RESOLUTIONS = {
@@ -27,8 +28,20 @@ class Litemetric
     week: 7*24*3600 # 1 week (lowest resolution)
   }
 
+  # :nodoc:
+  def self.options=(options)
+    # an ugly hack to pass options to a singleton
+    # need to rethink the whole singleton thing
+    @options = options
+  end
+  
+  def self.options
+    @options
+  end
+
   # :nodoc: 
   def initialize(options = {})
+    options = options.merge(Litemetric.options) if Litemetric.options
     init(options)
   end
   
@@ -41,36 +54,18 @@ class Litemetric
   
   ## event capturing
   ##################
-    
-  def capture(topic, event, key=event, value=nil)
-    if key.is_a? Array
-      key.each{|k| capture_single_key(topic, event, k, value)}
-    else
-      capture_single_key(topic, event, key, value)
-    end
-  end
   
-  def capture_single_key(topic, event, key=event, value=nil)
-    @mutex.synchronize do
-      time_slot = current_time_slot # should that be 5 minutes?
-      topic_slot = @metrics[topic]
-      if event_slot = topic_slot[event]
-        if key_slot = event_slot[key]
-          if key_slot[time_slot]
-            key_slot[time_slot][:count] += 1
-            key_slot[time_slot][:value] += value unless value.nil?
-          else # new time slot
-            key_slot[time_slot] = {count: 1, value: value}
-          end
-        else
-          event_slot[key] = {time_slot => {count: 1, value: value}}
-        end
-      else # new event
-        topic_slot[event] = {key => {time_slot => {count: 1, value: value}}}
-      end
-    end
+  def current_time_slot
+    (Time.now.to_i / 300) * 300
   end
   
+  def capture(topic, event, key=event, value=nil)
+    @collector.capture(topic, event, key, value, current_time_slot)
+  end
+      
+  def capture_snapshot(topic, state)
+    run_stmt(:capture_state, topic, Oj.dump(state))
+  end
 
   ## event reporting
   ##################
@@ -78,17 +73,48 @@ class Litemetric
   def topics
     run_stmt(:list_topics).to_a
   end
-
-  def event_names(resolution, topic)
-    run_stmt(:list_event_names, resolution, topic).to_a
+  
+  def topic_summaries(resolution, count, order, dir, search)
+    search = "%#{search}%" if search
+    if dir.downcase == "desc" 
+      run_stmt(:topics_summaries, resolution, count, order, search).to_a
+    else
+      run_stmt(:topics_summaries_asc, resolution, count, order, search).to_a
+    end
   end
-
-  def keys(resolution, topic, event_name)
-    run_stmt(:list_event_keys, resolution, topic, event_name).to_a
+    
+  def events_summaries(topic, resolution, order, dir, search, count)
+    search = "%#{search}%" if search
+    if dir.downcase == "desc" 
+      run_stmt_hash(:events_summaries, topic, resolution, order, search, count)
+    else
+      run_stmt_hash(:events_summaries_asc, topic, resolution, order, search, count)
+    end
+  end
+  
+  def keys_summaries(topic, event, resolution, order, dir, search, count)
+    search = "%#{search}%" if search
+    if dir.downcase == "desc" 
+      run_stmt_hash(:keys_summaries, topic, event, resolution, order, search, count).to_a
+    else
+      run_stmt_hash(:keys_summaries_asc, topic, event, resolution, order, search, count).to_a
+    end 
   end
 
-  def event_data(resolution, topic, event_name, key)
-    run_stmt(:list_events_by_key, resolution, topic, event_name, key).to_a
+  def topic_data_points(step, count, resolution, topic)
+    run_stmt(:topic_data_points, step, count, resolution, topic).to_a
+  end 
+
+  def event_data_points(step, count, resolution, topic, event)
+    run_stmt_hash(:event_data_points, step, count, resolution, topic, event).to_a
+  end 
+
+  def key_data_points(step, count, resolution, topic, event, key)
+    run_stmt_hash(:key_data_points, step, count, resolution, topic, event, key).to_a
+  end 
+
+  def snapshot(topic)
+    run_stmt(:snapshot, topic)[0].to_a
   end
   
   ## summarize data
@@ -107,9 +133,23 @@ class Litemetric
   ###################
 
   private
-
+  
+  def run_stmt_hash(stmt, *args)
+    res = run_stmt(stmt, *args)
+    cols = run_stmt_method(stmt, :columns)
+    hashes = []
+    res.each do | row |
+      hash = {}
+      row.each_with_index do |field, i|
+        hash[cols[i]] = field
+      end
+      hashes << hash
+    end
+    hashes
+  end
+  
   def exit_callback
-    puts "--- Litemetric detected an exit, flushing metrics"
+    STDERR.puts "--- Litemetric detected an exit, flushing metrics"
     @running = false
     flush
   end
@@ -118,9 +158,10 @@ class Litemetric
     super
     @metrics = {}
     @registered = {}
-    @flusher = create_flusher
-    @summarizer = create_summarizer
     @mutex = Litesupport::Mutex.new
+    @collector = Litemetric::Collector.new({dbpath: @options[:path]})
+    @summarizer = create_summarizer
+    @flusher = create_flusher
   end
 
   def current_time_slot
@@ -128,53 +169,20 @@ class Litemetric
   end
   
   def flush
-    to_delete = []
-    @conn.acquire do |conn|
-      conn.transaction(:immediate) do
-        @metrics.each_pair do |topic, event_hash|
-          event_hash.each_pair do |event, key_hash|
-            key_hash.each_pair do |key, time_hash|
-              time_hash.each_pair do |time, data|
-                conn.stmts[:capture_event].execute!(topic, event.to_s, key, time, data[:count], data[:value]) if data 
-                time_hash[time] = nil 
-                to_delete << [topic, event, key, time]
-              end
-            end      
-          end      
-        end
-      end
-    end
-    to_delete.each do |r| 
-      @metrics[r[0]][r[1]][r[2]].delete(r[3])
-      @metrics[r[0]][r[1]].delete(r[2]) if @metrics[r[0]][r[1]][r[2]].empty? 
-      @metrics[r[0]].delete(r[1]) if @metrics[r[0]][r[1]].empty? 
-    end
-  end  
+    @collector.flush
+  end
      
   def create_connection
-    conn = super
-    conn.wal_autocheckpoint = 10000
-    sql = YAML.load_file("#{__dir__}/litemetric.sql.yml")
-    version = conn.get_first_value("PRAGMA user_version")
-    sql["schema"].each_pair do |v, obj| 
-      if v > version
-        conn.transaction do 
-          obj.each{|k, s| conn.execute(s)}
-          conn.user_version = v
-        end
-      end
-    end 
-    sql["stmts"].each { |k, v| conn.stmts[k.to_sym] = conn.prepare(v) }
-    conn
+    super("#{__dir__}/litemetric.sql.yml") do |conn|
+      conn.wal_autocheckpoint = 10000 # checkpoint after 10000 pages are written
+    end
   end
   
   def create_flusher
     Litesupport.spawn do
       while @running do
         sleep @options[:flush_interval]
-        @mutex.synchronize do
-          flush
-        end
+        flush
       end      
     end 
   end
@@ -199,6 +207,16 @@ class Litemetric
     def collect_metrics
       @litemetric = Litemetric.instance
       @litemetric.register(metrics_identifier)
+      @snapshotter = create_snapshotter      
+    end
+
+    def create_snapshotter
+      Litesupport.spawn do
+        while @running do
+          sleep @litemetric.options[:snapshot_interval]
+          capture_snapshot
+        end      
+      end 
     end
 
     def metrics_identifier
@@ -211,18 +229,89 @@ class Litemetric
     end
     
     def measure(event, key=event)
-      return yield unless @litemetric
+      unless @litemetric
+        yield
+        return 0 
+      end
       t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC) 
-      res = yield
+      yield
       t2 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      value = (( t2 - t1 ) * 1000).round # capture time in milliseconds
+      value =  t2 - t1 
       capture(event, key, value)
-      res  
+      value # return value so other events can reuse it  
     end    
     
-    def snapshot
-      raise Litestack::NotImplementedError
+    def capture_snapshot
+      return unless @litemetric
+      state = snapshot if defined? snapshot
+      if state
+        @litemetric.capture_snapshot(metrics_identifier, state)
+      end
+    end
+        
+  end
+end
+
+class Litemetric
+
+  class Collector
+
+    include Litesupport::Liteconnection
+    
+    DEFAULT_OPTIONS = {
+      path: ":memory:",
+      sync: 1,
+      flush_interval:  3, # flush data every 1 minute
+      summarize_interval: 10, # summarize data every 1 minute
+      snapshot_interval: 1 # snapshot every 10 minutes
+    }
+
+    RESOLUTIONS = {
+      minute: 300, # 5 minutes (highest resolution)
+      hour: 3600, # 1 hour
+      day: 24*3600, # 1 day
+      week: 7*24*3600 # 1 week (lowest resolution)
+    }
+    
+    def initialize(options = {})
+      init(options)
+    end
+    
+    def capture(topic, event, key, value=nil, time=nil)
+      if key.is_a? Array
+        key.each{|k| capture_single_key(topic, event, k, value, time)}
+      else
+        capture_single_key(topic, event, key, value, time)
+      end
+    end
+        
+    def capture_single_key(topic, event, key, value, time=nil)
+      run_stmt(:capture_event, topic.to_s, event.to_s, key.to_s, time ,1, value)
+    end
+    
+    def flush
+      t = Time.now
+      limit = 1000 # migrate 1000 records at a time
+      count = run_stmt(:event_count)[0][0]
+      while count > 0
+        @conn.acquire do |conn|
+          conn.transaction(:immediate) do
+            conn.stmts[:migrate_events].execute!(limit)
+            conn.stmts[:delete_migrated_events].execute!(limit)
+            count = conn.stmts[:event_count].execute![0][0]
+          end         
+        end
+        sleep 0.005 #give other threads a chance to run
+      end
+    end
+
+    def create_connection
+      super("#{__dir__}/litemetric_collector.sql.yml") do |conn|
+        conn.execute("ATTACH ? as m", @options[:dbpath].to_s)      
+        conn.wal_autocheckpoint = 10000
+      end
     end
     
   end
+  
 end