litestack 0.2.2 → 0.2.6

data/lib/litestack/liteboard/views/topic.erb

@@ -0,0 +1,48 @@
+    <h5><a href="/?res=<%=@res%>">All Topics</a> > <%= @topic %></h5>
+    <% if @snapshot and not @snapshot.empty? and not @snapshot[0].nil?%>
+      <div><b>Snapshot</b> captured on <%= @snapshot[1] %></div>
+      <% @snapshot[0].each_pair do |k, v| %>
+        <table class ="table summary">
+          <tr>
+            <th colspan="2"><%=k.to_s.capitalize%></th>
+          </tr>
+          <% v.each do |ss| %>
+          <tr>
+            <td><%= ss[0].to_s.capitalize.gsub("_", " ") %></td>
+            <td><%= ss[1] %></td>
+          </tr>
+          <% end %>
+        </table>
+        <br/>
+      <% end %>
+    <% end %>
+    <div><b>Dynamic event data</b></div>
+    <div id="search"><form><input id="search-field" type="text" placeholder="Search events" onkeydown="search_kd(this)" onkeyup="search_ku(this)" value="<%=@search%>"/></form></div>    <table class="table sortable">
+      <tr>
+        <th width="16%" class="<%='sorted' if @order == 'name'%>"><a href="<%=topic_sort_url('name')%>">Event</a> <%=dir('name')%></th>
+        <th width="8%" class="<%='sorted' if @order == 'rcount'%>"><a href="<%=topic_sort_url('rcount')%>">Event Count</a> <%=dir('rcount')%></th>
+        <th width="8%" class="<%='sorted' if @order == 'ravg'%>"><a href="<%=topic_sort_url('ravg')%>">Avg Value</a> <%=dir('ravg')%></th>
+        <th width="8%" class="<%='sorted' if @order == 'rtotal'%>"><a href="<%=topic_sort_url('rtotal')%>">Total Value</a> <%=dir('rtotal')%></th>
+        <th width="8%" class="<%='sorted' if @order == 'rmin'%>"><a href="<%=topic_sort_url('rmin')%>">Min Value</a> <%=dir('rmin')%></th>
+        <th width="8%" class="<%='sorted' if @order == 'rmax'%>"><a href="<%=topic_sort_url('rmax')%>">Max Value</a> <%=dir('rmax')%></th>
+        <th width="22%">Events over time</th>
+        <th width="22%">Average value over time</th>
+      </tr>
+      <% @events.each do |event|%>
+      <tr>
+        <td title="<%=event[0]%>"><div class="label"><a href="/topics/<%=encode(@topic)%>/events/<%=encode(event[0])%>?res=<%=@res%>"><%=event[0]%></a></div></td>
+        <td><%=event[2]%></td>
+        <td><%="%0.2f" % [event[3]] if event[3] %></td>
+        <td><%="%0.2f" % [event[4]] if event[4] %></td>
+        <td><%="%0.2f" % [event[5]] if event[5] %></td>
+        <td><%="%0.2f" % [event[6]] if event[6] %></td>
+        <td class="chart"><span class="inlinecolumn hidden" data-label="Count"><%=Oj.dump(event[7]) if event[7]%></span></td>
+        <td class="chart"><span class="inlinecolumn hidden" data-label="Avg Value"><%=Oj.dump(event[8]) if event[8]%></span></td>
+      </tr>
+      <% end %>
+      <% if @events.empty? %>
+        <tr>
+         <td class="empty" colspan="9">No data to display</td>
+        </tr>
+      <% end %>
+      </table>

data/lib/litestack/litecable.rb

@@ -15,7 +15,7 @@ class Litecable
 
   DEFAULT_OPTIONS = {
     config_path: "./litecable.yml",
-    path: "./cable.db",
+    path: Litesupport.root.join("cable.sqlite3"),
     sync: 0,
     mmap_size: 16 * 1024 * 1024, # 16MB
     expire_after: 5, # remove messages older than 5 seconds

data/lib/litestack/litecache.rb

@@ -32,7 +32,7 @@ class Litecache
   #   sleep_interval: 1 -> 1 second of sleep between cleanup runs
 
   DEFAULT_OPTIONS = {
-    path: "./cache.db",
+    path: Litesupport.root.join("cache.sqlite3"),
     config_path: "./litecache.yml",
     sync: 0,
     expiry: 60 * 60 * 24 * 30, # one month 
@@ -64,8 +64,8 @@ class Litecache
   
   def initialize(options = {})
     options[:size] = DEFAULT_OPTIONS[:min_size] if options[:size] && options[:size] < DEFAULT_OPTIONS[:min_size]        
-    init(options)    
     @last_visited = {}
+    init(options)    
     collect_metrics if @options[:metrics]
   end
   
@@ -76,7 +76,7 @@ class Litecache
     @conn.acquire do |cache|
       begin
         cache.stmts[:setter].execute!(key, value, expires_in)
-        capture(:write, key)
+        capture(:set, key)
       rescue SQLite3::FullException
         cache.stmts[:extra_pruner].execute!(0.2)
         cache.execute("vacuum")
@@ -97,7 +97,7 @@ class Litecache
           cache.stmts[:inserter].execute!(key, value, expires_in)
           changes = cache.changes
         end
-        capture(:write, key)
+        capture(:set, key)
       rescue SQLite3::FullException
         cache.stmts[:extra_pruner].execute!(0.2)
         cache.execute("vacuum")
@@ -113,10 +113,10 @@ class Litecache
     key = key.to_s
     if record = @conn.acquire{|cache| cache.stmts[:getter].execute!(key)[0] }
       @last_visited[key] = true
-      capture(:hit, key)
+      capture(:get, key, 1)
       return record[1]
     end
-    capture(:miss, key)
+    capture(:get, key, 0)
     nil
   end
   
@@ -160,9 +160,9 @@ class Litecache
   end
   
   # return the actual size of the cache file
-  def size
-    run_stmt(:sizer)[0][0]
-  end
+  #def size
+  #  run_stmt(:sizer)[0][0]
+  #end
   
   # delete all key, value pairs in the cache
   def clear
@@ -177,9 +177,23 @@ class Litecache
   
   # return the maximum size of the cache
   def max_size
-    run_sql("SELECT s.page_size * c.max_page_count FROM pragma_page_size() as s, pragma_max_page_count() as c")[0][0]
+    run_sql("SELECT s.page_size * c.max_page_count FROM pragma_page_size() as s, pragma_max_page_count() as c")[0][0].to_f / (1024 * 1024)
   end
   
+  def snapshot
+    {
+      summary: {
+        path: path,
+        journal_mode: journal_mode,
+        synchronous: synchronous,
+        size: size,
+        max_size: max_size,
+        entries: count
+      }
+    }
+  end
+
+  
   # low level access to SQLite transactions, use with caution
   def transaction(mode, acquire=true)
     return cache.transaction(mode){yield} unless acquire
@@ -239,6 +253,6 @@ class Litecache
     end 
     sql["stmts"].each { |k, v| conn.stmts[k.to_sym] = conn.prepare(v) }
     conn
-  end
+  end  
   
 end

data/lib/litestack/litedb.rb

@@ -4,6 +4,9 @@ require_relative 'litesupport'
 # Litedb inherits from the SQLite3::Database class and adds a few initialization options
 class Litedb < ::SQLite3::Database
 
+  # add litemetric support
+  include Litemetric::Measurable
+  
   # overrride the original initilaizer to allow for connection configuration
   def initialize(file, options = {}, zfs = nil )
     if block_given?
@@ -15,6 +18,13 @@ class Litedb < ::SQLite3::Database
       super(file, options, zfs)
       init unless options[:noinit] == true
     end
+    @running = true
+    @collecting_metrics = options[:metrics]
+    collect_metrics if @collecting_metrics
+  end
+
+  def collecting_metrics?
+    @collecting_metrics
   end
 
   # enforce immediate mode to avoid deadlocks for a small performance penalty
@@ -22,6 +32,61 @@ class Litedb < ::SQLite3::Database
     super(mode)
   end
 
+  # return the size of the database file  
+  def size
+    execute("SELECT s.page_size * c.page_count FROM pragma_page_size() as s, pragma_page_count() as c")[0][0]
+  end
+  
+  # collect snapshot information
+  def snapshot
+    {
+      summary: {
+        path: filename,
+        journal_mode: journal_mode,
+        synchronous: synchronous,
+        size: size
+      }
+    }
+  end
+
+  # override prepare to return Litedb::Statement (and pass the sql to it)
+  def prepare(sql)
+    stmt = Litedb::Statement.new(self, sql)
+    stmt.sql = sql.strip.upcase
+    return stmt unless block_given?
+    begin
+      yield stmt
+    ensure
+      stmt.close unless stmt.closed?
+    end
+  end  
+
+  # override execute to capture metrics  
+  def execute(sql, bind_vars = [], *args, &block)
+    if bind_vars.nil? || !args.empty?
+      if args.empty?
+        bind_vars = []
+      else
+        bind_vars = [bind_vars] + args
+      end
+    end
+    
+    prepare(sql) do |stmt|
+      measure(stmt.stmt_type, stmt.sql) do
+        stmt.bind_params(bind_vars)
+        stmt = SQLite3::ResultSet.new self, stmt
+      end
+      if block_given?
+        stmt.each do |row|
+          yield row
+        end
+      else
+        stmt.to_a
+      end
+    end  
+   
+  end
+
   private
 
   # default connection configuration values 
@@ -41,7 +106,56 @@ class Litedb < ::SQLite3::Database
     # increase the local connection cache to 2000 pages
     self.cache_size = 2000
   end
-
+  
 end
 
+# the Litedb::Statement also inherits from SQLite3::Statement
+class Litedb::Statement < SQLite3::Statement      
+    
+  include Litemetric::Measurable
+
+  attr_accessor :sql
+
+  def initialize(db, sql)
+    super(db, sql)
+    collect_metrics if db.collecting_metrics?
+  end
 
+  def metrics_identifier
+    "Litedb" # overridden to match the parent class
+  end
+  
+  # return the type of the statement 
+  def stmt_type
+    @stmt_type ||= detect_stmt_type
+  end
+  
+  def detect_stmt_type
+    if @sql.start_with?("SEL") || @sql.start_with?("WITH")
+      "Read"
+    elsif @sql.start_with?("CRE") || @sql.start_with?("ALT") || @sql.start_with?("DRO")
+      "Schema change"
+    elsif @sql.start_with?("PRA") 
+      "Pragma"
+    else
+      "Write"
+    end        
+  end
+  
+  # overriding each to measure the query time (plus the processing time as well, sadly) 
+  def each
+    measure(stmt_type, @sql) do
+      super
+    end 
+  end
+  
+  # overriding execute to measure the query time  
+  def execute(*bind_vars)
+    res = nil
+    measure(stmt_type, @sql) do
+      res = super(*bind_vars)
+    end
+    res
+  end
+
+end

data/lib/litestack/litejob.rb

@@ -74,11 +74,11 @@ module Litejob
     end  
          
     def queue
-      @@queue ||= "default"
+      @queue_name ||= "default"
     end
     
     def queue=(queue_name)
-      @@queue = queue_name.to_s
+      @queue_name = queue_name.to_s
     end
 
     def options

data/lib/litestack/litejobqueue.rb

@@ -19,7 +19,7 @@ class Litejobqueue < Litequeue
   # can be overriden by passing new options in a hash 
   # to Litejobqueue.new, it will also be then passed to the underlying Litequeue object
   #   config_path: "./litejob.yml" -> were to find the configuration file (if any)
-  #   path: "./queue.db"
+  #   path: "./db/queue.db"
   #   mmap_size: 128 * 1024 * 1024 -> 128MB to be held in memory
   #   sync: 1 -> sync only when checkpointing
   #   queues: [["default", 1, "spawn"]] -> an array of queues to process 
@@ -32,7 +32,7 @@ class Litejobqueue < Litequeue
   # This can be particularly useful for long running, IO bound jobs. It is not recommended though for threaded environments, as it can result in creating many threads that may consudme a lot of memory. 
   DEFAULT_OPTIONS = {
     config_path: "./litejob.yml",
-    path: "./queue.db",
+    path: Litesupport.root.join("queue.sqlite3"),
     queues: [["default", 1]],
     workers: 5,
     retries: 5, 
@@ -79,6 +79,11 @@ class Litejobqueue < Litequeue
       pgroups[q[1]] << [q[0], q[2] == "spawn"]
     end
     @queues = pgroups.keys.sort.reverse.collect{|p| [p, pgroups[p]]}
+    collect_metrics if @options[:metrics]
+  end
+
+  def metrics_identifier
+    "Litejob" # overrides default identifier
   end
   
   # push a job to the queue
@@ -164,12 +169,7 @@ class Litejobqueue < Litequeue
   def job_finished
     Litesupport.synchronize(@mutex){@jobs_in_flight -= 1}
   end
-  
-  # return a hash encapsulating the info about the current jobqueue
-  def snapshot
-    info
-  end
-  
+    
   # optionally run a job in its own context
   def schedule(spawn = false, &block)
     if spawn

data/lib/litestack/litemetric.rb

@@ -13,11 +13,12 @@ class Litemetric
   
   DEFAULT_OPTIONS = {
     config_path: "./litemetric.yml",
-    path: "./metrics.db",
+    path: Litesupport.root.join("metrics.sqlite3"),
     sync: 1,
-    mmap_size: 16 * 1024 * 1024, # 16MB of memory to easily process 1 year worth of data
-    flush_interval: 10, # flush data every 1 minute
-    summarize_interval: 10 # summarize data every 1 minute
+    mmap_size: 128 * 1024 * 1024, # 16MB of memory to easily process 1 year worth of data
+    flush_interval:  10, # flush data every 10 seconds
+    summarize_interval: 30, # summarize data every 1/2 minute
+    snapshot_interval: 10*60 # snapshot every 10 minutes
   }
 
   RESOLUTIONS = {
@@ -27,8 +28,20 @@ class Litemetric
     week: 7*24*3600 # 1 week (lowest resolution)
   }
 
+  # :nodoc:
+  def self.options=(options)
+    # an ugly hack to pass options to a singleton
+    # need to rethink the whole singleton thing
+    @options = options
+  end
+  
+  def self.options
+    @options
+  end
+
   # :nodoc: 
   def initialize(options = {})
+    options = options.merge(Litemetric.options) if Litemetric.options
     init(options)
   end
   
@@ -41,36 +54,15 @@ class Litemetric
   
   ## event capturing
   ##################
-    
+  
+  
   def capture(topic, event, key=event, value=nil)
-    if key.is_a? Array
-      key.each{|k| capture_single_key(topic, event, k, value)}
-    else
-      capture_single_key(topic, event, key, value)
-    end
+    @collector.capture(topic, event, key, value)
   end
-  
-  def capture_single_key(topic, event, key=event, value=nil)
-    @mutex.synchronize do
-      time_slot = current_time_slot # should that be 5 minutes?
-      topic_slot = @metrics[topic]
-      if event_slot = topic_slot[event]
-        if key_slot = event_slot[key]
-          if key_slot[time_slot]
-            key_slot[time_slot][:count] += 1
-            key_slot[time_slot][:value] += value unless value.nil?
-          else # new time slot
-            key_slot[time_slot] = {count: 1, value: value}
-          end
-        else
-          event_slot[key] = {time_slot => {count: 1, value: value}}
-        end
-      else # new event
-        topic_slot[event] = {key => {time_slot => {count: 1, value: value}}}
-      end
-    end
+      
+  def capture_snapshot(topic, state)
+    run_stmt(:capture_state, topic, Oj.dump(state))
   end
-  
 
   ## event reporting
   ##################
@@ -78,17 +70,48 @@ class Litemetric
   def topics
     run_stmt(:list_topics).to_a
   end
-
-  def event_names(resolution, topic)
-    run_stmt(:list_event_names, resolution, topic).to_a
+  
+  def topic_summaries(resolution, count, order, dir, search)
+    search = "%#{search}%" if search
+    if dir.downcase == "desc" 
+      run_stmt(:topics_summaries, resolution, count, order, search).to_a
+    else
+      run_stmt(:topics_summaries_asc, resolution, count, order, search).to_a
+    end
   end
-
-  def keys(resolution, topic, event_name)
-    run_stmt(:list_event_keys, resolution, topic, event_name).to_a
+    
+  def events_summaries(topic, resolution, order, dir, search, count)
+    search = "%#{search}%" if search
+    if dir.downcase == "desc" 
+      run_stmt(:events_summaries, topic, resolution, order, search, count).to_a
+    else
+      run_stmt(:events_summaries_asc, topic, resolution, order, search, count).to_a
+    end
+  end
+  
+  def keys_summaries(topic, event, resolution, order, dir, search, count)
+    search = "%#{search}%" if search
+    if dir.downcase == "desc" 
+      run_stmt(:keys_summaries, topic, event, resolution, order, search, count).to_a
+    else
+      run_stmt(:keys_summaries_asc, topic, event, resolution, order, search, count).to_a
+    end 
   end
 
-  def event_data(resolution, topic, event_name, key)
-    run_stmt(:list_events_by_key, resolution, topic, event_name, key).to_a
+  def topic_data_points(step, count, resolution, topic)
+    run_stmt(:topic_data_points, step, count, resolution, topic).to_a
+  end 
+
+  def event_data_points(step, count, resolution, topic, event)
+    run_stmt(:event_data_points, step, count, resolution, topic, event).to_a
+  end 
+
+  def key_data_points(step, count, resolution, topic, event, key)
+    run_stmt(:key_data_points, step, count, resolution, topic, event, key).to_a
+  end 
+
+  def snapshot(topic)
+    run_stmt(:snapshot, topic)[0].to_a
   end
   
   ## summarize data
@@ -109,7 +132,7 @@ class Litemetric
   private
 
   def exit_callback
-    puts "--- Litemetric detected an exit, flushing metrics"
+    STDERR.puts "--- Litemetric detected an exit, flushing metrics"
     @running = false
     flush
   end
@@ -118,9 +141,10 @@ class Litemetric
     super
     @metrics = {}
     @registered = {}
-    @flusher = create_flusher
-    @summarizer = create_summarizer
     @mutex = Litesupport::Mutex.new
+    @collector = Litemetric::Collector.new({dbpath: @options[:path]})
+    @summarizer = create_summarizer
+    @flusher = create_flusher
   end
 
   def current_time_slot
@@ -128,39 +152,24 @@ class Litemetric
   end
   
   def flush
-    to_delete = []
-    @conn.acquire do |conn|
-      conn.transaction(:immediate) do
-        @metrics.each_pair do |topic, event_hash|
-          event_hash.each_pair do |event, key_hash|
-            key_hash.each_pair do |key, time_hash|
-              time_hash.each_pair do |time, data|
-                conn.stmts[:capture_event].execute!(topic, event.to_s, key, time, data[:count], data[:value]) if data 
-                time_hash[time] = nil 
-                to_delete << [topic, event, key, time]
-              end
-            end      
-          end      
-        end
-      end
-    end
-    to_delete.each do |r| 
-      @metrics[r[0]][r[1]][r[2]].delete(r[3])
-      @metrics[r[0]][r[1]].delete(r[2]) if @metrics[r[0]][r[1]][r[2]].empty? 
-      @metrics[r[0]].delete(r[1]) if @metrics[r[0]][r[1]].empty? 
-    end
-  end  
+    @collector.flush
+  end
      
   def create_connection
     conn = super
-    conn.wal_autocheckpoint = 10000
+    conn.wal_autocheckpoint = 10000 # checkpoint after 10000 pages are written
     sql = YAML.load_file("#{__dir__}/litemetric.sql.yml")
     version = conn.get_first_value("PRAGMA user_version")
     sql["schema"].each_pair do |v, obj| 
       if v > version
-        conn.transaction do 
-          obj.each{|k, s| conn.execute(s)}
-          conn.user_version = v
+        begin
+          conn.transaction do 
+            obj.each{|k, s| conn.execute(s)}
+            conn.user_version = v
+          end
+        rescue Exception => e
+          STDERR.puts e.message
+          raise e
         end
       end
     end 
@@ -172,9 +181,7 @@ class Litemetric
     Litesupport.spawn do
       while @running do
         sleep @options[:flush_interval]
-        @mutex.synchronize do
-          flush
-        end
+        flush
       end      
     end 
   end
@@ -199,6 +206,16 @@ class Litemetric
     def collect_metrics
       @litemetric = Litemetric.instance
       @litemetric.register(metrics_identifier)
+      @snapshotter = create_snapshotter      
+    end
+
+    def create_snapshotter
+      Litesupport.spawn do
+        while @running do
+          sleep @litemetric.options[:snapshot_interval]
+          capture_snapshot
+        end      
+      end 
     end
 
     def metrics_identifier
@@ -220,9 +237,104 @@ class Litemetric
       res  
     end    
     
-    def snapshot
-      raise Litestack::NotImplementedError
+    def capture_snapshot
+      return unless @litemetric
+      state = snapshot if defined? snapshot
+      if state
+        @litemetric.capture_snapshot(metrics_identifier, state)
+      end
+    end
+        
+  end
+end
+
+class Litemetric
+
+  class Collector
+
+    include Litesupport::Liteconnection
+    
+    DEFAULT_OPTIONS = {
+      path: ":memory:",
+      sync: 1,
+      flush_interval:  3, # flush data every 1 minute
+      summarize_interval: 10, # summarize data every 1 minute
+      snapshot_interval: 1 # snapshot every 10 minutes
+    }
+
+    RESOLUTIONS = {
+      minute: 300, # 5 minutes (highest resolution)
+      hour: 3600, # 1 hour
+      day: 24*3600, # 1 day
+      week: 7*24*3600 # 1 week (lowest resolution)
+    }
+    
+    def initialize(options = {})
+      init(options)
+    end
+    
+    def capture(topic, event, key, value=nil)
+      if key.is_a? Array
+        key.each{|k| capture_single_key(topic, event, k, value)}
+      else
+        capture_single_key(topic, event, key, value)
+      end
+    end
+        
+    def capture_single_key(topic, event, key, value)
+      run_stmt(:capture_event, topic.to_s, event.to_s, key.to_s, nil ,1, value)
+    end
+    
+    def flush
+      t = Time.now
+      limit = 1000 # migrate 1000 records at a time
+      count = run_stmt(:event_count)[0][0]
+      while count > 0
+        @conn.acquire do |conn|
+          conn.transaction(:immediate) do
+            conn.stmts[:migrate_events].execute!(limit)
+            conn.stmts[:delete_migrated_events].execute!(limit)
+            count = conn.stmts[:event_count].execute![0][0]
+          end         
+        end
+        sleep 0.005 #give other threads a chance to run
+      end
+    end
+
+    def create_connection
+      conn = super
+      conn.execute("ATTACH ? as m", @options[:dbpath].to_s)
+      conn.wal_autocheckpoint = 10000
+      sql = YAML.load_file("#{__dir__}/litemetric_collector.sql.yml")
+      version = conn.get_first_value("PRAGMA user_version")
+      sql["schema"].each_pair do |v, obj| 
+        if v > version
+          conn.transaction do 
+            obj.each do |k, s| 
+              begin
+                conn.execute(s)
+              rescue Exception => e
+                STDERR.puts "Error parsing #{k}"
+                STDERR.puts s
+                raise e
+              end
+            end
+            conn.user_version = v
+          end
+        end
+      end 
+      sql["stmts"].each do |k, v| 
+        begin
+          conn.stmts[k.to_sym] = conn.prepare(v) 
+        rescue Exception => e
+          STDERR.puts "Error parsing #{k}"
+          STDERR.puts v
+          raise e
+        end
+      end
+      conn
     end
     
   end
+  
 end