litestack 0.2.6 → 0.4.1

data/lib/litestack/liteboard/views/litedb.erb

@@ -0,0 +1,168 @@
+  <div class = "row">
+
+    <div class = "col">
+      <div class="card">
+        <div class="card-header">
+          Size
+        </div>
+        <div class="card-body">
+          <h1><%=round(@size)%>MB</h1>
+        </div>
+      </div>
+    </div>
+
+    <div class = "col">
+      <div class="card">
+        <div class="card-header">
+          Tables
+        </div>
+        <div class="card-body">
+          <h1><%=@tables%></h1>
+        </div>
+      </div>
+    </div>
+
+    <div class = "col">
+      <div class="card">
+        <div class="card-header">
+          Indexes
+        </div>
+        <div class="card-body">
+          <h1><%=@indexes%></h1>
+        </div>
+      </div>
+    </div>
+
+    <div class = "col">
+      <div class="card">
+        <div class="card-header">
+          Read queries
+        </div>
+        <div class="card-body">
+          <h1><%=format(@reads)%></h1>
+        </div>
+      </div>
+    </div>
+
+    <div class = "col">
+      <div class="card">
+        <div class="card-header">
+          Write queries
+        </div>
+        <div class="card-body">
+          <h1><%=format(@writes)%></h1>
+        </div>
+      </div>
+    </div>
+
+
+  </div>
+
+  <div class = "row">
+  &nbsp;<br/>
+  </div>
+
+  <div class = "row">
+
+    <div class = "col">
+      <div class="card">
+        <div class="card-header">
+          Reads vs writes over time
+        </div>
+        <div class="card-body">
+          <span class="hidden inlinecolumn"><%=Oj.dump([["Time", "Reads", "Writes"]] + @reads_vs_writes)%></span>
+        </div>
+      </div>
+    </div>
+
+    <div class = "col-6">
+      <div class="card">
+        <div class="card-header">
+          Reads vs writes
+        </div>
+        <div class="card-body">
+          <span class="hidden inlinepie"><%=[["name", "value"],["reads", @reads],["writes", @writes]]%></span>
+        </div>
+      </div>
+    </div>
+  
+  </div>
+  
+  <div class = "row">
+  &nbsp;<br/>
+  </div>
+
+  <div class = "row">
+
+    <div class = "col">
+      <div class="card">
+        <div class="card-header">
+          Query execution time over time
+        </div>
+        <div class="card-body">
+          <span class="hidden inlinecolumn"><%=Oj.dump([["Time", "Reads execution time",  {'role' => 'tooltip'}, "Writes execution time", {'role' => 'tooltip'}]] + @reads_vs_writes_times)%></span>
+        </div>
+      </div>
+    </div>
+
+    <div class = "col-6">
+      <div class="card">
+        <div class="card-header">
+          Query time
+        </div>
+        <div class="card-body">
+          <span class="hidden inlinepie"><%=[["name", "value"],["total reads time", @read_times],["total writes time", @write_times]]%></span>
+        </div>
+      </div>
+    </div>
+  
+  </div>
+  
+  <div class = "row">
+  &nbsp;<br/>
+  </div>
+
+  <div class = "row">
+
+    <div class = "col-6">
+      <div class="card">
+        <div class="card-header">
+          Slowest queries
+        </div>
+        <div class="card-body">
+          <table class="table">
+            <%@slowest.each do |r| %>
+            <tr>
+              <td><%=r['key']%></td>
+              <td align="right"><h6><%=format(r['ravg'].truncate(4))%>&nbsp;sec</h6></td>
+            </tr>
+            <% end %>
+          </table>
+        </div>
+      </div>
+    </div>
+
+    <div class = "col-6">
+      <div class="card">
+        <div class="card-header">
+          Most expensive queries (execution time x frequency)
+        </div>
+        <div class="card-body">
+          <table class="table">
+            <%@popular.each do |r| %>
+            <tr>
+              <td><%=r['key']%></td>
+              <td align="right"><h6><%=format(r['rtotal'].truncate(4))%>&nbsp;sec</h6></td>
+            </tr>
+            <% end %>
+          </table>
+        </div>
+      </div>
+    </div>
+
+  </div>
+
+  <div class = "row">
+  &nbsp;<br/>
+  </div>
+

data/lib/litestack/liteboard/views/litejob.erb

@@ -0,0 +1,151 @@
+  <div class = "row">
+
+    <div class = "col">
+      <div class="card">
+        <div class="card-header">
+          Size
+        </div>
+        <div class="card-body">
+          <h1><%=format(round(@size))%> MB</h1>
+        </div>
+      </div>
+    </div>
+
+    <div class = "col">
+      <div class="card">
+        <div class="card-header">
+          Active queues
+        </div>
+        <div class="card-body">
+          <h1><%=format(@queues.length)%></h1>
+        </div>
+      </div>
+    </div>
+
+    <div class = "col">
+      <div class="card">
+        <div class="card-header">
+          Outstanding jobs
+        </div>
+        <div class="card-body">
+          <h1><%=format(@jobs)%></h1>
+        </div>
+      </div>
+    </div>
+
+  </div>
+  
+  <div class = "row">
+  &nbsp;<br/>
+  </div>
+
+  
+  <div class="row">
+  
+    <div class = "col">
+      <div class="card">
+        <div class="card-header">
+          Processed jobs
+        </div>
+        <div class="card-body">
+          <h1><%=format(@processed_count)%> jobs</h1>
+        </div>
+      </div>
+    </div>
+    
+    <div class = "col">
+      <div class="card">
+        <div class="card-header">
+          Total processing time
+        </div>
+        <div class="card-body">
+          <h1><%=format(@processing_time)%> sec</h1>
+        </div>
+      </div>
+    </div>
+
+    <div class = "col">
+      <div class="card">
+        <div class="card-header">
+          Average processing time per job
+        </div>
+        <div class="card-body">
+          <h1><%=(@processing_time.to_f / @processed_count).truncate(4) rescue 0%> sec</h1>
+        </div>
+      </div>
+    </div>
+
+  </div>
+
+  <div class = "row">
+  &nbsp;<br/>
+  </div>
+
+  <div class="row">
+  
+    <div class = "col">
+      <div class="card">
+        <div class="card-header">
+          Processed jobs by queue
+        </div>
+        <div class="card-body">
+          <span class="hidden inlinepie">
+            <%=[["name", "value"]] + @processed_count_by_queue.to_a%>
+          </span>
+        </div>
+      </div>
+    </div>
+    
+    <div class = "col">
+      <div class="card">
+        <div class="card-header">
+          Job processing time by queue
+        </div>
+        <div class="card-body">
+          <span class="hidden inlinepie">
+            <%=[["name", "value"]] + @processing_time_by_queue.to_a%>
+          </span>
+        </div>
+      </div>
+    </div>
+
+  </div>
+
+  <div class = "row">
+  &nbsp;<br/>
+  </div>
+
+  <div class="row">
+  
+    <div class = "col">
+      <div class="card">
+        <div class="card-header">
+          Processed jobs by queue over time
+        </div>
+        <div class="card-body">
+          <span class="hidden inlinestackedcolumn">
+            <%=@processed_count_over_time_by_queues.to_a%>
+          </span>
+        </div>
+      </div>
+    </div>
+    
+    <div class = "col">
+      <div class="card">
+        <div class="card-header">
+          Job processing time by queue over time
+        </div>
+        <div class="card-body">
+          <span class="hidden inlinestackedcolumn">
+            <%=@processing_time_over_time_by_queues.to_a%>
+          </span>
+        </div>
+      </div>
+    </div>
+
+  </div>
+
+  <div class = "row">
+  &nbsp;<br/>
+  </div>
+

data/lib/litestack/litecable.rb

@@ -24,52 +24,54 @@ class Litecable
   }
   
   def initialize(options = {})  
-    @messages = []
+    @messages = Litesupport::Pool.new(1){[]}
     init(options)
+    collect_metrics if @options[:metrics]
   end
   
   # broadcast a message to a specific channel
   def broadcast(channel, payload=nil)
     # group meesages and only do broadcast every 10 ms
-    #run_stmt(:publish, channel.to_s, Oj.dump(payload), @pid)
     # but broadcast locally normally
-    @mutex.synchronize{ @messages << [channel.to_s, Oj.dump(payload)] }
+    @messages.acquire{|msgs| msgs << [channel.to_s, Oj.dump(payload)]}
+    capture(:broadcast, channel)
     local_broadcast(channel, payload) 
   end
   
   # subscribe to a channel, optionally providing a success callback proc
   def subscribe(channel, subscriber, success_callback = nil)
-    @mutex.synchronize do
-      @subscribers[channel] = {} unless @subscribers[channel]
-      @subscribers[channel][subscriber] = true
+    @subscribers.acquire do |subs|
+      subs[channel] = {} unless subs[channel]
+      subs[channel][subscriber] = true
     end
+    capture(:subscribe, channel)
   end
   
   # unsubscribe from a channel
   def unsubscribe(channel, subscriber)
-    @mutex.synchronize do
-      @subscribers[channel].delete(subscriber) rescue nil
-    end
+    @subscribers.acquire{|subs| subs[channel].delete(subscriber) rescue nil }
+    capture(:unsubscribe, channel)
   end
 
   private 
-    
+  
+  # broadcast the message to local subscribers  
   def local_broadcast(channel, payload=nil)
-    return unless @subscribers[channel]
     subscribers = []
-    @mutex.synchronize do
-      subscribers = @subscribers[channel].keys
+    @subscribers.acquire do |subs| 
+      return unless subs[channel]
+      subscribers = subs[channel].keys
     end
     subscribers.each do |subscriber|
       subscriber.call(payload)      
+      capture(:message, channel)
     end
   end  
       
   def setup
     super # create connection
     @pid = Process.pid
-    @subscribers = {}
-    @mutex = Litesupport::Mutex.new
+    @subscribers = Litesupport::Pool.new(1){{}}
     @running = true
     @listener = create_listener
     @pruner = create_pruner
@@ -78,16 +80,16 @@ class Litecable
   end
 
   def create_broadcaster
-    Litesupport.spawn do
+    Litescheduler.spawn do
       while @running do
-        @mutex.synchronize do
-          if @messages.length > 0
+        @messages.acquire do |msgs|
+          if msgs.length > 0
             run_sql("BEGIN IMMEDIATE")
-            while msg = @messages.shift
+            while msg = msgs.shift
               run_stmt(:publish, msg[0], msg[1], @pid)
             end
             run_sql("END")
-          end
+          end          
         end
         sleep 0.02
       end      
@@ -95,7 +97,7 @@ class Litecable
   end
 
   def create_pruner
-    Litesupport.spawn do
+    Litescheduler.spawn do
       while @running do
         run_stmt(:prune, @options[:expire_after])
         sleep @options[:expire_after]
@@ -104,10 +106,9 @@ class Litecable
   end
 
   def create_listener
-    Litesupport.spawn do
+    Litescheduler.spawn do
       while @running do
         @last_fetched_id ||= (run_stmt(:last_id)[0][0] || 0)
-        @logger.info @last_fetched_id
         run_stmt(:fetch, @last_fetched_id, @pid).to_a.each do |msg|
           @logger.info "RECEIVED #{msg}"
           @last_fetched_id = msg[0]
@@ -119,20 +120,9 @@ class Litecable
   end
 
   def create_connection
-    conn = super
-    conn.wal_autocheckpoint = 10000 
-    sql = YAML.load_file("#{__dir__}/litecable.sql.yml")
-    version = conn.get_first_value("PRAGMA user_version")
-    sql["schema"].each_pair do |v, obj| 
-      if v > version
-        conn.transaction do 
-          obj.each{|k, s| conn.execute(s)}
-          conn.user_version = v
-        end
-      end
-    end 
-    sql["stmts"].each { |k, v| conn.stmts[k.to_sym] = conn.prepare(v) }
-    conn
+    super("#{__dir__}/litecable.sql.yml") do |conn|
+      conn.wal_autocheckpoint = 10000 
+    end
   end
   
 end

data/lib/litestack/litecable.sql.yml

@@ -17,7 +17,7 @@ stmts:
 
   last_id: SELECT max(id) FROM messages
   
-  fetch: SELECT id, channel, value FROM messages WHERE id > $1 and pid != $2
+  fetch: SELECT id, channel, value, created_at FROM messages WHERE id > $1 and pid != $2
 
   prune: DELETE FROM messages WHERE created_at < (unixepoch() - $1)
 

data/lib/litestack/litecache.rb

@@ -212,7 +212,7 @@ class Litecache
   end
     
   def spawn_worker
-    Litesupport.spawn do
+    Litescheduler.spawn do
       while @running
         @conn.acquire do |cache|
           begin
@@ -236,23 +236,12 @@ class Litecache
   end
   
   def create_connection
-    conn = super 
-    conn.cache_size = 2000
-    conn.journal_size_limit = [(@options[:size]/2).to_i, @options[:min_size]].min
-    conn.max_page_count = (@options[:size] / conn.page_size).to_i
-    conn.case_sensitive_like = true
-    sql = YAML.load_file("#{__dir__}/litecache.sql.yml")
-    version = conn.get_first_value("PRAGMA user_version")
-    sql["schema"].each_pair do |v, obj| 
-      if v > version
-        conn.transaction do 
-          obj.each{|k, s| conn.execute(s)}
-          conn.user_version = v
-        end
-      end
-    end 
-    sql["stmts"].each { |k, v| conn.stmts[k.to_sym] = conn.prepare(v) }
-    conn
+    super("#{__dir__}/litecache.sql.yml") do |conn|
+      conn.cache_size = 2000
+      conn.journal_size_limit = [(@options[:size]/2).to_i, @options[:min_size]].min
+      conn.max_page_count = (@options[:size] / conn.page_size).to_i
+      conn.case_sensitive_like = true
+    end
   end  
   
 end

data/lib/litestack/litedb.rb

@@ -1,12 +1,21 @@
 # all components should require the support module
 require_relative 'litesupport'
 
+# all measurable components should require the litemetric class
+require_relative 'litemetric'
+
+# litedb in particular gets access to litesearch
+require_relative 'litesearch'
+
 # Litedb inherits from the SQLite3::Database class and adds a few initialization options
 class Litedb < ::SQLite3::Database
 
   # add litemetric support
   include Litemetric::Measurable
   
+  # add litesearch support
+  include Litesearch
+  
   # overrride the original initilaizer to allow for connection configuration
   def initialize(file, options = {}, zfs = nil )
     if block_given?
@@ -34,7 +43,11 @@ class Litedb < ::SQLite3::Database
 
   # return the size of the database file  
   def size
-    execute("SELECT s.page_size * c.page_count FROM pragma_page_size() as s, pragma_page_count() as c")[0][0]
+    execute("SELECT s.page_size * c.page_count FROM pragma_page_size() AS s, pragma_page_count() AS c")[0][0]
+  end
+  
+  def schema_object_count(type = nil)
+    execute("SELECT count(*) FROM SQLITE_MASTER WHERE iif(?1 IS NOT NULL, type = ?1, TRUE)", type)[0][0]
   end
   
   # collect snapshot information
@@ -44,7 +57,9 @@ class Litedb < ::SQLite3::Database
         path: filename,
         journal_mode: journal_mode,
         synchronous: synchronous,
-        size: size
+        size: size.to_f / (1024 * 1024),
+        tables: schema_object_count('table'), 
+        indexes: schema_object_count('index')
       }
     }
   end

data/lib/litestack/litejob.rb

@@ -68,9 +68,8 @@ module Litejob
       get_jobqueue
     end
        
-    def delete(id, queue_name=nil) 
-      queue_name ||= queue
-      get_jobqueue.delete(id, queue)
+    def delete(id) 
+      get_jobqueue.delete(id)
     end  
          
     def queue