litestack 0.2.6 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d6db833e61264ea2036338102d0ed1d3e9eb0a5d94a235db5a8f8d4cf390c65f
-  data.tar.gz: 5a48dd08c18ab39f2a19462025e351da8f3e2eedf55f779019cef3ed277e12bc
+  metadata.gz: a1125a2cb5a2d9ea277fb9a0774632eb297ed4d322882dd33776d8a8f1c6471e
+  data.tar.gz: 9cb917c999850b2bd668e8826a4be4f091ee4e8e0f963b54ed87f5b7a7b060d6
 SHA512:
-  metadata.gz: a6dd503c63cb0ca49092f9f612a6c087d07b13eefa083ca930931b956e5cf81357f130932e2514b340d2e240b73df377bf3f5a55516b7bf444bbe4149c60656c
-  data.tar.gz: a1749b7eba0f8d155fc5e18b7a2da8d26df25ee4bf954e8c15724b8cb8164fb9f23aeab9387793420714e27fd2e13d357a91ece891dfca49afb7dbf6f8a91564
+  metadata.gz: e51460d99e66732e3dcd72e9444f1d739df7ae37804eb9b308cdae0faa7c9c34d4e7554e8f9d0bf91905f7ae3e9b4d7470509a84653da2ee736582b268fe188a
+  data.tar.gz: 873b188bd6db924f1e56034c22f5cc465b01d653ac925a2d30de648271b0b64a705894667b90005bd3d457ad318d3ee4736b16a60fa128c34d894172bc9b8e16

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 ## [Unreleased]
 
+## [0.3.0] - 2023-08-13
+
+- Reworked the Litecable thread safety model
+- Fixed multiple litejob bugs (thanks Stephen Margheim)
+- Fixed Railtie dependency (thanks Marco Roth)
+- Litesupport fixes (thanks Stephen Margheim)
+- Much improved metrics reporting for Litedb, Litecache, Litejob & Litecable
+- Removed (for now, will come again later) litemetric reporting support for ad-hoc modules
+
 ## [0.2.6] - 2023-07-16
 
 - Much improved database location setting (thanks Brad Gessler)

data/README.md

@@ -183,7 +183,7 @@ production:
 
 ## Contributing
 
-Bug reports are welcome on GitHub at https://github.com/oldmoe/litestack. Please note that this is not an open contribution project and that we don't accept pull requests.
+Bug reports and pull requests are welcome on GitHub at https://github.com/oldmoe/litestack.
 
 ## License
 

data/lib/action_cable/subscription_adapter/litecable.rb

@@ -10,20 +10,10 @@ module ActionCable
       
       prepend ChannelPrefix
 
-      DEFAULT_OPTIONS = {
-        config_path: "./config/litecable.yml",
-        path: "./db/cable.db",
-        sync: 0, # no need to sync at all
-        mmap_size: 16 * 1024 * 1024, # 16MB of memory hold hot messages
-        expire_after: 10, # remove messages older than 10 seconds
-        listen_interval: 0.005, # check new messages every 5 milliseconds
-        metrics: false
-      }
-
       def initialize(server, logger=nil)
         @server = server
         @logger = server.logger
-        super(DEFAULT_OPTIONS.dup)
+        super({config_path: "./config/litecable.yml"})
       end
             
       def shutdown

data/lib/generators/litestack/install/templates/database.yml

@@ -6,10 +6,14 @@
 #
 # `Litesupport.root.join("data.sqlite3")` stores
 # application data in the path `./db/#{Rails.env}/data.sqlite3`
+#
+# idle_timeout should be set to zero, to avoid recycling sqlite connections
+# and losing the page cache
+# 
 default: &default
   adapter: litedb
   pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %>
-  timeout: 5000
+  idle_timeout: 0
   database: <%= Litesupport.root.join("data.sqlite3") %>
 
 development:

data/lib/litestack/liteboard/liteboard.rb

@@ -16,13 +16,21 @@ class Liteboard
     get "/", to: ->(env) do
       Liteboard.new(env).call(:index)
     end
-    
-    get "/topics/:topic", to: ->(env) do
-      Liteboard.new(env).call(:topic)
+
+    get "/topics/Litejob", to: ->(env) do
+      Liteboard.new(env).call(:litejob)
+    end
+
+    get "/topics/Litecache", to: ->(env) do
+      Liteboard.new(env).call(:litecache)
     end
     
-    get "/topics/:topic/events/:event", to: ->(env) do
-      Liteboard.new(env).call(:event)
+    get "/topics/Litedb", to: ->(env) do
+      Liteboard.new(env).call(:litedb)
+    end
+
+    get "/topics/Litecable", to: ->(env) do
+      Liteboard.new(env).call(:litecable)
     end
 
   end 
@@ -45,12 +53,6 @@ class Liteboard
     after(res)
   end
   
-  def render(tpl_name)
-    layout = Tilt.new("#{__dir__}/views/layout.erb")
-    tpl = Tilt.new("#{__dir__}/views/#{tpl_name.to_s}.erb")
-    res = layout.render(self){tpl.render(self)}
-  end
-
   def after(body=nil)
     [200, {'Cache-Control' => 'no-cache'}, [body]]    
   end
@@ -76,48 +78,150 @@ class Liteboard
     end
     @search = params(:search)
     @search = nil if @search == ''
+    @topics = @lm.topic_summaries(@resolution, @step * @count, @order, @dir, @search)
   end
   
   def index
     @order = 'topic' unless @order
-    topics = @lm.topics
-    @topics = @lm.topic_summaries(@resolution, @step * @count, @order, @dir, @search)
     @topics.each do |topic|
         data_points = @lm.topic_data_points(@step, @count, @resolution, topic[0])      
-        topic << data_points.collect{|r| [r[0],r[2]]}
+        topic << data_points.collect{|r| [r[0],r[2] || 0]}
     end    
     render :index
   end
   
-  def topic
+  def litecache
     @order = 'rcount' unless @order
-    @topic = params(:topic)
+    @topic = 'Litecache'
     @events = @lm.events_summaries(@topic, @resolution, @order, @dir, @search, @step * @count)
     @events.each do |event|
-      data_points = @lm.event_data_points(@step, @count, @resolution, @topic, event[0])
-      event << data_points.collect{|r| [r[0],r[2]]}
-      event << data_points.collect{|r| [r[0],r[3]]}
+      data_points = @lm.event_data_points(@step, @count, @resolution, @topic, event['name'])
+      event['counts'] = data_points.collect{|r| [r['rtime'],r['rcount']]}
+      event['values'] = data_points.collect{|r| [r['rtime'],r['ravg']]}
     end
-    @snapshot = @lm.snapshot(@topic)
-    if @snapshot.empty?
-      @snapshot = []
-    else
-      @snapshot[0] = Oj.load(@snapshot[0]) unless @snapshot[0].nil?
+    @snapshot = read_snapshot(@topic)
+    @size = @snapshot[0][:summary][:size] rescue 0
+    @max_size = @snapshot[0][:summary][:max_size] rescue 0
+    @full = (@size / @max_size)*100 rescue 0
+    @entries = @snapshot[0][:summary][:entries] rescue 0
+    @gets = @events.find{|t| t['name'] == 'get'}
+    @sets = @events.find{|t| t['name'] == 'set'}
+    @reads = @gets['rcount'] rescue 0
+    @writes = @sets['rcount'] rescue 0
+    @hitrate = @gets['ravg'] rescue 0
+    @hits = @reads * @hitrate
+    @misses = @reads - @hits
+    @reads_vs_writes = @gets['counts'].collect.with_index{|obj, i| obj.clone << @sets['counts'][i][1] } rescue []
+    @hits_vs_misses = @gets['values'].collect.with_index{|obj, i| [obj[0], obj[1].to_f * @gets['counts'][i][1].to_f, (1 - obj[1].to_f) * @gets['counts'][i][1].to_f] } rescue []
+    @top_reads = @lm.keys_summaries(@topic, 'get', @resolution, @order, @dir, nil, @step * @count).first(8)
+    @top_writes = @lm.keys_summaries(@topic, 'set', @resolution, @order, @dir, nil, @step * @count).first(8)
+    render :litecache
+  end
+  
+  def litedb
+    @order = 'rcount' unless @order
+    @topic = 'Litedb'
+    @events = @lm.events_summaries(@topic, @resolution, @order, @dir, @search, @step * @count)
+    @events.each do |event|
+      data_points = @lm.event_data_points(@step, @count, @resolution, @topic, event['name'])
+      event['counts'] = data_points.collect{|r| [r['rtime'],r['rcount'] || 0]}
+      event['values'] = data_points.collect{|r| [r['rtime'],r['rtotal'] || 0]}
     end
-    render :topic
+    @snapshot = read_snapshot(@topic)
+    @size = @snapshot[0][:summary][:size] rescue 0
+    @tables = @snapshot[0][:summary][:tables] rescue 0
+    @indexes = @snapshot[0][:summary][:indexes] rescue 0
+    @gets = @events.find{|t| t['name'] == 'Read'}
+    @sets = @events.find{|t| t['name'] == 'Write'}
+    @reads = @gets['rcount'] rescue 0
+    @writes = @sets['rcount'] rescue 0
+    @time = @gets['ravg'] rescue 0
+    @reads_vs_writes = @gets['counts'].collect.with_index{|obj, i| obj.clone << @sets['counts'][i][1] } rescue []
+    @reads_vs_writes_times = @gets['values'].collect.with_index{|obj, i| [obj[0], obj[1], @sets['values'][i][1].to_f] } rescue []
+    @read_times = @gets['rtotal'] rescue 0
+    @write_times = @sets['rtotal'] rescue 0
+    @slowest = @lm.keys_summaries(@topic, 'Read', @resolution, 'ravg', 'desc', nil, @step * @count).first(8)
+    @slowest += @lm.keys_summaries(@topic, 'Write', @resolution, 'ravg', 'desc', nil, @step * @count).first(8)
+    @slowest = @slowest.sort{|a, b| a['ravg'] <=> b['ravg']}.reverse.first(8) 
+    @popular = @lm.keys_summaries(@topic, 'Read', @resolution, 'rtotal', 'desc', nil, @step * @count).first(8)
+    @popular += @lm.keys_summaries(@topic, 'Write', @resolution, 'rtotal', 'desc', nil, @step * @count).first(8)
+    @popular = @popular.sort{|a, b| a['rtotal'] <=> b['rtotal']}.reverse.first(8) 
+    render :litedb
   end
   
-  def event
+  def litejob
     @order = 'rcount' unless @order
-    @topic = params(:topic)
-    @event = params(:event)
-    @keys = @lm.keys_summaries(@topic, @event, @resolution, @order, @dir, @search, @step * @count)  
-    @keys.each do |key|
-      data_points = @lm.key_data_points(@step, @count, @resolution, @topic, @event, key[0])
-      key << data_points.collect{|r| [r[0],r[2]]}
-      key << data_points.collect{|r| [r[0],r[3]]}
-    end    
-    render :event
+    @topic = 'Litejob'
+    @events = @lm.events_summaries(@topic, @resolution, @order, @dir, @search, @step * @count)
+    @events.each do |event|
+      data_points = @lm.event_data_points(@step, @count, @resolution, @topic, event['name'])
+      event['counts'] = data_points.collect{|r| [r['rtime'],r['rcount'] || 0]}
+      event['values'] = data_points.collect{|r| [r['rtime'],r['rtotal'] || 0]}
+    end
+    @snapshot = read_snapshot(@topic)
+    @size = @snapshot[0][:summary][:size] rescue 0
+    @jobs = @snapshot[0][:summary][:jobs] rescue 0
+    @queues = @snapshot[0][:queues] rescue {}
+    @processed_jobs = @events.find{|e|e['name'] == 'perform'}
+    @processed_count = @processed_jobs['rcount'] rescue 0
+    @processing_time = @processed_jobs['rtotal'] rescue 0 
+    keys_summaries = @lm.keys_summaries(@topic, 'perform', @resolution, 'rcount', 'desc', nil, @step * @count)
+    @processed_count_by_queue = keys_summaries.collect{|r|[r['key'], r['rcount']]}
+    @processing_time_by_queue = keys_summaries.collect{|r|[r['key'], r['rtotal']]} #.sort{|r1, r2| r1['rtotal'] > r2['rtotal'] }
+    @processed_count_over_time = @events.find{|e| e['name'] == 'perform'}['counts'] rescue []
+    @processing_time_over_time = @events.find{|e| e['name'] == 'perform'}['values'] rescue []
+    @processed_count_over_time_by_queues = [] 
+    @processing_time_over_time_by_queues = []
+    keys = ['Time']
+    keys_summaries.each_with_index do |summary,i|
+      key = summary['key']
+      keys << key
+      data_points = @lm.key_data_points(@step, @count, @resolution, @topic, 'perform', key)
+      if i == 0
+        data_points.each do |dp|        
+          @processed_count_over_time_by_queues << [dp['rtime']] 
+          @processing_time_over_time_by_queues << [dp['rtime']] 
+        end
+      end
+      data_points.each_with_index do |dp, j|
+        @processed_count_over_time_by_queues[j] << (dp['rcount'] || 0)
+        @processing_time_over_time_by_queues[j] << (dp['rtotal'] || 0)
+      end
+    end
+    @processed_count_over_time_by_queues.unshift(keys)
+    @processing_time_over_time_by_queues.unshift(keys)      
+    render :litejob
+  end
+  
+  def litecable
+    @order = 'rcount' unless @order
+    @topic = 'Litecable'
+    @events = @lm.events_summaries(@topic, @resolution, @order, @dir, @search, @step * @count)
+    @events.each do |event|
+      data_points = @lm.event_data_points(@step, @count, @resolution, @topic, event['name'])
+      event['counts'] = data_points.collect{|r| [r['rtime'],r['rcount'] || 0]}
+    end
+
+    @subscription_count = @events.find{|t| t['name'] == 'subscribe'}['rcount'] rescue 0
+    @broadcast_count = @events.find{|t| t['name'] == 'broadcast'}['rcount'] rescue 0
+    @message_count = @events.find{|t| t['name'] == 'message'}['rcount'] rescue 0
+
+    @subscriptions_over_time = @events.find{|t| t['name'] == 'subscribe'}['counts'] rescue []
+    @broadcasts_over_time = @events.find{|t| t['name'] == 'broadcast'}['counts'] rescue []
+    @messages_over_time = @events.find{|t| t['name'] == 'message'}['counts'] rescue []
+    @messages_over_time = @messages_over_time.collect.with_index{|msg, i| [msg[0], @broadcasts_over_time[i][1], msg[1]]}
+    
+    @top_subscribed_channels = @lm.keys_summaries(@topic, 'subscribe', @resolution, @order, @dir, @search, @step * @count).first(8)
+    @top_messaged_channels = @lm.keys_summaries(@topic, 'message', @resolution, @order, @dir, @search, @step * @count).first(8)
+    render :litecable
+  end
+  
+  def index_url
+    "/?res=#{@res}&order=#{@order}&dir=#{@dir}&search=#{@search}"
+  end
+  
+  def topic_url(topic)
+    "/topics/#{encode(topic)}?res=#{@res}&order=#{@order}&dir=#{@dir}&search=#{@search}"
   end
 
   def index_sort_url(field)
@@ -156,10 +260,43 @@ class Liteboard
     URI.encode_uri_component(text)
   end
   
+  def round(float)
+    return 0 unless float.is_a? Numeric
+    ((float * 100).round).to_f / 100
+  end
+  
+  def format(float)
+    string = float.to_s
+    whole, decimal = string.split('.')
+    whole = whole.split('').reverse.each_slice(3).map(&:join).join(',').reverse
+    whole = [whole, decimal].join('.') if decimal
+    whole
+  end
+  
   def self.app
    @@app
   end
 
+  private
+  
+  def read_snapshot(topic)
+    snapshot = @lm.snapshot(topic)
+    if snapshot.empty?
+      snapshot = []
+    else
+      snapshot[0] = Oj.load(snapshot[0]) unless snapshot[0].nil?
+    end
+    snapshot  
+  end
+  
+  def render(tpl_name)
+    layout = Tilt.new("#{__dir__}/views/layout.erb")
+    tpl_path = "#{__dir__}/views/#{tpl_name.to_s}.erb"
+    tpl = Tilt.new(tpl_path)
+    res = layout.render(self){tpl.render(self)}
+  end
+
+
 end
 
 

data/lib/litestack/liteboard/views/index.erb

@@ -1,22 +1,54 @@
 
-    <h5>All topics</h5>
-    <div id="search"><form><input id="search-field" type="text" placeholder="Search topics" onkeydown="search_kd(this)" onkeyup="search_ku(this)" value="<%=@search%>"/></form></div>
-    <table class="sortable table">
-      <tr>
-        <th width="20%" class="<%='sorted' if @order == 'topic'%>"><a href="<%=index_sort_url('topic')%>">Topics</a>  <%=dir('topic')%></th>
-        <th width="20%" class="<%='sorted' if @order == 'rcount'%>"><a href="<%=index_sort_url('rcount')%>">Total events</a>  <%=dir('rcount')%></th>
-        <th width="40%">Events over time</th>
-      </tr>
-      <% @topics.each do |topic|%>
-        <tr>
-          <td title="<%=topic[0]%>"><div class="label"><a href="./topics/<%=encode(topic[0])%>?res=<%=@res%>"><%=topic[0]%></a></div></td>
-          <td><%=topic[3]%></td>
-          <td class="chart"><span class="inlinecolumn hidden" data-label="Count"><%=Oj.dump(topic[4]) if topic[4]%></span></td>
-        </tr>
-      <%end%>
-      <% if @topics.empty? %>
-        <tr>
-         <td class="empty" colspan="5">No data to display</td>
-        </tr>
-      <% end %>
 
+ 
+
+<div class="container">
+   <% @topics.each do |topic|%>
+
+    <div class = "row justify-content-center">
+
+    <div class = "col-6">
+      <div class="card">
+        <div class="card-header">
+          <a href="./topics/<%=encode(topic[0])%>?res=<%=@res%>"><%=topic[0]%></a>
+        </div>
+        <div class="card-body">
+          <div class="container">
+            <div class= "row">
+              <div class= "col">
+                <h1><%=topic[3]%> <span class="fs-4">events</span></h1>
+              </div>
+              <div class= "col">
+                <span class="inlineminicolumn hidden" data-label="Count"><%=Oj.dump(topic[4].unshift(['Time', 'Count'])) if topic[4]%></span>
+              </div>
+            </div>
+          </div>
+        </div>
+      </div>
+    </div>
+
+  </div>
+
+  <div class = "row">
+  &nbsp;<br/>
+  </div>
+
+  
+  <%end%>
+  <% if @topics.empty? %>
+    <div class = "row justify-content-center">
+
+    <div class = "col-6">
+      <div class="card">
+        <div class="card-header">
+          Topics
+        </div>
+        <div class="card-body justify-content-center">
+          No data to display
+        </div>
+      </div>
+    </div>
+
+  </div>
+  <%end%>
+</div>