litestack 0.2.6 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/BENCHMARKS.md +11 -0
  3. data/CHANGELOG.md +19 -0
  4. data/Gemfile +2 -0
  5. data/README.md +1 -1
  6. data/assets/event_page.png +0 -0
  7. data/assets/index_page.png +0 -0
  8. data/assets/topic_page.png +0 -0
  9. data/bench/bench_jobs_rails.rb +1 -1
  10. data/bench/bench_jobs_raw.rb +1 -1
  11. data/bench/uljob.rb +1 -1
  12. data/lib/action_cable/subscription_adapter/litecable.rb +1 -11
  13. data/lib/active_support/cache/litecache.rb +1 -1
  14. data/lib/generators/litestack/install/templates/database.yml +5 -1
  15. data/lib/litestack/liteboard/liteboard.rb +172 -35
  16. data/lib/litestack/liteboard/views/index.erb +52 -20
  17. data/lib/litestack/liteboard/views/layout.erb +189 -38
  18. data/lib/litestack/liteboard/views/litecable.erb +118 -0
  19. data/lib/litestack/liteboard/views/litecache.erb +144 -0
  20. data/lib/litestack/liteboard/views/litedb.erb +168 -0
  21. data/lib/litestack/liteboard/views/litejob.erb +151 -0
  22. data/lib/litestack/litecable.rb +27 -37
  23. data/lib/litestack/litecable.sql.yml +1 -1
  24. data/lib/litestack/litecache.rb +7 -18
  25. data/lib/litestack/litedb.rb +17 -2
  26. data/lib/litestack/litejob.rb +2 -3
  27. data/lib/litestack/litejobqueue.rb +51 -48
  28. data/lib/litestack/litemetric.rb +46 -69
  29. data/lib/litestack/litemetric.sql.yml +14 -12
  30. data/lib/litestack/litemetric_collector.sql.yml +4 -4
  31. data/lib/litestack/litequeue.rb +9 -20
  32. data/lib/litestack/litescheduler.rb +84 -0
  33. data/lib/litestack/litesearch/index.rb +230 -0
  34. data/lib/litestack/litesearch/model.rb +178 -0
  35. data/lib/litestack/litesearch/schema.rb +193 -0
  36. data/lib/litestack/litesearch/schema_adapters/backed_adapter.rb +147 -0
  37. data/lib/litestack/litesearch/schema_adapters/basic_adapter.rb +128 -0
  38. data/lib/litestack/litesearch/schema_adapters/contentless_adapter.rb +17 -0
  39. data/lib/litestack/litesearch/schema_adapters/standalone_adapter.rb +33 -0
  40. data/lib/litestack/litesearch/schema_adapters.rb +9 -0
  41. data/lib/litestack/litesearch.rb +37 -0
  42. data/lib/litestack/litesupport.rb +55 -125
  43. data/lib/litestack/version.rb +1 -1
  44. data/lib/litestack.rb +2 -1
  45. data/lib/sequel/adapters/litedb.rb +3 -2
  46. metadata +20 -3
@@ -0,0 +1,168 @@
1
+ <div class = "row">
2
+
3
+ <div class = "col">
4
+ <div class="card">
5
+ <div class="card-header">
6
+ Size
7
+ </div>
8
+ <div class="card-body">
9
+ <h1><%=round(@size)%>MB</h1>
10
+ </div>
11
+ </div>
12
+ </div>
13
+
14
+ <div class = "col">
15
+ <div class="card">
16
+ <div class="card-header">
17
+ Tables
18
+ </div>
19
+ <div class="card-body">
20
+ <h1><%=@tables%></h1>
21
+ </div>
22
+ </div>
23
+ </div>
24
+
25
+ <div class = "col">
26
+ <div class="card">
27
+ <div class="card-header">
28
+ Indexes
29
+ </div>
30
+ <div class="card-body">
31
+ <h1><%=@indexes%></h1>
32
+ </div>
33
+ </div>
34
+ </div>
35
+
36
+ <div class = "col">
37
+ <div class="card">
38
+ <div class="card-header">
39
+ Read queries
40
+ </div>
41
+ <div class="card-body">
42
+ <h1><%=format(@reads)%></h1>
43
+ </div>
44
+ </div>
45
+ </div>
46
+
47
+ <div class = "col">
48
+ <div class="card">
49
+ <div class="card-header">
50
+ Write queries
51
+ </div>
52
+ <div class="card-body">
53
+ <h1><%=format(@writes)%></h1>
54
+ </div>
55
+ </div>
56
+ </div>
57
+
58
+
59
+ </div>
60
+
61
+ <div class = "row">
62
+ &nbsp;<br/>
63
+ </div>
64
+
65
+ <div class = "row">
66
+
67
+ <div class = "col">
68
+ <div class="card">
69
+ <div class="card-header">
70
+ Reads vs writes over time
71
+ </div>
72
+ <div class="card-body">
73
+ <span class="hidden inlinecolumn"><%=Oj.dump([["Time", "Reads", "Writes"]] + @reads_vs_writes)%></span>
74
+ </div>
75
+ </div>
76
+ </div>
77
+
78
+ <div class = "col-6">
79
+ <div class="card">
80
+ <div class="card-header">
81
+ Reads vs writes
82
+ </div>
83
+ <div class="card-body">
84
+ <span class="hidden inlinepie"><%=[["name", "value"],["reads", @reads],["writes", @writes]]%></span>
85
+ </div>
86
+ </div>
87
+ </div>
88
+
89
+ </div>
90
+
91
+ <div class = "row">
92
+ &nbsp;<br/>
93
+ </div>
94
+
95
+ <div class = "row">
96
+
97
+ <div class = "col">
98
+ <div class="card">
99
+ <div class="card-header">
100
+ Query execution time over time
101
+ </div>
102
+ <div class="card-body">
103
+ <span class="hidden inlinecolumn"><%=Oj.dump([["Time", "Reads execution time", {'role' => 'tooltip'}, "Writes execution time", {'role' => 'tooltip'}]] + @reads_vs_writes_times)%></span>
104
+ </div>
105
+ </div>
106
+ </div>
107
+
108
+ <div class = "col-6">
109
+ <div class="card">
110
+ <div class="card-header">
111
+ Query time
112
+ </div>
113
+ <div class="card-body">
114
+ <span class="hidden inlinepie"><%=[["name", "value"],["total reads time", @read_times],["total writes time", @write_times]]%></span>
115
+ </div>
116
+ </div>
117
+ </div>
118
+
119
+ </div>
120
+
121
+ <div class = "row">
122
+ &nbsp;<br/>
123
+ </div>
124
+
125
+ <div class = "row">
126
+
127
+ <div class = "col-6">
128
+ <div class="card">
129
+ <div class="card-header">
130
+ Slowest queries
131
+ </div>
132
+ <div class="card-body">
133
+ <table class="table">
134
+ <%@slowest.each do |r| %>
135
+ <tr>
136
+ <td><%=r['key']%></td>
137
+ <td align="right"><h6><%=format(r['ravg'].truncate(4))%>&nbsp;sec</h6></td>
138
+ </tr>
139
+ <% end %>
140
+ </table>
141
+ </div>
142
+ </div>
143
+ </div>
144
+
145
+ <div class = "col-6">
146
+ <div class="card">
147
+ <div class="card-header">
148
+ Most expensive queries (execution time x frequency)
149
+ </div>
150
+ <div class="card-body">
151
+ <table class="table">
152
+ <%@popular.each do |r| %>
153
+ <tr>
154
+ <td><%=r['key']%></td>
155
+ <td align="right"><h6><%=format(r['rtotal'].truncate(4))%>&nbsp;sec</h6></td>
156
+ </tr>
157
+ <% end %>
158
+ </table>
159
+ </div>
160
+ </div>
161
+ </div>
162
+
163
+ </div>
164
+
165
+ <div class = "row">
166
+ &nbsp;<br/>
167
+ </div>
168
+
@@ -0,0 +1,151 @@
1
+ <div class = "row">
2
+
3
+ <div class = "col">
4
+ <div class="card">
5
+ <div class="card-header">
6
+ Size
7
+ </div>
8
+ <div class="card-body">
9
+ <h1><%=format(round(@size))%> MB</h1>
10
+ </div>
11
+ </div>
12
+ </div>
13
+
14
+ <div class = "col">
15
+ <div class="card">
16
+ <div class="card-header">
17
+ Active queues
18
+ </div>
19
+ <div class="card-body">
20
+ <h1><%=format(@queues.length)%></h1>
21
+ </div>
22
+ </div>
23
+ </div>
24
+
25
+ <div class = "col">
26
+ <div class="card">
27
+ <div class="card-header">
28
+ Outstanding jobs
29
+ </div>
30
+ <div class="card-body">
31
+ <h1><%=format(@jobs)%></h1>
32
+ </div>
33
+ </div>
34
+ </div>
35
+
36
+ </div>
37
+
38
+ <div class = "row">
39
+ &nbsp;<br/>
40
+ </div>
41
+
42
+
43
+ <div class="row">
44
+
45
+ <div class = "col">
46
+ <div class="card">
47
+ <div class="card-header">
48
+ Processed jobs
49
+ </div>
50
+ <div class="card-body">
51
+ <h1><%=format(@processed_count)%> jobs</h1>
52
+ </div>
53
+ </div>
54
+ </div>
55
+
56
+ <div class = "col">
57
+ <div class="card">
58
+ <div class="card-header">
59
+ Total processing time
60
+ </div>
61
+ <div class="card-body">
62
+ <h1><%=format(@processing_time)%> sec</h1>
63
+ </div>
64
+ </div>
65
+ </div>
66
+
67
+ <div class = "col">
68
+ <div class="card">
69
+ <div class="card-header">
70
+ Average processing time per job
71
+ </div>
72
+ <div class="card-body">
73
+ <h1><%=(@processing_time.to_f / @processed_count).truncate(4) rescue 0%> sec</h1>
74
+ </div>
75
+ </div>
76
+ </div>
77
+
78
+ </div>
79
+
80
+ <div class = "row">
81
+ &nbsp;<br/>
82
+ </div>
83
+
84
+ <div class="row">
85
+
86
+ <div class = "col">
87
+ <div class="card">
88
+ <div class="card-header">
89
+ Processed jobs by queue
90
+ </div>
91
+ <div class="card-body">
92
+ <span class="hidden inlinepie">
93
+ <%=[["name", "value"]] + @processed_count_by_queue.to_a%>
94
+ </span>
95
+ </div>
96
+ </div>
97
+ </div>
98
+
99
+ <div class = "col">
100
+ <div class="card">
101
+ <div class="card-header">
102
+ Job processing time by queue
103
+ </div>
104
+ <div class="card-body">
105
+ <span class="hidden inlinepie">
106
+ <%=[["name", "value"]] + @processing_time_by_queue.to_a%>
107
+ </span>
108
+ </div>
109
+ </div>
110
+ </div>
111
+
112
+ </div>
113
+
114
+ <div class = "row">
115
+ &nbsp;<br/>
116
+ </div>
117
+
118
+ <div class="row">
119
+
120
+ <div class = "col">
121
+ <div class="card">
122
+ <div class="card-header">
123
+ Processed jobs by queue over time
124
+ </div>
125
+ <div class="card-body">
126
+ <span class="hidden inlinestackedcolumn">
127
+ <%=@processed_count_over_time_by_queues.to_a%>
128
+ </span>
129
+ </div>
130
+ </div>
131
+ </div>
132
+
133
+ <div class = "col">
134
+ <div class="card">
135
+ <div class="card-header">
136
+ Job processing time by queue over time
137
+ </div>
138
+ <div class="card-body">
139
+ <span class="hidden inlinestackedcolumn">
140
+ <%=@processing_time_over_time_by_queues.to_a%>
141
+ </span>
142
+ </div>
143
+ </div>
144
+ </div>
145
+
146
+ </div>
147
+
148
+ <div class = "row">
149
+ &nbsp;<br/>
150
+ </div>
151
+
@@ -24,52 +24,54 @@ class Litecable
24
24
  }
25
25
 
26
26
  def initialize(options = {})
27
- @messages = []
27
+ @messages = Litesupport::Pool.new(1){[]}
28
28
  init(options)
29
+ collect_metrics if @options[:metrics]
29
30
  end
30
31
 
31
32
  # broadcast a message to a specific channel
32
33
  def broadcast(channel, payload=nil)
33
34
  # group meesages and only do broadcast every 10 ms
34
- #run_stmt(:publish, channel.to_s, Oj.dump(payload), @pid)
35
35
  # but broadcast locally normally
36
- @mutex.synchronize{ @messages << [channel.to_s, Oj.dump(payload)] }
36
+ @messages.acquire{|msgs| msgs << [channel.to_s, Oj.dump(payload)]}
37
+ capture(:broadcast, channel)
37
38
  local_broadcast(channel, payload)
38
39
  end
39
40
 
40
41
  # subscribe to a channel, optionally providing a success callback proc
41
42
  def subscribe(channel, subscriber, success_callback = nil)
42
- @mutex.synchronize do
43
- @subscribers[channel] = {} unless @subscribers[channel]
44
- @subscribers[channel][subscriber] = true
43
+ @subscribers.acquire do |subs|
44
+ subs[channel] = {} unless subs[channel]
45
+ subs[channel][subscriber] = true
45
46
  end
47
+ capture(:subscribe, channel)
46
48
  end
47
49
 
48
50
  # unsubscribe from a channel
49
51
  def unsubscribe(channel, subscriber)
50
- @mutex.synchronize do
51
- @subscribers[channel].delete(subscriber) rescue nil
52
- end
52
+ @subscribers.acquire{|subs| subs[channel].delete(subscriber) rescue nil }
53
+ capture(:unsubscribe, channel)
53
54
  end
54
55
 
55
56
  private
56
-
57
+
58
+ # broadcast the message to local subscribers
57
59
  def local_broadcast(channel, payload=nil)
58
- return unless @subscribers[channel]
59
60
  subscribers = []
60
- @mutex.synchronize do
61
- subscribers = @subscribers[channel].keys
61
+ @subscribers.acquire do |subs|
62
+ return unless subs[channel]
63
+ subscribers = subs[channel].keys
62
64
  end
63
65
  subscribers.each do |subscriber|
64
66
  subscriber.call(payload)
67
+ capture(:message, channel)
65
68
  end
66
69
  end
67
70
 
68
71
  def setup
69
72
  super # create connection
70
73
  @pid = Process.pid
71
- @subscribers = {}
72
- @mutex = Litesupport::Mutex.new
74
+ @subscribers = Litesupport::Pool.new(1){{}}
73
75
  @running = true
74
76
  @listener = create_listener
75
77
  @pruner = create_pruner
@@ -78,16 +80,16 @@ class Litecable
78
80
  end
79
81
 
80
82
  def create_broadcaster
81
- Litesupport.spawn do
83
+ Litescheduler.spawn do
82
84
  while @running do
83
- @mutex.synchronize do
84
- if @messages.length > 0
85
+ @messages.acquire do |msgs|
86
+ if msgs.length > 0
85
87
  run_sql("BEGIN IMMEDIATE")
86
- while msg = @messages.shift
88
+ while msg = msgs.shift
87
89
  run_stmt(:publish, msg[0], msg[1], @pid)
88
90
  end
89
91
  run_sql("END")
90
- end
92
+ end
91
93
  end
92
94
  sleep 0.02
93
95
  end
@@ -95,7 +97,7 @@ class Litecable
95
97
  end
96
98
 
97
99
  def create_pruner
98
- Litesupport.spawn do
100
+ Litescheduler.spawn do
99
101
  while @running do
100
102
  run_stmt(:prune, @options[:expire_after])
101
103
  sleep @options[:expire_after]
@@ -104,10 +106,9 @@ class Litecable
104
106
  end
105
107
 
106
108
  def create_listener
107
- Litesupport.spawn do
109
+ Litescheduler.spawn do
108
110
  while @running do
109
111
  @last_fetched_id ||= (run_stmt(:last_id)[0][0] || 0)
110
- @logger.info @last_fetched_id
111
112
  run_stmt(:fetch, @last_fetched_id, @pid).to_a.each do |msg|
112
113
  @logger.info "RECEIVED #{msg}"
113
114
  @last_fetched_id = msg[0]
@@ -119,20 +120,9 @@ class Litecable
119
120
  end
120
121
 
121
122
  def create_connection
122
- conn = super
123
- conn.wal_autocheckpoint = 10000
124
- sql = YAML.load_file("#{__dir__}/litecable.sql.yml")
125
- version = conn.get_first_value("PRAGMA user_version")
126
- sql["schema"].each_pair do |v, obj|
127
- if v > version
128
- conn.transaction do
129
- obj.each{|k, s| conn.execute(s)}
130
- conn.user_version = v
131
- end
132
- end
133
- end
134
- sql["stmts"].each { |k, v| conn.stmts[k.to_sym] = conn.prepare(v) }
135
- conn
123
+ super("#{__dir__}/litecable.sql.yml") do |conn|
124
+ conn.wal_autocheckpoint = 10000
125
+ end
136
126
  end
137
127
 
138
128
  end
@@ -17,7 +17,7 @@ stmts:
17
17
 
18
18
  last_id: SELECT max(id) FROM messages
19
19
 
20
- fetch: SELECT id, channel, value FROM messages WHERE id > $1 and pid != $2
20
+ fetch: SELECT id, channel, value, created_at FROM messages WHERE id > $1 and pid != $2
21
21
 
22
22
  prune: DELETE FROM messages WHERE created_at < (unixepoch() - $1)
23
23
 
@@ -212,7 +212,7 @@ class Litecache
212
212
  end
213
213
 
214
214
  def spawn_worker
215
- Litesupport.spawn do
215
+ Litescheduler.spawn do
216
216
  while @running
217
217
  @conn.acquire do |cache|
218
218
  begin
@@ -236,23 +236,12 @@ class Litecache
236
236
  end
237
237
 
238
238
  def create_connection
239
- conn = super
240
- conn.cache_size = 2000
241
- conn.journal_size_limit = [(@options[:size]/2).to_i, @options[:min_size]].min
242
- conn.max_page_count = (@options[:size] / conn.page_size).to_i
243
- conn.case_sensitive_like = true
244
- sql = YAML.load_file("#{__dir__}/litecache.sql.yml")
245
- version = conn.get_first_value("PRAGMA user_version")
246
- sql["schema"].each_pair do |v, obj|
247
- if v > version
248
- conn.transaction do
249
- obj.each{|k, s| conn.execute(s)}
250
- conn.user_version = v
251
- end
252
- end
253
- end
254
- sql["stmts"].each { |k, v| conn.stmts[k.to_sym] = conn.prepare(v) }
255
- conn
239
+ super("#{__dir__}/litecache.sql.yml") do |conn|
240
+ conn.cache_size = 2000
241
+ conn.journal_size_limit = [(@options[:size]/2).to_i, @options[:min_size]].min
242
+ conn.max_page_count = (@options[:size] / conn.page_size).to_i
243
+ conn.case_sensitive_like = true
244
+ end
256
245
  end
257
246
 
258
247
  end
@@ -1,12 +1,21 @@
1
1
  # all components should require the support module
2
2
  require_relative 'litesupport'
3
3
 
4
+ # all measurable components should require the litemetric class
5
+ require_relative 'litemetric'
6
+
7
+ # litedb in particular gets access to litesearch
8
+ require_relative 'litesearch'
9
+
4
10
  # Litedb inherits from the SQLite3::Database class and adds a few initialization options
5
11
  class Litedb < ::SQLite3::Database
6
12
 
7
13
  # add litemetric support
8
14
  include Litemetric::Measurable
9
15
 
16
+ # add litesearch support
17
+ include Litesearch
18
+
10
19
  # overrride the original initilaizer to allow for connection configuration
11
20
  def initialize(file, options = {}, zfs = nil )
12
21
  if block_given?
@@ -34,7 +43,11 @@ class Litedb < ::SQLite3::Database
34
43
 
35
44
  # return the size of the database file
36
45
  def size
37
- execute("SELECT s.page_size * c.page_count FROM pragma_page_size() as s, pragma_page_count() as c")[0][0]
46
+ execute("SELECT s.page_size * c.page_count FROM pragma_page_size() AS s, pragma_page_count() AS c")[0][0]
47
+ end
48
+
49
+ def schema_object_count(type = nil)
50
+ execute("SELECT count(*) FROM SQLITE_MASTER WHERE iif(?1 IS NOT NULL, type = ?1, TRUE)", type)[0][0]
38
51
  end
39
52
 
40
53
  # collect snapshot information
@@ -44,7 +57,9 @@ class Litedb < ::SQLite3::Database
44
57
  path: filename,
45
58
  journal_mode: journal_mode,
46
59
  synchronous: synchronous,
47
- size: size
60
+ size: size.to_f / (1024 * 1024),
61
+ tables: schema_object_count('table'),
62
+ indexes: schema_object_count('index')
48
63
  }
49
64
  }
50
65
  end
@@ -68,9 +68,8 @@ module Litejob
68
68
  get_jobqueue
69
69
  end
70
70
 
71
- def delete(id, queue_name=nil)
72
- queue_name ||= queue
73
- get_jobqueue.delete(id, queue)
71
+ def delete(id)
72
+ get_jobqueue.delete(id)
74
73
  end
75
74
 
76
75
  def queue