litestack 0.2.6 → 0.4.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/BENCHMARKS.md +11 -0
  3. data/CHANGELOG.md +19 -0
  4. data/Gemfile +2 -0
  5. data/README.md +1 -1
  6. data/assets/event_page.png +0 -0
  7. data/assets/index_page.png +0 -0
  8. data/assets/topic_page.png +0 -0
  9. data/bench/bench_jobs_rails.rb +1 -1
  10. data/bench/bench_jobs_raw.rb +1 -1
  11. data/bench/uljob.rb +1 -1
  12. data/lib/action_cable/subscription_adapter/litecable.rb +1 -11
  13. data/lib/active_support/cache/litecache.rb +1 -1
  14. data/lib/generators/litestack/install/templates/database.yml +5 -1
  15. data/lib/litestack/liteboard/liteboard.rb +172 -35
  16. data/lib/litestack/liteboard/views/index.erb +52 -20
  17. data/lib/litestack/liteboard/views/layout.erb +189 -38
  18. data/lib/litestack/liteboard/views/litecable.erb +118 -0
  19. data/lib/litestack/liteboard/views/litecache.erb +144 -0
  20. data/lib/litestack/liteboard/views/litedb.erb +168 -0
  21. data/lib/litestack/liteboard/views/litejob.erb +151 -0
  22. data/lib/litestack/litecable.rb +27 -37
  23. data/lib/litestack/litecable.sql.yml +1 -1
  24. data/lib/litestack/litecache.rb +7 -18
  25. data/lib/litestack/litedb.rb +17 -2
  26. data/lib/litestack/litejob.rb +2 -3
  27. data/lib/litestack/litejobqueue.rb +51 -48
  28. data/lib/litestack/litemetric.rb +46 -69
  29. data/lib/litestack/litemetric.sql.yml +14 -12
  30. data/lib/litestack/litemetric_collector.sql.yml +4 -4
  31. data/lib/litestack/litequeue.rb +9 -20
  32. data/lib/litestack/litescheduler.rb +84 -0
  33. data/lib/litestack/litesearch/index.rb +230 -0
  34. data/lib/litestack/litesearch/model.rb +178 -0
  35. data/lib/litestack/litesearch/schema.rb +193 -0
  36. data/lib/litestack/litesearch/schema_adapters/backed_adapter.rb +147 -0
  37. data/lib/litestack/litesearch/schema_adapters/basic_adapter.rb +128 -0
  38. data/lib/litestack/litesearch/schema_adapters/contentless_adapter.rb +17 -0
  39. data/lib/litestack/litesearch/schema_adapters/standalone_adapter.rb +33 -0
  40. data/lib/litestack/litesearch/schema_adapters.rb +9 -0
  41. data/lib/litestack/litesearch.rb +37 -0
  42. data/lib/litestack/litesupport.rb +55 -125
  43. data/lib/litestack/version.rb +1 -1
  44. data/lib/litestack.rb +2 -1
  45. data/lib/sequel/adapters/litedb.rb +3 -2
  46. metadata +20 -3
@@ -0,0 +1,168 @@
1
+ <div class = "row">
2
+
3
+ <div class = "col">
4
+ <div class="card">
5
+ <div class="card-header">
6
+ Size
7
+ </div>
8
+ <div class="card-body">
9
+ <h1><%=round(@size)%>MB</h1>
10
+ </div>
11
+ </div>
12
+ </div>
13
+
14
+ <div class = "col">
15
+ <div class="card">
16
+ <div class="card-header">
17
+ Tables
18
+ </div>
19
+ <div class="card-body">
20
+ <h1><%=@tables%></h1>
21
+ </div>
22
+ </div>
23
+ </div>
24
+
25
+ <div class = "col">
26
+ <div class="card">
27
+ <div class="card-header">
28
+ Indexes
29
+ </div>
30
+ <div class="card-body">
31
+ <h1><%=@indexes%></h1>
32
+ </div>
33
+ </div>
34
+ </div>
35
+
36
+ <div class = "col">
37
+ <div class="card">
38
+ <div class="card-header">
39
+ Read queries
40
+ </div>
41
+ <div class="card-body">
42
+ <h1><%=format(@reads)%></h1>
43
+ </div>
44
+ </div>
45
+ </div>
46
+
47
+ <div class = "col">
48
+ <div class="card">
49
+ <div class="card-header">
50
+ Write queries
51
+ </div>
52
+ <div class="card-body">
53
+ <h1><%=format(@writes)%></h1>
54
+ </div>
55
+ </div>
56
+ </div>
57
+
58
+
59
+ </div>
60
+
61
+ <div class = "row">
62
+ &nbsp;<br/>
63
+ </div>
64
+
65
+ <div class = "row">
66
+
67
+ <div class = "col">
68
+ <div class="card">
69
+ <div class="card-header">
70
+ Reads vs writes over time
71
+ </div>
72
+ <div class="card-body">
73
+ <span class="hidden inlinecolumn"><%=Oj.dump([["Time", "Reads", "Writes"]] + @reads_vs_writes)%></span>
74
+ </div>
75
+ </div>
76
+ </div>
77
+
78
+ <div class = "col-6">
79
+ <div class="card">
80
+ <div class="card-header">
81
+ Reads vs writes
82
+ </div>
83
+ <div class="card-body">
84
+ <span class="hidden inlinepie"><%=[["name", "value"],["reads", @reads],["writes", @writes]]%></span>
85
+ </div>
86
+ </div>
87
+ </div>
88
+
89
+ </div>
90
+
91
+ <div class = "row">
92
+ &nbsp;<br/>
93
+ </div>
94
+
95
+ <div class = "row">
96
+
97
+ <div class = "col">
98
+ <div class="card">
99
+ <div class="card-header">
100
+ Query execution time over time
101
+ </div>
102
+ <div class="card-body">
103
+ <span class="hidden inlinecolumn"><%=Oj.dump([["Time", "Reads execution time", {'role' => 'tooltip'}, "Writes execution time", {'role' => 'tooltip'}]] + @reads_vs_writes_times)%></span>
104
+ </div>
105
+ </div>
106
+ </div>
107
+
108
+ <div class = "col-6">
109
+ <div class="card">
110
+ <div class="card-header">
111
+ Query time
112
+ </div>
113
+ <div class="card-body">
114
+ <span class="hidden inlinepie"><%=[["name", "value"],["total reads time", @read_times],["total writes time", @write_times]]%></span>
115
+ </div>
116
+ </div>
117
+ </div>
118
+
119
+ </div>
120
+
121
+ <div class = "row">
122
+ &nbsp;<br/>
123
+ </div>
124
+
125
+ <div class = "row">
126
+
127
+ <div class = "col-6">
128
+ <div class="card">
129
+ <div class="card-header">
130
+ Slowest queries
131
+ </div>
132
+ <div class="card-body">
133
+ <table class="table">
134
+ <%@slowest.each do |r| %>
135
+ <tr>
136
+ <td><%=r['key']%></td>
137
+ <td align="right"><h6><%=format(r['ravg'].truncate(4))%>&nbsp;sec</h6></td>
138
+ </tr>
139
+ <% end %>
140
+ </table>
141
+ </div>
142
+ </div>
143
+ </div>
144
+
145
+ <div class = "col-6">
146
+ <div class="card">
147
+ <div class="card-header">
148
+ Most expensive queries (execution time x frequency)
149
+ </div>
150
+ <div class="card-body">
151
+ <table class="table">
152
+ <%@popular.each do |r| %>
153
+ <tr>
154
+ <td><%=r['key']%></td>
155
+ <td align="right"><h6><%=format(r['rtotal'].truncate(4))%>&nbsp;sec</h6></td>
156
+ </tr>
157
+ <% end %>
158
+ </table>
159
+ </div>
160
+ </div>
161
+ </div>
162
+
163
+ </div>
164
+
165
+ <div class = "row">
166
+ &nbsp;<br/>
167
+ </div>
168
+
@@ -0,0 +1,151 @@
1
+ <div class = "row">
2
+
3
+ <div class = "col">
4
+ <div class="card">
5
+ <div class="card-header">
6
+ Size
7
+ </div>
8
+ <div class="card-body">
9
+ <h1><%=format(round(@size))%> MB</h1>
10
+ </div>
11
+ </div>
12
+ </div>
13
+
14
+ <div class = "col">
15
+ <div class="card">
16
+ <div class="card-header">
17
+ Active queues
18
+ </div>
19
+ <div class="card-body">
20
+ <h1><%=format(@queues.length)%></h1>
21
+ </div>
22
+ </div>
23
+ </div>
24
+
25
+ <div class = "col">
26
+ <div class="card">
27
+ <div class="card-header">
28
+ Outstanding jobs
29
+ </div>
30
+ <div class="card-body">
31
+ <h1><%=format(@jobs)%></h1>
32
+ </div>
33
+ </div>
34
+ </div>
35
+
36
+ </div>
37
+
38
+ <div class = "row">
39
+ &nbsp;<br/>
40
+ </div>
41
+
42
+
43
+ <div class="row">
44
+
45
+ <div class = "col">
46
+ <div class="card">
47
+ <div class="card-header">
48
+ Processed jobs
49
+ </div>
50
+ <div class="card-body">
51
+ <h1><%=format(@processed_count)%> jobs</h1>
52
+ </div>
53
+ </div>
54
+ </div>
55
+
56
+ <div class = "col">
57
+ <div class="card">
58
+ <div class="card-header">
59
+ Total processing time
60
+ </div>
61
+ <div class="card-body">
62
+ <h1><%=format(@processing_time)%> sec</h1>
63
+ </div>
64
+ </div>
65
+ </div>
66
+
67
+ <div class = "col">
68
+ <div class="card">
69
+ <div class="card-header">
70
+ Average processing time per job
71
+ </div>
72
+ <div class="card-body">
73
+ <h1><%=(@processing_time.to_f / @processed_count).truncate(4) rescue 0%> sec</h1>
74
+ </div>
75
+ </div>
76
+ </div>
77
+
78
+ </div>
79
+
80
+ <div class = "row">
81
+ &nbsp;<br/>
82
+ </div>
83
+
84
+ <div class="row">
85
+
86
+ <div class = "col">
87
+ <div class="card">
88
+ <div class="card-header">
89
+ Processed jobs by queue
90
+ </div>
91
+ <div class="card-body">
92
+ <span class="hidden inlinepie">
93
+ <%=[["name", "value"]] + @processed_count_by_queue.to_a%>
94
+ </span>
95
+ </div>
96
+ </div>
97
+ </div>
98
+
99
+ <div class = "col">
100
+ <div class="card">
101
+ <div class="card-header">
102
+ Job processing time by queue
103
+ </div>
104
+ <div class="card-body">
105
+ <span class="hidden inlinepie">
106
+ <%=[["name", "value"]] + @processing_time_by_queue.to_a%>
107
+ </span>
108
+ </div>
109
+ </div>
110
+ </div>
111
+
112
+ </div>
113
+
114
+ <div class = "row">
115
+ &nbsp;<br/>
116
+ </div>
117
+
118
+ <div class="row">
119
+
120
+ <div class = "col">
121
+ <div class="card">
122
+ <div class="card-header">
123
+ Processed jobs by queue over time
124
+ </div>
125
+ <div class="card-body">
126
+ <span class="hidden inlinestackedcolumn">
127
+ <%=@processed_count_over_time_by_queues.to_a%>
128
+ </span>
129
+ </div>
130
+ </div>
131
+ </div>
132
+
133
+ <div class = "col">
134
+ <div class="card">
135
+ <div class="card-header">
136
+ Job processing time by queue over time
137
+ </div>
138
+ <div class="card-body">
139
+ <span class="hidden inlinestackedcolumn">
140
+ <%=@processing_time_over_time_by_queues.to_a%>
141
+ </span>
142
+ </div>
143
+ </div>
144
+ </div>
145
+
146
+ </div>
147
+
148
+ <div class = "row">
149
+ &nbsp;<br/>
150
+ </div>
151
+
@@ -24,52 +24,54 @@ class Litecable
24
24
  }
25
25
 
26
26
  def initialize(options = {})
27
- @messages = []
27
+ @messages = Litesupport::Pool.new(1){[]}
28
28
  init(options)
29
+ collect_metrics if @options[:metrics]
29
30
  end
30
31
 
31
32
  # broadcast a message to a specific channel
32
33
  def broadcast(channel, payload=nil)
33
34
  # group meesages and only do broadcast every 10 ms
34
- #run_stmt(:publish, channel.to_s, Oj.dump(payload), @pid)
35
35
  # but broadcast locally normally
36
- @mutex.synchronize{ @messages << [channel.to_s, Oj.dump(payload)] }
36
+ @messages.acquire{|msgs| msgs << [channel.to_s, Oj.dump(payload)]}
37
+ capture(:broadcast, channel)
37
38
  local_broadcast(channel, payload)
38
39
  end
39
40
 
40
41
  # subscribe to a channel, optionally providing a success callback proc
41
42
  def subscribe(channel, subscriber, success_callback = nil)
42
- @mutex.synchronize do
43
- @subscribers[channel] = {} unless @subscribers[channel]
44
- @subscribers[channel][subscriber] = true
43
+ @subscribers.acquire do |subs|
44
+ subs[channel] = {} unless subs[channel]
45
+ subs[channel][subscriber] = true
45
46
  end
47
+ capture(:subscribe, channel)
46
48
  end
47
49
 
48
50
  # unsubscribe from a channel
49
51
  def unsubscribe(channel, subscriber)
50
- @mutex.synchronize do
51
- @subscribers[channel].delete(subscriber) rescue nil
52
- end
52
+ @subscribers.acquire{|subs| subs[channel].delete(subscriber) rescue nil }
53
+ capture(:unsubscribe, channel)
53
54
  end
54
55
 
55
56
  private
56
-
57
+
58
+ # broadcast the message to local subscribers
57
59
  def local_broadcast(channel, payload=nil)
58
- return unless @subscribers[channel]
59
60
  subscribers = []
60
- @mutex.synchronize do
61
- subscribers = @subscribers[channel].keys
61
+ @subscribers.acquire do |subs|
62
+ return unless subs[channel]
63
+ subscribers = subs[channel].keys
62
64
  end
63
65
  subscribers.each do |subscriber|
64
66
  subscriber.call(payload)
67
+ capture(:message, channel)
65
68
  end
66
69
  end
67
70
 
68
71
  def setup
69
72
  super # create connection
70
73
  @pid = Process.pid
71
- @subscribers = {}
72
- @mutex = Litesupport::Mutex.new
74
+ @subscribers = Litesupport::Pool.new(1){{}}
73
75
  @running = true
74
76
  @listener = create_listener
75
77
  @pruner = create_pruner
@@ -78,16 +80,16 @@ class Litecable
78
80
  end
79
81
 
80
82
  def create_broadcaster
81
- Litesupport.spawn do
83
+ Litescheduler.spawn do
82
84
  while @running do
83
- @mutex.synchronize do
84
- if @messages.length > 0
85
+ @messages.acquire do |msgs|
86
+ if msgs.length > 0
85
87
  run_sql("BEGIN IMMEDIATE")
86
- while msg = @messages.shift
88
+ while msg = msgs.shift
87
89
  run_stmt(:publish, msg[0], msg[1], @pid)
88
90
  end
89
91
  run_sql("END")
90
- end
92
+ end
91
93
  end
92
94
  sleep 0.02
93
95
  end
@@ -95,7 +97,7 @@ class Litecable
95
97
  end
96
98
 
97
99
  def create_pruner
98
- Litesupport.spawn do
100
+ Litescheduler.spawn do
99
101
  while @running do
100
102
  run_stmt(:prune, @options[:expire_after])
101
103
  sleep @options[:expire_after]
@@ -104,10 +106,9 @@ class Litecable
104
106
  end
105
107
 
106
108
  def create_listener
107
- Litesupport.spawn do
109
+ Litescheduler.spawn do
108
110
  while @running do
109
111
  @last_fetched_id ||= (run_stmt(:last_id)[0][0] || 0)
110
- @logger.info @last_fetched_id
111
112
  run_stmt(:fetch, @last_fetched_id, @pid).to_a.each do |msg|
112
113
  @logger.info "RECEIVED #{msg}"
113
114
  @last_fetched_id = msg[0]
@@ -119,20 +120,9 @@ class Litecable
119
120
  end
120
121
 
121
122
  def create_connection
122
- conn = super
123
- conn.wal_autocheckpoint = 10000
124
- sql = YAML.load_file("#{__dir__}/litecable.sql.yml")
125
- version = conn.get_first_value("PRAGMA user_version")
126
- sql["schema"].each_pair do |v, obj|
127
- if v > version
128
- conn.transaction do
129
- obj.each{|k, s| conn.execute(s)}
130
- conn.user_version = v
131
- end
132
- end
133
- end
134
- sql["stmts"].each { |k, v| conn.stmts[k.to_sym] = conn.prepare(v) }
135
- conn
123
+ super("#{__dir__}/litecable.sql.yml") do |conn|
124
+ conn.wal_autocheckpoint = 10000
125
+ end
136
126
  end
137
127
 
138
128
  end
@@ -17,7 +17,7 @@ stmts:
17
17
 
18
18
  last_id: SELECT max(id) FROM messages
19
19
 
20
- fetch: SELECT id, channel, value FROM messages WHERE id > $1 and pid != $2
20
+ fetch: SELECT id, channel, value, created_at FROM messages WHERE id > $1 and pid != $2
21
21
 
22
22
  prune: DELETE FROM messages WHERE created_at < (unixepoch() - $1)
23
23
 
@@ -212,7 +212,7 @@ class Litecache
212
212
  end
213
213
 
214
214
  def spawn_worker
215
- Litesupport.spawn do
215
+ Litescheduler.spawn do
216
216
  while @running
217
217
  @conn.acquire do |cache|
218
218
  begin
@@ -236,23 +236,12 @@ class Litecache
236
236
  end
237
237
 
238
238
  def create_connection
239
- conn = super
240
- conn.cache_size = 2000
241
- conn.journal_size_limit = [(@options[:size]/2).to_i, @options[:min_size]].min
242
- conn.max_page_count = (@options[:size] / conn.page_size).to_i
243
- conn.case_sensitive_like = true
244
- sql = YAML.load_file("#{__dir__}/litecache.sql.yml")
245
- version = conn.get_first_value("PRAGMA user_version")
246
- sql["schema"].each_pair do |v, obj|
247
- if v > version
248
- conn.transaction do
249
- obj.each{|k, s| conn.execute(s)}
250
- conn.user_version = v
251
- end
252
- end
253
- end
254
- sql["stmts"].each { |k, v| conn.stmts[k.to_sym] = conn.prepare(v) }
255
- conn
239
+ super("#{__dir__}/litecache.sql.yml") do |conn|
240
+ conn.cache_size = 2000
241
+ conn.journal_size_limit = [(@options[:size]/2).to_i, @options[:min_size]].min
242
+ conn.max_page_count = (@options[:size] / conn.page_size).to_i
243
+ conn.case_sensitive_like = true
244
+ end
256
245
  end
257
246
 
258
247
  end
@@ -1,12 +1,21 @@
1
1
  # all components should require the support module
2
2
  require_relative 'litesupport'
3
3
 
4
+ # all measurable components should require the litemetric class
5
+ require_relative 'litemetric'
6
+
7
+ # litedb in particular gets access to litesearch
8
+ require_relative 'litesearch'
9
+
4
10
  # Litedb inherits from the SQLite3::Database class and adds a few initialization options
5
11
  class Litedb < ::SQLite3::Database
6
12
 
7
13
  # add litemetric support
8
14
  include Litemetric::Measurable
9
15
 
16
+ # add litesearch support
17
+ include Litesearch
18
+
10
19
  # overrride the original initilaizer to allow for connection configuration
11
20
  def initialize(file, options = {}, zfs = nil )
12
21
  if block_given?
@@ -34,7 +43,11 @@ class Litedb < ::SQLite3::Database
34
43
 
35
44
  # return the size of the database file
36
45
  def size
37
- execute("SELECT s.page_size * c.page_count FROM pragma_page_size() as s, pragma_page_count() as c")[0][0]
46
+ execute("SELECT s.page_size * c.page_count FROM pragma_page_size() AS s, pragma_page_count() AS c")[0][0]
47
+ end
48
+
49
+ def schema_object_count(type = nil)
50
+ execute("SELECT count(*) FROM SQLITE_MASTER WHERE iif(?1 IS NOT NULL, type = ?1, TRUE)", type)[0][0]
38
51
  end
39
52
 
40
53
  # collect snapshot information
@@ -44,7 +57,9 @@ class Litedb < ::SQLite3::Database
44
57
  path: filename,
45
58
  journal_mode: journal_mode,
46
59
  synchronous: synchronous,
47
- size: size
60
+ size: size.to_f / (1024 * 1024),
61
+ tables: schema_object_count('table'),
62
+ indexes: schema_object_count('index')
48
63
  }
49
64
  }
50
65
  end
@@ -68,9 +68,8 @@ module Litejob
68
68
  get_jobqueue
69
69
  end
70
70
 
71
- def delete(id, queue_name=nil)
72
- queue_name ||= queue
73
- get_jobqueue.delete(id, queue)
71
+ def delete(id)
72
+ get_jobqueue.delete(id)
74
73
  end
75
74
 
76
75
  def queue