litestack 0.4.1 → 0.4.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (66) hide show
  1. checksums.yaml +4 -4
  2. data/.standard.yml +3 -0
  3. data/BENCHMARKS.md +23 -7
  4. data/CHANGELOG.md +11 -0
  5. data/Gemfile +1 -7
  6. data/Gemfile.lock +92 -0
  7. data/README.md +120 -6
  8. data/ROADMAP.md +45 -0
  9. data/Rakefile +3 -1
  10. data/WHYLITESTACK.md +1 -1
  11. data/assets/litecache_metrics.png +0 -0
  12. data/assets/litedb_metrics.png +0 -0
  13. data/assets/litemetric_logo_teal.png +0 -0
  14. data/assets/litesearch_logo_teal.png +0 -0
  15. data/bench/bench.rb +17 -10
  16. data/bench/bench_cache_rails.rb +10 -13
  17. data/bench/bench_cache_raw.rb +17 -22
  18. data/bench/bench_jobs_rails.rb +18 -12
  19. data/bench/bench_jobs_raw.rb +17 -10
  20. data/bench/bench_queue.rb +4 -6
  21. data/bench/rails_job.rb +5 -7
  22. data/bench/skjob.rb +4 -4
  23. data/bench/uljob.rb +6 -6
  24. data/lib/action_cable/subscription_adapter/litecable.rb +5 -8
  25. data/lib/active_job/queue_adapters/litejob_adapter.rb +6 -8
  26. data/lib/active_record/connection_adapters/litedb_adapter.rb +65 -75
  27. data/lib/active_support/cache/litecache.rb +38 -41
  28. data/lib/generators/litestack/install/install_generator.rb +3 -3
  29. data/lib/generators/litestack/install/templates/database.yml +7 -1
  30. data/lib/litestack/liteboard/liteboard.rb +269 -149
  31. data/lib/litestack/litecable.rb +41 -37
  32. data/lib/litestack/litecable.sql.yml +22 -11
  33. data/lib/litestack/litecache.rb +79 -88
  34. data/lib/litestack/litecache.sql.yml +81 -22
  35. data/lib/litestack/litecache.yml +1 -1
  36. data/lib/litestack/litedb.rb +35 -40
  37. data/lib/litestack/litejob.rb +30 -29
  38. data/lib/litestack/litejobqueue.rb +63 -65
  39. data/lib/litestack/litemetric.rb +80 -92
  40. data/lib/litestack/litemetric.sql.yml +244 -234
  41. data/lib/litestack/litemetric_collector.sql.yml +38 -41
  42. data/lib/litestack/litequeue.rb +39 -41
  43. data/lib/litestack/litequeue.sql.yml +39 -31
  44. data/lib/litestack/litescheduler.rb +15 -15
  45. data/lib/litestack/litesearch/index.rb +93 -63
  46. data/lib/litestack/litesearch/model.rb +66 -65
  47. data/lib/litestack/litesearch/schema.rb +53 -56
  48. data/lib/litestack/litesearch/schema_adapters/backed_adapter.rb +46 -50
  49. data/lib/litestack/litesearch/schema_adapters/basic_adapter.rb +44 -35
  50. data/lib/litestack/litesearch/schema_adapters/contentless_adapter.rb +3 -6
  51. data/lib/litestack/litesearch/schema_adapters/standalone_adapter.rb +7 -9
  52. data/lib/litestack/litesearch/schema_adapters.rb +4 -9
  53. data/lib/litestack/litesearch.rb +6 -9
  54. data/lib/litestack/litesupport.rb +76 -86
  55. data/lib/litestack/railtie.rb +1 -1
  56. data/lib/litestack/version.rb +2 -2
  57. data/lib/litestack.rb +6 -4
  58. data/lib/railties/rails/commands/dbconsole.rb +11 -15
  59. data/lib/sequel/adapters/litedb.rb +16 -21
  60. data/lib/sequel/adapters/shared/litedb.rb +168 -168
  61. data/scripts/build_metrics.rb +91 -0
  62. data/scripts/test_cable.rb +30 -0
  63. data/scripts/test_job_retry.rb +33 -0
  64. data/scripts/test_metrics.rb +60 -0
  65. data/template.rb +2 -2
  66. metadata +101 -6
@@ -1,31 +1,30 @@
1
1
  # frozen_stringe_literal: true
2
2
 
3
- require 'singleton'
3
+ require "singleton"
4
4
 
5
- require_relative './litesupport'
5
+ require_relative "./litesupport"
6
6
 
7
7
  # this class is a singleton
8
8
  # and should remain so
9
9
  class Litemetric
10
-
11
10
  include Singleton
12
11
  include Litesupport::Liteconnection
13
-
12
+
14
13
  DEFAULT_OPTIONS = {
15
14
  config_path: "./litemetric.yml",
16
15
  path: Litesupport.root.join("metrics.sqlite3"),
17
16
  sync: 1,
18
17
  mmap_size: 128 * 1024 * 1024, # 16MB of memory to easily process 1 year worth of data
19
- flush_interval: 10, # flush data every 10 seconds
18
+ flush_interval: 10, # flush data every 10 seconds
20
19
  summarize_interval: 30, # summarize data every 1/2 minute
21
- snapshot_interval: 10*60 # snapshot every 10 minutes
20
+ snapshot_interval: 10 * 60 # snapshot every 10 minutes
22
21
  }
23
22
 
24
23
  RESOLUTIONS = {
25
24
  minute: 300, # 5 minutes (highest resolution)
26
25
  hour: 3600, # 1 hour
27
- day: 24*3600, # 1 day
28
- week: 7*24*3600 # 1 week (lowest resolution)
26
+ day: 24 * 3600, # 1 day
27
+ week: 7 * 24 * 3600 # 1 week (lowest resolution)
29
28
  }
30
29
 
31
30
  # :nodoc:
@@ -34,111 +33,111 @@ class Litemetric
34
33
  # need to rethink the whole singleton thing
35
34
  @options = options
36
35
  end
37
-
36
+
38
37
  def self.options
39
38
  @options
40
39
  end
41
40
 
42
- # :nodoc:
41
+ # :nodoc:
43
42
  def initialize(options = {})
44
43
  options = options.merge(Litemetric.options) if Litemetric.options
45
44
  init(options)
46
45
  end
47
-
46
+
48
47
  # registers a class for metrics to be collected
49
48
  def register(identifier)
50
- @registered[identifier] = true
49
+ @registered[identifier] = true
51
50
  @metrics[identifier] = {} unless @metrics[identifier]
52
51
  run_stmt(:register_topic, identifier) # it is safe to call register topic multiple times with the same identifier
53
52
  end
54
-
53
+
55
54
  ## event capturing
56
55
  ##################
57
-
56
+
58
57
  def current_time_slot
59
58
  (Time.now.to_i / 300) * 300
60
59
  end
61
-
62
- def capture(topic, event, key=event, value=nil)
60
+
61
+ def capture(topic, event, key = event, value = nil)
63
62
  @collector.capture(topic, event, key, value, current_time_slot)
64
63
  end
65
-
64
+
66
65
  def capture_snapshot(topic, state)
67
66
  run_stmt(:capture_state, topic, Oj.dump(state))
68
67
  end
69
68
 
70
69
  ## event reporting
71
70
  ##################
72
-
71
+
73
72
  def topics
74
73
  run_stmt(:list_topics).to_a
75
74
  end
76
-
75
+
77
76
  def topic_summaries(resolution, count, order, dir, search)
78
77
  search = "%#{search}%" if search
79
- if dir.downcase == "desc"
78
+ if dir.downcase == "desc"
80
79
  run_stmt(:topics_summaries, resolution, count, order, search).to_a
81
80
  else
82
81
  run_stmt(:topics_summaries_asc, resolution, count, order, search).to_a
83
82
  end
84
83
  end
85
-
84
+
86
85
  def events_summaries(topic, resolution, order, dir, search, count)
87
86
  search = "%#{search}%" if search
88
- if dir.downcase == "desc"
87
+ if dir.downcase == "desc"
89
88
  run_stmt_hash(:events_summaries, topic, resolution, order, search, count)
90
89
  else
91
90
  run_stmt_hash(:events_summaries_asc, topic, resolution, order, search, count)
92
91
  end
93
92
  end
94
-
93
+
95
94
  def keys_summaries(topic, event, resolution, order, dir, search, count)
96
95
  search = "%#{search}%" if search
97
- if dir.downcase == "desc"
96
+ if dir.downcase == "desc"
98
97
  run_stmt_hash(:keys_summaries, topic, event, resolution, order, search, count).to_a
99
98
  else
100
99
  run_stmt_hash(:keys_summaries_asc, topic, event, resolution, order, search, count).to_a
101
- end
100
+ end
102
101
  end
103
102
 
104
103
  def topic_data_points(step, count, resolution, topic)
105
104
  run_stmt(:topic_data_points, step, count, resolution, topic).to_a
106
- end
105
+ end
107
106
 
108
107
  def event_data_points(step, count, resolution, topic, event)
109
108
  run_stmt_hash(:event_data_points, step, count, resolution, topic, event).to_a
110
- end
109
+ end
111
110
 
112
111
  def key_data_points(step, count, resolution, topic, event, key)
113
112
  run_stmt_hash(:key_data_points, step, count, resolution, topic, event, key).to_a
114
- end
113
+ end
115
114
 
116
115
  def snapshot(topic)
117
116
  run_stmt(:snapshot, topic)[0].to_a
118
117
  end
119
-
118
+
120
119
  ## summarize data
121
- #################
120
+ #################
122
121
 
123
122
  def summarize
124
- run_stmt(:summarize_events, RESOLUTIONS[:hour], "hour", "minute")
125
- run_stmt(:summarize_events, RESOLUTIONS[:day], "day", "hour")
123
+ run_stmt(:summarize_events, RESOLUTIONS[:hour], "hour", "minute")
124
+ run_stmt(:summarize_events, RESOLUTIONS[:day], "day", "hour")
126
125
  run_stmt(:summarize_events, RESOLUTIONS[:week], "week", "day")
127
- run_stmt(:delete_events, "minute", RESOLUTIONS[:hour]*1)
128
- run_stmt(:delete_events, "hour", RESOLUTIONS[:day]*1)
129
- run_stmt(:delete_events, "day", RESOLUTIONS[:week]*1)
126
+ run_stmt(:delete_events, "minute", RESOLUTIONS[:hour] * 1)
127
+ run_stmt(:delete_events, "hour", RESOLUTIONS[:day] * 1)
128
+ run_stmt(:delete_events, "day", RESOLUTIONS[:week] * 1)
130
129
  end
131
130
 
132
131
  ## background stuff
133
132
  ###################
134
133
 
135
134
  private
136
-
135
+
137
136
  def run_stmt_hash(stmt, *args)
138
137
  res = run_stmt(stmt, *args)
139
138
  cols = run_stmt_method(stmt, :columns)
140
139
  hashes = []
141
- res.each do | row |
140
+ res.each do |row|
142
141
  hash = {}
143
142
  row.each_with_index do |field, i|
144
143
  hash[cols[i]] = field
@@ -147,11 +146,12 @@ class Litemetric
147
146
  end
148
147
  hashes
149
148
  end
150
-
149
+
151
150
  def exit_callback
152
- STDERR.puts "--- Litemetric detected an exit, flushing metrics"
151
+ return unless @collector.count > 0
152
+ warn "--- Litemetric detected an exit, flushing metrics"
153
153
  @running = false
154
- flush
154
+ @collector.flush
155
155
  end
156
156
 
157
157
  def setup
@@ -164,38 +164,29 @@ class Litemetric
164
164
  @flusher = create_flusher
165
165
  end
166
166
 
167
- def current_time_slot
168
- (Time.now.to_i / 300) * 300 # every 5 minutes
169
- end
170
-
171
- def flush
172
- @collector.flush
173
- end
174
-
175
167
  def create_connection
176
168
  super("#{__dir__}/litemetric.sql.yml") do |conn|
177
169
  conn.wal_autocheckpoint = 10000 # checkpoint after 10000 pages are written
178
170
  end
179
171
  end
180
-
172
+
181
173
  def create_flusher
182
174
  Litescheduler.spawn do
183
- while @running do
175
+ while @running
184
176
  sleep @options[:flush_interval]
185
- flush
186
- end
187
- end
177
+ @collector.flush
178
+ end
179
+ end
188
180
  end
189
-
181
+
190
182
  def create_summarizer
191
183
  Litescheduler.spawn do
192
- while @running do
184
+ while @running
193
185
  sleep @options[:summarize_interval]
194
186
  summarize
195
- end
196
- end
187
+ end
188
+ end
197
189
  end
198
-
199
190
  end
200
191
 
201
192
  ## Measurable Module
@@ -203,44 +194,43 @@ end
203
194
 
204
195
  class Litemetric
205
196
  module Measurable
206
-
207
197
  def collect_metrics
208
198
  @litemetric = Litemetric.instance
209
199
  @litemetric.register(metrics_identifier)
210
- @snapshotter = create_snapshotter
200
+ @snapshotter = create_snapshotter
211
201
  end
212
202
 
213
203
  def create_snapshotter
214
204
  Litescheduler.spawn do
215
- while @running do
205
+ while @running
216
206
  sleep @litemetric.options[:snapshot_interval]
217
207
  capture_snapshot
218
- end
219
- end
208
+ end
209
+ end
220
210
  end
221
211
 
222
212
  def metrics_identifier
223
213
  self.class.name # override in included classes
224
214
  end
225
215
 
226
- def capture(event, key=event, value=nil)
216
+ def capture(event, key = event, value = nil)
227
217
  return unless @litemetric
228
218
  @litemetric.capture(metrics_identifier, event, key, value)
229
219
  end
230
-
231
- def measure(event, key=event)
220
+
221
+ def measure(event, key = event)
232
222
  unless @litemetric
233
223
  yield
234
- return 0
224
+ return 0
235
225
  end
236
- t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
226
+ t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
237
227
  yield
238
228
  t2 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
239
- value = t2 - t1
229
+ value = t2 - t1
240
230
  capture(event, key, value)
241
- value # return value so other events can reuse it
242
- end
243
-
231
+ value # return value so other events can reuse it
232
+ end
233
+
244
234
  def capture_snapshot
245
235
  return unless @litemetric
246
236
  state = snapshot if defined? snapshot
@@ -248,20 +238,17 @@ class Litemetric
248
238
  @litemetric.capture_snapshot(metrics_identifier, state)
249
239
  end
250
240
  end
251
-
252
241
  end
253
242
  end
254
243
 
255
244
  class Litemetric
256
-
257
245
  class Collector
258
-
259
246
  include Litesupport::Liteconnection
260
-
247
+
261
248
  DEFAULT_OPTIONS = {
262
249
  path: ":memory:",
263
250
  sync: 1,
264
- flush_interval: 3, # flush data every 1 minute
251
+ flush_interval: 3, # flush data every 1 minute
265
252
  summarize_interval: 10, # summarize data every 1 minute
266
253
  snapshot_interval: 1 # snapshot every 10 minutes
267
254
  }
@@ -269,28 +256,31 @@ class Litemetric
269
256
  RESOLUTIONS = {
270
257
  minute: 300, # 5 minutes (highest resolution)
271
258
  hour: 3600, # 1 hour
272
- day: 24*3600, # 1 day
273
- week: 7*24*3600 # 1 week (lowest resolution)
259
+ day: 24 * 3600, # 1 day
260
+ week: 7 * 24 * 3600 # 1 week (lowest resolution)
274
261
  }
275
-
262
+
276
263
  def initialize(options = {})
277
264
  init(options)
278
265
  end
279
-
280
- def capture(topic, event, key, value=nil, time=nil)
266
+
267
+ def capture(topic, event, key, value = nil, time = nil)
281
268
  if key.is_a? Array
282
- key.each{|k| capture_single_key(topic, event, k, value, time)}
269
+ key.each { |k| capture_single_key(topic, event, k, value, time) }
283
270
  else
284
271
  capture_single_key(topic, event, key, value, time)
285
272
  end
286
273
  end
287
-
288
- def capture_single_key(topic, event, key, value, time=nil)
289
- run_stmt(:capture_event, topic.to_s, event.to_s, key.to_s, time ,1, value)
274
+
275
+ def capture_single_key(topic, event, key, value, time = nil)
276
+ run_stmt(:capture_event, topic.to_s, event.to_s, key.to_s, time, 1, value)
290
277
  end
291
278
 
279
+ def count
280
+ run_stmt(:event_count)[0][0]
281
+ end
282
+
292
283
  def flush
293
- t = Time.now
294
284
  limit = 1000 # migrate 1000 records at a time
295
285
  count = run_stmt(:event_count)[0][0]
296
286
  while count > 0
@@ -299,19 +289,17 @@ class Litemetric
299
289
  conn.stmts[:migrate_events].execute!(limit)
300
290
  conn.stmts[:delete_migrated_events].execute!(limit)
301
291
  count = conn.stmts[:event_count].execute![0][0]
302
- end
292
+ end
303
293
  end
304
- sleep 0.005 #give other threads a chance to run
294
+ sleep 0.005 # give other threads a chance to run
305
295
  end
306
296
  end
307
297
 
308
298
  def create_connection
309
299
  super("#{__dir__}/litemetric_collector.sql.yml") do |conn|
310
- conn.execute("ATTACH ? as m", @options[:dbpath].to_s)
300
+ conn.execute("ATTACH ? as m", @options[:dbpath].to_s)
311
301
  conn.wal_autocheckpoint = 10000
312
302
  end
313
303
  end
314
-
315
304
  end
316
-
317
305
  end