litestack 0.2.3 → 0.2.6

Sign up to get free protection for your applications and to get access to all the features.
@@ -2,68 +2,336 @@ schema:
2
2
  1:
3
3
  create_topics: >
4
4
  CREATE TABLE IF NOT EXISTS topics(
5
- name text PRIMARY KEY NOT NULL
6
- ) WITHOUT ROWID;
5
+ name TEXT PRIMARY KEY NOT NULL,
6
+ state TEXT,
7
+ updated_at INTEGER
8
+ );
9
+
7
10
  create_events: >
8
11
  CREATE TABLE IF NOT EXISTS events(
9
- topic text NOT NULL references topics(name) ON DELETE CASCADE,
10
- name TEXT NOT NULL,
11
- key TEXT NOT NULL,
12
+ topic TEXT NOT NULL references topics(name) ON DELETE CASCADE,
13
+ name TEXT DEFAULT('___') NOT NULL ON CONFLICT REPLACE,
14
+ key TEXT DEFAULT('___') NOT NULL ON CONFLICT REPLACE,
12
15
  count INTEGER DEFAULT(0) NOT NULL ON CONFLICT REPLACE,
13
16
  value INTEGER,
14
17
  minimum INTEGER,
15
18
  maximum INTEGER,
16
19
  created_at INTEGER DEFAULT((unixepoch()/300*300)) NOT NULL,
17
20
  resolution TEXT DEFAULT('minute') NOT NULL,
18
- PRIMARY KEY(resolution, topic, name, key, created_at)
19
- ) WITHOUT ROWID;
20
- create_index_on_event: CREATE INDEX IF NOT EXISTS events_by_resolution ON events(resolution, created_at);
21
+ PRIMARY KEY(resolution, created_at, topic, name, key)
22
+ );
23
+
24
+ create_topic_index_on_events: CREATE INDEX events_topic_index ON events (resolution, created_at, topic) WHERE name = '___';
25
+ create_event_index_on_events: CREATE INDEX events_event_index ON events (resolution, created_at , topic, name) WHERE key = '___';
26
+
27
+ create_events_search: >
28
+ CREATE VIRTUAL TABLE IF NOT EXISTS events_search USING fts5 (
29
+ topic,
30
+ name,
31
+ key,
32
+ resolution,
33
+ content=events,
34
+ tokeniz="trigram"
35
+ )
36
+
37
+ create_events_insert_trigger: >
38
+ CREATE TRIGGER events_insert AFTER INSERT ON events
39
+ BEGIN
40
+ INSERT INTO
41
+ events_search(rowid, topic, name, key, resolution)
42
+ VALUES
43
+ (new.rowid, new.topic, new.name, new.key, new.resolution);
44
+ END;
45
+
46
+ create_events_delete_trigger: >
47
+ CREATE TRIGGER events_delete AFTER DELETE ON events
48
+ BEGIN
49
+ INSERT INTO
50
+ events_search(events_search, rowid, topic, name, key, resolution)
51
+ VALUES
52
+ ('delete', old.rowid, old.topic, old.name, old.key, old.resolution);
53
+ END;
54
+
55
+ create_events_update_trigger: >
56
+ CREATE TRIGGER events_update AFTER UPDATE ON events
57
+ BEGIN
58
+ INSERT INTO
59
+ events_search(events_search, rowid, topic, name, key, resolution)
60
+ VALUES
61
+ ('delete', old.rowid, old.topic, old.name, old.key, old.resolution);
62
+ INSERT INTO
63
+ events_search(rowid, topic, name, key, resolution)
64
+ VALUES
65
+ (new.rowid, new.topic, new.name, new.key, new.resolution);
66
+ END;
67
+
68
+ rebuild_index: >
69
+ INSERT INTO events_search(events_search) VALUES ('rebuild');
70
+
71
+
72
+
21
73
 
22
74
  stmts:
23
- # register topic
24
- register_topic: INSERT INTO topics VALUES (?) ON CONFLICT DO NOTHING;
75
+
76
+ register_topic: >
77
+ INSERT INTO topics (name, updated_at) VALUES (?, unixepoch())
78
+ ON CONFLICT DO
79
+ UPDATE SET updated_at = unixepoch();
80
+
81
+ capture_state: >
82
+ UPDATE topics
83
+ SET state = ?2, updated_at = unixepoch()
84
+ WHERE name = ?1
85
+
86
+ snapshot: >
87
+ SELECT state, datetime(updated_at, 'unixepoch')
88
+ FROM topics
89
+ WHERE name = ?1
25
90
 
26
91
  capture_event: >
27
- INSERT INTO events(topic, name, key, created_at, count, value, minimum, maximum) VALUES ($1, $2, $3, $4, $5, $6, $6, $6)
92
+ INSERT INTO events(topic, name, key, created_at, count, value, minimum, maximum) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)
28
93
  ON CONFLICT DO
29
94
  UPDATE SET count = count + EXCLUDED.count, value = value + EXCLUDED.value, minimum = min(minimum, EXCLUDED.minimum), maximum = max(maximum, EXCLUDED.maximum)
30
95
 
31
96
  # requires an index on (resolution, created_at)
32
97
  summarize_events: >
33
- INSERT INTO events (topic, name, key, count, value, minimum, maximum, created_at, resolution ) SELECT
98
+ INSERT INTO
99
+ events (topic, name, key, count, value, minimum, maximum, created_at, resolution )
100
+ SELECT
34
101
  topic,
35
102
  name,
36
103
  key,
37
- sum(count) as count,
38
- sum(value) as value,
39
- min(minimum) as minimum,
40
- max(maximum) as maximum,
41
- (created_at/$1)*$1 as created,
42
- $2
43
- FROM events WHERE resolution = $3 AND created_at < (unixepoch()/$1)*$1 GROUP BY topic, name, key, created ON CONFLICT DO UPDATE
44
- SET count = count + EXCLUDED.count, value = value + EXCLUDED.value, minimum = min(minimum, EXCLUDED.minimum), maximum = max(maximum, EXCLUDED.maximum);
104
+ sum(count),
105
+ sum(value),
106
+ min(minimum),
107
+ max(maximum),
108
+ (created_at/(?1))*?1 as created,
109
+ ?2
110
+ FROM events
111
+ WHERE resolution = ?3
112
+ GROUP BY topic, name, key, created
113
+ ON CONFLICT DO
114
+ UPDATE SET count = EXCLUDED.count, value = EXCLUDED.value, minimum = EXCLUDED.minimum, maximum = EXCLUDED.maximum
45
115
 
46
- # requires an index on (resolution, created_at)
47
- delete_events: DELETE FROM events WHERE resolution = $3 AND created_at < (unixepoch() - $4);
116
+ # requires an index on (resolution, created_at) #TODO: fix deletion boundary (use the date mapping)
117
+ delete_events: DELETE FROM events WHERE resolution = ?1 AND (created_at /(?2)) * ?2 < ((unixepoch()/(?2)) * ?2 - ?2);
48
118
 
49
119
  # select topics from the topics table
50
- list_topics: SELECT name FROM topics;
51
-
52
- # requires an index on (resolution, topic, name)
53
- list_event_names: >
54
- SELECT name, sum(count) as count, count(distinct name) as name, sum(value) as value, min(minimum), max(maximum)
55
- FROM events WHERE resolution = ? AND topic = ? GROUP BY name ORDER BY count;
120
+ list_topics: SELECT name FROM topics ORDER BY name ASC;
56
121
 
57
- # requires an index on (resolution, topic, name, key)
58
- list_event_keys: >
59
- SELECT key, sum(count) as count, sum(value) as value, min(minimum), max(maximum)
60
- FROM events WHERE resolution = ? AND topic = ? AND name = ? GROUP BY key ORDER BY count;
122
+ # select topics, their event counts, total values, min and max values
123
+ topics_summaries: >
124
+ SELECT
125
+ topic,
126
+ count(distinct name) as events,
127
+ count(distinct key) as keys,
128
+ sum(count) as rcount
129
+ from events
130
+ where resolution = ?1
131
+ AND created_at <= unixepoch()
132
+ AND created_at >= unixepoch() -?2
133
+ AND name = '___'
134
+ AND iif(length(?4) > 0, rowid IN (SELECT rowid FROM events_search WHERE topic LIKE ?4), TRUE)
135
+ GROUP BY topic
136
+ ORDER BY
137
+ iif(?3 = 'events', events,
138
+ iif(?3 = 'keys', keys,
139
+ iif(?3 = 'rcount', rcount, topic)))
140
+ DESC;
61
141
 
62
- # requires an index on (resolution, topic, name, key, created_at)
63
- list_events_by_key: >
64
- SELECT * FROM events WHERE resolution = $1 AND topic = $2 AND name = $3 AND key = $4 ORDER BY created_at ASC;
65
-
66
- # requires an index on (resolution, topic, name, key, created_at)
67
- list_all_events: >
68
- SELECT * FROM events WHERE resolution = ? AND topic = ? ORDER BY name, key, created_at ASC;
142
+ topics_summaries_asc: >
143
+ SELECT
144
+ topic,
145
+ count(distinct name) as events,
146
+ count(distinct key) as keys,
147
+ sum(count) as rcount
148
+ from events
149
+ where resolution = ?1
150
+ AND created_at <= unixepoch()
151
+ AND created_at >= unixepoch() -?2
152
+ AND name = '___'
153
+ AND iif(length(?4) > 0, rowid IN (SELECT rowid FROM events_search WHERE topic LIKE ?4), TRUE)
154
+ GROUP BY topic
155
+ ORDER BY
156
+ iif(?3 = 'events', events,
157
+ iif(?3 = 'keys', keys,
158
+ iif(?3 = 'rcount', rcount, topic)))
159
+ ASC;
160
+
161
+ events_summaries: >
162
+ SELECT
163
+ name,
164
+ count(DISTINCT key) AS keys,
165
+ sum(count) as rcount,
166
+ cast(sum(value) as double) / sum(count) as ravg,
167
+ sum(value) AS rtotal,
168
+ min(minimum) AS rmin,
169
+ max(maximum) AS rmax
170
+ FROM events
171
+ WHERE topic = ?1 AND resolution = ?2
172
+ AND created_at <= unixepoch()
173
+ AND created_at >= unixepoch() - ?5
174
+ AND name != '___'
175
+ AND key = '___'
176
+ AND iif(length(?4) > 0, rowid IN (SELECT rowid FROM events_search WHERE name LIKE ?4), TRUE)
177
+ GROUP BY name
178
+ ORDER BY
179
+ iif(?3 = 'rcount', rcount,
180
+ iif(?3 = 'ravg', ravg,
181
+ iif(?3 = 'rtotal', rtotal,
182
+ iif(?3 = 'rmin', rmin,
183
+ iif(?3 = 'rmax', rmax,
184
+ iif(?3 = 'keys', keys,
185
+ iif(?3 = 'name', name, rcount)))))))
186
+ DESC
187
+ LIMIT 20;
188
+
189
+ events_summaries_asc: >
190
+ SELECT
191
+ name,
192
+ count(DISTINCT key) AS keys,
193
+ sum(count) as rcount,
194
+ cast(sum(value) as double) / sum(count) as ravg,
195
+ sum(value) AS rtotal,
196
+ min(minimum) AS rmin,
197
+ max(maximum) AS rmax
198
+ FROM events
199
+ WHERE topic = ?1 AND resolution = ?2
200
+ AND created_at <= unixepoch()
201
+ AND created_at >= unixepoch() -?5
202
+ AND name != '___'
203
+ AND key = '___'
204
+ AND iif(length(?4) > 0, rowid IN (SELECT rowid FROM events_search WHERE name LIKE ?4), TRUE)
205
+ GROUP BY name
206
+ ORDER BY
207
+ iif(?3 = 'rcount', rcount,
208
+ iif(?3 = 'ravg', ravg,
209
+ iif(?3 = 'rtotal', rtotal,
210
+ iif(?3 = 'rmin', rmin,
211
+ iif(?3 = 'rmax', rmax,
212
+ iif(?3 = 'keys', keys,
213
+ iif(?3 = 'name', name, rcount)))))))
214
+ ASC
215
+ LIMIT 20;
216
+
69
217
 
218
+ keys_summaries: >
219
+ SELECT
220
+ key,
221
+ sum(count) AS rcount,
222
+ cast(sum(value) AS double) / sum(count) AS ravg,
223
+ sum(value) AS rtotal,
224
+ min(minimum) AS rmin,
225
+ max(maximum) AS rmax
226
+ FROM events
227
+ WHERE topic = ?1 AND name = ?2 AND resolution = ?3
228
+ AND created_at <= unixepoch()
229
+ AND created_at >= unixepoch() -?6
230
+ AND key != '___'
231
+ AND iif(length(?5) > 0, rowid IN (SELECT rowid FROM events_search WHERE key LIKE ?5), TRUE)
232
+ GROUP BY key
233
+ ORDER BY
234
+ iif(?4 = 'rcount', rcount,
235
+ iif(?4 = 'ravg', ravg,
236
+ iif(?4 = 'rtotal', rtotal,
237
+ iif(?4 = 'rmin', rmin,
238
+ iif(?4 = 'rmax', rmax,
239
+ iif(?4 = 'key', key, rcount))))))
240
+ DESC
241
+ LIMIT 20;
242
+
243
+ keys_summaries_asc: >
244
+ SELECT
245
+ key,
246
+ sum(count) AS rcount,
247
+ cast(sum(value) AS double) / sum(count) AS ravg,
248
+ sum(value) AS rtotal,
249
+ min(minimum) AS rmin,
250
+ max(maximum) AS rmax
251
+ FROM events
252
+ WHERE topic = ?1 AND name = ?2 AND resolution = ?3
253
+ AND created_at <= unixepoch()
254
+ AND created_at >= unixepoch() -?6
255
+ AND key != '___'
256
+ AND iif(length(?5) > 0, rowid IN (SELECT rowid FROM events_search WHERE key LIKE ?5), TRUE)
257
+ GROUP BY key
258
+ ORDER BY
259
+ iif(?4 = 'rcount', rcount,
260
+ iif(?4 = 'ravg', ravg,
261
+ iif(?4 = 'rtotal', rtotal,
262
+ iif(?4 = 'rmin', rmin,
263
+ iif(?4 = 'rmax', rmax,
264
+ iif(?4 = 'key', key, rcount))))))
265
+ ASC
266
+ LIMIT 20;
267
+
268
+ # list data points at the topic, event, or key level
269
+ # needs: duration, limit, resolution, topic, name (optional), key (optional)
270
+
271
+ topic_data_points: >
272
+ SELECT
273
+ datetime(ts.slot, 'unixepoch') AS rtime,
274
+ ts.slot AS unixtime,
275
+ format('%.2f', sum(events.count)) AS rcount,
276
+ format('%.2f', avg(events.value/events.count)) AS ravg
277
+ FROM (
278
+ WITH RECURSIVE timeslot(x) AS (
279
+ SELECT (unixepoch()/(?1))*(?1)
280
+ UNION ALL
281
+ SELECT x - (?1) FROM timeslot LIMIT ?2
282
+ ) SELECT x as slot FROM timeslot
283
+ ) AS ts
284
+ LEFT OUTER JOIN events
285
+ ON ts.slot = events.created_at
286
+ AND events.resolution = ?3
287
+ AND events.topic = ?4
288
+ AND events.created_at <= UNIXEPOCH()
289
+ AND events.name = '___'
290
+ GROUP BY ts.slot
291
+ ORDER BY ts.slot ASC;
292
+
293
+ event_data_points: >
294
+ SELECT
295
+ datetime(ts.slot, 'unixepoch') AS rtime,
296
+ ts.slot AS unixtime,
297
+ format('%.2f', sum(events.count)) AS rcount,
298
+ format('%.2f', avg(events.value/events.count)) AS ravg
299
+ FROM (
300
+ WITH RECURSIVE timeslot(x) AS (
301
+ SELECT (unixepoch()/(?1))*(?1)
302
+ UNION ALL
303
+ SELECT x - (?1) FROM timeslot LIMIT ?2
304
+ ) SELECT x as slot FROM timeslot
305
+ ) AS ts
306
+ LEFT OUTER JOIN events
307
+ ON ts.slot = events.created_at
308
+ AND events.resolution = ?3
309
+ AND events.topic = ?4
310
+ AND events.name = ?5
311
+ AND events.key = '___'
312
+ AND events.created_at <= UNIXEPOCH()
313
+ GROUP BY ts.slot
314
+ ORDER BY ts.slot ASC;
315
+
316
+ key_data_points: >
317
+ SELECT
318
+ datetime(ts.slot, 'unixepoch') AS rtime,
319
+ ts.slot AS unixtime,
320
+ format('%.2f', sum(events.count)) AS rcount,
321
+ format('%.2f', avg(events.value/events.count)) AS ravg
322
+ FROM (
323
+ WITH RECURSIVE timeslot(x) AS (
324
+ SELECT (unixepoch()/(?1))*(?1)
325
+ UNION ALL
326
+ SELECT x - (?1) FROM timeslot LIMIT ?2
327
+ ) SELECT x as slot FROM timeslot
328
+ ) AS ts
329
+ LEFT OUTER JOIN events
330
+ ON ts.slot = events.created_at
331
+ AND events.resolution = ?3
332
+ AND events.topic = ?4
333
+ AND events.name = ?5
334
+ AND events.key = ?6
335
+ AND events.created_at <= UNIXEPOCH()
336
+ GROUP BY ts.slot
337
+ ORDER BY ts.slot ASC;
@@ -0,0 +1,56 @@
1
+ schema:
2
+ 1:
3
+
4
+ create_events: >
5
+ CREATE TABLE IF NOT EXISTS local_events(
6
+ topic TEXT NOT NULL,
7
+ name TEXT DEFAULT('___') NOT NULL ON CONFLICT REPLACE,
8
+ key TEXT DEFAULT('___') NOT NULL ON CONFLICT REPLACE,
9
+ count INTEGER DEFAULT(0) NOT NULL ON CONFLICT REPLACE,
10
+ value INTEGER,
11
+ minimum INTEGER,
12
+ maximum INTEGER,
13
+ created_at INTEGER DEFAULT((unixepoch()/300*300)) NOT NULL ON CONFLICT REPLACE,
14
+ resolution TEXT DEFAULT('minute') NOT NULL,
15
+ PRIMARY KEY(resolution, created_at, topic, name, key)
16
+ );
17
+
18
+ stmts:
19
+
20
+ capture_event: >
21
+ INSERT INTO local_events(topic, name, key, created_at, count, value, minimum, maximum)
22
+ VALUES
23
+ (?1, ?2, ?3, ?4, ?5, ?6, ?6, ?6),
24
+ (?1, ?2, '___', ?4, ?5, ?6, ?6, ?6),
25
+ (?1, '___', '___', ?4, ?5, ?6, ?6, ?6)
26
+ ON CONFLICT DO UPDATE
27
+ SET
28
+ count = count + EXCLUDED.count,
29
+ value = value + EXCLUDED.value,
30
+ minimum = min(minimum, EXCLUDED.minimum),
31
+ maximum = max(maximum, EXCLUDED.maximum)
32
+
33
+ migrate_events:
34
+ INSERT INTO m.events(topic, name, key, created_at, count, value, minimum, maximum)
35
+ SELECT topic, name, key, created_at, count, value, minimum, maximum
36
+ FROM local_events
37
+ ORDER BY resolution, created_at ASC, topic, name, key
38
+ LIMIT ?
39
+ ON CONFLICT DO UPDATE
40
+ SET
41
+ count = count + EXCLUDED.count,
42
+ value = value + EXCLUDED.value,
43
+ minimum = min(minimum, EXCLUDED.minimum),
44
+ maximum = max(maximum, EXCLUDED.maximum)
45
+
46
+ delete_migrated_events:
47
+ DELETE FROM local_events WHERE rowid IN (
48
+ SELECT rowid
49
+ FROM local_events
50
+ ORDER BY resolution, created_at ASC, topic, name, key
51
+ LIMIT ?
52
+ )
53
+
54
+ event_count:
55
+ SELECT count(*) FROM local_events;
56
+
@@ -23,7 +23,7 @@ class Litequeue
23
23
  include Litesupport::Liteconnection
24
24
 
25
25
  DEFAULT_OPTIONS = {
26
- path: "./queue.db",
26
+ path: Litesupport.root.join("queue.sqlite3"),
27
27
  mmap_size: 32 * 1024,
28
28
  sync: 0
29
29
  }
@@ -87,20 +87,30 @@ class Litequeue
87
87
  end
88
88
 
89
89
  # return the size of the queue file on disk
90
- def size
91
- run_sql("SELECT size.page_size * count.page_count FROM pragma_page_size() AS size, pragma_page_count() AS count")[0][0]
92
- end
90
+ #def size
91
+ # run_sql("SELECT size.page_size * count.page_count FROM pragma_page_size() AS size, pragma_page_count() AS count")[0][0]
92
+ #end
93
93
 
94
94
  def queues_info
95
- run_sql("SELECT name, count(*) AS count, avg(unixepoch() - created_at), min(unixepoch() - created_at), max(unixepoch() - created_at) FROM queue GROUP BY name ORDER BY count DESC ")
95
+ run_stmt(:info)
96
96
  end
97
97
 
98
- def info
99
- counts = {}
98
+ def snapshot
99
+ queues = {}
100
100
  queues_info.each do |qc|
101
- counts[qc[0]] = {count: qc[1], time_in_queue: {avg: qc[2], min: qc[3], max: qc[4]}}
101
+ #queues[qc[0]] = {count: qc[1], time_in_queue: {avg: qc[2], min: qc[3], max: qc[4]}}
102
+ queues[qc[0]] = qc[1]
102
103
  end
103
- {size: size, count: count, info: counts}
104
+ {
105
+ summary: {
106
+ path: path,
107
+ journal_mode: journal_mode,
108
+ synchronous: synchronous,
109
+ size: size,
110
+ jobs: count
111
+ },
112
+ queues: queues
113
+ }
104
114
  end
105
115
 
106
116
  private
@@ -110,7 +120,7 @@ class Litequeue
110
120
  conn.wal_autocheckpoint = 10000
111
121
  sql = YAML.load_file("#{__dir__}/litequeue.sql.yml")
112
122
  version = conn.get_first_value("PRAGMA user_version")
113
- sql["schema"].each_pair do |v, obj|
123
+ sql["schema"].each_pair do |v, obj|
114
124
  if v > version
115
125
  conn.transaction(:immediate) do
116
126
  obj.each{|k, s| conn.execute(s)}
@@ -31,4 +31,15 @@ stmts:
31
31
  RETURNING id, value
32
32
 
33
33
  delete: DELETE FROM queue WHERE id = $1 RETURNING value
34
+
35
+ info: >
36
+ SELECT
37
+ name,
38
+ count(*) AS count,
39
+ avg(unixepoch() - created_at) AS avg,
40
+ min(unixepoch() - created_at) AS min,
41
+ max(unixepoch() - created_at) AS max
42
+ FROM queue
43
+ GROUP BY name
44
+ ORDER BY count DESC
34
45