litestack 0.1.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/BENCHMARKS.md +1 -1
- data/CHANGELOG.md +20 -3
- data/README.md +28 -1
- data/assets/litecable_logo_teal.png +0 -0
- data/bench/bench_cache_raw.rb +18 -2
- data/bench/bench_jobs_rails.rb +20 -14
- data/bench/bench_jobs_raw.rb +0 -2
- data/lib/action_cable/subscription_adapter/litecable.rb +36 -0
- data/lib/active_job/queue_adapters/litejob_adapter.rb +14 -10
- data/lib/litestack/litecable.rb +138 -0
- data/lib/litestack/litecable.sql.yml +24 -0
- data/lib/litestack/litecache.rb +56 -62
- data/lib/litestack/litecache.sql.yml +28 -0
- data/lib/litestack/litecache.yml +7 -0
- data/lib/litestack/litejob.rb +20 -11
- data/lib/litestack/litejobqueue.rb +122 -44
- data/lib/litestack/litemetric.rb +228 -0
- data/lib/litestack/litemetric.sql.yml +69 -0
- data/lib/litestack/litequeue.rb +57 -29
- data/lib/litestack/litequeue.sql.yml +34 -0
- data/lib/litestack/litesupport.rb +155 -6
- data/lib/litestack/metrics_app.rb +5 -0
- data/lib/litestack/version.rb +1 -1
- data/lib/litestack.rb +19 -10
- metadata +13 -6
- data/bench/bench_rails.rb +0 -81
- data/bench/bench_raw.rb +0 -72
- data/lib/active_job/queue_adapters/ultralite_adapter.rb +0 -49
@@ -0,0 +1,228 @@
|
|
1
|
+
# frozen_stringe_literal: true
|
2
|
+
|
3
|
+
require 'singleton'
|
4
|
+
|
5
|
+
require_relative './litesupport'
|
6
|
+
|
7
|
+
# this class is a singleton
|
8
|
+
# and should remain so
|
9
|
+
class Litemetric
|
10
|
+
|
11
|
+
include Singleton
|
12
|
+
include Litesupport::Liteconnection
|
13
|
+
|
14
|
+
DEFAULT_OPTIONS = {
|
15
|
+
config_path: "./litemetric.yml",
|
16
|
+
path: "./metrics.db",
|
17
|
+
sync: 1,
|
18
|
+
mmap_size: 16 * 1024 * 1024, # 16MB of memory to easily process 1 year worth of data
|
19
|
+
flush_interval: 10, # flush data every 1 minute
|
20
|
+
summarize_interval: 10 # summarize data every 1 minute
|
21
|
+
}
|
22
|
+
|
23
|
+
RESOLUTIONS = {
|
24
|
+
minute: 300, # 5 minutes (highest resolution)
|
25
|
+
hour: 3600, # 1 hour
|
26
|
+
day: 24*3600, # 1 day
|
27
|
+
week: 7*24*3600 # 1 week (lowest resolution)
|
28
|
+
}
|
29
|
+
|
30
|
+
# :nodoc:
|
31
|
+
def initialize(options = {})
|
32
|
+
init(options)
|
33
|
+
end
|
34
|
+
|
35
|
+
# registers a class for metrics to be collected
|
36
|
+
def register(identifier)
|
37
|
+
@registered[identifier] = true
|
38
|
+
@metrics[identifier] = {} unless @metrics[identifier]
|
39
|
+
run_stmt(:register_topic, identifier) # it is safe to call register topic multiple times with the same identifier
|
40
|
+
end
|
41
|
+
|
42
|
+
## event capturing
|
43
|
+
##################
|
44
|
+
|
45
|
+
def capture(topic, event, key=event, value=nil)
|
46
|
+
if key.is_a? Array
|
47
|
+
key.each{|k| capture_single_key(topic, event, k, value)}
|
48
|
+
else
|
49
|
+
capture_single_key(topic, event, key, value)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def capture_single_key(topic, event, key=event, value=nil)
|
54
|
+
@mutex.synchronize do
|
55
|
+
time_slot = current_time_slot # should that be 5 minutes?
|
56
|
+
topic_slot = @metrics[topic]
|
57
|
+
if event_slot = topic_slot[event]
|
58
|
+
if key_slot = event_slot[key]
|
59
|
+
if key_slot[time_slot]
|
60
|
+
key_slot[time_slot][:count] += 1
|
61
|
+
key_slot[time_slot][:value] += value unless value.nil?
|
62
|
+
else # new time slot
|
63
|
+
key_slot[time_slot] = {count: 1, value: value}
|
64
|
+
end
|
65
|
+
else
|
66
|
+
event_slot[key] = {time_slot => {count: 1, value: value}}
|
67
|
+
end
|
68
|
+
else # new event
|
69
|
+
topic_slot[event] = {key => {time_slot => {count: 1, value: value}}}
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
|
75
|
+
## event reporting
|
76
|
+
##################
|
77
|
+
|
78
|
+
def topics
|
79
|
+
run_stmt(:list_topics).to_a
|
80
|
+
end
|
81
|
+
|
82
|
+
def event_names(resolution, topic)
|
83
|
+
run_stmt(:list_event_names, resolution, topic).to_a
|
84
|
+
end
|
85
|
+
|
86
|
+
def keys(resolution, topic, event_name)
|
87
|
+
run_stmt(:list_event_keys, resolution, topic, event_name).to_a
|
88
|
+
end
|
89
|
+
|
90
|
+
def event_data(resolution, topic, event_name, key)
|
91
|
+
run_stmt(:list_events_by_key, resolution, topic, event_name, key).to_a
|
92
|
+
end
|
93
|
+
|
94
|
+
## summarize data
|
95
|
+
#################
|
96
|
+
|
97
|
+
def summarize
|
98
|
+
run_stmt(:summarize_events, RESOLUTIONS[:hour], "hour", "minute")
|
99
|
+
run_stmt(:summarize_events, RESOLUTIONS[:day], "day", "hour")
|
100
|
+
run_stmt(:summarize_events, RESOLUTIONS[:week], "week", "day")
|
101
|
+
run_stmt(:delete_events, "minute", RESOLUTIONS[:hour]*1)
|
102
|
+
run_stmt(:delete_events, "hour", RESOLUTIONS[:day]*1)
|
103
|
+
run_stmt(:delete_events, "day", RESOLUTIONS[:week]*1)
|
104
|
+
end
|
105
|
+
|
106
|
+
## background stuff
|
107
|
+
###################
|
108
|
+
|
109
|
+
private
|
110
|
+
|
111
|
+
def exit_callback
|
112
|
+
puts "--- Litemetric detected an exit, flushing metrics"
|
113
|
+
@running = false
|
114
|
+
flush
|
115
|
+
end
|
116
|
+
|
117
|
+
def setup
|
118
|
+
super
|
119
|
+
@metrics = {}
|
120
|
+
@registered = {}
|
121
|
+
@flusher = create_flusher
|
122
|
+
@summarizer = create_summarizer
|
123
|
+
@mutex = Litesupport::Mutex.new
|
124
|
+
end
|
125
|
+
|
126
|
+
def current_time_slot
|
127
|
+
(Time.now.to_i / 300) * 300 # every 5 minutes
|
128
|
+
end
|
129
|
+
|
130
|
+
def flush
|
131
|
+
to_delete = []
|
132
|
+
@conn.acquire do |conn|
|
133
|
+
conn.transaction(:immediate) do
|
134
|
+
@metrics.each_pair do |topic, event_hash|
|
135
|
+
event_hash.each_pair do |event, key_hash|
|
136
|
+
key_hash.each_pair do |key, time_hash|
|
137
|
+
time_hash.each_pair do |time, data|
|
138
|
+
conn.stmts[:capture_event].execute!(topic, event.to_s, key, time, data[:count], data[:value]) if data
|
139
|
+
time_hash[time] = nil
|
140
|
+
to_delete << [topic, event, key, time]
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
147
|
+
to_delete.each do |r|
|
148
|
+
@metrics[r[0]][r[1]][r[2]].delete(r[3])
|
149
|
+
@metrics[r[0]][r[1]].delete(r[2]) if @metrics[r[0]][r[1]][r[2]].empty?
|
150
|
+
@metrics[r[0]].delete(r[1]) if @metrics[r[0]][r[1]].empty?
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
def create_connection
|
155
|
+
conn = super
|
156
|
+
conn.wal_autocheckpoint = 10000
|
157
|
+
sql = YAML.load_file("#{__dir__}/litemetric.sql.yml")
|
158
|
+
version = conn.get_first_value("PRAGMA user_version")
|
159
|
+
sql["schema"].each_pair do |v, obj|
|
160
|
+
if v > version
|
161
|
+
conn.transaction do
|
162
|
+
obj.each{|k, s| conn.execute(s)}
|
163
|
+
conn.user_version = v
|
164
|
+
end
|
165
|
+
end
|
166
|
+
end
|
167
|
+
sql["stmts"].each { |k, v| conn.stmts[k.to_sym] = conn.prepare(v) }
|
168
|
+
conn
|
169
|
+
end
|
170
|
+
|
171
|
+
def create_flusher
|
172
|
+
Litesupport.spawn do
|
173
|
+
while @running do
|
174
|
+
sleep @options[:flush_interval]
|
175
|
+
@mutex.synchronize do
|
176
|
+
flush
|
177
|
+
end
|
178
|
+
end
|
179
|
+
end
|
180
|
+
end
|
181
|
+
|
182
|
+
def create_summarizer
|
183
|
+
Litesupport.spawn do
|
184
|
+
while @running do
|
185
|
+
sleep @options[:summarize_interval]
|
186
|
+
summarize
|
187
|
+
end
|
188
|
+
end
|
189
|
+
end
|
190
|
+
|
191
|
+
end
|
192
|
+
|
193
|
+
## Measurable Module
|
194
|
+
####################
|
195
|
+
|
196
|
+
class Litemetric
|
197
|
+
module Measurable
|
198
|
+
|
199
|
+
def collect_metrics
|
200
|
+
@litemetric = Litemetric.instance
|
201
|
+
@litemetric.register(metrics_identifier)
|
202
|
+
end
|
203
|
+
|
204
|
+
def metrics_identifier
|
205
|
+
self.class.name # override in included classes
|
206
|
+
end
|
207
|
+
|
208
|
+
def capture(event, key=event, value=nil)
|
209
|
+
return unless @litemetric
|
210
|
+
@litemetric.capture(metrics_identifier, event, key, value)
|
211
|
+
end
|
212
|
+
|
213
|
+
def measure(event, key=event)
|
214
|
+
return yield unless @litemetric
|
215
|
+
t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
216
|
+
res = yield
|
217
|
+
t2 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
218
|
+
value = (( t2 - t1 ) * 1000).round # capture time in milliseconds
|
219
|
+
capture(event, key, value)
|
220
|
+
res
|
221
|
+
end
|
222
|
+
|
223
|
+
def snapshot
|
224
|
+
raise Litestack::NotImplementedError
|
225
|
+
end
|
226
|
+
|
227
|
+
end
|
228
|
+
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
schema:
|
2
|
+
1:
|
3
|
+
create_topics: >
|
4
|
+
CREATE TABLE IF NOT EXISTS topics(
|
5
|
+
name text PRIMARY KEY NOT NULL
|
6
|
+
) WITHOUT ROWID;
|
7
|
+
create_events: >
|
8
|
+
CREATE TABLE IF NOT EXISTS events(
|
9
|
+
topic text NOT NULL references topics(name) ON DELETE CASCADE,
|
10
|
+
name TEXT NOT NULL,
|
11
|
+
key TEXT NOT NULL,
|
12
|
+
count INTEGER DEFAULT(0) NOT NULL ON CONFLICT REPLACE,
|
13
|
+
value INTEGER,
|
14
|
+
minimum INTEGER,
|
15
|
+
maximum INTEGER,
|
16
|
+
created_at INTEGER DEFAULT((unixepoch()/300*300)) NOT NULL,
|
17
|
+
resolution TEXT DEFAULT('minute') NOT NULL,
|
18
|
+
PRIMARY KEY(resolution, topic, name, key, created_at)
|
19
|
+
) WITHOUT ROWID;
|
20
|
+
create_index_on_event: CREATE INDEX IF NOT EXISTS events_by_resolution ON events(resolution, created_at);
|
21
|
+
|
22
|
+
stmts:
|
23
|
+
# register topic
|
24
|
+
register_topic: INSERT INTO topics VALUES (?) ON CONFLICT DO NOTHING;
|
25
|
+
|
26
|
+
capture_event: >
|
27
|
+
INSERT INTO events(topic, name, key, created_at, count, value, minimum, maximum) VALUES ($1, $2, $3, $4, $5, $6, $6, $6)
|
28
|
+
ON CONFLICT DO
|
29
|
+
UPDATE SET count = count + EXCLUDED.count, value = value + EXCLUDED.value, minimum = min(minimum, EXCLUDED.minimum), maximum = max(maximum, EXCLUDED.maximum)
|
30
|
+
|
31
|
+
# requires an index on (resolution, created_at)
|
32
|
+
summarize_events: >
|
33
|
+
INSERT INTO events (topic, name, key, count, value, minimum, maximum, created_at, resolution ) SELECT
|
34
|
+
topic,
|
35
|
+
name,
|
36
|
+
key,
|
37
|
+
sum(count) as count,
|
38
|
+
sum(value) as value,
|
39
|
+
min(minimum) as minimum,
|
40
|
+
max(maximum) as maximum,
|
41
|
+
(created_at/$1)*$1 as created,
|
42
|
+
$2
|
43
|
+
FROM events WHERE resolution = $3 AND created_at < (unixepoch()/$1)*$1 GROUP BY topic, name, key, created ON CONFLICT DO UPDATE
|
44
|
+
SET count = count + EXCLUDED.count, value = value + EXCLUDED.value, minimum = min(minimum, EXCLUDED.minimum), maximum = max(maximum, EXCLUDED.maximum);
|
45
|
+
|
46
|
+
# requires an index on (resolution, created_at)
|
47
|
+
delete_events: DELETE FROM events WHERE resolution = $3 AND created_at < (unixepoch() - $4);
|
48
|
+
|
49
|
+
# select topics from the topics table
|
50
|
+
list_topics: SELECT name FROM topics;
|
51
|
+
|
52
|
+
# requires an index on (resolution, topic, name)
|
53
|
+
list_event_names: >
|
54
|
+
SELECT name, sum(count) as count, count(distinct name) as name, sum(value) as value, min(minimum), max(maximum)
|
55
|
+
FROM events WHERE resolution = ? AND topic = ? GROUP BY name ORDER BY count;
|
56
|
+
|
57
|
+
# requires an index on (resolution, topic, name, key)
|
58
|
+
list_event_keys: >
|
59
|
+
SELECT key, sum(count) as count, sum(value) as value, min(minimum), max(maximum)
|
60
|
+
FROM events WHERE resolution = ? AND topic = ? AND name = ? GROUP BY key ORDER BY count;
|
61
|
+
|
62
|
+
# requires an index on (resolution, topic, name, key, created_at)
|
63
|
+
list_events_by_key: >
|
64
|
+
SELECT * FROM events WHERE resolution = $1 AND topic = $2 AND name = $3 AND key = $4 ORDER BY created_at ASC;
|
65
|
+
|
66
|
+
# requires an index on (resolution, topic, name, key, created_at)
|
67
|
+
list_all_events: >
|
68
|
+
SELECT * FROM events WHERE resolution = ? AND topic = ? ORDER BY name, key, created_at ASC;
|
69
|
+
|
data/lib/litestack/litequeue.rb
CHANGED
@@ -3,6 +3,8 @@
|
|
3
3
|
# all components should require the support module
|
4
4
|
require_relative 'litesupport'
|
5
5
|
|
6
|
+
#require 'securerandom'
|
7
|
+
|
6
8
|
##
|
7
9
|
#Litequeue is a simple queueing system for Ruby applications that allows you to push and pop values from a queue. It provides a straightforward API for creating and managing named queues, and for adding and removing values from those queues. Additionally, it offers options for scheduling pops at a certain time in the future, which can be useful for delaying processing until a later time.
|
8
10
|
#
|
@@ -18,10 +20,12 @@ class Litequeue
|
|
18
20
|
# mmap_size: 128 * 1024 * 1024 -> 128MB to be held in memory
|
19
21
|
# sync: 1 -> sync only when checkpointing
|
20
22
|
|
23
|
+
include Litesupport::Liteconnection
|
24
|
+
|
21
25
|
DEFAULT_OPTIONS = {
|
22
26
|
path: "./queue.db",
|
23
27
|
mmap_size: 32 * 1024,
|
24
|
-
sync:
|
28
|
+
sync: 0
|
25
29
|
}
|
26
30
|
|
27
31
|
# create a new instance of the litequeue object
|
@@ -33,8 +37,7 @@ class Litequeue
|
|
33
37
|
# queue.pop # => "somevalue"
|
34
38
|
|
35
39
|
def initialize(options = {})
|
36
|
-
|
37
|
-
@queue = Litesupport::Pool.new(1){create_db} # delegate the db creation to the litepool
|
40
|
+
init(options)
|
38
41
|
end
|
39
42
|
|
40
43
|
# push an item to the queue, optionally specifying the queue name (defaults to default) and after how many seconds it should be ready to pop (defaults to zero)
|
@@ -45,17 +48,23 @@ class Litequeue
|
|
45
48
|
# also bring back the synchronize block, to prevent
|
46
49
|
# a race condition if a thread hits the busy handler
|
47
50
|
# before the current thread proceeds after a backoff
|
48
|
-
|
49
|
-
|
51
|
+
#id = SecureRandom.uuid # this is somehow expensive, can we improve?
|
52
|
+
run_stmt(:push, queue, delay, value)[0]
|
53
|
+
end
|
54
|
+
|
55
|
+
def repush(id, value, delay=0, queue='default')
|
56
|
+
run_stmt(:repush, id, queue, delay, value)[0]
|
50
57
|
end
|
51
58
|
|
52
59
|
alias_method :"<<", :push
|
60
|
+
alias_method :"<<<", :repush
|
53
61
|
|
54
62
|
# pop an item from the queue, optionally with a specific queue name (default queue name is 'default')
|
55
63
|
def pop(queue='default', limit = 1)
|
56
|
-
res =
|
57
|
-
|
58
|
-
|
64
|
+
res = run_stmt(:pop, queue, limit)
|
65
|
+
return res[0] if res.length == 1
|
66
|
+
return nil if res.empty?
|
67
|
+
res
|
59
68
|
end
|
60
69
|
|
61
70
|
# delete an item from the queue
|
@@ -63,42 +72,61 @@ class Litequeue
|
|
63
72
|
# id = queue.push("somevalue")
|
64
73
|
# queue.delete(id) # => "somevalue"
|
65
74
|
# queue.pop # => nil
|
66
|
-
def delete(id
|
67
|
-
|
68
|
-
result = @queue.acquire{|q| q.stmts[:delete].execute!(queue, fire_at.to_i, id)[0] }
|
75
|
+
def delete(id)
|
76
|
+
result = run_stmt(:delete, id)[0]
|
69
77
|
end
|
70
78
|
|
71
79
|
# deletes all the entries in all queues, or if a queue name is given, deletes all entries in that specific queue
|
72
80
|
def clear(queue=nil)
|
73
|
-
|
81
|
+
run_sql("DELETE FROM queue WHERE iif(?, name = ?, 1)", queue)
|
74
82
|
end
|
75
83
|
|
76
84
|
# returns a count of entries in all queues, or if a queue name is given, reutrns the count of entries in that queue
|
77
85
|
def count(queue=nil)
|
78
|
-
|
86
|
+
run_sql("SELECT count(*) FROM queue WHERE iif(?, name = ?, 1)", queue)[0][0]
|
79
87
|
end
|
80
88
|
|
81
89
|
# return the size of the queue file on disk
|
82
90
|
def size
|
83
|
-
|
91
|
+
run_sql("SELECT size.page_size * count.page_count FROM pragma_page_size() AS size, pragma_page_count() AS count")[0][0]
|
84
92
|
end
|
85
|
-
|
86
|
-
private
|
87
93
|
|
88
|
-
def
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
94
|
+
def queues_info
|
95
|
+
run_sql("SELECT name, count(*) AS count, avg(unixepoch() - created_at), min(unixepoch() - created_at), max(unixepoch() - created_at) FROM queue GROUP BY name ORDER BY count DESC ")
|
96
|
+
end
|
97
|
+
|
98
|
+
def info
|
99
|
+
counts = {}
|
100
|
+
queues_info.each do |qc|
|
101
|
+
counts[qc[0]] = {count: qc[1], time_in_queue: {avg: qc[2], min: qc[3], max: qc[4]}}
|
102
|
+
end
|
103
|
+
{size: size, count: count, info: counts}
|
98
104
|
end
|
99
105
|
|
106
|
+
private
|
107
|
+
|
108
|
+
def create_connection
|
109
|
+
conn = super
|
110
|
+
conn.wal_autocheckpoint = 10000
|
111
|
+
sql = YAML.load_file("#{__dir__}/litequeue.sql.yml")
|
112
|
+
version = conn.get_first_value("PRAGMA user_version")
|
113
|
+
sql["schema"].each_pair do |v, obj|
|
114
|
+
if v > version
|
115
|
+
conn.transaction(:immediate) do
|
116
|
+
obj.each{|k, s| conn.execute(s)}
|
117
|
+
conn.user_version = v
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
121
|
+
sql["stmts"].each { |k, v| conn.stmts[k.to_sym] = conn.prepare(v) }
|
122
|
+
# check if there is an old database and convert entries to the new format
|
123
|
+
if conn.get_first_value("select count(*) from sqlite_master where name = '_ul_queue_'") == 1
|
124
|
+
conn.transaction(:immediate) do
|
125
|
+
conn.execute("INSERT INTO queue(fire_at, name, value, created_at) SELECT fire_at, queue, value, created_at FROM _ul_queue_")
|
126
|
+
conn.execute("DROP TABLE _ul_queue_")
|
127
|
+
end
|
128
|
+
end
|
129
|
+
conn
|
130
|
+
end
|
100
131
|
|
101
132
|
end
|
102
|
-
|
103
|
-
|
104
|
-
|
@@ -0,0 +1,34 @@
|
|
1
|
+
schema:
|
2
|
+
1:
|
3
|
+
create_table_queue: >
|
4
|
+
CREATE TABLE IF NOT EXISTS queue(
|
5
|
+
id TEXT PRIMARY KEY DEFAULT(hex(randomblob(32))) NOT NULL ON CONFLICT REPLACE,
|
6
|
+
name TEXT DEFAULT('default') NOT NULL ON CONFLICT REPLACE,
|
7
|
+
fire_at INTEGER DEFAULT(unixepoch()) NOT NULL ON CONFLICT REPLACE,
|
8
|
+
value TEXT,
|
9
|
+
created_at INTEGER DEFAULT(unixepoch()) NOT NULL ON CONFLICT REPLACE
|
10
|
+
) WITHOUT ROWID
|
11
|
+
|
12
|
+
create_index_queue_by_name: >
|
13
|
+
CREATE INDEX IF NOT EXISTS idx_queue_by_name ON queue(name, fire_at ASC)
|
14
|
+
|
15
|
+
stmts:
|
16
|
+
|
17
|
+
push: INSERT INTO queue(id, name, fire_at, value) VALUES (hex(randomblob(32)), $1, (unixepoch() + $2), $3) RETURNING id, name
|
18
|
+
|
19
|
+
repush: INSERT INTO queue(id, name, fire_at, value) VALUES (?, ?, (unixepoch() + ?), ?) RETURNING name
|
20
|
+
|
21
|
+
pop: >
|
22
|
+
DELETE FROM queue
|
23
|
+
WHERE (name, fire_at, id)
|
24
|
+
IN (
|
25
|
+
SELECT name, fire_at, id FROM queue
|
26
|
+
WHERE name = ifnull($1, 'default')
|
27
|
+
AND fire_at <= (unixepoch())
|
28
|
+
ORDER BY fire_at ASC
|
29
|
+
LIMIT ifnull($2, 1)
|
30
|
+
)
|
31
|
+
RETURNING id, value
|
32
|
+
|
33
|
+
delete: DELETE FROM queue WHERE id = $1 RETURNING value
|
34
|
+
|
@@ -1,5 +1,9 @@
|
|
1
|
+
# frozen_stringe_literal: true
|
2
|
+
|
1
3
|
require 'sqlite3'
|
2
|
-
require '
|
4
|
+
require 'logger'
|
5
|
+
require 'oj'
|
6
|
+
require 'yaml'
|
3
7
|
|
4
8
|
module Litesupport
|
5
9
|
|
@@ -38,8 +42,8 @@ module Litesupport
|
|
38
42
|
end
|
39
43
|
# we should never reach here
|
40
44
|
end
|
41
|
-
|
42
|
-
def self.
|
45
|
+
|
46
|
+
def self.context
|
43
47
|
if environment == :fiber || environment == :poylphony
|
44
48
|
Fiber.current.storage
|
45
49
|
else
|
@@ -47,8 +51,12 @@ module Litesupport
|
|
47
51
|
end
|
48
52
|
end
|
49
53
|
|
50
|
-
def self.
|
51
|
-
|
54
|
+
def self.current_context
|
55
|
+
if environment == :fiber || environment == :poylphony
|
56
|
+
Fiber.current
|
57
|
+
else
|
58
|
+
Thread.current
|
59
|
+
end
|
52
60
|
end
|
53
61
|
|
54
62
|
# switch the execution context to allow others to run
|
@@ -87,7 +95,7 @@ module Litesupport
|
|
87
95
|
# common db object options
|
88
96
|
def self.create_db(path)
|
89
97
|
db = SQLite3::Database.new(path)
|
90
|
-
db.busy_handler{ switch || sleep(0.
|
98
|
+
db.busy_handler{ switch || sleep(0.0001) }
|
91
99
|
db.journal_mode = "WAL"
|
92
100
|
db.instance_variable_set(:@stmts, {})
|
93
101
|
class << db
|
@@ -111,6 +119,21 @@ module Litesupport
|
|
111
119
|
end
|
112
120
|
|
113
121
|
end
|
122
|
+
|
123
|
+
module Forkable
|
124
|
+
|
125
|
+
def _fork(*args)
|
126
|
+
ppid = Process.pid
|
127
|
+
result = super
|
128
|
+
if Process.pid != ppid
|
129
|
+
# trigger a restart of all connections owned by Litesupport::Pool
|
130
|
+
end
|
131
|
+
result
|
132
|
+
end
|
133
|
+
|
134
|
+
end
|
135
|
+
|
136
|
+
#::Process.singleton_class.prepend(::Litesupport::Forkable)
|
114
137
|
|
115
138
|
class Pool
|
116
139
|
|
@@ -126,6 +149,7 @@ module Litesupport
|
|
126
149
|
end
|
127
150
|
|
128
151
|
def acquire
|
152
|
+
# check for pid changes
|
129
153
|
acquired = false
|
130
154
|
result = nil
|
131
155
|
while !acquired do
|
@@ -148,5 +172,130 @@ module Litesupport
|
|
148
172
|
end
|
149
173
|
|
150
174
|
end
|
175
|
+
|
176
|
+
module ForkListener
|
177
|
+
def self.listeners
|
178
|
+
@listeners ||= []
|
179
|
+
end
|
180
|
+
|
181
|
+
def self.listen(&block)
|
182
|
+
listeners << block
|
183
|
+
end
|
184
|
+
end
|
185
|
+
|
186
|
+
module Forkable
|
187
|
+
|
188
|
+
def _fork(*args)
|
189
|
+
ppid = Process.pid
|
190
|
+
result = super
|
191
|
+
if Process.pid != ppid && [:threaded, :iodine].include?(Litesupport.environment)
|
192
|
+
ForkListener.listeners.each{|l| l.call }
|
193
|
+
end
|
194
|
+
result
|
195
|
+
end
|
196
|
+
|
197
|
+
end
|
151
198
|
|
199
|
+
module Liteconnection
|
200
|
+
|
201
|
+
include Forkable
|
202
|
+
|
203
|
+
# close, setup, run_stmt and run_sql assume a single connection was created
|
204
|
+
def close
|
205
|
+
@running = false
|
206
|
+
@conn.acquire do |q|
|
207
|
+
q.stmts.each_pair {|k, v| q.stmts[k].close }
|
208
|
+
q.close
|
209
|
+
end
|
210
|
+
end
|
211
|
+
|
212
|
+
private # all methods are private
|
213
|
+
|
214
|
+
def init(options = {})
|
215
|
+
#c configure the object, loading options from the appropriate location
|
216
|
+
configure(options)
|
217
|
+
# setup connections and background threads
|
218
|
+
setup
|
219
|
+
# handle process exiting
|
220
|
+
at_exit do
|
221
|
+
exit_callback
|
222
|
+
end
|
223
|
+
# handle forking (restart connections and background threads)
|
224
|
+
Litesupport::ForkListener.listen do
|
225
|
+
setup
|
226
|
+
end
|
227
|
+
end
|
228
|
+
|
229
|
+
def configure(options = {})
|
230
|
+
# detect environment (production, development, etc.)
|
231
|
+
env = "development"
|
232
|
+
if defined? Rails
|
233
|
+
env = ENV["RAILS_ENV"]
|
234
|
+
elsif ENV["RACK_ENV"]
|
235
|
+
env = ENV["RACK_ENV"]
|
236
|
+
elsif ENV["APP_ENV"]
|
237
|
+
env = ENV["RACK_ENV"]
|
238
|
+
end
|
239
|
+
defaults = self.class::DEFAULT_OPTIONS rescue {}
|
240
|
+
@options = defaults.merge(options)
|
241
|
+
config = YAML.load_file(@options[:config_path]) rescue {} # an empty hash won't hurt
|
242
|
+
config = config[env] if config[env] # if there is a config for the current environment defined then use it, otherwise use the top level declaration
|
243
|
+
config.keys.each do |k| # symbolize keys
|
244
|
+
config[k.to_sym] = config[k]
|
245
|
+
config.delete k
|
246
|
+
end
|
247
|
+
@options.merge!(config)
|
248
|
+
@options.merge!(options) # make sure options passed to initialize trump everything else
|
249
|
+
end
|
250
|
+
|
251
|
+
def setup
|
252
|
+
@conn = create_pooled_connection
|
253
|
+
@logger = create_logger
|
254
|
+
@running = true
|
255
|
+
end
|
256
|
+
|
257
|
+
def create_logger
|
258
|
+
@options[:logger] = nil unless @options[:logger]
|
259
|
+
return @options[:logger] if @options[:logger].respond_to? :info
|
260
|
+
return Logger.new(STDOUT) if @options[:logger] == 'STDOUT'
|
261
|
+
return Logger.new(STDERR) if @options[:logger] == 'STDERR'
|
262
|
+
return Logger.new(@options[:logger]) if @options[:logger].is_a? String
|
263
|
+
return Logger.new(IO::NULL)
|
264
|
+
end
|
265
|
+
|
266
|
+
def exit_callback
|
267
|
+
close
|
268
|
+
end
|
269
|
+
|
270
|
+
def run_stmt(stmt, *args)
|
271
|
+
@conn.acquire{|q| q.stmts[stmt].execute!(*args) }
|
272
|
+
end
|
273
|
+
|
274
|
+
def run_sql(sql, *args)
|
275
|
+
@conn.acquire{|q| q.execute(sql, *args) }
|
276
|
+
end
|
277
|
+
|
278
|
+
def create_pooled_connection(count = 1)
|
279
|
+
Litesupport::Pool.new(1){create_connection}
|
280
|
+
end
|
281
|
+
|
282
|
+
# common db object options
|
283
|
+
def create_connection
|
284
|
+
conn = SQLite3::Database.new(@options[:path])
|
285
|
+
conn.busy_handler{ Litesupport.switch || sleep(rand * 0.002) }
|
286
|
+
conn.journal_mode = "WAL"
|
287
|
+
conn.synchronous = @options[:sync] || 1
|
288
|
+
conn.mmap_size = @options[:mmap_size] || 0
|
289
|
+
conn.instance_variable_set(:@stmts, {})
|
290
|
+
class << conn
|
291
|
+
attr_reader :stmts
|
292
|
+
end
|
293
|
+
conn
|
294
|
+
end
|
295
|
+
|
296
|
+
end
|
297
|
+
|
152
298
|
end
|
299
|
+
|
300
|
+
Process.singleton_class.prepend(Litesupport::Forkable)
|
301
|
+
|
data/lib/litestack/version.rb
CHANGED