litestack 0.1.7 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/BENCHMARKS.md +1 -1
- data/CHANGELOG.md +20 -3
- data/README.md +28 -1
- data/assets/litecable_logo_teal.png +0 -0
- data/bench/bench_cache_raw.rb +18 -2
- data/bench/bench_jobs_rails.rb +20 -14
- data/bench/bench_jobs_raw.rb +0 -2
- data/lib/action_cable/subscription_adapter/litecable.rb +36 -0
- data/lib/active_job/queue_adapters/litejob_adapter.rb +14 -10
- data/lib/litestack/litecable.rb +138 -0
- data/lib/litestack/litecable.sql.yml +24 -0
- data/lib/litestack/litecache.rb +56 -62
- data/lib/litestack/litecache.sql.yml +28 -0
- data/lib/litestack/litecache.yml +7 -0
- data/lib/litestack/litejob.rb +20 -11
- data/lib/litestack/litejobqueue.rb +122 -44
- data/lib/litestack/litemetric.rb +228 -0
- data/lib/litestack/litemetric.sql.yml +69 -0
- data/lib/litestack/litequeue.rb +57 -29
- data/lib/litestack/litequeue.sql.yml +34 -0
- data/lib/litestack/litesupport.rb +155 -6
- data/lib/litestack/metrics_app.rb +5 -0
- data/lib/litestack/version.rb +1 -1
- data/lib/litestack.rb +19 -10
- metadata +13 -6
- data/bench/bench_rails.rb +0 -81
- data/bench/bench_raw.rb +0 -72
- data/lib/active_job/queue_adapters/ultralite_adapter.rb +0 -49
@@ -0,0 +1,228 @@
|
|
1
|
+
# frozen_stringe_literal: true
|
2
|
+
|
3
|
+
require 'singleton'
|
4
|
+
|
5
|
+
require_relative './litesupport'
|
6
|
+
|
7
|
+
# this class is a singleton
|
8
|
+
# and should remain so
|
9
|
+
class Litemetric
|
10
|
+
|
11
|
+
include Singleton
|
12
|
+
include Litesupport::Liteconnection
|
13
|
+
|
14
|
+
DEFAULT_OPTIONS = {
|
15
|
+
config_path: "./litemetric.yml",
|
16
|
+
path: "./metrics.db",
|
17
|
+
sync: 1,
|
18
|
+
mmap_size: 16 * 1024 * 1024, # 16MB of memory to easily process 1 year worth of data
|
19
|
+
flush_interval: 10, # flush data every 1 minute
|
20
|
+
summarize_interval: 10 # summarize data every 1 minute
|
21
|
+
}
|
22
|
+
|
23
|
+
RESOLUTIONS = {
|
24
|
+
minute: 300, # 5 minutes (highest resolution)
|
25
|
+
hour: 3600, # 1 hour
|
26
|
+
day: 24*3600, # 1 day
|
27
|
+
week: 7*24*3600 # 1 week (lowest resolution)
|
28
|
+
}
|
29
|
+
|
30
|
+
# :nodoc:
|
31
|
+
def initialize(options = {})
|
32
|
+
init(options)
|
33
|
+
end
|
34
|
+
|
35
|
+
# registers a class for metrics to be collected
|
36
|
+
def register(identifier)
|
37
|
+
@registered[identifier] = true
|
38
|
+
@metrics[identifier] = {} unless @metrics[identifier]
|
39
|
+
run_stmt(:register_topic, identifier) # it is safe to call register topic multiple times with the same identifier
|
40
|
+
end
|
41
|
+
|
42
|
+
## event capturing
|
43
|
+
##################
|
44
|
+
|
45
|
+
def capture(topic, event, key=event, value=nil)
|
46
|
+
if key.is_a? Array
|
47
|
+
key.each{|k| capture_single_key(topic, event, k, value)}
|
48
|
+
else
|
49
|
+
capture_single_key(topic, event, key, value)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def capture_single_key(topic, event, key=event, value=nil)
|
54
|
+
@mutex.synchronize do
|
55
|
+
time_slot = current_time_slot # should that be 5 minutes?
|
56
|
+
topic_slot = @metrics[topic]
|
57
|
+
if event_slot = topic_slot[event]
|
58
|
+
if key_slot = event_slot[key]
|
59
|
+
if key_slot[time_slot]
|
60
|
+
key_slot[time_slot][:count] += 1
|
61
|
+
key_slot[time_slot][:value] += value unless value.nil?
|
62
|
+
else # new time slot
|
63
|
+
key_slot[time_slot] = {count: 1, value: value}
|
64
|
+
end
|
65
|
+
else
|
66
|
+
event_slot[key] = {time_slot => {count: 1, value: value}}
|
67
|
+
end
|
68
|
+
else # new event
|
69
|
+
topic_slot[event] = {key => {time_slot => {count: 1, value: value}}}
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
|
75
|
+
## event reporting
|
76
|
+
##################
|
77
|
+
|
78
|
+
def topics
|
79
|
+
run_stmt(:list_topics).to_a
|
80
|
+
end
|
81
|
+
|
82
|
+
def event_names(resolution, topic)
|
83
|
+
run_stmt(:list_event_names, resolution, topic).to_a
|
84
|
+
end
|
85
|
+
|
86
|
+
def keys(resolution, topic, event_name)
|
87
|
+
run_stmt(:list_event_keys, resolution, topic, event_name).to_a
|
88
|
+
end
|
89
|
+
|
90
|
+
def event_data(resolution, topic, event_name, key)
|
91
|
+
run_stmt(:list_events_by_key, resolution, topic, event_name, key).to_a
|
92
|
+
end
|
93
|
+
|
94
|
+
## summarize data
|
95
|
+
#################
|
96
|
+
|
97
|
+
def summarize
|
98
|
+
run_stmt(:summarize_events, RESOLUTIONS[:hour], "hour", "minute")
|
99
|
+
run_stmt(:summarize_events, RESOLUTIONS[:day], "day", "hour")
|
100
|
+
run_stmt(:summarize_events, RESOLUTIONS[:week], "week", "day")
|
101
|
+
run_stmt(:delete_events, "minute", RESOLUTIONS[:hour]*1)
|
102
|
+
run_stmt(:delete_events, "hour", RESOLUTIONS[:day]*1)
|
103
|
+
run_stmt(:delete_events, "day", RESOLUTIONS[:week]*1)
|
104
|
+
end
|
105
|
+
|
106
|
+
## background stuff
|
107
|
+
###################
|
108
|
+
|
109
|
+
private
|
110
|
+
|
111
|
+
def exit_callback
|
112
|
+
puts "--- Litemetric detected an exit, flushing metrics"
|
113
|
+
@running = false
|
114
|
+
flush
|
115
|
+
end
|
116
|
+
|
117
|
+
def setup
|
118
|
+
super
|
119
|
+
@metrics = {}
|
120
|
+
@registered = {}
|
121
|
+
@flusher = create_flusher
|
122
|
+
@summarizer = create_summarizer
|
123
|
+
@mutex = Litesupport::Mutex.new
|
124
|
+
end
|
125
|
+
|
126
|
+
def current_time_slot
|
127
|
+
(Time.now.to_i / 300) * 300 # every 5 minutes
|
128
|
+
end
|
129
|
+
|
130
|
+
def flush
|
131
|
+
to_delete = []
|
132
|
+
@conn.acquire do |conn|
|
133
|
+
conn.transaction(:immediate) do
|
134
|
+
@metrics.each_pair do |topic, event_hash|
|
135
|
+
event_hash.each_pair do |event, key_hash|
|
136
|
+
key_hash.each_pair do |key, time_hash|
|
137
|
+
time_hash.each_pair do |time, data|
|
138
|
+
conn.stmts[:capture_event].execute!(topic, event.to_s, key, time, data[:count], data[:value]) if data
|
139
|
+
time_hash[time] = nil
|
140
|
+
to_delete << [topic, event, key, time]
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
147
|
+
to_delete.each do |r|
|
148
|
+
@metrics[r[0]][r[1]][r[2]].delete(r[3])
|
149
|
+
@metrics[r[0]][r[1]].delete(r[2]) if @metrics[r[0]][r[1]][r[2]].empty?
|
150
|
+
@metrics[r[0]].delete(r[1]) if @metrics[r[0]][r[1]].empty?
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
def create_connection
|
155
|
+
conn = super
|
156
|
+
conn.wal_autocheckpoint = 10000
|
157
|
+
sql = YAML.load_file("#{__dir__}/litemetric.sql.yml")
|
158
|
+
version = conn.get_first_value("PRAGMA user_version")
|
159
|
+
sql["schema"].each_pair do |v, obj|
|
160
|
+
if v > version
|
161
|
+
conn.transaction do
|
162
|
+
obj.each{|k, s| conn.execute(s)}
|
163
|
+
conn.user_version = v
|
164
|
+
end
|
165
|
+
end
|
166
|
+
end
|
167
|
+
sql["stmts"].each { |k, v| conn.stmts[k.to_sym] = conn.prepare(v) }
|
168
|
+
conn
|
169
|
+
end
|
170
|
+
|
171
|
+
def create_flusher
|
172
|
+
Litesupport.spawn do
|
173
|
+
while @running do
|
174
|
+
sleep @options[:flush_interval]
|
175
|
+
@mutex.synchronize do
|
176
|
+
flush
|
177
|
+
end
|
178
|
+
end
|
179
|
+
end
|
180
|
+
end
|
181
|
+
|
182
|
+
def create_summarizer
|
183
|
+
Litesupport.spawn do
|
184
|
+
while @running do
|
185
|
+
sleep @options[:summarize_interval]
|
186
|
+
summarize
|
187
|
+
end
|
188
|
+
end
|
189
|
+
end
|
190
|
+
|
191
|
+
end
|
192
|
+
|
193
|
+
## Measurable Module
|
194
|
+
####################
|
195
|
+
|
196
|
+
class Litemetric
|
197
|
+
module Measurable
|
198
|
+
|
199
|
+
def collect_metrics
|
200
|
+
@litemetric = Litemetric.instance
|
201
|
+
@litemetric.register(metrics_identifier)
|
202
|
+
end
|
203
|
+
|
204
|
+
def metrics_identifier
|
205
|
+
self.class.name # override in included classes
|
206
|
+
end
|
207
|
+
|
208
|
+
def capture(event, key=event, value=nil)
|
209
|
+
return unless @litemetric
|
210
|
+
@litemetric.capture(metrics_identifier, event, key, value)
|
211
|
+
end
|
212
|
+
|
213
|
+
def measure(event, key=event)
|
214
|
+
return yield unless @litemetric
|
215
|
+
t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
216
|
+
res = yield
|
217
|
+
t2 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
218
|
+
value = (( t2 - t1 ) * 1000).round # capture time in milliseconds
|
219
|
+
capture(event, key, value)
|
220
|
+
res
|
221
|
+
end
|
222
|
+
|
223
|
+
def snapshot
|
224
|
+
raise Litestack::NotImplementedError
|
225
|
+
end
|
226
|
+
|
227
|
+
end
|
228
|
+
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
schema:
|
2
|
+
1:
|
3
|
+
create_topics: >
|
4
|
+
CREATE TABLE IF NOT EXISTS topics(
|
5
|
+
name text PRIMARY KEY NOT NULL
|
6
|
+
) WITHOUT ROWID;
|
7
|
+
create_events: >
|
8
|
+
CREATE TABLE IF NOT EXISTS events(
|
9
|
+
topic text NOT NULL references topics(name) ON DELETE CASCADE,
|
10
|
+
name TEXT NOT NULL,
|
11
|
+
key TEXT NOT NULL,
|
12
|
+
count INTEGER DEFAULT(0) NOT NULL ON CONFLICT REPLACE,
|
13
|
+
value INTEGER,
|
14
|
+
minimum INTEGER,
|
15
|
+
maximum INTEGER,
|
16
|
+
created_at INTEGER DEFAULT((unixepoch()/300*300)) NOT NULL,
|
17
|
+
resolution TEXT DEFAULT('minute') NOT NULL,
|
18
|
+
PRIMARY KEY(resolution, topic, name, key, created_at)
|
19
|
+
) WITHOUT ROWID;
|
20
|
+
create_index_on_event: CREATE INDEX IF NOT EXISTS events_by_resolution ON events(resolution, created_at);
|
21
|
+
|
22
|
+
stmts:
|
23
|
+
# register topic
|
24
|
+
register_topic: INSERT INTO topics VALUES (?) ON CONFLICT DO NOTHING;
|
25
|
+
|
26
|
+
capture_event: >
|
27
|
+
INSERT INTO events(topic, name, key, created_at, count, value, minimum, maximum) VALUES ($1, $2, $3, $4, $5, $6, $6, $6)
|
28
|
+
ON CONFLICT DO
|
29
|
+
UPDATE SET count = count + EXCLUDED.count, value = value + EXCLUDED.value, minimum = min(minimum, EXCLUDED.minimum), maximum = max(maximum, EXCLUDED.maximum)
|
30
|
+
|
31
|
+
# requires an index on (resolution, created_at)
|
32
|
+
summarize_events: >
|
33
|
+
INSERT INTO events (topic, name, key, count, value, minimum, maximum, created_at, resolution ) SELECT
|
34
|
+
topic,
|
35
|
+
name,
|
36
|
+
key,
|
37
|
+
sum(count) as count,
|
38
|
+
sum(value) as value,
|
39
|
+
min(minimum) as minimum,
|
40
|
+
max(maximum) as maximum,
|
41
|
+
(created_at/$1)*$1 as created,
|
42
|
+
$2
|
43
|
+
FROM events WHERE resolution = $3 AND created_at < (unixepoch()/$1)*$1 GROUP BY topic, name, key, created ON CONFLICT DO UPDATE
|
44
|
+
SET count = count + EXCLUDED.count, value = value + EXCLUDED.value, minimum = min(minimum, EXCLUDED.minimum), maximum = max(maximum, EXCLUDED.maximum);
|
45
|
+
|
46
|
+
# requires an index on (resolution, created_at)
|
47
|
+
delete_events: DELETE FROM events WHERE resolution = $3 AND created_at < (unixepoch() - $4);
|
48
|
+
|
49
|
+
# select topics from the topics table
|
50
|
+
list_topics: SELECT name FROM topics;
|
51
|
+
|
52
|
+
# requires an index on (resolution, topic, name)
|
53
|
+
list_event_names: >
|
54
|
+
SELECT name, sum(count) as count, count(distinct name) as name, sum(value) as value, min(minimum), max(maximum)
|
55
|
+
FROM events WHERE resolution = ? AND topic = ? GROUP BY name ORDER BY count;
|
56
|
+
|
57
|
+
# requires an index on (resolution, topic, name, key)
|
58
|
+
list_event_keys: >
|
59
|
+
SELECT key, sum(count) as count, sum(value) as value, min(minimum), max(maximum)
|
60
|
+
FROM events WHERE resolution = ? AND topic = ? AND name = ? GROUP BY key ORDER BY count;
|
61
|
+
|
62
|
+
# requires an index on (resolution, topic, name, key, created_at)
|
63
|
+
list_events_by_key: >
|
64
|
+
SELECT * FROM events WHERE resolution = $1 AND topic = $2 AND name = $3 AND key = $4 ORDER BY created_at ASC;
|
65
|
+
|
66
|
+
# requires an index on (resolution, topic, name, key, created_at)
|
67
|
+
list_all_events: >
|
68
|
+
SELECT * FROM events WHERE resolution = ? AND topic = ? ORDER BY name, key, created_at ASC;
|
69
|
+
|
data/lib/litestack/litequeue.rb
CHANGED
@@ -3,6 +3,8 @@
|
|
3
3
|
# all components should require the support module
|
4
4
|
require_relative 'litesupport'
|
5
5
|
|
6
|
+
#require 'securerandom'
|
7
|
+
|
6
8
|
##
|
7
9
|
#Litequeue is a simple queueing system for Ruby applications that allows you to push and pop values from a queue. It provides a straightforward API for creating and managing named queues, and for adding and removing values from those queues. Additionally, it offers options for scheduling pops at a certain time in the future, which can be useful for delaying processing until a later time.
|
8
10
|
#
|
@@ -18,10 +20,12 @@ class Litequeue
|
|
18
20
|
# mmap_size: 128 * 1024 * 1024 -> 128MB to be held in memory
|
19
21
|
# sync: 1 -> sync only when checkpointing
|
20
22
|
|
23
|
+
include Litesupport::Liteconnection
|
24
|
+
|
21
25
|
DEFAULT_OPTIONS = {
|
22
26
|
path: "./queue.db",
|
23
27
|
mmap_size: 32 * 1024,
|
24
|
-
sync:
|
28
|
+
sync: 0
|
25
29
|
}
|
26
30
|
|
27
31
|
# create a new instance of the litequeue object
|
@@ -33,8 +37,7 @@ class Litequeue
|
|
33
37
|
# queue.pop # => "somevalue"
|
34
38
|
|
35
39
|
def initialize(options = {})
|
36
|
-
|
37
|
-
@queue = Litesupport::Pool.new(1){create_db} # delegate the db creation to the litepool
|
40
|
+
init(options)
|
38
41
|
end
|
39
42
|
|
40
43
|
# push an item to the queue, optionally specifying the queue name (defaults to default) and after how many seconds it should be ready to pop (defaults to zero)
|
@@ -45,17 +48,23 @@ class Litequeue
|
|
45
48
|
# also bring back the synchronize block, to prevent
|
46
49
|
# a race condition if a thread hits the busy handler
|
47
50
|
# before the current thread proceeds after a backoff
|
48
|
-
|
49
|
-
|
51
|
+
#id = SecureRandom.uuid # this is somehow expensive, can we improve?
|
52
|
+
run_stmt(:push, queue, delay, value)[0]
|
53
|
+
end
|
54
|
+
|
55
|
+
def repush(id, value, delay=0, queue='default')
|
56
|
+
run_stmt(:repush, id, queue, delay, value)[0]
|
50
57
|
end
|
51
58
|
|
52
59
|
alias_method :"<<", :push
|
60
|
+
alias_method :"<<<", :repush
|
53
61
|
|
54
62
|
# pop an item from the queue, optionally with a specific queue name (default queue name is 'default')
|
55
63
|
def pop(queue='default', limit = 1)
|
56
|
-
res =
|
57
|
-
|
58
|
-
|
64
|
+
res = run_stmt(:pop, queue, limit)
|
65
|
+
return res[0] if res.length == 1
|
66
|
+
return nil if res.empty?
|
67
|
+
res
|
59
68
|
end
|
60
69
|
|
61
70
|
# delete an item from the queue
|
@@ -63,42 +72,61 @@ class Litequeue
|
|
63
72
|
# id = queue.push("somevalue")
|
64
73
|
# queue.delete(id) # => "somevalue"
|
65
74
|
# queue.pop # => nil
|
66
|
-
def delete(id
|
67
|
-
|
68
|
-
result = @queue.acquire{|q| q.stmts[:delete].execute!(queue, fire_at.to_i, id)[0] }
|
75
|
+
def delete(id)
|
76
|
+
result = run_stmt(:delete, id)[0]
|
69
77
|
end
|
70
78
|
|
71
79
|
# deletes all the entries in all queues, or if a queue name is given, deletes all entries in that specific queue
|
72
80
|
def clear(queue=nil)
|
73
|
-
|
81
|
+
run_sql("DELETE FROM queue WHERE iif(?, name = ?, 1)", queue)
|
74
82
|
end
|
75
83
|
|
76
84
|
# returns a count of entries in all queues, or if a queue name is given, reutrns the count of entries in that queue
|
77
85
|
def count(queue=nil)
|
78
|
-
|
86
|
+
run_sql("SELECT count(*) FROM queue WHERE iif(?, name = ?, 1)", queue)[0][0]
|
79
87
|
end
|
80
88
|
|
81
89
|
# return the size of the queue file on disk
|
82
90
|
def size
|
83
|
-
|
91
|
+
run_sql("SELECT size.page_size * count.page_count FROM pragma_page_size() AS size, pragma_page_count() AS count")[0][0]
|
84
92
|
end
|
85
|
-
|
86
|
-
private
|
87
93
|
|
88
|
-
def
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
94
|
+
def queues_info
|
95
|
+
run_sql("SELECT name, count(*) AS count, avg(unixepoch() - created_at), min(unixepoch() - created_at), max(unixepoch() - created_at) FROM queue GROUP BY name ORDER BY count DESC ")
|
96
|
+
end
|
97
|
+
|
98
|
+
def info
|
99
|
+
counts = {}
|
100
|
+
queues_info.each do |qc|
|
101
|
+
counts[qc[0]] = {count: qc[1], time_in_queue: {avg: qc[2], min: qc[3], max: qc[4]}}
|
102
|
+
end
|
103
|
+
{size: size, count: count, info: counts}
|
98
104
|
end
|
99
105
|
|
106
|
+
private
|
107
|
+
|
108
|
+
def create_connection
|
109
|
+
conn = super
|
110
|
+
conn.wal_autocheckpoint = 10000
|
111
|
+
sql = YAML.load_file("#{__dir__}/litequeue.sql.yml")
|
112
|
+
version = conn.get_first_value("PRAGMA user_version")
|
113
|
+
sql["schema"].each_pair do |v, obj|
|
114
|
+
if v > version
|
115
|
+
conn.transaction(:immediate) do
|
116
|
+
obj.each{|k, s| conn.execute(s)}
|
117
|
+
conn.user_version = v
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
121
|
+
sql["stmts"].each { |k, v| conn.stmts[k.to_sym] = conn.prepare(v) }
|
122
|
+
# check if there is an old database and convert entries to the new format
|
123
|
+
if conn.get_first_value("select count(*) from sqlite_master where name = '_ul_queue_'") == 1
|
124
|
+
conn.transaction(:immediate) do
|
125
|
+
conn.execute("INSERT INTO queue(fire_at, name, value, created_at) SELECT fire_at, queue, value, created_at FROM _ul_queue_")
|
126
|
+
conn.execute("DROP TABLE _ul_queue_")
|
127
|
+
end
|
128
|
+
end
|
129
|
+
conn
|
130
|
+
end
|
100
131
|
|
101
132
|
end
|
102
|
-
|
103
|
-
|
104
|
-
|
@@ -0,0 +1,34 @@
|
|
1
|
+
schema:
|
2
|
+
1:
|
3
|
+
create_table_queue: >
|
4
|
+
CREATE TABLE IF NOT EXISTS queue(
|
5
|
+
id TEXT PRIMARY KEY DEFAULT(hex(randomblob(32))) NOT NULL ON CONFLICT REPLACE,
|
6
|
+
name TEXT DEFAULT('default') NOT NULL ON CONFLICT REPLACE,
|
7
|
+
fire_at INTEGER DEFAULT(unixepoch()) NOT NULL ON CONFLICT REPLACE,
|
8
|
+
value TEXT,
|
9
|
+
created_at INTEGER DEFAULT(unixepoch()) NOT NULL ON CONFLICT REPLACE
|
10
|
+
) WITHOUT ROWID
|
11
|
+
|
12
|
+
create_index_queue_by_name: >
|
13
|
+
CREATE INDEX IF NOT EXISTS idx_queue_by_name ON queue(name, fire_at ASC)
|
14
|
+
|
15
|
+
stmts:
|
16
|
+
|
17
|
+
push: INSERT INTO queue(id, name, fire_at, value) VALUES (hex(randomblob(32)), $1, (unixepoch() + $2), $3) RETURNING id, name
|
18
|
+
|
19
|
+
repush: INSERT INTO queue(id, name, fire_at, value) VALUES (?, ?, (unixepoch() + ?), ?) RETURNING name
|
20
|
+
|
21
|
+
pop: >
|
22
|
+
DELETE FROM queue
|
23
|
+
WHERE (name, fire_at, id)
|
24
|
+
IN (
|
25
|
+
SELECT name, fire_at, id FROM queue
|
26
|
+
WHERE name = ifnull($1, 'default')
|
27
|
+
AND fire_at <= (unixepoch())
|
28
|
+
ORDER BY fire_at ASC
|
29
|
+
LIMIT ifnull($2, 1)
|
30
|
+
)
|
31
|
+
RETURNING id, value
|
32
|
+
|
33
|
+
delete: DELETE FROM queue WHERE id = $1 RETURNING value
|
34
|
+
|
@@ -1,5 +1,9 @@
|
|
1
|
+
# frozen_stringe_literal: true
|
2
|
+
|
1
3
|
require 'sqlite3'
|
2
|
-
require '
|
4
|
+
require 'logger'
|
5
|
+
require 'oj'
|
6
|
+
require 'yaml'
|
3
7
|
|
4
8
|
module Litesupport
|
5
9
|
|
@@ -38,8 +42,8 @@ module Litesupport
|
|
38
42
|
end
|
39
43
|
# we should never reach here
|
40
44
|
end
|
41
|
-
|
42
|
-
def self.
|
45
|
+
|
46
|
+
def self.context
|
43
47
|
if environment == :fiber || environment == :poylphony
|
44
48
|
Fiber.current.storage
|
45
49
|
else
|
@@ -47,8 +51,12 @@ module Litesupport
|
|
47
51
|
end
|
48
52
|
end
|
49
53
|
|
50
|
-
def self.
|
51
|
-
|
54
|
+
def self.current_context
|
55
|
+
if environment == :fiber || environment == :poylphony
|
56
|
+
Fiber.current
|
57
|
+
else
|
58
|
+
Thread.current
|
59
|
+
end
|
52
60
|
end
|
53
61
|
|
54
62
|
# switch the execution context to allow others to run
|
@@ -87,7 +95,7 @@ module Litesupport
|
|
87
95
|
# common db object options
|
88
96
|
def self.create_db(path)
|
89
97
|
db = SQLite3::Database.new(path)
|
90
|
-
db.busy_handler{ switch || sleep(0.
|
98
|
+
db.busy_handler{ switch || sleep(0.0001) }
|
91
99
|
db.journal_mode = "WAL"
|
92
100
|
db.instance_variable_set(:@stmts, {})
|
93
101
|
class << db
|
@@ -111,6 +119,21 @@ module Litesupport
|
|
111
119
|
end
|
112
120
|
|
113
121
|
end
|
122
|
+
|
123
|
+
module Forkable
|
124
|
+
|
125
|
+
def _fork(*args)
|
126
|
+
ppid = Process.pid
|
127
|
+
result = super
|
128
|
+
if Process.pid != ppid
|
129
|
+
# trigger a restart of all connections owned by Litesupport::Pool
|
130
|
+
end
|
131
|
+
result
|
132
|
+
end
|
133
|
+
|
134
|
+
end
|
135
|
+
|
136
|
+
#::Process.singleton_class.prepend(::Litesupport::Forkable)
|
114
137
|
|
115
138
|
class Pool
|
116
139
|
|
@@ -126,6 +149,7 @@ module Litesupport
|
|
126
149
|
end
|
127
150
|
|
128
151
|
def acquire
|
152
|
+
# check for pid changes
|
129
153
|
acquired = false
|
130
154
|
result = nil
|
131
155
|
while !acquired do
|
@@ -148,5 +172,130 @@ module Litesupport
|
|
148
172
|
end
|
149
173
|
|
150
174
|
end
|
175
|
+
|
176
|
+
module ForkListener
|
177
|
+
def self.listeners
|
178
|
+
@listeners ||= []
|
179
|
+
end
|
180
|
+
|
181
|
+
def self.listen(&block)
|
182
|
+
listeners << block
|
183
|
+
end
|
184
|
+
end
|
185
|
+
|
186
|
+
module Forkable
|
187
|
+
|
188
|
+
def _fork(*args)
|
189
|
+
ppid = Process.pid
|
190
|
+
result = super
|
191
|
+
if Process.pid != ppid && [:threaded, :iodine].include?(Litesupport.environment)
|
192
|
+
ForkListener.listeners.each{|l| l.call }
|
193
|
+
end
|
194
|
+
result
|
195
|
+
end
|
196
|
+
|
197
|
+
end
|
151
198
|
|
199
|
+
module Liteconnection
|
200
|
+
|
201
|
+
include Forkable
|
202
|
+
|
203
|
+
# close, setup, run_stmt and run_sql assume a single connection was created
|
204
|
+
def close
|
205
|
+
@running = false
|
206
|
+
@conn.acquire do |q|
|
207
|
+
q.stmts.each_pair {|k, v| q.stmts[k].close }
|
208
|
+
q.close
|
209
|
+
end
|
210
|
+
end
|
211
|
+
|
212
|
+
private # all methods are private
|
213
|
+
|
214
|
+
def init(options = {})
|
215
|
+
#c configure the object, loading options from the appropriate location
|
216
|
+
configure(options)
|
217
|
+
# setup connections and background threads
|
218
|
+
setup
|
219
|
+
# handle process exiting
|
220
|
+
at_exit do
|
221
|
+
exit_callback
|
222
|
+
end
|
223
|
+
# handle forking (restart connections and background threads)
|
224
|
+
Litesupport::ForkListener.listen do
|
225
|
+
setup
|
226
|
+
end
|
227
|
+
end
|
228
|
+
|
229
|
+
def configure(options = {})
|
230
|
+
# detect environment (production, development, etc.)
|
231
|
+
env = "development"
|
232
|
+
if defined? Rails
|
233
|
+
env = ENV["RAILS_ENV"]
|
234
|
+
elsif ENV["RACK_ENV"]
|
235
|
+
env = ENV["RACK_ENV"]
|
236
|
+
elsif ENV["APP_ENV"]
|
237
|
+
env = ENV["RACK_ENV"]
|
238
|
+
end
|
239
|
+
defaults = self.class::DEFAULT_OPTIONS rescue {}
|
240
|
+
@options = defaults.merge(options)
|
241
|
+
config = YAML.load_file(@options[:config_path]) rescue {} # an empty hash won't hurt
|
242
|
+
config = config[env] if config[env] # if there is a config for the current environment defined then use it, otherwise use the top level declaration
|
243
|
+
config.keys.each do |k| # symbolize keys
|
244
|
+
config[k.to_sym] = config[k]
|
245
|
+
config.delete k
|
246
|
+
end
|
247
|
+
@options.merge!(config)
|
248
|
+
@options.merge!(options) # make sure options passed to initialize trump everything else
|
249
|
+
end
|
250
|
+
|
251
|
+
def setup
|
252
|
+
@conn = create_pooled_connection
|
253
|
+
@logger = create_logger
|
254
|
+
@running = true
|
255
|
+
end
|
256
|
+
|
257
|
+
def create_logger
|
258
|
+
@options[:logger] = nil unless @options[:logger]
|
259
|
+
return @options[:logger] if @options[:logger].respond_to? :info
|
260
|
+
return Logger.new(STDOUT) if @options[:logger] == 'STDOUT'
|
261
|
+
return Logger.new(STDERR) if @options[:logger] == 'STDERR'
|
262
|
+
return Logger.new(@options[:logger]) if @options[:logger].is_a? String
|
263
|
+
return Logger.new(IO::NULL)
|
264
|
+
end
|
265
|
+
|
266
|
+
def exit_callback
|
267
|
+
close
|
268
|
+
end
|
269
|
+
|
270
|
+
def run_stmt(stmt, *args)
|
271
|
+
@conn.acquire{|q| q.stmts[stmt].execute!(*args) }
|
272
|
+
end
|
273
|
+
|
274
|
+
def run_sql(sql, *args)
|
275
|
+
@conn.acquire{|q| q.execute(sql, *args) }
|
276
|
+
end
|
277
|
+
|
278
|
+
def create_pooled_connection(count = 1)
|
279
|
+
Litesupport::Pool.new(1){create_connection}
|
280
|
+
end
|
281
|
+
|
282
|
+
# common db object options
|
283
|
+
def create_connection
|
284
|
+
conn = SQLite3::Database.new(@options[:path])
|
285
|
+
conn.busy_handler{ Litesupport.switch || sleep(rand * 0.002) }
|
286
|
+
conn.journal_mode = "WAL"
|
287
|
+
conn.synchronous = @options[:sync] || 1
|
288
|
+
conn.mmap_size = @options[:mmap_size] || 0
|
289
|
+
conn.instance_variable_set(:@stmts, {})
|
290
|
+
class << conn
|
291
|
+
attr_reader :stmts
|
292
|
+
end
|
293
|
+
conn
|
294
|
+
end
|
295
|
+
|
296
|
+
end
|
297
|
+
|
152
298
|
end
|
299
|
+
|
300
|
+
Process.singleton_class.prepend(Litesupport::Forkable)
|
301
|
+
|
data/lib/litestack/version.rb
CHANGED