litestack 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/BENCHMARKS.md +4 -4
- data/lib/litestack/litecache.rb +59 -67
- data/lib/litestack/litejob.rb +6 -6
- data/lib/litestack/litejobqueue.rb +7 -5
- data/lib/litestack/litequeue.rb +1 -5
- data/lib/litestack/litesupport.rb +4 -1
- data/lib/litestack/version.rb +1 -1
- data/lib/sequel/adapters/litedb.rb +8 -3
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 88034778fbac75441d1a9ed2e066ad6ba1aa4962eb48629de1d9ba7ae85f3468
|
4
|
+
data.tar.gz: d6ac66ffd1e78856bd74aa1d88163b95581494aaa481de88b673d84b5a8b8ff6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 37043f1eab519ea41e81e5b9cb6b20798ea5f6e7ed7ce99d8105454067129e7f49f4b8357b0855948e6df51bf6d466966802085b67943b1f9416ee90f443502a
|
7
|
+
data.tar.gz: 602d0e03d53eeb8e780c31b2f9bb1b3add550035c3cb3fd07f1d14fe94d94aea50a93dea19ab819454465c7824249bb40f7768ec4eebf6662af713218175071d
|
data/BENCHMARKS.md
CHANGED
@@ -16,7 +16,7 @@ This produces
|
|
16
16
|
SELECT * FROM posts WHERE id = ?
|
17
17
|
```
|
18
18
|
|
19
|
-
|
|
19
|
+
|Prcoesses|AR:PG|AR:litedb|Sequel:PG|Sequel:litedb|
|
20
20
|
|-:|-:|-:|-:|-:|
|
21
21
|
|1|1.3K q/s|6.5K q/s|1.8K q/s|17.4K q/s|
|
22
22
|
|2|2.6K q/s|13.9K q/s|3.5K q/s|33.2K q/s|
|
@@ -33,7 +33,7 @@ This produces
|
|
33
33
|
SELECT * FROM posts WHERE user_id = ? LIMIT 5
|
34
34
|
```
|
35
35
|
|
36
|
-
|
|
36
|
+
|Prcoesses|AR:PG|AR:litedb|Sequel:PG|Sequel:litedb|
|
37
37
|
|-:|-:|-:|-:|-:|
|
38
38
|
|1|345 q/s|482 q/s|937 q/s|1.1K q/s|
|
39
39
|
|2|751 q/s|848 q/s|1.3K q/s|2.3K q/s|
|
@@ -51,14 +51,14 @@ This produces
|
|
51
51
|
Update posts SET updated_at = ? WHERE id = ?
|
52
52
|
```
|
53
53
|
|
54
|
-
|
|
54
|
+
|Prcoesses|AR:PG|AR:litedb|Sequel:PG|Sequel:litedb|
|
55
55
|
|-:|-:|-:|-:|-:|
|
56
56
|
|1|125 q/s|484 q/s|129 q/s|2.1K q/s|
|
57
57
|
|2|265 q/s|576 q/s|333 q/s|2.5K q/s|
|
58
58
|
|4|481 q/s|693 q/s|704 q/s|2.3K q/s|
|
59
59
|
|8|898 q/s|748 q/s|1.2K q/s|2.4K q/s|
|
60
60
|
|
61
|
-
It is clear the Litedb enjoys a
|
61
|
+
It is clear the Litedb enjoys a significant advantage for reads and is very competitive for updates until many processes are relentlessly trying to write at the same time non stop.
|
62
62
|
For most applications, even with higher level of concurrency, Litedb will scale super well for reads and provide a very good baseline for writes.
|
63
63
|
|
64
64
|
> 
|
data/lib/litestack/litecache.rb
CHANGED
@@ -58,21 +58,23 @@ class Litecache
|
|
58
58
|
def initialize(options = {})
|
59
59
|
@options = DEFAULT_OPTIONS.merge(options)
|
60
60
|
@options[:size] = @options[:min_size] if @options[:size] < @options[:min_size]
|
61
|
-
@
|
62
|
-
|
63
|
-
:
|
64
|
-
:
|
65
|
-
:
|
66
|
-
:
|
67
|
-
:
|
68
|
-
:
|
69
|
-
:
|
70
|
-
:
|
71
|
-
:
|
72
|
-
:
|
73
|
-
:
|
74
|
-
:sizer => @cache.prepare("SELECT size.page_size * count.page_count FROM pragma_page_size() AS size, pragma_page_count() AS count")
|
61
|
+
@sql = {
|
62
|
+
:pruner => "DELETE FROM data WHERE expires_in <= $1",
|
63
|
+
:extra_pruner => "DELETE FROM data WHERE id IN (SELECT id FROM data ORDER BY last_used ASC LIMIT (SELECT CAST((count(*) * $1) AS int) FROM data))",
|
64
|
+
:limited_pruner => "DELETE FROM data WHERE id IN (SELECT id FROM data ORDER BY last_used asc limit $1)",
|
65
|
+
:toucher => "UPDATE data SET last_used = unixepoch('now') WHERE id = $1",
|
66
|
+
:setter => "INSERT into data (id, value, expires_in, last_used) VALUES ($1, $2, unixepoch('now') + $3, unixepoch('now')) on conflict(id) do UPDATE SET value = excluded.value, last_used = excluded.last_used, expires_in = excluded.expires_in",
|
67
|
+
:inserter => "INSERT into data (id, value, expires_in, last_used) VALUES ($1, $2, unixepoch('now') + $3, unixepoch('now')) on conflict(id) do UPDATE SET value = excluded.value, last_used = excluded.last_used, expires_in = excluded.expires_in WHERE id = $1 and expires_in <= unixepoch('now')",
|
68
|
+
:finder => "SELECT id FROM data WHERE id = $1",
|
69
|
+
:getter => "SELECT id, value, expires_in FROM data WHERE id = $1",
|
70
|
+
:deleter => "delete FROM data WHERE id = $1 returning value",
|
71
|
+
:incrementer => "INSERT into data (id, value, expires_in, last_used) VALUES ($1, $2, unixepoch('now') + $3, unixepoch('now')) on conflict(id) do UPDATE SET value = cast(value AS int) + cast(excluded.value as int), last_used = excluded.last_used, expires_in = excluded.expires_in",
|
72
|
+
:counter => "SELECT count(*) FROM data",
|
73
|
+
:sizer => "SELECT size.page_size * count.page_count FROM pragma_page_size() AS size, pragma_page_count() AS count"
|
75
74
|
}
|
75
|
+
@cache = create_store
|
76
|
+
@stmts = {}
|
77
|
+
@sql.each_pair{|k, v| @stmts[k] = @cache.prepare(v)}
|
76
78
|
@stats = {hit: 0, miss: 0}
|
77
79
|
@last_visited = {}
|
78
80
|
@running = true
|
@@ -83,14 +85,12 @@ class Litecache
|
|
83
85
|
def set(key, value, expires_in = nil)
|
84
86
|
key = key.to_s
|
85
87
|
expires_in = @options[:expires_in] if expires_in.nil? or expires_in.zero?
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
retry
|
93
|
-
end
|
88
|
+
begin
|
89
|
+
@stmts[:setter].execute!(key, value, expires_in)
|
90
|
+
rescue SQLite3::FullException
|
91
|
+
@stmts[:extra_pruner].execute!(0.2)
|
92
|
+
@cache.execute("vacuum")
|
93
|
+
retry
|
94
94
|
end
|
95
95
|
return true
|
96
96
|
end
|
@@ -99,17 +99,15 @@ class Litecache
|
|
99
99
|
def set_unless_exists(key, value, expires_in = nil)
|
100
100
|
key = key.to_s
|
101
101
|
expires_in = @options[:expires_in] if expires_in.nil? or expires_in.zero?
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
changes = @cache.changes
|
107
|
-
end
|
108
|
-
rescue SQLite3::FullException
|
109
|
-
@stmts[:extra_pruner].execute!(0.2)
|
110
|
-
@cache.execute("vacuum")
|
111
|
-
retry
|
102
|
+
begin
|
103
|
+
transaction(:immediate) do
|
104
|
+
@stmts[:inserter].execute!(key, value, expires_in)
|
105
|
+
changes = @cache.changes
|
112
106
|
end
|
107
|
+
rescue SQLite3::FullException
|
108
|
+
@stmts[:extra_pruner].execute!(0.2)
|
109
|
+
@cache.execute("vacuum")
|
110
|
+
retry
|
113
111
|
end
|
114
112
|
return changes > 0
|
115
113
|
end
|
@@ -118,11 +116,7 @@ class Litecache
|
|
118
116
|
# if the key doesn't exist or it is expired then null will be returned
|
119
117
|
def get(key)
|
120
118
|
key = key.to_s
|
121
|
-
record =
|
122
|
-
Litesupport.synchronize do
|
123
|
-
record = @stmts[:getter].execute!(key)[0]
|
124
|
-
end
|
125
|
-
if record
|
119
|
+
if record = @stmts[:getter].execute!(key)[0]
|
126
120
|
@last_visited[key] = true
|
127
121
|
@stats[:hit] +=1
|
128
122
|
return record[1]
|
@@ -133,18 +127,14 @@ class Litecache
|
|
133
127
|
|
134
128
|
# delete a key, value pair from the cache
|
135
129
|
def delete(key)
|
136
|
-
|
137
|
-
|
138
|
-
return @cache.changes > 0
|
139
|
-
end
|
130
|
+
@stmts[:deleter].execute!(key)
|
131
|
+
return @cache.changes > 0
|
140
132
|
end
|
141
133
|
|
142
134
|
# increment an integer value by amount, optionally add an expiry value (in seconds)
|
143
135
|
def increment(key, amount, expires_in = nil)
|
144
136
|
expires_in = @expires_in unless expires_in
|
145
|
-
|
146
|
-
@stmts[:incrementer].execute!(key.to_s, amount, expires_in)
|
147
|
-
end
|
137
|
+
@stmts[:incrementer].execute!(key.to_s, amount, expires_in)
|
148
138
|
end
|
149
139
|
|
150
140
|
# decrement an integer value by amount, optionally add an expiry value (in seconds)
|
@@ -154,43 +144,35 @@ class Litecache
|
|
154
144
|
|
155
145
|
# delete all entries in the cache up limit (ordered by LRU), if no limit is provided approximately 20% of the entries will be deleted
|
156
146
|
def prune(limit=nil)
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
@stmts[:pruner].execute!
|
164
|
-
end
|
147
|
+
if limit and limit.is_a? Integer
|
148
|
+
@stmts[:limited_pruner].execute!(limit)
|
149
|
+
elsif limit and limit.is_a? Float
|
150
|
+
@stmts[:extra_pruner].execute!(limit)
|
151
|
+
else
|
152
|
+
@stmts[:pruner].execute!
|
165
153
|
end
|
166
154
|
end
|
167
155
|
|
168
156
|
# return the number of key, value pairs in the cache
|
169
157
|
def count
|
170
|
-
|
171
|
-
@stmts[:counter].execute!.to_a[0][0]
|
172
|
-
end
|
158
|
+
@stmts[:counter].execute!.to_a[0][0]
|
173
159
|
end
|
174
160
|
|
175
161
|
# return the actual size of the cache file
|
176
162
|
def size
|
177
|
-
|
178
|
-
@stmts[:sizer].execute!.to_a[0][0]
|
179
|
-
end
|
163
|
+
@stmts[:sizer].execute!.to_a[0][0]
|
180
164
|
end
|
181
165
|
|
182
166
|
# delete all key, value pairs in the cache
|
183
167
|
def clear
|
184
|
-
|
185
|
-
@cache.execute("delete FROM data")
|
186
|
-
end
|
168
|
+
@cache.execute("delete FROM data")
|
187
169
|
end
|
188
170
|
|
189
171
|
# close the connection to the cache file
|
190
172
|
def close
|
191
173
|
@running = false
|
192
174
|
#Litesupport.synchronize do
|
193
|
-
|
175
|
+
@cache.close
|
194
176
|
#end
|
195
177
|
end
|
196
178
|
|
@@ -219,21 +201,31 @@ class Litecache
|
|
219
201
|
|
220
202
|
def spawn_worker
|
221
203
|
Litesupport.spawn do
|
204
|
+
# create a specific cache instance for this worker
|
205
|
+
# to overcome SQLite3 Database is locked error
|
206
|
+
cache = create_store
|
207
|
+
stmts = {}
|
208
|
+
[:toucher, :pruner, :extra_pruner].each do |stmt|
|
209
|
+
stmts[stmt] = cache.prepare(@sql[stmt])
|
210
|
+
end
|
222
211
|
while @running
|
223
212
|
Litesupport.synchronize do
|
224
213
|
begin
|
225
|
-
|
226
|
-
@last_visited.delete_if do |k|
|
227
|
-
|
214
|
+
cache.transaction(:immediate) do
|
215
|
+
@last_visited.delete_if do |k| # there is a race condition here, but not a serious one
|
216
|
+
stmts[:toucher].execute!(k) || true
|
228
217
|
end
|
229
|
-
|
218
|
+
stmts[:pruner].execute!
|
230
219
|
end
|
220
|
+
rescue SQLite3::BusyException
|
221
|
+
retry
|
231
222
|
rescue SQLite3::FullException
|
232
|
-
|
223
|
+
stmts[:extra_pruner].execute!(0.2)
|
233
224
|
end
|
234
225
|
end
|
235
226
|
sleep @options[:sleep_interval]
|
236
227
|
end
|
228
|
+
cache.close
|
237
229
|
end
|
238
230
|
end
|
239
231
|
|
data/lib/litestack/litejob.rb
CHANGED
@@ -43,21 +43,21 @@ module Litejob
|
|
43
43
|
private
|
44
44
|
def self.included(klass)
|
45
45
|
klass.extend(ClassMethods)
|
46
|
-
klass.
|
46
|
+
klass.get_jobqueue
|
47
47
|
end
|
48
48
|
|
49
49
|
module ClassMethods
|
50
50
|
def perform_async(*params)
|
51
|
-
|
51
|
+
get_jobqueue.push(self.name, params, 0, queue)
|
52
52
|
end
|
53
53
|
|
54
54
|
def perform_at(time, *params)
|
55
55
|
delay = time - Time.now.to_i
|
56
|
-
|
56
|
+
get_jobqueue.push(self.name, params, delay, queue)
|
57
57
|
end
|
58
58
|
|
59
59
|
def perfrom_in(delay, *params)
|
60
|
-
|
60
|
+
get_jobqueue.push(self.name, params, delay, queue)
|
61
61
|
end
|
62
62
|
|
63
63
|
def options
|
@@ -76,8 +76,8 @@ module Litejob
|
|
76
76
|
@@queue_name = queue_name.to_s
|
77
77
|
end
|
78
78
|
|
79
|
-
def
|
80
|
-
Litejobqueue.
|
79
|
+
def get_jobqueue
|
80
|
+
Litejobqueue.jobqueue(options)
|
81
81
|
end
|
82
82
|
end
|
83
83
|
|
@@ -41,7 +41,7 @@ class Litejobqueue
|
|
41
41
|
|
42
42
|
# a method that returns a single instance of the job queue
|
43
43
|
# for use by Litejob
|
44
|
-
def self.
|
44
|
+
def self.jobqueue(options = {})
|
45
45
|
@@queue ||= Litesupport.synchronize{self.new(options)}
|
46
46
|
end
|
47
47
|
|
@@ -58,8 +58,8 @@ class Litejobqueue
|
|
58
58
|
def initialize(options = {})
|
59
59
|
@options = DEFAULT_OPTIONS.merge(options)
|
60
60
|
@worker_sleep_index = 0
|
61
|
-
config = YAML.load_file(@options[:config_path]) rescue {} #an empty hash won't hurt
|
62
|
-
config.
|
61
|
+
config = YAML.load_file(@options[:config_path]) rescue {} # an empty hash won't hurt
|
62
|
+
config.keys.each do |k| # symbolize keys
|
63
63
|
config[k.to_sym] = config[k]
|
64
64
|
config.delete k
|
65
65
|
end
|
@@ -116,14 +116,16 @@ class Litejobqueue
|
|
116
116
|
# create a worker according to environment
|
117
117
|
def create_worker
|
118
118
|
Litesupport.spawn do
|
119
|
-
|
119
|
+
# we create a queue object specific to this worker here
|
120
|
+
# this way we can survive potential SQLite3 Database is locked errors
|
121
|
+
queue = Litequeue.new(@options)
|
120
122
|
loop do
|
121
123
|
processed = 0
|
122
124
|
@queues.each do |level| # iterate through the levels
|
123
125
|
level[1].each do |q| # iterate through the queues in the level
|
124
126
|
index = 0
|
125
127
|
max = level[0]
|
126
|
-
while index < max && payload =
|
128
|
+
while index < max && payload = queue.pop(q[0])
|
127
129
|
processed += 1
|
128
130
|
index += 1
|
129
131
|
begin
|
data/lib/litestack/litequeue.rb
CHANGED
@@ -50,11 +50,7 @@ class Litequeue
|
|
50
50
|
|
51
51
|
# pop an item from the queue, optionally with a specific queue name (default queue name is 'default')
|
52
52
|
def pop(queue='default')
|
53
|
-
|
54
|
-
Litesupport.synchronize do
|
55
|
-
result = @pop.execute!(queue)[0]
|
56
|
-
end
|
57
|
-
result
|
53
|
+
@pop.execute!(queue)[0]
|
58
54
|
end
|
59
55
|
|
60
56
|
# delete an item from the queue
|
@@ -37,11 +37,14 @@ module Litesupport
|
|
37
37
|
def self.switch
|
38
38
|
if self.environment == :fiber
|
39
39
|
Fiber.scheduler.yield
|
40
|
+
true
|
40
41
|
elsif self.environment == :polyphony
|
41
42
|
Fiber.current.schedule
|
42
43
|
Thread.current.switch_fiber
|
44
|
+
true
|
43
45
|
else
|
44
46
|
# do nothing in case of thread, switching will auto-happen
|
47
|
+
false
|
45
48
|
end
|
46
49
|
end
|
47
50
|
|
@@ -66,7 +69,7 @@ module Litesupport
|
|
66
69
|
# common db object options
|
67
70
|
def self.create_db(path)
|
68
71
|
db = SQLite3::Database.new(path)
|
69
|
-
db.busy_handler{ sleep
|
72
|
+
db.busy_handler{ switch || sleep(0.001) }
|
70
73
|
db.journal_mode = "WAL"
|
71
74
|
db
|
72
75
|
end
|
data/lib/litestack/version.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
require_relative '../../litestack/litedb'
|
2
2
|
require 'sequel'
|
3
3
|
require 'sequel/adapters/sqlite'
|
4
|
+
#require 'shared/litedb'
|
4
5
|
|
5
6
|
module Sequel
|
6
7
|
module Litedb
|
@@ -12,15 +13,19 @@ module Sequel
|
|
12
13
|
|
13
14
|
set_adapter_scheme :litedb
|
14
15
|
|
15
|
-
|
16
16
|
def connect(server)
|
17
|
+
|
18
|
+
Sequel.extension :fiber_concurrency if [:fiber, :polyphony].include? Litesupport.environment
|
19
|
+
|
17
20
|
opts = server_opts(server)
|
18
21
|
opts[:database] = ':memory:' if blank_object?(opts[:database])
|
19
22
|
sqlite3_opts = {}
|
20
23
|
sqlite3_opts[:readonly] = typecast_value_boolean(opts[:readonly]) if opts.has_key?(:readonly)
|
21
24
|
db = ::Litedb.new(opts[:database].to_s, sqlite3_opts)
|
25
|
+
|
26
|
+
self.transaction_mode = :immediate
|
22
27
|
|
23
|
-
|
28
|
+
if sqlite_version >= 104
|
24
29
|
db.extended_result_codes = true
|
25
30
|
end
|
26
31
|
|
@@ -29,8 +34,8 @@ module Sequel
|
|
29
34
|
class << db
|
30
35
|
attr_reader :prepared_statements
|
31
36
|
end
|
32
|
-
db.instance_variable_set(:@prepared_statements, {})
|
33
37
|
|
38
|
+
db.instance_variable_set(:@prepared_statements, {})
|
34
39
|
db
|
35
40
|
end
|
36
41
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: litestack
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Mohamed Hassan
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-02-
|
11
|
+
date: 2023-02-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sqlite3
|