lowkiq 1.0.1 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +4 -4
- data/LICENSE.md +10 -0
- data/README.md +23 -3
- data/docker-compose.yml +1 -1
- data/lib/lowkiq.rb +6 -2
- data/lib/lowkiq/queue/fetch.rb +2 -2
- data/lib/lowkiq/queue/keys.rb +16 -4
- data/lib/lowkiq/queue/queue.rb +103 -55
- data/lib/lowkiq/script.rb +42 -0
- data/lib/lowkiq/server.rb +4 -0
- data/lib/lowkiq/shard_handler.rb +3 -3
- data/lib/lowkiq/version.rb +1 -1
- data/lib/lowkiq/web.rb +2 -0
- data/lowkiq.gemspec +1 -0
- metadata +7 -5
- data/lib/lowkiq/queue/marshal.rb +0 -23
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 765d6d3c54595e43fe428927bab38b8e797e3727bd4bc229009133e3a3df2356
|
4
|
+
data.tar.gz: d7dcf125b3e9734f749f1c0ae385f2b48e4fb1b75f2574082bfecc95b6122666
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 987fe866751990b60dbdac866a2789c85f8a3ef5743170d77a7b03a88c368a2f94ce051d9def1e037f36aa42b3b404eb3fd5e0acce00fa760f54223b387a7451
|
7
|
+
data.tar.gz: c39de8f08d3b98a4eb4ffa586435d1935c3bf803b936591106ff97eb38486323c34a7e5c25f24f122592a08b2661df2f201802508f4c6a3b61ecdda2a04acfe3
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
lowkiq (1.0.
|
4
|
+
lowkiq (1.0.5)
|
5
5
|
connection_pool (~> 2.2, >= 2.2.2)
|
6
6
|
rack (>= 1.5.0)
|
7
7
|
redis (>= 4.0.1, < 5)
|
@@ -9,13 +9,13 @@ PATH
|
|
9
9
|
GEM
|
10
10
|
remote: https://rubygems.org/
|
11
11
|
specs:
|
12
|
-
connection_pool (2.2.
|
12
|
+
connection_pool (2.2.3)
|
13
13
|
diff-lcs (1.3)
|
14
14
|
rack (2.2.2)
|
15
15
|
rack-test (1.1.0)
|
16
16
|
rack (>= 1.0, < 3)
|
17
17
|
rake (12.3.3)
|
18
|
-
redis (4.
|
18
|
+
redis (4.2.2)
|
19
19
|
rspec (3.9.0)
|
20
20
|
rspec-core (~> 3.9.0)
|
21
21
|
rspec-expectations (~> 3.9.0)
|
@@ -42,4 +42,4 @@ DEPENDENCIES
|
|
42
42
|
rspec-mocks (~> 3.8)
|
43
43
|
|
44
44
|
BUNDLED WITH
|
45
|
-
2.1.
|
45
|
+
2.1.4
|
data/LICENSE.md
CHANGED
@@ -1,3 +1,13 @@
|
|
1
|
+
This software is dual-licensed under the LGPL version 3 or under the Licence Agreement.
|
2
|
+
Recipients can choose the terms under which they want to use or distribute the
|
3
|
+
software.
|
4
|
+
|
5
|
+
Copyright © BIA-Technologies Limited Liability Company (OOO)
|
6
|
+
|
7
|
+
# The LGPL Version 3 (LGPL-3.0)
|
8
|
+
|
9
|
+
https://www.gnu.org/licenses/lgpl-3.0.html
|
10
|
+
|
1
11
|
# Licence Agreement
|
2
12
|
On granting a non-exclusive right to use open source software
|
3
13
|
|
data/README.md
CHANGED
@@ -16,6 +16,7 @@ Ordered background jobs processing
|
|
16
16
|
* [Api](#api)
|
17
17
|
* [Ring app](#ring-app)
|
18
18
|
* [Configuration](#configuration)
|
19
|
+
* [Performance](#performance)
|
19
20
|
* [Execution](#execution)
|
20
21
|
* [Shutdown](#shutdown)
|
21
22
|
* [Debug](#debug)
|
@@ -93,8 +94,9 @@ Furthermore, Lowkiq's queues are reliable. Only Sidekiq Pro or plugins can add s
|
|
93
94
|
This [benchmark](examples/benchmark) shows overhead on redis usage.
|
94
95
|
This is the results for 5 threads, 100,000 blank jobs:
|
95
96
|
|
96
|
-
+ lowkiq:
|
97
|
-
+
|
97
|
+
+ lowkiq: 155 sec or 1.55 ms per job
|
98
|
+
+ lowkiq +hiredis: 80 sec or 0.80 ms per job
|
99
|
+
+ sidekiq: 15 sec or 0.15 ms per job
|
98
100
|
|
99
101
|
This difference is related to different queues structure.
|
100
102
|
Sidekiq uses one list for all workers and fetches the job entirely for O(1).
|
@@ -268,7 +270,7 @@ ATestWorker.perform_async 1000.times.map { |id| { payload: {id: id} } }
|
|
268
270
|
|
269
271
|
## Configuration
|
270
272
|
|
271
|
-
|
273
|
+
Options and their default values are:
|
272
274
|
|
273
275
|
+ `Lowkiq.poll_interval = 1` - delay in seconds between queue polling for new jobs.
|
274
276
|
Used only if the queue was empty at previous cycle or error was occured.
|
@@ -281,6 +283,8 @@ Default options and values are:
|
|
281
283
|
+ `Lowkiq.build_scheduler = ->() { Lowkiq.build_lag_scheduler }` is a scheduler
|
282
284
|
+ `Lowkiq.build_splitter = ->() { Lowkiq.build_default_splitter }` is a splitter
|
283
285
|
+ `Lowkiq.last_words = ->(ex) {}` is an exception handler of descendants of `StandardError` caused the process stop
|
286
|
+
+ `Lowkiq.dump_payload = Marshal.method :dump`
|
287
|
+
+ `Lowkiq.load_payload = Marshal.method :load`
|
284
288
|
|
285
289
|
```ruby
|
286
290
|
$logger = Logger.new(STDOUT)
|
@@ -301,6 +305,22 @@ Lowkiq.server_middlewares << -> (worker, batch, &block) do
|
|
301
305
|
end
|
302
306
|
```
|
303
307
|
|
308
|
+
## Performance
|
309
|
+
|
310
|
+
Use [hiredis](https://github.com/redis/hiredis-rb) for better performance.
|
311
|
+
|
312
|
+
```ruby
|
313
|
+
# Gemfile
|
314
|
+
|
315
|
+
gem "hiredis"
|
316
|
+
```
|
317
|
+
|
318
|
+
```ruby
|
319
|
+
# config
|
320
|
+
|
321
|
+
Lowkiq.redis = ->() { Redis.new url: ENV.fetch('REDIS_URL'), driver: :hiredis }
|
322
|
+
```
|
323
|
+
|
304
324
|
## Execution
|
305
325
|
|
306
326
|
`lowkiq -r ./path_to_app`
|
data/docker-compose.yml
CHANGED
data/lib/lowkiq.rb
CHANGED
@@ -4,9 +4,11 @@ require "zlib"
|
|
4
4
|
require "json"
|
5
5
|
require "ostruct"
|
6
6
|
require "optparse"
|
7
|
+
require "digest"
|
7
8
|
|
8
9
|
require "lowkiq/version"
|
9
10
|
require "lowkiq/utils"
|
11
|
+
require "lowkiq/script"
|
10
12
|
|
11
13
|
require "lowkiq/extend_tracker"
|
12
14
|
require "lowkiq/option_parser"
|
@@ -19,7 +21,6 @@ require "lowkiq/schedulers/seq"
|
|
19
21
|
|
20
22
|
require "lowkiq/server"
|
21
23
|
|
22
|
-
require "lowkiq/queue/marshal"
|
23
24
|
require "lowkiq/queue/keys"
|
24
25
|
require "lowkiq/queue/fetch"
|
25
26
|
require "lowkiq/queue/queue"
|
@@ -40,7 +41,8 @@ module Lowkiq
|
|
40
41
|
:redis, :client_pool_size, :pool_timeout,
|
41
42
|
:server_middlewares, :on_server_init,
|
42
43
|
:build_scheduler, :build_splitter,
|
43
|
-
:last_words
|
44
|
+
:last_words,
|
45
|
+
:dump_payload, :load_payload
|
44
46
|
|
45
47
|
def server_redis_pool
|
46
48
|
@server_redis_pool ||= ConnectionPool.new(size: threads_per_node, timeout: pool_timeout, &redis)
|
@@ -108,4 +110,6 @@ module Lowkiq
|
|
108
110
|
self.build_scheduler = ->() { Lowkiq.build_lag_scheduler }
|
109
111
|
self.build_splitter = ->() { Lowkiq.build_default_splitter }
|
110
112
|
self.last_words = ->(ex) {}
|
113
|
+
self.dump_payload = ::Marshal.method :dump
|
114
|
+
self.load_payload = ::Marshal.method :load
|
111
115
|
end
|
data/lib/lowkiq/queue/fetch.rb
CHANGED
@@ -21,7 +21,7 @@ module Lowkiq
|
|
21
21
|
id: x[0],
|
22
22
|
perform_in: x[1][0],
|
23
23
|
retry_count: x[1][1],
|
24
|
-
payloads: x[1][2].map { |(payload, score)| [
|
24
|
+
payloads: x[1][2].map { |(payload, score)| [Lowkiq.load_payload.call(payload), score] },
|
25
25
|
error: x[1][3],
|
26
26
|
}.compact
|
27
27
|
end.compact
|
@@ -41,7 +41,7 @@ module Lowkiq
|
|
41
41
|
{
|
42
42
|
id: x[0],
|
43
43
|
updated_at: x[1][0],
|
44
|
-
payloads: x[1][1].map { |(payload, score)| [
|
44
|
+
payloads: x[1][1].map { |(payload, score)| [Lowkiq.load_payload.call(payload), score] },
|
45
45
|
error: x[1][2],
|
46
46
|
}.compact
|
47
47
|
end.compact
|
data/lib/lowkiq/queue/keys.rb
CHANGED
@@ -39,14 +39,26 @@ module Lowkiq
|
|
39
39
|
[@prefix, :errors].join(':')
|
40
40
|
end
|
41
41
|
|
42
|
-
def processing_key(shard)
|
43
|
-
[@prefix, :processing, shard].join(':')
|
44
|
-
end
|
45
|
-
|
46
42
|
def processing_length_by_shard_hash
|
47
43
|
[@prefix, :processing_length_by_shard].join(':')
|
48
44
|
end
|
49
45
|
|
46
|
+
def processing_ids_with_perform_in_hash(shard)
|
47
|
+
[@prefix, :processing, :ids_with_perform_in, shard].join(':')
|
48
|
+
end
|
49
|
+
|
50
|
+
def processing_ids_with_retry_count_hash(shard)
|
51
|
+
[@prefix, :processing, :ids_with_retry_count, shard].join(':')
|
52
|
+
end
|
53
|
+
|
54
|
+
def processing_payloads_zset(id)
|
55
|
+
[@prefix, :processing, :payloads, id].join(':')
|
56
|
+
end
|
57
|
+
|
58
|
+
def processing_errors_hash(shard)
|
59
|
+
[@prefix, :processing, :errors, shard].join(':')
|
60
|
+
end
|
61
|
+
|
50
62
|
def morgue_all_ids_lex_zset
|
51
63
|
[@prefix, :morgue, :all_ids_lex].join(':')
|
52
64
|
end
|
data/lib/lowkiq/queue/queue.rb
CHANGED
@@ -29,7 +29,7 @@ module Lowkiq
|
|
29
29
|
redis.zadd @keys.all_ids_scored_by_retry_count_zset, retry_count, id, nx: true
|
30
30
|
|
31
31
|
redis.zadd @keys.ids_scored_by_perform_in_zset(shard), perform_in, id, nx: true
|
32
|
-
redis.zadd @keys.payloads_zset(id), score,
|
32
|
+
redis.zadd @keys.payloads_zset(id), score, Lowkiq.dump_payload.call(payload), nx: true
|
33
33
|
end
|
34
34
|
end
|
35
35
|
end
|
@@ -37,44 +37,56 @@ module Lowkiq
|
|
37
37
|
|
38
38
|
def pop(shard, limit:)
|
39
39
|
@pool.with do |redis|
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
limit: [0, limit]
|
45
|
-
|
46
|
-
if ids.empty?
|
47
|
-
redis.unwatch
|
48
|
-
return []
|
49
|
-
end
|
40
|
+
ids = redis.zrangebyscore @keys.ids_scored_by_perform_in_zset(shard),
|
41
|
+
0, @timestamp.call,
|
42
|
+
limit: [0, limit]
|
43
|
+
return [] if ids.empty?
|
50
44
|
|
51
|
-
|
45
|
+
res = redis.multi do |redis|
|
46
|
+
redis.hset @keys.processing_length_by_shard_hash, shard, ids.length
|
52
47
|
|
53
|
-
|
54
|
-
|
55
|
-
redis.
|
56
|
-
|
48
|
+
ids.each do |id|
|
49
|
+
redis.zrem @keys.all_ids_lex_zset, id
|
50
|
+
redis.zrem @keys.ids_scored_by_perform_in_zset(shard), id
|
51
|
+
|
52
|
+
Script.zremhset redis,
|
53
|
+
@keys.all_ids_scored_by_perform_in_zset,
|
54
|
+
@keys.processing_ids_with_perform_in_hash(shard),
|
55
|
+
id
|
56
|
+
Script.zremhset redis,
|
57
|
+
@keys.all_ids_scored_by_retry_count_zset,
|
58
|
+
@keys.processing_ids_with_retry_count_hash(shard),
|
59
|
+
id
|
60
|
+
redis.rename @keys.payloads_zset(id),
|
61
|
+
@keys.processing_payloads_zset(id)
|
62
|
+
Script.hmove redis,
|
63
|
+
@keys.errors_hash,
|
64
|
+
@keys.processing_errors_hash(shard),
|
65
|
+
id
|
57
66
|
end
|
58
|
-
|
67
|
+
processing_data_pipeline(redis, shard, ids)
|
68
|
+
end
|
59
69
|
|
60
|
-
|
70
|
+
res.shift 1 + ids.length * 6
|
71
|
+
processing_data_build res, ids
|
61
72
|
end
|
62
73
|
end
|
63
74
|
|
64
75
|
def push_back(batch)
|
65
76
|
@pool.with do |redis|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
77
|
+
timestamp = @timestamp.call
|
78
|
+
redis.multi do |redis|
|
79
|
+
batch.each do |job|
|
80
|
+
id = job.fetch(:id)
|
81
|
+
perform_in = job.fetch(:perform_in, timestamp)
|
82
|
+
retry_count = job.fetch(:retry_count, -1)
|
83
|
+
payloads = job.fetch(:payloads).map do |(payload, score)|
|
84
|
+
[score, Lowkiq.dump_payload.call(payload)]
|
85
|
+
end
|
86
|
+
error = job.fetch(:error, nil)
|
74
87
|
|
75
|
-
|
88
|
+
shard = id_to_shard id
|
76
89
|
|
77
|
-
redis.multi do
|
78
90
|
redis.zadd @keys.all_ids_lex_zset, 0, id
|
79
91
|
redis.zadd @keys.all_ids_scored_by_perform_in_zset, perform_in, id
|
80
92
|
redis.zadd @keys.all_ids_scored_by_retry_count_zset, retry_count, id
|
@@ -88,40 +100,52 @@ module Lowkiq
|
|
88
100
|
end
|
89
101
|
end
|
90
102
|
|
91
|
-
def ack(shard, result = nil)
|
103
|
+
def ack(shard, data, result = nil)
|
104
|
+
ids = data.map { |job| job[:id] }
|
105
|
+
length = ids.length
|
106
|
+
|
92
107
|
@pool.with do |redis|
|
93
|
-
length = redis.hget(@keys.processing_length_by_shard_hash, shard).to_i
|
94
108
|
redis.multi do
|
95
|
-
redis.del
|
109
|
+
redis.del @keys.processing_ids_with_perform_in_hash(shard)
|
110
|
+
redis.del @keys.processing_ids_with_retry_count_hash(shard)
|
111
|
+
redis.del @keys.processing_errors_hash(shard)
|
112
|
+
ids.each do |id|
|
113
|
+
redis.del @keys.processing_payloads_zset(id)
|
114
|
+
end
|
96
115
|
redis.hdel @keys.processing_length_by_shard_hash, shard
|
97
|
-
|
98
116
|
redis.incrby @keys.processed_key, length if result == :success
|
99
|
-
redis.incrby @keys.failed_key,
|
117
|
+
redis.incrby @keys.failed_key, length if result == :fail
|
100
118
|
end
|
101
119
|
end
|
102
120
|
end
|
103
121
|
|
104
122
|
def processing_data(shard)
|
105
|
-
|
106
|
-
redis.
|
107
|
-
|
108
|
-
return [] if data.nil?
|
123
|
+
@pool.with do |redis|
|
124
|
+
ids = redis.hkeys @keys.processing_ids_with_perform_in_hash(shard)
|
125
|
+
return [] if ids.empty?
|
109
126
|
|
110
|
-
|
127
|
+
res = redis.multi do |redis|
|
128
|
+
processing_data_pipeline redis, shard, ids
|
129
|
+
end
|
130
|
+
|
131
|
+
processing_data_build res, ids
|
132
|
+
end
|
111
133
|
end
|
112
134
|
|
113
135
|
def push_to_morgue(batch)
|
114
136
|
@pool.with do |redis|
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
137
|
+
timestamp = @timestamp.call
|
138
|
+
redis.multi do
|
139
|
+
batch.each do |job|
|
140
|
+
id = job.fetch(:id)
|
141
|
+
payloads = job.fetch(:payloads).map do |(payload, score)|
|
142
|
+
[score, Lowkiq.dump_payload.call(payload)]
|
143
|
+
end
|
144
|
+
error = job.fetch(:error, nil)
|
145
|
+
|
121
146
|
|
122
|
-
redis.multi do
|
123
147
|
redis.zadd @keys.morgue_all_ids_lex_zset, 0, id
|
124
|
-
redis.zadd @keys.morgue_all_ids_scored_by_updated_at_zset,
|
148
|
+
redis.zadd @keys.morgue_all_ids_scored_by_updated_at_zset, timestamp, id
|
125
149
|
redis.zadd @keys.morgue_payloads_zset(id), payloads, nx: true
|
126
150
|
|
127
151
|
redis.hset @keys.morgue_errors_hash, id, error unless error.nil?
|
@@ -146,7 +170,15 @@ module Lowkiq
|
|
146
170
|
def delete(ids)
|
147
171
|
@pool.with do |redis|
|
148
172
|
redis.multi do
|
149
|
-
|
173
|
+
ids.each do |id|
|
174
|
+
shard = id_to_shard id
|
175
|
+
redis.zrem @keys.all_ids_lex_zset, id
|
176
|
+
redis.zrem @keys.all_ids_scored_by_perform_in_zset, id
|
177
|
+
redis.zrem @keys.all_ids_scored_by_retry_count_zset, id
|
178
|
+
redis.zrem @keys.ids_scored_by_perform_in_zset(shard), id
|
179
|
+
redis.del @keys.payloads_zset(id)
|
180
|
+
redis.hdel @keys.errors_hash, id
|
181
|
+
end
|
150
182
|
end
|
151
183
|
end
|
152
184
|
end
|
@@ -161,17 +193,33 @@ module Lowkiq
|
|
161
193
|
Zlib.crc32(id.to_s) % @shards_count
|
162
194
|
end
|
163
195
|
|
164
|
-
def
|
196
|
+
def processing_data_pipeline(redis, shard, ids)
|
197
|
+
redis.hgetall @keys.processing_ids_with_perform_in_hash(shard)
|
198
|
+
redis.hgetall @keys.processing_ids_with_retry_count_hash(shard)
|
199
|
+
redis.hgetall @keys.processing_errors_hash(shard)
|
200
|
+
|
165
201
|
ids.each do |id|
|
166
|
-
|
167
|
-
redis.zrem @keys.all_ids_lex_zset, id
|
168
|
-
redis.zrem @keys.all_ids_scored_by_perform_in_zset, id
|
169
|
-
redis.zrem @keys.all_ids_scored_by_retry_count_zset, id
|
170
|
-
redis.zrem @keys.ids_scored_by_perform_in_zset(shard), id
|
171
|
-
redis.del @keys.payloads_zset(id)
|
172
|
-
redis.hdel @keys.errors_hash, id
|
202
|
+
redis.zrange @keys.processing_payloads_zset(id), 0, -1, with_scores: true
|
173
203
|
end
|
174
204
|
end
|
205
|
+
|
206
|
+
def processing_data_build(arr, ids)
|
207
|
+
ids_with_perform_in = arr.shift
|
208
|
+
ids_with_retry_count = arr.shift
|
209
|
+
errors = arr.shift
|
210
|
+
payloads = arr
|
211
|
+
|
212
|
+
ids.zip(payloads).map do |(id, payloads)|
|
213
|
+
next if payloads.empty?
|
214
|
+
{
|
215
|
+
id: id,
|
216
|
+
perform_in: ids_with_perform_in[id].to_f,
|
217
|
+
retry_count: ids_with_retry_count[id].to_f,
|
218
|
+
payloads: payloads.map { |(payload, score)| [Lowkiq.load_payload.call(payload), score] },
|
219
|
+
error: errors[id]
|
220
|
+
}.compact
|
221
|
+
end.compact
|
222
|
+
end
|
175
223
|
end
|
176
224
|
end
|
177
225
|
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
module Lowkiq
|
2
|
+
module Script
|
3
|
+
module_function
|
4
|
+
|
5
|
+
ALL = {
|
6
|
+
hmove: <<-LUA,
|
7
|
+
local source = KEYS[1]
|
8
|
+
local destination = KEYS[2]
|
9
|
+
local key = ARGV[1]
|
10
|
+
local value = redis.call('hget', source, key)
|
11
|
+
if value then
|
12
|
+
redis.call('hdel', source, key)
|
13
|
+
redis.call('hset', destination, key, value)
|
14
|
+
end
|
15
|
+
LUA
|
16
|
+
zremhset: <<-LUA
|
17
|
+
local source = KEYS[1]
|
18
|
+
local destination = KEYS[2]
|
19
|
+
local member = ARGV[1]
|
20
|
+
local score = redis.call('zscore', source, member)
|
21
|
+
if score then
|
22
|
+
redis.call('zrem', source, member)
|
23
|
+
redis.call('hset', destination, member, score)
|
24
|
+
end
|
25
|
+
LUA
|
26
|
+
}.transform_values { |v| { sha: Digest::SHA1.hexdigest(v), source: v } }.freeze
|
27
|
+
|
28
|
+
def load!(redis)
|
29
|
+
ALL.each do |_, item|
|
30
|
+
redis.script(:load, item[:source])
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def hmove(redis, source, destination, key)
|
35
|
+
redis.evalsha ALL[:hmove][:sha], keys: [source, destination], argv: [key]
|
36
|
+
end
|
37
|
+
|
38
|
+
def zremhset(redis, source, destination, member)
|
39
|
+
redis.evalsha ALL[:zremhset][:sha], keys: [source, destination], argv: [member]
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
data/lib/lowkiq/server.rb
CHANGED
data/lib/lowkiq/shard_handler.rb
CHANGED
@@ -31,7 +31,7 @@ module Lowkiq
|
|
31
31
|
@worker.perform batch
|
32
32
|
end
|
33
33
|
|
34
|
-
@queue.ack @shard_index, :success
|
34
|
+
@queue.ack @shard_index, data, :success
|
35
35
|
true
|
36
36
|
rescue => ex
|
37
37
|
fail! data, ex
|
@@ -39,7 +39,7 @@ module Lowkiq
|
|
39
39
|
|
40
40
|
@queue.push_back back
|
41
41
|
@queue.push_to_morgue morgue
|
42
|
-
@queue.ack @shard_index, :fail
|
42
|
+
@queue.ack @shard_index, data, :fail
|
43
43
|
false
|
44
44
|
end
|
45
45
|
end
|
@@ -48,7 +48,7 @@ module Lowkiq
|
|
48
48
|
data = @queue.processing_data @shard_index
|
49
49
|
return if data.nil?
|
50
50
|
@queue.push_back data
|
51
|
-
@queue.ack @shard_index
|
51
|
+
@queue.ack @shard_index, data
|
52
52
|
end
|
53
53
|
|
54
54
|
private
|
data/lib/lowkiq/version.rb
CHANGED
data/lib/lowkiq/web.rb
CHANGED
@@ -28,6 +28,7 @@ module Lowkiq
|
|
28
28
|
|
29
29
|
APP = Rack::Builder.new do
|
30
30
|
map "/api" do
|
31
|
+
use Rack::ContentType, "application/json"
|
31
32
|
run Api
|
32
33
|
end
|
33
34
|
|
@@ -35,6 +36,7 @@ module Lowkiq
|
|
35
36
|
run Rack::File.new ASSETS, { 'Cache-Control' => 'public, max-age=86400' }
|
36
37
|
end
|
37
38
|
|
39
|
+
use Rack::ContentType, "text/html"
|
38
40
|
run HTML
|
39
41
|
end
|
40
42
|
|
data/lowkiq.gemspec
CHANGED
@@ -11,6 +11,7 @@ Gem::Specification.new do |spec|
|
|
11
11
|
spec.summary = %q{Lowkiq}
|
12
12
|
spec.description = %q{Lowkiq}
|
13
13
|
spec.homepage = "https://github.com/bia-technologies/lowkiq"
|
14
|
+
spec.licenses = ['LGPL', 'EULA']
|
14
15
|
|
15
16
|
# Specify which files should be added to the gem when it is released.
|
16
17
|
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: lowkiq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Mikhail Kuzmin
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-10-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: redis
|
@@ -162,7 +162,6 @@ files:
|
|
162
162
|
- lib/lowkiq/queue/actions.rb
|
163
163
|
- lib/lowkiq/queue/fetch.rb
|
164
164
|
- lib/lowkiq/queue/keys.rb
|
165
|
-
- lib/lowkiq/queue/marshal.rb
|
166
165
|
- lib/lowkiq/queue/queries.rb
|
167
166
|
- lib/lowkiq/queue/queue.rb
|
168
167
|
- lib/lowkiq/queue/queue_metrics.rb
|
@@ -170,6 +169,7 @@ files:
|
|
170
169
|
- lib/lowkiq/redis_info.rb
|
171
170
|
- lib/lowkiq/schedulers/lag.rb
|
172
171
|
- lib/lowkiq/schedulers/seq.rb
|
172
|
+
- lib/lowkiq/script.rb
|
173
173
|
- lib/lowkiq/server.rb
|
174
174
|
- lib/lowkiq/shard_handler.rb
|
175
175
|
- lib/lowkiq/splitters/by_node.rb
|
@@ -182,7 +182,9 @@ files:
|
|
182
182
|
- lib/lowkiq/worker.rb
|
183
183
|
- lowkiq.gemspec
|
184
184
|
homepage: https://github.com/bia-technologies/lowkiq
|
185
|
-
licenses:
|
185
|
+
licenses:
|
186
|
+
- LGPL
|
187
|
+
- EULA
|
186
188
|
metadata: {}
|
187
189
|
post_install_message:
|
188
190
|
rdoc_options: []
|
@@ -199,7 +201,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
199
201
|
- !ruby/object:Gem::Version
|
200
202
|
version: '0'
|
201
203
|
requirements: []
|
202
|
-
rubygems_version: 3.1.
|
204
|
+
rubygems_version: 3.1.4
|
203
205
|
signing_key:
|
204
206
|
specification_version: 4
|
205
207
|
summary: Lowkiq
|
data/lib/lowkiq/queue/marshal.rb
DELETED
@@ -1,23 +0,0 @@
|
|
1
|
-
module Lowkiq
|
2
|
-
module Queue
|
3
|
-
module Marshal
|
4
|
-
class << self
|
5
|
-
def dump_payload(data)
|
6
|
-
::Marshal.dump data
|
7
|
-
end
|
8
|
-
|
9
|
-
def load_payload(str)
|
10
|
-
::Marshal.load str
|
11
|
-
end
|
12
|
-
|
13
|
-
def dump_data(data)
|
14
|
-
::Marshal.dump data
|
15
|
-
end
|
16
|
-
|
17
|
-
def load_data(str)
|
18
|
-
::Marshal.load str
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
22
|
-
end
|
23
|
-
end
|