jobba 1.5.0 → 1.8.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.github/workflows/tests.yml +42 -0
- data/README.md +19 -0
- data/Rakefile +1 -1
- data/jobba.gemspec +3 -3
- data/lib/jobba/clause.rb +85 -1
- data/lib/jobba/clause_factory.rb +4 -1
- data/lib/jobba/id_clause.rb +8 -0
- data/lib/jobba/query.rb +108 -57
- data/lib/jobba/redis_with_expiration.rb +68 -0
- data/lib/jobba/state.rb +1 -6
- data/lib/jobba/status.rb +11 -8
- data/lib/jobba/statuses.rb +1 -1
- data/lib/jobba/utils.rb +41 -0
- data/lib/jobba/version.rb +1 -1
- data/lib/jobba.rb +28 -3
- metadata +23 -24
- data/.ruby-version +0 -1
- data/.travis.yml +0 -15
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 489a463b81e0d1dc9d7f8c79b1aacd25ab66358cea92a2692e4428e0f7b30a54
|
4
|
+
data.tar.gz: c8eb29ef8b368390b1f78eec40a04aaa06cc4dee3b45453355d9109eace62fa7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 16abf20ee8428d89b11c1e76315e957a357bf4f4381b4c072efdfd90365d5863a558f2a92f0b051fda32bde52f58d012b17000a5602050be77e9202a609f7cd3
|
7
|
+
data.tar.gz: 6b63cffc85fa57a36b454913a016b6b745e7975c3741868644735505315068086f01e5dd07cb1613304dafce896a4e537f2c169c55a4926ea0d833b4949efabd
|
@@ -0,0 +1,42 @@
|
|
1
|
+
name: Tests
|
2
|
+
|
3
|
+
env:
|
4
|
+
USE_REAL_REDIS: true
|
5
|
+
REDIS_HOST: redis
|
6
|
+
REDIS_PORT: 6379
|
7
|
+
on:
|
8
|
+
pull_request:
|
9
|
+
push:
|
10
|
+
branches:
|
11
|
+
- master
|
12
|
+
jobs:
|
13
|
+
tests:
|
14
|
+
timeout-minutes: 30
|
15
|
+
runs-on: ubuntu-18.04
|
16
|
+
strategy:
|
17
|
+
fail-fast: false
|
18
|
+
matrix:
|
19
|
+
os: [ubuntu]
|
20
|
+
ruby: [ '2.5', '2.6', '2.7' ]
|
21
|
+
services:
|
22
|
+
# Label used to access the service container
|
23
|
+
redis:
|
24
|
+
# Docker Hub image
|
25
|
+
image: redis
|
26
|
+
options: >-
|
27
|
+
--health-cmd "redis-cli ping"
|
28
|
+
--health-interval 10s
|
29
|
+
--health-timeout 5s
|
30
|
+
--health-retries 5
|
31
|
+
ports:
|
32
|
+
# Maps port 6379 on service container to the host
|
33
|
+
- 6379:6379
|
34
|
+
steps:
|
35
|
+
- uses: actions/checkout@v2
|
36
|
+
- uses: ruby/setup-ruby@v1
|
37
|
+
with:
|
38
|
+
ruby-version: ${{ matrix.ruby }}
|
39
|
+
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
|
40
|
+
- name: Test
|
41
|
+
run: |
|
42
|
+
bundle exec rake spec
|
data/README.md
CHANGED
@@ -439,6 +439,17 @@ Jobba.where(...).run.count # These pull data back to Ruby and count in Ruby
|
|
439
439
|
Jobba.where(...).run.empty?
|
440
440
|
```
|
441
441
|
|
442
|
+
## Pagination
|
443
|
+
|
444
|
+
Pagination is supported with an ActiveRecord-like interface. You can call `.limit(x)` and `.offset(y)` on
|
445
|
+
queries, e.g.
|
446
|
+
|
447
|
+
```ruby
|
448
|
+
Jobba.where(state: :succeeded).limit(10).offset(20).to_a
|
449
|
+
```
|
450
|
+
|
451
|
+
Specifying a limit does not guarantee that you'll get that many elements back, as there may not be that many left in the result.
|
452
|
+
|
442
453
|
## Notes
|
443
454
|
|
444
455
|
### Times
|
@@ -449,6 +460,8 @@ Note that, in operations having to do with time, this gem ignores anything beyon
|
|
449
460
|
|
450
461
|
Jobba strives to do all of its operations as efficiently as possible using built-in Redis operations. If you find a place where the efficiency can be improved, please submit an issue or a pull request.
|
451
462
|
|
463
|
+
Single-clause queries (those with one `where` call) have been optimized. `Jobba.all` is a single-clause query. If you have lots of IDs, try to get by with single-clause queries. Multi-clause queries (including `count`) have to copy sets into temporary working sets where query clauses are ANDed together. This can be expensive for large datasets.
|
464
|
+
|
452
465
|
### Write from one; Read from many
|
453
466
|
|
454
467
|
Jobba assumes that any job is being run at one time by only one worker. Jobba makes no accomodations for multiple processes updating a Status at the same time; multiple processes reading of a Status are fine of course.
|
@@ -463,6 +476,12 @@ $> USE_REAL_REDIS=true rspec
|
|
463
476
|
|
464
477
|
Travis runs the specs with both `fakeredis` and real Redis.
|
465
478
|
|
479
|
+
Clauses need to implement three methods:
|
480
|
+
|
481
|
+
1. `to_new_set` - puts the IDs indicated by the clause into a new sorted set in redis
|
482
|
+
2. `result_ids` - used to get the IDs indicated by the clause when the clause is the only one in the query
|
483
|
+
3. `result_count` - used to get the count of IDs indicated by the clause when the clause is the only one in the query
|
484
|
+
|
466
485
|
## TODO
|
467
486
|
|
468
487
|
1. Provide job min, max, and average durations.
|
data/Rakefile
CHANGED
data/jobba.gemspec
CHANGED
@@ -19,11 +19,11 @@ Gem::Specification.new do |spec|
|
|
19
19
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
20
20
|
spec.require_paths = ["lib"]
|
21
21
|
|
22
|
-
spec.add_runtime_dependency "redis"
|
22
|
+
spec.add_runtime_dependency "redis"
|
23
|
+
spec.add_runtime_dependency "oj"
|
23
24
|
spec.add_runtime_dependency "redis-namespace"
|
24
25
|
|
25
|
-
spec.add_development_dependency "
|
26
|
-
spec.add_development_dependency "rake", "~> 10.0"
|
26
|
+
spec.add_development_dependency "rake"
|
27
27
|
spec.add_development_dependency "rspec"
|
28
28
|
spec.add_development_dependency "byebug"
|
29
29
|
spec.add_development_dependency "fakeredis"
|
data/lib/jobba/clause.rb
CHANGED
@@ -4,7 +4,9 @@ class Jobba::Clause
|
|
4
4
|
include Jobba::Common
|
5
5
|
|
6
6
|
# if `keys` or `suffixes` is an array, all entries will be included in the resulting set
|
7
|
-
def initialize(prefix: nil, suffixes: nil, keys: nil, min: nil, max: nil
|
7
|
+
def initialize(prefix: nil, suffixes: nil, keys: nil, min: nil, max: nil,
|
8
|
+
keys_contain_only_unique_ids: false)
|
9
|
+
|
8
10
|
if keys.nil? && prefix.nil? && suffixes.nil?
|
9
11
|
raise ArgumentError, "Either `keys` or both `prefix` and `suffix` must be specified.", caller
|
10
12
|
end
|
@@ -22,6 +24,8 @@ class Jobba::Clause
|
|
22
24
|
|
23
25
|
@min = min
|
24
26
|
@max = max
|
27
|
+
|
28
|
+
@keys_contain_only_unique_ids = keys_contain_only_unique_ids
|
25
29
|
end
|
26
30
|
|
27
31
|
def to_new_set
|
@@ -41,4 +45,84 @@ class Jobba::Clause
|
|
41
45
|
new_key
|
42
46
|
end
|
43
47
|
|
48
|
+
def result_ids(offset: nil, limit: nil)
|
49
|
+
# If we have one key and it is sorted, we can let redis return limited IDs,
|
50
|
+
# so handle that case specially.
|
51
|
+
|
52
|
+
id_data =
|
53
|
+
if @keys.one?
|
54
|
+
# offset and limit may or may not be used, so have to do again below
|
55
|
+
get_members(key: @keys.first, offset: offset, limit: limit)
|
56
|
+
else
|
57
|
+
ids = @keys.flat_map do |key|
|
58
|
+
# don't do limiting here -- doesn't make sense til we collect all the members
|
59
|
+
get_members(key: key)[:ids]
|
60
|
+
end
|
61
|
+
|
62
|
+
ids.sort!
|
63
|
+
ids.uniq! unless @keys_contain_only_unique_ids
|
64
|
+
|
65
|
+
{ids: ids, is_limited: false}
|
66
|
+
end
|
67
|
+
|
68
|
+
if !offset.nil? && !limit.nil? && id_data[:is_limited] == false
|
69
|
+
id_data[:ids].slice(offset, limit)
|
70
|
+
else
|
71
|
+
id_data[:ids]
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def get_members(key:, offset: nil, limit: nil)
|
76
|
+
if sorted_key?(key)
|
77
|
+
min = @min.nil? ? "-inf" : "#{@min}"
|
78
|
+
max = @max.nil? ? "+inf" : "#{@max}"
|
79
|
+
|
80
|
+
options = {}
|
81
|
+
is_limited = false
|
82
|
+
|
83
|
+
if !offset.nil? && !limit.nil?
|
84
|
+
options[:limit] = [offset, limit]
|
85
|
+
is_limited = true
|
86
|
+
end
|
87
|
+
|
88
|
+
ids = redis.zrangebyscore(key, min, max, options)
|
89
|
+
|
90
|
+
{ids: ids, is_limited: is_limited}
|
91
|
+
else
|
92
|
+
ids = redis.smembers(key)
|
93
|
+
ids.sort!
|
94
|
+
{ids: ids, is_limited: false}
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
def result_count(offset: nil, limit: nil)
|
99
|
+
if @keys.one? || @keys_contain_only_unique_ids
|
100
|
+
# can count each key on its own using fast redis ops and add them up
|
101
|
+
nonlimited_count = @keys.map do |key|
|
102
|
+
if sorted_key?(key)
|
103
|
+
if @min.nil? && @max.nil?
|
104
|
+
redis.zcard(key)
|
105
|
+
else
|
106
|
+
min = @min.nil? ? "-inf" : "#{@min}"
|
107
|
+
max = @max.nil? ? "+inf" : "#{@max}"
|
108
|
+
|
109
|
+
redis.zcount(key, min, max)
|
110
|
+
end
|
111
|
+
else
|
112
|
+
redis.scard(key)
|
113
|
+
end
|
114
|
+
end.reduce(:+)
|
115
|
+
|
116
|
+
Jobba::Utils.limited_count(nonlimited_count: nonlimited_count,
|
117
|
+
offset: offset, limit: limit)
|
118
|
+
else
|
119
|
+
# Because we need to get a count of uniq members, have to do a full query
|
120
|
+
result_ids(offset: offset, limit: limit).count
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
def sorted_key?(key)
|
125
|
+
key.match(/_at$/)
|
126
|
+
end
|
127
|
+
|
44
128
|
end
|
data/lib/jobba/clause_factory.rb
CHANGED
@@ -65,7 +65,10 @@ class Jobba::ClauseFactory
|
|
65
65
|
}.uniq
|
66
66
|
|
67
67
|
validate_state_name!(state)
|
68
|
-
|
68
|
+
|
69
|
+
# An ID is in only one state at a time, so we can tell `Clause` that
|
70
|
+
# info via `keys_contain_only_unique_ids` -- helps it be more efficient
|
71
|
+
Jobba::Clause.new(keys: state, keys_contain_only_unique_ids: true)
|
69
72
|
end
|
70
73
|
|
71
74
|
def self.validate_state_name!(state_name)
|
data/lib/jobba/id_clause.rb
CHANGED
@@ -11,4 +11,12 @@ class Jobba::IdClause
|
|
11
11
|
redis.zadd(new_key, @ids.collect{|id| [0, id]}) if @ids.any?
|
12
12
|
new_key
|
13
13
|
end
|
14
|
+
|
15
|
+
def result_ids(offset: nil, limit: nil)
|
16
|
+
@ids.map(&:to_s).slice(offset || 0, limit || @ids.count)
|
17
|
+
end
|
18
|
+
|
19
|
+
def result_count(offset: nil, limit: nil)
|
20
|
+
Jobba::Utils.limited_count(nonlimited_count: @ids.count, offset: offset, limit: limit)
|
21
|
+
end
|
14
22
|
end
|
data/lib/jobba/query.rb
CHANGED
@@ -6,6 +6,8 @@ class Jobba::Query
|
|
6
6
|
|
7
7
|
include Jobba::Common
|
8
8
|
|
9
|
+
attr_reader :_limit, :_offset
|
10
|
+
|
9
11
|
def where(options)
|
10
12
|
options.each do |kk,vv|
|
11
13
|
clauses.push(Jobba::ClauseFactory.new_clause(kk,vv))
|
@@ -14,8 +16,19 @@ class Jobba::Query
|
|
14
16
|
self
|
15
17
|
end
|
16
18
|
|
19
|
+
def limit(number)
|
20
|
+
@_limit = number
|
21
|
+
@_offset ||= 0
|
22
|
+
self
|
23
|
+
end
|
24
|
+
|
25
|
+
def offset(number)
|
26
|
+
@_offset = number
|
27
|
+
self
|
28
|
+
end
|
29
|
+
|
17
30
|
def count
|
18
|
-
_run(
|
31
|
+
_run(CountStatuses.new(self))
|
19
32
|
end
|
20
33
|
|
21
34
|
def empty?
|
@@ -39,7 +52,7 @@ class Jobba::Query
|
|
39
52
|
end
|
40
53
|
|
41
54
|
def run
|
42
|
-
_run(
|
55
|
+
_run(GetStatuses.new(self))
|
43
56
|
end
|
44
57
|
|
45
58
|
protected
|
@@ -50,73 +63,111 @@ class Jobba::Query
|
|
50
63
|
@clauses = []
|
51
64
|
end
|
52
65
|
|
53
|
-
class
|
54
|
-
attr_reader :
|
66
|
+
class Operations
|
67
|
+
attr_reader :query, :redis
|
68
|
+
|
69
|
+
def initialize(query)
|
70
|
+
@query = query
|
71
|
+
@redis = query.redis
|
72
|
+
end
|
55
73
|
|
56
|
-
|
57
|
-
|
58
|
-
|
74
|
+
# Standalone method that gives the final result when the query is one clause
|
75
|
+
def handle_single_clause(clause)
|
76
|
+
raise "AbstractMethod"
|
59
77
|
end
|
60
78
|
|
61
|
-
|
62
|
-
|
79
|
+
# When the query is multiple clauses, this method is called on the final set
|
80
|
+
# that represents the ANDing of all clauses. It is called inside a `redis.multi`
|
81
|
+
# block.
|
82
|
+
def multi_clause_last_redis_op(result_set)
|
83
|
+
raise "AbstractMethod"
|
84
|
+
end
|
85
|
+
|
86
|
+
# Called on the output from the redis multi block for multi-clause queries.
|
87
|
+
def multi_clause_postprocess(redis_output)
|
88
|
+
raise "AbstractMethod"
|
63
89
|
end
|
64
90
|
end
|
65
91
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
},
|
70
|
-
->(ids) {
|
92
|
+
class GetStatuses < Operations
|
93
|
+
def handle_single_clause(clause)
|
94
|
+
ids = clause.result_ids(limit: query._limit, offset: query._offset)
|
71
95
|
Jobba::Statuses.new(ids)
|
72
|
-
|
73
|
-
)
|
74
|
-
|
75
|
-
COUNT_STATUSES = RunBlocks.new(
|
76
|
-
->(working_set, redis) {
|
77
|
-
redis.zcard(working_set)
|
78
|
-
},
|
79
|
-
nil
|
80
|
-
)
|
81
|
-
|
82
|
-
def _run(run_blocks)
|
83
|
-
# Each clause in a query is converted to a sorted set (which may be filtered,
|
84
|
-
# e.g. in the case of timestamp clauses) and then the sets are successively
|
85
|
-
# intersected.
|
86
|
-
#
|
87
|
-
# Different users of this method have different uses for the final "working"
|
88
|
-
# set. Because we want to bundle all of the creations and intersections of
|
89
|
-
# clause sets into one call to Redis (via a `multi` block), we have users
|
90
|
-
# of this method provide a final block to run on the working set within
|
91
|
-
# Redis (and within the `multi` call) and then another block to run on
|
92
|
-
# the output of the first block.
|
93
|
-
|
94
|
-
multi_result = redis.multi do
|
95
|
-
load_default_clause if clauses.empty?
|
96
|
-
working_set = nil
|
97
|
-
|
98
|
-
clauses.each do |clause|
|
99
|
-
clause_set = clause.to_new_set
|
100
|
-
|
101
|
-
if working_set.nil?
|
102
|
-
working_set = clause_set
|
103
|
-
else
|
104
|
-
redis.zinterstore(working_set, [working_set, clause_set], weights: [0, 0])
|
105
|
-
redis.del(clause_set)
|
106
|
-
end
|
107
|
-
end
|
96
|
+
end
|
108
97
|
|
109
|
-
|
110
|
-
|
98
|
+
def multi_clause_last_redis_op(result_set)
|
99
|
+
start = query._offset || 0
|
100
|
+
stop = query._limit.nil? ? -1 : start + query._limit - 1
|
101
|
+
redis.zrange(result_set, start, stop)
|
102
|
+
end
|
111
103
|
|
112
|
-
|
104
|
+
def multi_clause_postprocess(ids)
|
105
|
+
Jobba::Statuses.new(ids)
|
113
106
|
end
|
107
|
+
end
|
114
108
|
|
115
|
-
|
109
|
+
class CountStatuses < Operations
|
110
|
+
def handle_single_clause(clause)
|
111
|
+
clause.result_count(limit: query._limit, offset: query._offset)
|
112
|
+
end
|
113
|
+
|
114
|
+
def multi_clause_last_redis_op(result_set)
|
115
|
+
redis.zcard(result_set)
|
116
|
+
end
|
117
|
+
|
118
|
+
def multi_clause_postprocess(redis_output)
|
119
|
+
Jobba::Utils.limited_count(nonlimited_count: redis_output, offset: query._offset, limit: query._limit)
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
def _run(operations)
|
124
|
+
if _limit.nil? && !_offset.nil?
|
125
|
+
raise ArgumentError, "`limit` must be set if `offset` is set", caller
|
126
|
+
end
|
116
127
|
|
117
|
-
|
118
|
-
|
119
|
-
|
128
|
+
load_default_clause if clauses.empty?
|
129
|
+
|
130
|
+
if clauses.one?
|
131
|
+
# We can make specialized calls that don't need intermediate copies of sets
|
132
|
+
# to be made (which are costly)
|
133
|
+
operations.handle_single_clause(clauses.first)
|
134
|
+
else
|
135
|
+
# Each clause in a query is converted to a sorted set (which may be filtered,
|
136
|
+
# e.g. in the case of timestamp clauses) and then the sets are successively
|
137
|
+
# intersected.
|
138
|
+
#
|
139
|
+
# Different users of this method have different uses for the final "working"
|
140
|
+
# set. Because we want to bundle all of the creations and intersections of
|
141
|
+
# clause sets into one call to Redis (via a `multi` block), we have users
|
142
|
+
# of this method provide a final block to run on the working set within
|
143
|
+
# Redis (and within the `multi` call) and then another block to run on
|
144
|
+
# the output of the first block.
|
145
|
+
#
|
146
|
+
# This code also works for the single clause case, but it is less efficient
|
147
|
+
|
148
|
+
multi_result = redis.multi do
|
149
|
+
|
150
|
+
working_set = nil
|
151
|
+
|
152
|
+
clauses.each do |clause|
|
153
|
+
clause_set = clause.to_new_set
|
154
|
+
|
155
|
+
if working_set.nil?
|
156
|
+
working_set = clause_set
|
157
|
+
else
|
158
|
+
redis.zinterstore(working_set, [working_set, clause_set], weights: [0, 0])
|
159
|
+
redis.del(clause_set)
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
# This is later accessed as `multi_result[-2]` since it is the second to last output
|
164
|
+
operations.multi_clause_last_redis_op(working_set)
|
165
|
+
|
166
|
+
redis.del(working_set)
|
167
|
+
end
|
168
|
+
|
169
|
+
operations.multi_clause_postprocess(multi_result[-2])
|
170
|
+
end
|
120
171
|
end
|
121
172
|
|
122
173
|
def load_default_clause
|
@@ -0,0 +1,68 @@
|
|
1
|
+
# This class provides Redis commands that automatically set key expiration.
|
2
|
+
# The only commands modified are commands that:
|
3
|
+
# 1. Take their (only) key as the first argument
|
4
|
+
# AND
|
5
|
+
# 2. Modify said key
|
6
|
+
# AND
|
7
|
+
# 3. Don't already set the key expiration by themselves
|
8
|
+
class Jobba::RedisWithExpiration < SimpleDelegator
|
9
|
+
# 68.years in seconds
|
10
|
+
EXPIRES_IN = 2145916800
|
11
|
+
|
12
|
+
EXPIRE_METHODS = [
|
13
|
+
:append,
|
14
|
+
:decr,
|
15
|
+
:decrby,
|
16
|
+
:hdel,
|
17
|
+
:hincrby,
|
18
|
+
:hincrbyfloat,
|
19
|
+
:hmset,
|
20
|
+
:hset,
|
21
|
+
:hsetnx,
|
22
|
+
:incr,
|
23
|
+
:incrby,
|
24
|
+
:incrbyfloat,
|
25
|
+
:linsert,
|
26
|
+
:lpop,
|
27
|
+
:lpush,
|
28
|
+
:lpushx,
|
29
|
+
:lrem,
|
30
|
+
:lset,
|
31
|
+
:ltrim,
|
32
|
+
:mapped_hmset,
|
33
|
+
:migrate,
|
34
|
+
:move,
|
35
|
+
:pfadd,
|
36
|
+
:pfmerge,
|
37
|
+
:restore,
|
38
|
+
:rpop,
|
39
|
+
:rpush,
|
40
|
+
:rpushx,
|
41
|
+
:sadd,
|
42
|
+
:set,
|
43
|
+
:setbit,
|
44
|
+
:setnx,
|
45
|
+
:setrange,
|
46
|
+
:sinterstore,
|
47
|
+
:smove,
|
48
|
+
:spop,
|
49
|
+
:srem,
|
50
|
+
:sunionstore,
|
51
|
+
:zadd,
|
52
|
+
:zincrby,
|
53
|
+
:zinterstore,
|
54
|
+
:zrem,
|
55
|
+
:zremrangebyrank,
|
56
|
+
:zremrangebyscore,
|
57
|
+
:zunionstore
|
58
|
+
]
|
59
|
+
|
60
|
+
EXPIRE_METHODS.each do |method|
|
61
|
+
define_method method do |key, *args|
|
62
|
+
result = super key, *args
|
63
|
+
# Only set expiration if the command (seems to have) succeeded
|
64
|
+
expire key, EXPIRES_IN if result
|
65
|
+
result
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
data/lib/jobba/state.rb
CHANGED
data/lib/jobba/status.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
|
-
require 'json'
|
2
1
|
require 'ostruct'
|
2
|
+
require 'oj'
|
3
3
|
|
4
4
|
module Jobba
|
5
5
|
class Status
|
@@ -163,7 +163,11 @@ module Jobba
|
|
163
163
|
end
|
164
164
|
|
165
165
|
def normalize_for_json(input)
|
166
|
-
|
166
|
+
Oj.load(convert_to_json(input))
|
167
|
+
end
|
168
|
+
|
169
|
+
def convert_to_json(value)
|
170
|
+
Oj.dump(value, mode: :custom, use_to_json: true)
|
167
171
|
end
|
168
172
|
|
169
173
|
def delete
|
@@ -219,7 +223,7 @@ module Jobba
|
|
219
223
|
def archive_attempt!
|
220
224
|
archived_job_key = job_key(attempt)
|
221
225
|
redis.rename(job_key, archived_job_key)
|
222
|
-
redis.hset(archived_job_key, :id, "#{id}:#{attempt}"
|
226
|
+
redis.hset(archived_job_key, :id, convert_to_json("#{id}:#{attempt}"))
|
223
227
|
delete_locally!
|
224
228
|
end
|
225
229
|
|
@@ -246,12 +250,12 @@ module Jobba
|
|
246
250
|
|
247
251
|
if attrs[:persist]
|
248
252
|
redis.multi do
|
249
|
-
set(
|
253
|
+
set(
|
250
254
|
id: id,
|
251
255
|
progress: progress,
|
252
256
|
errors: errors,
|
253
257
|
attempt: attempt
|
254
|
-
|
258
|
+
)
|
255
259
|
move_to_state!(state)
|
256
260
|
end
|
257
261
|
end
|
@@ -260,7 +264,7 @@ module Jobba
|
|
260
264
|
|
261
265
|
def load_from_json_encoded_attrs(attribute_name)
|
262
266
|
json = (@json_encoded_attrs || {})[attribute_name]
|
263
|
-
attribute = json.nil? ? nil :
|
267
|
+
attribute = json.nil? ? nil : Oj.load(json)
|
264
268
|
|
265
269
|
case attribute_name
|
266
270
|
when 'state'
|
@@ -298,8 +302,7 @@ module Jobba
|
|
298
302
|
key = kv_array[0]
|
299
303
|
value = kv_array[1]
|
300
304
|
value = Jobba::Utils.time_to_usec_int(value) if value.is_a?(::Time)
|
301
|
-
|
302
|
-
[key, value.to_json]
|
305
|
+
[key, convert_to_json(value)]
|
303
306
|
end
|
304
307
|
|
305
308
|
Jobba.redis.hmset(job_key, *redis_key_value_array)
|
data/lib/jobba/statuses.rb
CHANGED
data/lib/jobba/utils.rb
CHANGED
@@ -26,4 +26,45 @@ module Jobba::Utils
|
|
26
26
|
"temp:#{SecureRandom.hex(10)}"
|
27
27
|
end
|
28
28
|
|
29
|
+
def self.limited_count(nonlimited_count:, offset:, limit:)
|
30
|
+
raise(ArgumentError, "`limit` cannot be negative") if !limit.nil? && limit < 0
|
31
|
+
raise(ArgumentError, "`offset` cannot be negative") if !offset.nil? && offset < 0
|
32
|
+
|
33
|
+
# If we get a count of an array or set that doesn't take into account
|
34
|
+
# specified offsets and limits (what we call a `nonlimited_count`, but
|
35
|
+
# we need the count to effectively have been done with an offset and
|
36
|
+
# limit, this method calculates that limited count.
|
37
|
+
#
|
38
|
+
# This can happen when it is more efficient to calculate an unlimited
|
39
|
+
# count and then limit it after the fact.
|
40
|
+
#
|
41
|
+
# E.g.
|
42
|
+
#
|
43
|
+
# Get count of
|
44
|
+
# array = [a b c d e f g]
|
45
|
+
# where
|
46
|
+
# offset = 4
|
47
|
+
# limit = 5
|
48
|
+
#
|
49
|
+
# nonlimited_count = 7
|
50
|
+
#
|
51
|
+
# The limited array includes the highlighted (^) elements
|
52
|
+
# array = [a b c d e f g]
|
53
|
+
# ^ ^ ^ ^ ^
|
54
|
+
# Element `e` is the first element indicated by an offset of 4. The
|
55
|
+
# limit of 5 then causes us to take the rest of the elements in the array.
|
56
|
+
# The limit here is effectively 3 since there are only 3 elements left.
|
57
|
+
#
|
58
|
+
# So the limited_count is 3.
|
59
|
+
|
60
|
+
first_position_counted = offset || 0
|
61
|
+
|
62
|
+
# The `min` here is to make sure we don't go beyond the end of the array. The `- 1`
|
63
|
+
# is because we are getting a zero-indexed position from a count.
|
64
|
+
last_position_counted = [first_position_counted + (limit || nonlimited_count), nonlimited_count].min - 1
|
65
|
+
|
66
|
+
# Guard against first position being after last position by forcing min of 0
|
67
|
+
[last_position_counted - first_position_counted + 1, 0].max
|
68
|
+
end
|
69
|
+
|
29
70
|
end
|
data/lib/jobba/version.rb
CHANGED
data/lib/jobba.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
require 'delegate'
|
1
2
|
require 'forwardable'
|
2
3
|
require 'securerandom'
|
3
4
|
|
@@ -10,6 +11,7 @@ require "jobba/time"
|
|
10
11
|
require "jobba/utils"
|
11
12
|
require "jobba/configuration"
|
12
13
|
require "jobba/common"
|
14
|
+
require "jobba/redis_with_expiration"
|
13
15
|
require "jobba/state"
|
14
16
|
require "jobba/status"
|
15
17
|
require "jobba/statuses"
|
@@ -32,10 +34,33 @@ module Jobba
|
|
32
34
|
end
|
33
35
|
|
34
36
|
def self.redis
|
35
|
-
@redis ||=
|
36
|
-
|
37
|
-
|
37
|
+
@redis ||= Jobba::RedisWithExpiration.new(
|
38
|
+
Redis::Namespace.new(
|
39
|
+
configuration.namespace,
|
40
|
+
redis: Redis.new(configuration.redis_options || {})
|
41
|
+
)
|
38
42
|
)
|
39
43
|
end
|
40
44
|
|
45
|
+
def self.cleanup(seconds_ago: 60*60*24*30*12, batch_size: 1000)
|
46
|
+
start_time = Jobba::Time.now
|
47
|
+
delete_before = start_time - seconds_ago
|
48
|
+
|
49
|
+
jobs_count = 0
|
50
|
+
loop do
|
51
|
+
jobs = where(recorded_at: { before: delete_before }).limit(batch_size).to_a
|
52
|
+
jobs.each(&:delete!)
|
53
|
+
|
54
|
+
num_jobs = jobs.size
|
55
|
+
jobs_count += num_jobs
|
56
|
+
break if jobs.size < batch_size
|
57
|
+
end
|
58
|
+
jobs_count
|
59
|
+
end
|
60
|
+
|
61
|
+
# Clears the whole shebang! USE WITH CARE!
|
62
|
+
def self.clear_all_jobba_data!
|
63
|
+
cleanup(seconds_ago: 0)
|
64
|
+
end
|
65
|
+
|
41
66
|
end
|
metadata
CHANGED
@@ -1,31 +1,31 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: jobba
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.8.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- JP Slavinsky
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-10-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: redis
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '
|
19
|
+
version: '0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - "
|
24
|
+
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '
|
26
|
+
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
28
|
+
name: oj
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - ">="
|
@@ -39,33 +39,33 @@ dependencies:
|
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: redis-namespace
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- - "
|
45
|
+
- - ">="
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '
|
48
|
-
type: :
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- - "
|
52
|
+
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: '
|
54
|
+
version: '0'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: rake
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- - "
|
59
|
+
- - ">="
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: '
|
61
|
+
version: '0'
|
62
62
|
type: :development
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- - "
|
66
|
+
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version: '
|
68
|
+
version: '0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: rspec
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -115,11 +115,10 @@ executables: []
|
|
115
115
|
extensions: []
|
116
116
|
extra_rdoc_files: []
|
117
117
|
files:
|
118
|
+
- ".github/workflows/tests.yml"
|
118
119
|
- ".gitignore"
|
119
120
|
- ".rspec"
|
120
121
|
- ".ruby-gemset"
|
121
|
-
- ".ruby-version"
|
122
|
-
- ".travis.yml"
|
123
122
|
- Gemfile
|
124
123
|
- LICENSE.txt
|
125
124
|
- README.md
|
@@ -135,6 +134,7 @@ files:
|
|
135
134
|
- lib/jobba/exceptions.rb
|
136
135
|
- lib/jobba/id_clause.rb
|
137
136
|
- lib/jobba/query.rb
|
137
|
+
- lib/jobba/redis_with_expiration.rb
|
138
138
|
- lib/jobba/state.rb
|
139
139
|
- lib/jobba/status.rb
|
140
140
|
- lib/jobba/statuses.rb
|
@@ -145,7 +145,7 @@ homepage: https://github.com/openstax/jobba
|
|
145
145
|
licenses:
|
146
146
|
- MIT
|
147
147
|
metadata: {}
|
148
|
-
post_install_message:
|
148
|
+
post_install_message:
|
149
149
|
rdoc_options: []
|
150
150
|
require_paths:
|
151
151
|
- lib
|
@@ -160,9 +160,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
160
160
|
- !ruby/object:Gem::Version
|
161
161
|
version: '0'
|
162
162
|
requirements: []
|
163
|
-
|
164
|
-
|
165
|
-
signing_key:
|
163
|
+
rubygems_version: 3.1.4
|
164
|
+
signing_key:
|
166
165
|
specification_version: 4
|
167
166
|
summary: Redis-based background job status tracking.
|
168
167
|
test_files: []
|
data/.ruby-version
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
2.2.3
|
data/.travis.yml
DELETED