jobba 1.5.0 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.github/workflows/tests.yml +42 -0
- data/README.md +19 -0
- data/Rakefile +1 -1
- data/jobba.gemspec +3 -3
- data/lib/jobba/clause.rb +85 -1
- data/lib/jobba/clause_factory.rb +4 -1
- data/lib/jobba/id_clause.rb +8 -0
- data/lib/jobba/query.rb +108 -57
- data/lib/jobba/redis_with_expiration.rb +68 -0
- data/lib/jobba/state.rb +1 -6
- data/lib/jobba/status.rb +11 -8
- data/lib/jobba/statuses.rb +1 -1
- data/lib/jobba/utils.rb +41 -0
- data/lib/jobba/version.rb +1 -1
- data/lib/jobba.rb +28 -3
- metadata +23 -24
- data/.ruby-version +0 -1
- data/.travis.yml +0 -15
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 489a463b81e0d1dc9d7f8c79b1aacd25ab66358cea92a2692e4428e0f7b30a54
|
4
|
+
data.tar.gz: c8eb29ef8b368390b1f78eec40a04aaa06cc4dee3b45453355d9109eace62fa7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 16abf20ee8428d89b11c1e76315e957a357bf4f4381b4c072efdfd90365d5863a558f2a92f0b051fda32bde52f58d012b17000a5602050be77e9202a609f7cd3
|
7
|
+
data.tar.gz: 6b63cffc85fa57a36b454913a016b6b745e7975c3741868644735505315068086f01e5dd07cb1613304dafce896a4e537f2c169c55a4926ea0d833b4949efabd
|
@@ -0,0 +1,42 @@
|
|
1
|
+
name: Tests
|
2
|
+
|
3
|
+
env:
|
4
|
+
USE_REAL_REDIS: true
|
5
|
+
REDIS_HOST: redis
|
6
|
+
REDIS_PORT: 6379
|
7
|
+
on:
|
8
|
+
pull_request:
|
9
|
+
push:
|
10
|
+
branches:
|
11
|
+
- master
|
12
|
+
jobs:
|
13
|
+
tests:
|
14
|
+
timeout-minutes: 30
|
15
|
+
runs-on: ubuntu-18.04
|
16
|
+
strategy:
|
17
|
+
fail-fast: false
|
18
|
+
matrix:
|
19
|
+
os: [ubuntu]
|
20
|
+
ruby: [ '2.5', '2.6', '2.7' ]
|
21
|
+
services:
|
22
|
+
# Label used to access the service container
|
23
|
+
redis:
|
24
|
+
# Docker Hub image
|
25
|
+
image: redis
|
26
|
+
options: >-
|
27
|
+
--health-cmd "redis-cli ping"
|
28
|
+
--health-interval 10s
|
29
|
+
--health-timeout 5s
|
30
|
+
--health-retries 5
|
31
|
+
ports:
|
32
|
+
# Maps port 6379 on service container to the host
|
33
|
+
- 6379:6379
|
34
|
+
steps:
|
35
|
+
- uses: actions/checkout@v2
|
36
|
+
- uses: ruby/setup-ruby@v1
|
37
|
+
with:
|
38
|
+
ruby-version: ${{ matrix.ruby }}
|
39
|
+
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
|
40
|
+
- name: Test
|
41
|
+
run: |
|
42
|
+
bundle exec rake spec
|
data/README.md
CHANGED
@@ -439,6 +439,17 @@ Jobba.where(...).run.count # These pull data back to Ruby and count in Ruby
|
|
439
439
|
Jobba.where(...).run.empty?
|
440
440
|
```
|
441
441
|
|
442
|
+
## Pagination
|
443
|
+
|
444
|
+
Pagination is supported with an ActiveRecord-like interface. You can call `.limit(x)` and `.offset(y)` on
|
445
|
+
queries, e.g.
|
446
|
+
|
447
|
+
```ruby
|
448
|
+
Jobba.where(state: :succeeded).limit(10).offset(20).to_a
|
449
|
+
```
|
450
|
+
|
451
|
+
Specifying a limit does not guarantee that you'll get that many elements back, as there may not be that many left in the result.
|
452
|
+
|
442
453
|
## Notes
|
443
454
|
|
444
455
|
### Times
|
@@ -449,6 +460,8 @@ Note that, in operations having to do with time, this gem ignores anything beyon
|
|
449
460
|
|
450
461
|
Jobba strives to do all of its operations as efficiently as possible using built-in Redis operations. If you find a place where the efficiency can be improved, please submit an issue or a pull request.
|
451
462
|
|
463
|
+
Single-clause queries (those with one `where` call) have been optimized. `Jobba.all` is a single-clause query. If you have lots of IDs, try to get by with single-clause queries. Multi-clause queries (including `count`) have to copy sets into temporary working sets where query clauses are ANDed together. This can be expensive for large datasets.
|
464
|
+
|
452
465
|
### Write from one; Read from many
|
453
466
|
|
454
467
|
Jobba assumes that any job is being run at one time by only one worker. Jobba makes no accomodations for multiple processes updating a Status at the same time; multiple processes reading of a Status are fine of course.
|
@@ -463,6 +476,12 @@ $> USE_REAL_REDIS=true rspec
|
|
463
476
|
|
464
477
|
Travis runs the specs with both `fakeredis` and real Redis.
|
465
478
|
|
479
|
+
Clauses need to implement three methods:
|
480
|
+
|
481
|
+
1. `to_new_set` - puts the IDs indicated by the clause into a new sorted set in redis
|
482
|
+
2. `result_ids` - used to get the IDs indicated by the clause when the clause is the only one in the query
|
483
|
+
3. `result_count` - used to get the count of IDs indicated by the clause when the clause is the only one in the query
|
484
|
+
|
466
485
|
## TODO
|
467
486
|
|
468
487
|
1. Provide job min, max, and average durations.
|
data/Rakefile
CHANGED
data/jobba.gemspec
CHANGED
@@ -19,11 +19,11 @@ Gem::Specification.new do |spec|
|
|
19
19
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
20
20
|
spec.require_paths = ["lib"]
|
21
21
|
|
22
|
-
spec.add_runtime_dependency "redis"
|
22
|
+
spec.add_runtime_dependency "redis"
|
23
|
+
spec.add_runtime_dependency "oj"
|
23
24
|
spec.add_runtime_dependency "redis-namespace"
|
24
25
|
|
25
|
-
spec.add_development_dependency "
|
26
|
-
spec.add_development_dependency "rake", "~> 10.0"
|
26
|
+
spec.add_development_dependency "rake"
|
27
27
|
spec.add_development_dependency "rspec"
|
28
28
|
spec.add_development_dependency "byebug"
|
29
29
|
spec.add_development_dependency "fakeredis"
|
data/lib/jobba/clause.rb
CHANGED
@@ -4,7 +4,9 @@ class Jobba::Clause
|
|
4
4
|
include Jobba::Common
|
5
5
|
|
6
6
|
# if `keys` or `suffixes` is an array, all entries will be included in the resulting set
|
7
|
-
def initialize(prefix: nil, suffixes: nil, keys: nil, min: nil, max: nil
|
7
|
+
def initialize(prefix: nil, suffixes: nil, keys: nil, min: nil, max: nil,
|
8
|
+
keys_contain_only_unique_ids: false)
|
9
|
+
|
8
10
|
if keys.nil? && prefix.nil? && suffixes.nil?
|
9
11
|
raise ArgumentError, "Either `keys` or both `prefix` and `suffix` must be specified.", caller
|
10
12
|
end
|
@@ -22,6 +24,8 @@ class Jobba::Clause
|
|
22
24
|
|
23
25
|
@min = min
|
24
26
|
@max = max
|
27
|
+
|
28
|
+
@keys_contain_only_unique_ids = keys_contain_only_unique_ids
|
25
29
|
end
|
26
30
|
|
27
31
|
def to_new_set
|
@@ -41,4 +45,84 @@ class Jobba::Clause
|
|
41
45
|
new_key
|
42
46
|
end
|
43
47
|
|
48
|
+
def result_ids(offset: nil, limit: nil)
|
49
|
+
# If we have one key and it is sorted, we can let redis return limited IDs,
|
50
|
+
# so handle that case specially.
|
51
|
+
|
52
|
+
id_data =
|
53
|
+
if @keys.one?
|
54
|
+
# offset and limit may or may not be used, so have to do again below
|
55
|
+
get_members(key: @keys.first, offset: offset, limit: limit)
|
56
|
+
else
|
57
|
+
ids = @keys.flat_map do |key|
|
58
|
+
# don't do limiting here -- doesn't make sense til we collect all the members
|
59
|
+
get_members(key: key)[:ids]
|
60
|
+
end
|
61
|
+
|
62
|
+
ids.sort!
|
63
|
+
ids.uniq! unless @keys_contain_only_unique_ids
|
64
|
+
|
65
|
+
{ids: ids, is_limited: false}
|
66
|
+
end
|
67
|
+
|
68
|
+
if !offset.nil? && !limit.nil? && id_data[:is_limited] == false
|
69
|
+
id_data[:ids].slice(offset, limit)
|
70
|
+
else
|
71
|
+
id_data[:ids]
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def get_members(key:, offset: nil, limit: nil)
|
76
|
+
if sorted_key?(key)
|
77
|
+
min = @min.nil? ? "-inf" : "#{@min}"
|
78
|
+
max = @max.nil? ? "+inf" : "#{@max}"
|
79
|
+
|
80
|
+
options = {}
|
81
|
+
is_limited = false
|
82
|
+
|
83
|
+
if !offset.nil? && !limit.nil?
|
84
|
+
options[:limit] = [offset, limit]
|
85
|
+
is_limited = true
|
86
|
+
end
|
87
|
+
|
88
|
+
ids = redis.zrangebyscore(key, min, max, options)
|
89
|
+
|
90
|
+
{ids: ids, is_limited: is_limited}
|
91
|
+
else
|
92
|
+
ids = redis.smembers(key)
|
93
|
+
ids.sort!
|
94
|
+
{ids: ids, is_limited: false}
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
def result_count(offset: nil, limit: nil)
|
99
|
+
if @keys.one? || @keys_contain_only_unique_ids
|
100
|
+
# can count each key on its own using fast redis ops and add them up
|
101
|
+
nonlimited_count = @keys.map do |key|
|
102
|
+
if sorted_key?(key)
|
103
|
+
if @min.nil? && @max.nil?
|
104
|
+
redis.zcard(key)
|
105
|
+
else
|
106
|
+
min = @min.nil? ? "-inf" : "#{@min}"
|
107
|
+
max = @max.nil? ? "+inf" : "#{@max}"
|
108
|
+
|
109
|
+
redis.zcount(key, min, max)
|
110
|
+
end
|
111
|
+
else
|
112
|
+
redis.scard(key)
|
113
|
+
end
|
114
|
+
end.reduce(:+)
|
115
|
+
|
116
|
+
Jobba::Utils.limited_count(nonlimited_count: nonlimited_count,
|
117
|
+
offset: offset, limit: limit)
|
118
|
+
else
|
119
|
+
# Because we need to get a count of uniq members, have to do a full query
|
120
|
+
result_ids(offset: offset, limit: limit).count
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
def sorted_key?(key)
|
125
|
+
key.match(/_at$/)
|
126
|
+
end
|
127
|
+
|
44
128
|
end
|
data/lib/jobba/clause_factory.rb
CHANGED
@@ -65,7 +65,10 @@ class Jobba::ClauseFactory
|
|
65
65
|
}.uniq
|
66
66
|
|
67
67
|
validate_state_name!(state)
|
68
|
-
|
68
|
+
|
69
|
+
# An ID is in only one state at a time, so we can tell `Clause` that
|
70
|
+
# info via `keys_contain_only_unique_ids` -- helps it be more efficient
|
71
|
+
Jobba::Clause.new(keys: state, keys_contain_only_unique_ids: true)
|
69
72
|
end
|
70
73
|
|
71
74
|
def self.validate_state_name!(state_name)
|
data/lib/jobba/id_clause.rb
CHANGED
@@ -11,4 +11,12 @@ class Jobba::IdClause
|
|
11
11
|
redis.zadd(new_key, @ids.collect{|id| [0, id]}) if @ids.any?
|
12
12
|
new_key
|
13
13
|
end
|
14
|
+
|
15
|
+
def result_ids(offset: nil, limit: nil)
|
16
|
+
@ids.map(&:to_s).slice(offset || 0, limit || @ids.count)
|
17
|
+
end
|
18
|
+
|
19
|
+
def result_count(offset: nil, limit: nil)
|
20
|
+
Jobba::Utils.limited_count(nonlimited_count: @ids.count, offset: offset, limit: limit)
|
21
|
+
end
|
14
22
|
end
|
data/lib/jobba/query.rb
CHANGED
@@ -6,6 +6,8 @@ class Jobba::Query
|
|
6
6
|
|
7
7
|
include Jobba::Common
|
8
8
|
|
9
|
+
attr_reader :_limit, :_offset
|
10
|
+
|
9
11
|
def where(options)
|
10
12
|
options.each do |kk,vv|
|
11
13
|
clauses.push(Jobba::ClauseFactory.new_clause(kk,vv))
|
@@ -14,8 +16,19 @@ class Jobba::Query
|
|
14
16
|
self
|
15
17
|
end
|
16
18
|
|
19
|
+
def limit(number)
|
20
|
+
@_limit = number
|
21
|
+
@_offset ||= 0
|
22
|
+
self
|
23
|
+
end
|
24
|
+
|
25
|
+
def offset(number)
|
26
|
+
@_offset = number
|
27
|
+
self
|
28
|
+
end
|
29
|
+
|
17
30
|
def count
|
18
|
-
_run(
|
31
|
+
_run(CountStatuses.new(self))
|
19
32
|
end
|
20
33
|
|
21
34
|
def empty?
|
@@ -39,7 +52,7 @@ class Jobba::Query
|
|
39
52
|
end
|
40
53
|
|
41
54
|
def run
|
42
|
-
_run(
|
55
|
+
_run(GetStatuses.new(self))
|
43
56
|
end
|
44
57
|
|
45
58
|
protected
|
@@ -50,73 +63,111 @@ class Jobba::Query
|
|
50
63
|
@clauses = []
|
51
64
|
end
|
52
65
|
|
53
|
-
class
|
54
|
-
attr_reader :
|
66
|
+
class Operations
|
67
|
+
attr_reader :query, :redis
|
68
|
+
|
69
|
+
def initialize(query)
|
70
|
+
@query = query
|
71
|
+
@redis = query.redis
|
72
|
+
end
|
55
73
|
|
56
|
-
|
57
|
-
|
58
|
-
|
74
|
+
# Standalone method that gives the final result when the query is one clause
|
75
|
+
def handle_single_clause(clause)
|
76
|
+
raise "AbstractMethod"
|
59
77
|
end
|
60
78
|
|
61
|
-
|
62
|
-
|
79
|
+
# When the query is multiple clauses, this method is called on the final set
|
80
|
+
# that represents the ANDing of all clauses. It is called inside a `redis.multi`
|
81
|
+
# block.
|
82
|
+
def multi_clause_last_redis_op(result_set)
|
83
|
+
raise "AbstractMethod"
|
84
|
+
end
|
85
|
+
|
86
|
+
# Called on the output from the redis multi block for multi-clause queries.
|
87
|
+
def multi_clause_postprocess(redis_output)
|
88
|
+
raise "AbstractMethod"
|
63
89
|
end
|
64
90
|
end
|
65
91
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
},
|
70
|
-
->(ids) {
|
92
|
+
class GetStatuses < Operations
|
93
|
+
def handle_single_clause(clause)
|
94
|
+
ids = clause.result_ids(limit: query._limit, offset: query._offset)
|
71
95
|
Jobba::Statuses.new(ids)
|
72
|
-
|
73
|
-
)
|
74
|
-
|
75
|
-
COUNT_STATUSES = RunBlocks.new(
|
76
|
-
->(working_set, redis) {
|
77
|
-
redis.zcard(working_set)
|
78
|
-
},
|
79
|
-
nil
|
80
|
-
)
|
81
|
-
|
82
|
-
def _run(run_blocks)
|
83
|
-
# Each clause in a query is converted to a sorted set (which may be filtered,
|
84
|
-
# e.g. in the case of timestamp clauses) and then the sets are successively
|
85
|
-
# intersected.
|
86
|
-
#
|
87
|
-
# Different users of this method have different uses for the final "working"
|
88
|
-
# set. Because we want to bundle all of the creations and intersections of
|
89
|
-
# clause sets into one call to Redis (via a `multi` block), we have users
|
90
|
-
# of this method provide a final block to run on the working set within
|
91
|
-
# Redis (and within the `multi` call) and then another block to run on
|
92
|
-
# the output of the first block.
|
93
|
-
|
94
|
-
multi_result = redis.multi do
|
95
|
-
load_default_clause if clauses.empty?
|
96
|
-
working_set = nil
|
97
|
-
|
98
|
-
clauses.each do |clause|
|
99
|
-
clause_set = clause.to_new_set
|
100
|
-
|
101
|
-
if working_set.nil?
|
102
|
-
working_set = clause_set
|
103
|
-
else
|
104
|
-
redis.zinterstore(working_set, [working_set, clause_set], weights: [0, 0])
|
105
|
-
redis.del(clause_set)
|
106
|
-
end
|
107
|
-
end
|
96
|
+
end
|
108
97
|
|
109
|
-
|
110
|
-
|
98
|
+
def multi_clause_last_redis_op(result_set)
|
99
|
+
start = query._offset || 0
|
100
|
+
stop = query._limit.nil? ? -1 : start + query._limit - 1
|
101
|
+
redis.zrange(result_set, start, stop)
|
102
|
+
end
|
111
103
|
|
112
|
-
|
104
|
+
def multi_clause_postprocess(ids)
|
105
|
+
Jobba::Statuses.new(ids)
|
113
106
|
end
|
107
|
+
end
|
114
108
|
|
115
|
-
|
109
|
+
class CountStatuses < Operations
|
110
|
+
def handle_single_clause(clause)
|
111
|
+
clause.result_count(limit: query._limit, offset: query._offset)
|
112
|
+
end
|
113
|
+
|
114
|
+
def multi_clause_last_redis_op(result_set)
|
115
|
+
redis.zcard(result_set)
|
116
|
+
end
|
117
|
+
|
118
|
+
def multi_clause_postprocess(redis_output)
|
119
|
+
Jobba::Utils.limited_count(nonlimited_count: redis_output, offset: query._offset, limit: query._limit)
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
def _run(operations)
|
124
|
+
if _limit.nil? && !_offset.nil?
|
125
|
+
raise ArgumentError, "`limit` must be set if `offset` is set", caller
|
126
|
+
end
|
116
127
|
|
117
|
-
|
118
|
-
|
119
|
-
|
128
|
+
load_default_clause if clauses.empty?
|
129
|
+
|
130
|
+
if clauses.one?
|
131
|
+
# We can make specialized calls that don't need intermediate copies of sets
|
132
|
+
# to be made (which are costly)
|
133
|
+
operations.handle_single_clause(clauses.first)
|
134
|
+
else
|
135
|
+
# Each clause in a query is converted to a sorted set (which may be filtered,
|
136
|
+
# e.g. in the case of timestamp clauses) and then the sets are successively
|
137
|
+
# intersected.
|
138
|
+
#
|
139
|
+
# Different users of this method have different uses for the final "working"
|
140
|
+
# set. Because we want to bundle all of the creations and intersections of
|
141
|
+
# clause sets into one call to Redis (via a `multi` block), we have users
|
142
|
+
# of this method provide a final block to run on the working set within
|
143
|
+
# Redis (and within the `multi` call) and then another block to run on
|
144
|
+
# the output of the first block.
|
145
|
+
#
|
146
|
+
# This code also works for the single clause case, but it is less efficient
|
147
|
+
|
148
|
+
multi_result = redis.multi do
|
149
|
+
|
150
|
+
working_set = nil
|
151
|
+
|
152
|
+
clauses.each do |clause|
|
153
|
+
clause_set = clause.to_new_set
|
154
|
+
|
155
|
+
if working_set.nil?
|
156
|
+
working_set = clause_set
|
157
|
+
else
|
158
|
+
redis.zinterstore(working_set, [working_set, clause_set], weights: [0, 0])
|
159
|
+
redis.del(clause_set)
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
# This is later accessed as `multi_result[-2]` since it is the second to last output
|
164
|
+
operations.multi_clause_last_redis_op(working_set)
|
165
|
+
|
166
|
+
redis.del(working_set)
|
167
|
+
end
|
168
|
+
|
169
|
+
operations.multi_clause_postprocess(multi_result[-2])
|
170
|
+
end
|
120
171
|
end
|
121
172
|
|
122
173
|
def load_default_clause
|
@@ -0,0 +1,68 @@
|
|
1
|
+
# This class provides Redis commands that automatically set key expiration.
|
2
|
+
# The only commands modified are commands that:
|
3
|
+
# 1. Take their (only) key as the first argument
|
4
|
+
# AND
|
5
|
+
# 2. Modify said key
|
6
|
+
# AND
|
7
|
+
# 3. Don't already set the key expiration by themselves
|
8
|
+
class Jobba::RedisWithExpiration < SimpleDelegator
|
9
|
+
# 68.years in seconds
|
10
|
+
EXPIRES_IN = 2145916800
|
11
|
+
|
12
|
+
EXPIRE_METHODS = [
|
13
|
+
:append,
|
14
|
+
:decr,
|
15
|
+
:decrby,
|
16
|
+
:hdel,
|
17
|
+
:hincrby,
|
18
|
+
:hincrbyfloat,
|
19
|
+
:hmset,
|
20
|
+
:hset,
|
21
|
+
:hsetnx,
|
22
|
+
:incr,
|
23
|
+
:incrby,
|
24
|
+
:incrbyfloat,
|
25
|
+
:linsert,
|
26
|
+
:lpop,
|
27
|
+
:lpush,
|
28
|
+
:lpushx,
|
29
|
+
:lrem,
|
30
|
+
:lset,
|
31
|
+
:ltrim,
|
32
|
+
:mapped_hmset,
|
33
|
+
:migrate,
|
34
|
+
:move,
|
35
|
+
:pfadd,
|
36
|
+
:pfmerge,
|
37
|
+
:restore,
|
38
|
+
:rpop,
|
39
|
+
:rpush,
|
40
|
+
:rpushx,
|
41
|
+
:sadd,
|
42
|
+
:set,
|
43
|
+
:setbit,
|
44
|
+
:setnx,
|
45
|
+
:setrange,
|
46
|
+
:sinterstore,
|
47
|
+
:smove,
|
48
|
+
:spop,
|
49
|
+
:srem,
|
50
|
+
:sunionstore,
|
51
|
+
:zadd,
|
52
|
+
:zincrby,
|
53
|
+
:zinterstore,
|
54
|
+
:zrem,
|
55
|
+
:zremrangebyrank,
|
56
|
+
:zremrangebyscore,
|
57
|
+
:zunionstore
|
58
|
+
]
|
59
|
+
|
60
|
+
EXPIRE_METHODS.each do |method|
|
61
|
+
define_method method do |key, *args|
|
62
|
+
result = super key, *args
|
63
|
+
# Only set expiration if the command (seems to have) succeeded
|
64
|
+
expire key, EXPIRES_IN if result
|
65
|
+
result
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
data/lib/jobba/state.rb
CHANGED
data/lib/jobba/status.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
|
-
require 'json'
|
2
1
|
require 'ostruct'
|
2
|
+
require 'oj'
|
3
3
|
|
4
4
|
module Jobba
|
5
5
|
class Status
|
@@ -163,7 +163,11 @@ module Jobba
|
|
163
163
|
end
|
164
164
|
|
165
165
|
def normalize_for_json(input)
|
166
|
-
|
166
|
+
Oj.load(convert_to_json(input))
|
167
|
+
end
|
168
|
+
|
169
|
+
def convert_to_json(value)
|
170
|
+
Oj.dump(value, mode: :custom, use_to_json: true)
|
167
171
|
end
|
168
172
|
|
169
173
|
def delete
|
@@ -219,7 +223,7 @@ module Jobba
|
|
219
223
|
def archive_attempt!
|
220
224
|
archived_job_key = job_key(attempt)
|
221
225
|
redis.rename(job_key, archived_job_key)
|
222
|
-
redis.hset(archived_job_key, :id, "#{id}:#{attempt}"
|
226
|
+
redis.hset(archived_job_key, :id, convert_to_json("#{id}:#{attempt}"))
|
223
227
|
delete_locally!
|
224
228
|
end
|
225
229
|
|
@@ -246,12 +250,12 @@ module Jobba
|
|
246
250
|
|
247
251
|
if attrs[:persist]
|
248
252
|
redis.multi do
|
249
|
-
set(
|
253
|
+
set(
|
250
254
|
id: id,
|
251
255
|
progress: progress,
|
252
256
|
errors: errors,
|
253
257
|
attempt: attempt
|
254
|
-
|
258
|
+
)
|
255
259
|
move_to_state!(state)
|
256
260
|
end
|
257
261
|
end
|
@@ -260,7 +264,7 @@ module Jobba
|
|
260
264
|
|
261
265
|
def load_from_json_encoded_attrs(attribute_name)
|
262
266
|
json = (@json_encoded_attrs || {})[attribute_name]
|
263
|
-
attribute = json.nil? ? nil :
|
267
|
+
attribute = json.nil? ? nil : Oj.load(json)
|
264
268
|
|
265
269
|
case attribute_name
|
266
270
|
when 'state'
|
@@ -298,8 +302,7 @@ module Jobba
|
|
298
302
|
key = kv_array[0]
|
299
303
|
value = kv_array[1]
|
300
304
|
value = Jobba::Utils.time_to_usec_int(value) if value.is_a?(::Time)
|
301
|
-
|
302
|
-
[key, value.to_json]
|
305
|
+
[key, convert_to_json(value)]
|
303
306
|
end
|
304
307
|
|
305
308
|
Jobba.redis.hmset(job_key, *redis_key_value_array)
|
data/lib/jobba/statuses.rb
CHANGED
data/lib/jobba/utils.rb
CHANGED
@@ -26,4 +26,45 @@ module Jobba::Utils
|
|
26
26
|
"temp:#{SecureRandom.hex(10)}"
|
27
27
|
end
|
28
28
|
|
29
|
+
def self.limited_count(nonlimited_count:, offset:, limit:)
|
30
|
+
raise(ArgumentError, "`limit` cannot be negative") if !limit.nil? && limit < 0
|
31
|
+
raise(ArgumentError, "`offset` cannot be negative") if !offset.nil? && offset < 0
|
32
|
+
|
33
|
+
# If we get a count of an array or set that doesn't take into account
|
34
|
+
# specified offsets and limits (what we call a `nonlimited_count`, but
|
35
|
+
# we need the count to effectively have been done with an offset and
|
36
|
+
# limit, this method calculates that limited count.
|
37
|
+
#
|
38
|
+
# This can happen when it is more efficient to calculate an unlimited
|
39
|
+
# count and then limit it after the fact.
|
40
|
+
#
|
41
|
+
# E.g.
|
42
|
+
#
|
43
|
+
# Get count of
|
44
|
+
# array = [a b c d e f g]
|
45
|
+
# where
|
46
|
+
# offset = 4
|
47
|
+
# limit = 5
|
48
|
+
#
|
49
|
+
# nonlimited_count = 7
|
50
|
+
#
|
51
|
+
# The limited array includes the highlighted (^) elements
|
52
|
+
# array = [a b c d e f g]
|
53
|
+
# ^ ^ ^ ^ ^
|
54
|
+
# Element `e` is the first element indicated by an offset of 4. The
|
55
|
+
# limit of 5 then causes us to take the rest of the elements in the array.
|
56
|
+
# The limit here is effectively 3 since there are only 3 elements left.
|
57
|
+
#
|
58
|
+
# So the limited_count is 3.
|
59
|
+
|
60
|
+
first_position_counted = offset || 0
|
61
|
+
|
62
|
+
# The `min` here is to make sure we don't go beyond the end of the array. The `- 1`
|
63
|
+
# is because we are getting a zero-indexed position from a count.
|
64
|
+
last_position_counted = [first_position_counted + (limit || nonlimited_count), nonlimited_count].min - 1
|
65
|
+
|
66
|
+
# Guard against first position being after last position by forcing min of 0
|
67
|
+
[last_position_counted - first_position_counted + 1, 0].max
|
68
|
+
end
|
69
|
+
|
29
70
|
end
|
data/lib/jobba/version.rb
CHANGED
data/lib/jobba.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
require 'delegate'
|
1
2
|
require 'forwardable'
|
2
3
|
require 'securerandom'
|
3
4
|
|
@@ -10,6 +11,7 @@ require "jobba/time"
|
|
10
11
|
require "jobba/utils"
|
11
12
|
require "jobba/configuration"
|
12
13
|
require "jobba/common"
|
14
|
+
require "jobba/redis_with_expiration"
|
13
15
|
require "jobba/state"
|
14
16
|
require "jobba/status"
|
15
17
|
require "jobba/statuses"
|
@@ -32,10 +34,33 @@ module Jobba
|
|
32
34
|
end
|
33
35
|
|
34
36
|
def self.redis
|
35
|
-
@redis ||=
|
36
|
-
|
37
|
-
|
37
|
+
@redis ||= Jobba::RedisWithExpiration.new(
|
38
|
+
Redis::Namespace.new(
|
39
|
+
configuration.namespace,
|
40
|
+
redis: Redis.new(configuration.redis_options || {})
|
41
|
+
)
|
38
42
|
)
|
39
43
|
end
|
40
44
|
|
45
|
+
def self.cleanup(seconds_ago: 60*60*24*30*12, batch_size: 1000)
|
46
|
+
start_time = Jobba::Time.now
|
47
|
+
delete_before = start_time - seconds_ago
|
48
|
+
|
49
|
+
jobs_count = 0
|
50
|
+
loop do
|
51
|
+
jobs = where(recorded_at: { before: delete_before }).limit(batch_size).to_a
|
52
|
+
jobs.each(&:delete!)
|
53
|
+
|
54
|
+
num_jobs = jobs.size
|
55
|
+
jobs_count += num_jobs
|
56
|
+
break if jobs.size < batch_size
|
57
|
+
end
|
58
|
+
jobs_count
|
59
|
+
end
|
60
|
+
|
61
|
+
# Clears the whole shebang! USE WITH CARE!
|
62
|
+
def self.clear_all_jobba_data!
|
63
|
+
cleanup(seconds_ago: 0)
|
64
|
+
end
|
65
|
+
|
41
66
|
end
|
metadata
CHANGED
@@ -1,31 +1,31 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: jobba
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.8.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- JP Slavinsky
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-10-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: redis
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '
|
19
|
+
version: '0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - "
|
24
|
+
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '
|
26
|
+
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
28
|
+
name: oj
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - ">="
|
@@ -39,33 +39,33 @@ dependencies:
|
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: redis-namespace
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- - "
|
45
|
+
- - ">="
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '
|
48
|
-
type: :
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- - "
|
52
|
+
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: '
|
54
|
+
version: '0'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: rake
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- - "
|
59
|
+
- - ">="
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: '
|
61
|
+
version: '0'
|
62
62
|
type: :development
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- - "
|
66
|
+
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version: '
|
68
|
+
version: '0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: rspec
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -115,11 +115,10 @@ executables: []
|
|
115
115
|
extensions: []
|
116
116
|
extra_rdoc_files: []
|
117
117
|
files:
|
118
|
+
- ".github/workflows/tests.yml"
|
118
119
|
- ".gitignore"
|
119
120
|
- ".rspec"
|
120
121
|
- ".ruby-gemset"
|
121
|
-
- ".ruby-version"
|
122
|
-
- ".travis.yml"
|
123
122
|
- Gemfile
|
124
123
|
- LICENSE.txt
|
125
124
|
- README.md
|
@@ -135,6 +134,7 @@ files:
|
|
135
134
|
- lib/jobba/exceptions.rb
|
136
135
|
- lib/jobba/id_clause.rb
|
137
136
|
- lib/jobba/query.rb
|
137
|
+
- lib/jobba/redis_with_expiration.rb
|
138
138
|
- lib/jobba/state.rb
|
139
139
|
- lib/jobba/status.rb
|
140
140
|
- lib/jobba/statuses.rb
|
@@ -145,7 +145,7 @@ homepage: https://github.com/openstax/jobba
|
|
145
145
|
licenses:
|
146
146
|
- MIT
|
147
147
|
metadata: {}
|
148
|
-
post_install_message:
|
148
|
+
post_install_message:
|
149
149
|
rdoc_options: []
|
150
150
|
require_paths:
|
151
151
|
- lib
|
@@ -160,9 +160,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
160
160
|
- !ruby/object:Gem::Version
|
161
161
|
version: '0'
|
162
162
|
requirements: []
|
163
|
-
|
164
|
-
|
165
|
-
signing_key:
|
163
|
+
rubygems_version: 3.1.4
|
164
|
+
signing_key:
|
166
165
|
specification_version: 4
|
167
166
|
summary: Redis-based background job status tracking.
|
168
167
|
test_files: []
|
data/.ruby-version
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
2.2.3
|
data/.travis.yml
DELETED