riverqueue 0.6.1 → 0.8.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/client.rb +300 -0
- data/lib/driver.rb +46 -0
- data/lib/insert_opts.rb +146 -0
- data/lib/job.rb +193 -0
- data/lib/riverqueue.rb +1 -1
- data/lib/unique_bitmask.rb +41 -0
- metadata +8 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 46f45a18aa1b8884c93295ba1b023751c826d8feeac212949e18f388f3a6b6cc
|
4
|
+
data.tar.gz: 679dde3151af410e5e8f8949d80750d68f247e300af1e6f93b43270d81dc5d40
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a7507d5e78f99ef0bb6cda3a91479da386bb5e66c18c40a26c98ed5aec42bb3b5425b3753e797759e4ca4117f9a01636d9784fd1b27d160af215e5e5216ac998
|
7
|
+
data.tar.gz: a65b3a772b4d85b823256fe2bebb656a816cff1ba87777c77d9393670dbdfe3be383095bb49f3446d8110a143cb1ba68b9c7530d64da44a9e9093265be9aa8ba
|
data/lib/client.rb
ADDED
@@ -0,0 +1,300 @@
|
|
1
|
+
require "digest"
|
2
|
+
require "time"
|
3
|
+
|
4
|
+
module River
|
5
|
+
# Default number of maximum attempts for a job.
|
6
|
+
MAX_ATTEMPTS_DEFAULT = 25
|
7
|
+
|
8
|
+
# Default priority for a job.
|
9
|
+
PRIORITY_DEFAULT = 1
|
10
|
+
|
11
|
+
# Default queue for a job.
|
12
|
+
QUEUE_DEFAULT = "default"
|
13
|
+
|
14
|
+
# Provides a client for River that inserts jobs. Unlike the Go version of the
|
15
|
+
# River client, this one can insert jobs only. Jobs can only be worked from Go
|
16
|
+
# code, so job arg kinds and JSON encoding details must be shared between Ruby
|
17
|
+
# and Go code.
|
18
|
+
#
|
19
|
+
# Used in conjunction with a River driver like:
|
20
|
+
#
|
21
|
+
# DB = Sequel.connect(...)
|
22
|
+
# client = River::Client.new(River::Driver::Sequel.new(DB))
|
23
|
+
#
|
24
|
+
# River drivers are found in separate gems like `riverqueue-sequel` to help
|
25
|
+
# minimize transient dependencies.
|
26
|
+
class Client
|
27
|
+
def initialize(driver)
|
28
|
+
@driver = driver
|
29
|
+
@time_now_utc = -> { Time.now.utc } # for test time stubbing
|
30
|
+
end
|
31
|
+
|
32
|
+
# Inserts a new job for work given a job args implementation and insertion
|
33
|
+
# options (which may be omitted).
|
34
|
+
#
|
35
|
+
# With job args only:
|
36
|
+
#
|
37
|
+
# insert_res = client.insert(SimpleArgs.new(job_num: 1))
|
38
|
+
# insert_res.job # inserted job row
|
39
|
+
#
|
40
|
+
# With insert opts:
|
41
|
+
#
|
42
|
+
# insert_res = client.insert(SimpleArgs.new(job_num: 1), insert_opts: InsertOpts.new(queue: "high_priority"))
|
43
|
+
# insert_res.job # inserted job row
|
44
|
+
#
|
45
|
+
# Job arg implementations are expected to respond to:
|
46
|
+
#
|
47
|
+
# * `#kind`: A string that uniquely identifies the job in the database.
|
48
|
+
# * `#to_json`: Encodes the args to JSON for persistence in the database.
|
49
|
+
# Must match encoding an args struct on the Go side to be workable.
|
50
|
+
#
|
51
|
+
# They may also respond to `#insert_opts` which is expected to return an
|
52
|
+
# `InsertOpts` that contains options that will apply to all jobs of this
|
53
|
+
# kind. Insertion options provided as an argument to `#insert` override
|
54
|
+
# those returned by job args.
|
55
|
+
#
|
56
|
+
# For example:
|
57
|
+
#
|
58
|
+
# class SimpleArgs
|
59
|
+
# attr_accessor :job_num
|
60
|
+
#
|
61
|
+
# def initialize(job_num:)
|
62
|
+
# self.job_num = job_num
|
63
|
+
# end
|
64
|
+
#
|
65
|
+
# def kind = "simple"
|
66
|
+
#
|
67
|
+
# def to_json = JSON.dump({job_num: job_num})
|
68
|
+
# end
|
69
|
+
#
|
70
|
+
# See also JobArgsHash for an easy way to insert a job from a hash.
|
71
|
+
#
|
72
|
+
# Returns an instance of InsertResult.
|
73
|
+
def insert(args, insert_opts: EMPTY_INSERT_OPTS)
|
74
|
+
insert_params = make_insert_params(args, insert_opts)
|
75
|
+
insert_and_check_unique_job(insert_params)
|
76
|
+
end
|
77
|
+
|
78
|
+
# Inserts many new jobs as part of a single batch operation for improved
|
79
|
+
# efficiency.
|
80
|
+
#
|
81
|
+
# Takes an array of job args or InsertManyParams which encapsulate job args
|
82
|
+
# and a paired InsertOpts.
|
83
|
+
#
|
84
|
+
# With job args:
|
85
|
+
#
|
86
|
+
# num_inserted = client.insert_many([
|
87
|
+
# SimpleArgs.new(job_num: 1),
|
88
|
+
# SimpleArgs.new(job_num: 2)
|
89
|
+
# ])
|
90
|
+
#
|
91
|
+
# With InsertManyParams:
|
92
|
+
#
|
93
|
+
# num_inserted = client.insert_many([
|
94
|
+
# River::InsertManyParams.new(SimpleArgs.new(job_num: 1), insert_opts: InsertOpts.new(max_attempts: 5)),
|
95
|
+
# River::InsertManyParams.new(SimpleArgs.new(job_num: 2), insert_opts: InsertOpts.new(queue: "high_priority"))
|
96
|
+
# ])
|
97
|
+
#
|
98
|
+
# Job arg implementations are expected to respond to:
|
99
|
+
#
|
100
|
+
# * `#kind`: A string that uniquely identifies the job in the database.
|
101
|
+
# * `#to_json`: Encodes the args to JSON for persistence in the database.
|
102
|
+
# Must match encoding an args struct on the Go side to be workable.
|
103
|
+
#
|
104
|
+
# For example:
|
105
|
+
#
|
106
|
+
# class SimpleArgs
|
107
|
+
# attr_accessor :job_num
|
108
|
+
#
|
109
|
+
# def initialize(job_num:)
|
110
|
+
# self.job_num = job_num
|
111
|
+
# end
|
112
|
+
#
|
113
|
+
# def kind = "simple"
|
114
|
+
#
|
115
|
+
# def to_json = JSON.dump({job_num: job_num})
|
116
|
+
# end
|
117
|
+
#
|
118
|
+
# See also JobArgsHash for an easy way to insert a job from a hash.
|
119
|
+
#
|
120
|
+
# Returns the number of jobs inserted.
|
121
|
+
def insert_many(args)
|
122
|
+
all_params = args.map do |arg|
|
123
|
+
if arg.is_a?(InsertManyParams)
|
124
|
+
make_insert_params(arg.args, arg.insert_opts || EMPTY_INSERT_OPTS)
|
125
|
+
else # jobArgs
|
126
|
+
make_insert_params(arg, EMPTY_INSERT_OPTS)
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
@driver.job_insert_many(all_params)
|
131
|
+
.map do |job, unique_skipped_as_duplicate|
|
132
|
+
InsertResult.new(job, unique_skipped_as_duplicated: unique_skipped_as_duplicate)
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
# Default states that are used during a unique insert. Can be overridden by
|
137
|
+
# setting UniqueOpts#by_state.
|
138
|
+
DEFAULT_UNIQUE_STATES = [
|
139
|
+
JOB_STATE_AVAILABLE,
|
140
|
+
JOB_STATE_COMPLETED,
|
141
|
+
JOB_STATE_PENDING,
|
142
|
+
JOB_STATE_RETRYABLE,
|
143
|
+
JOB_STATE_RUNNING,
|
144
|
+
JOB_STATE_SCHEDULED
|
145
|
+
].freeze
|
146
|
+
private_constant :DEFAULT_UNIQUE_STATES
|
147
|
+
|
148
|
+
REQUIRED_UNIQUE_STATES = [
|
149
|
+
JOB_STATE_AVAILABLE,
|
150
|
+
JOB_STATE_PENDING,
|
151
|
+
JOB_STATE_RUNNING,
|
152
|
+
JOB_STATE_SCHEDULED
|
153
|
+
].freeze
|
154
|
+
private_constant :REQUIRED_UNIQUE_STATES
|
155
|
+
|
156
|
+
EMPTY_INSERT_OPTS = InsertOpts.new.freeze
|
157
|
+
private_constant :EMPTY_INSERT_OPTS
|
158
|
+
|
159
|
+
private def insert_and_check_unique_job(insert_params)
|
160
|
+
job, unique_skipped_as_duplicate = @driver.job_insert(insert_params)
|
161
|
+
InsertResult.new(job, unique_skipped_as_duplicated: unique_skipped_as_duplicate)
|
162
|
+
end
|
163
|
+
|
164
|
+
private def make_insert_params(args, insert_opts)
|
165
|
+
raise "args should respond to `#kind`" if !args.respond_to?(:kind)
|
166
|
+
|
167
|
+
# ~all objects in Ruby respond to `#to_json`, so check non-nil instead.
|
168
|
+
args_json = args.to_json
|
169
|
+
raise "args should return non-nil from `#to_json`" if !args_json
|
170
|
+
|
171
|
+
args_insert_opts = if args.respond_to?(:insert_opts)
|
172
|
+
args_with_insert_opts = args #: _JobArgsWithInsertOpts # rubocop:disable Layout/LeadingCommentSpace
|
173
|
+
args_with_insert_opts.insert_opts || EMPTY_INSERT_OPTS
|
174
|
+
else
|
175
|
+
EMPTY_INSERT_OPTS
|
176
|
+
end
|
177
|
+
|
178
|
+
scheduled_at = insert_opts.scheduled_at || args_insert_opts.scheduled_at
|
179
|
+
|
180
|
+
insert_params = Driver::JobInsertParams.new(
|
181
|
+
encoded_args: args_json,
|
182
|
+
kind: args.kind,
|
183
|
+
max_attempts: insert_opts.max_attempts || args_insert_opts.max_attempts || MAX_ATTEMPTS_DEFAULT,
|
184
|
+
priority: insert_opts.priority || args_insert_opts.priority || PRIORITY_DEFAULT,
|
185
|
+
queue: insert_opts.queue || args_insert_opts.queue || QUEUE_DEFAULT,
|
186
|
+
scheduled_at: scheduled_at&.utc || Time.now,
|
187
|
+
state: scheduled_at ? JOB_STATE_SCHEDULED : JOB_STATE_AVAILABLE,
|
188
|
+
tags: validate_tags(insert_opts.tags || args_insert_opts.tags || [])
|
189
|
+
)
|
190
|
+
|
191
|
+
unique_opts = insert_opts.unique_opts || args_insert_opts.unique_opts
|
192
|
+
if unique_opts
|
193
|
+
unique_key, unique_states = make_unique_key_and_bitmask(insert_params, unique_opts)
|
194
|
+
insert_params.unique_key = unique_key
|
195
|
+
insert_params.unique_states = unique_states
|
196
|
+
end
|
197
|
+
insert_params
|
198
|
+
end
|
199
|
+
|
200
|
+
private def make_unique_key_and_bitmask(insert_params, unique_opts)
|
201
|
+
unique_key = ""
|
202
|
+
|
203
|
+
# It's extremely important here that this unique key format and algorithm
|
204
|
+
# match the one in the main River library _exactly_. Don't change them
|
205
|
+
# unless they're updated everywhere.
|
206
|
+
unless unique_opts.exclude_kind
|
207
|
+
unique_key += "&kind=#{insert_params.kind}"
|
208
|
+
end
|
209
|
+
|
210
|
+
if unique_opts.by_args
|
211
|
+
parsed_args = JSON.parse(insert_params.encoded_args)
|
212
|
+
filtered_args = if unique_opts.by_args.is_a?(Array)
|
213
|
+
parsed_args.select { |k, _| unique_opts.by_args.include?(k) }
|
214
|
+
else
|
215
|
+
parsed_args
|
216
|
+
end
|
217
|
+
|
218
|
+
encoded_args = JSON.generate(filtered_args.sort.to_h)
|
219
|
+
unique_key += "&args=#{encoded_args}"
|
220
|
+
end
|
221
|
+
|
222
|
+
if unique_opts.by_period
|
223
|
+
lower_period_bound = truncate_time(@time_now_utc.call, unique_opts.by_period).utc
|
224
|
+
|
225
|
+
unique_key += "&period=#{lower_period_bound.strftime("%FT%TZ")}"
|
226
|
+
end
|
227
|
+
|
228
|
+
if unique_opts.by_queue
|
229
|
+
unique_key += "&queue=#{insert_params.queue}"
|
230
|
+
end
|
231
|
+
|
232
|
+
unique_key_hash = Digest::SHA256.digest(unique_key)
|
233
|
+
unique_states = validate_unique_states(unique_opts.by_state || DEFAULT_UNIQUE_STATES)
|
234
|
+
|
235
|
+
[unique_key_hash, UniqueBitmask.from_states(unique_states)]
|
236
|
+
end
|
237
|
+
|
238
|
+
# Truncates the given time down to the interval. For example:
|
239
|
+
#
|
240
|
+
# Thu Jan 15 21:26:36 UTC 2024 @ 15 minutes ->
|
241
|
+
# Thu Jan 15 21:15:00 UTC 2024
|
242
|
+
private def truncate_time(time, interval_seconds)
|
243
|
+
Time.at((time.to_f / interval_seconds).floor * interval_seconds)
|
244
|
+
end
|
245
|
+
|
246
|
+
# Moves an integer that may occupy the entire uint64 space to one that's
|
247
|
+
# bounded within int64. Allows overflow.
|
248
|
+
private def uint64_to_int64(int)
|
249
|
+
[int].pack("Q").unpack1("q") #: Integer # rubocop:disable Layout/LeadingCommentSpace
|
250
|
+
end
|
251
|
+
|
252
|
+
TAG_RE = /\A[\w][\w\-]+[\w]\z/
|
253
|
+
private_constant :TAG_RE
|
254
|
+
|
255
|
+
private def validate_tags(tags)
|
256
|
+
tags.each do |tag|
|
257
|
+
raise ArgumentError, "tags should be 255 characters or less" if tag.length > 255
|
258
|
+
raise ArgumentError, "tag should match regex #{TAG_RE.inspect}" unless TAG_RE.match(tag)
|
259
|
+
end
|
260
|
+
end
|
261
|
+
|
262
|
+
private def validate_unique_states(states)
|
263
|
+
REQUIRED_UNIQUE_STATES.each do |required_state|
|
264
|
+
raise ArgumentError, "by_state should include required state #{required_state}" unless states.include?(required_state)
|
265
|
+
end
|
266
|
+
states
|
267
|
+
end
|
268
|
+
end
|
269
|
+
|
270
|
+
# A single job to insert that's part of an #insert_many batch insert. Unlike
|
271
|
+
# sending raw job args, supports an InsertOpts to pair with the job.
|
272
|
+
class InsertManyParams
|
273
|
+
# Job args to insert.
|
274
|
+
attr_reader :args
|
275
|
+
|
276
|
+
# Insertion options to use with the insert.
|
277
|
+
attr_reader :insert_opts
|
278
|
+
|
279
|
+
def initialize(args, insert_opts: nil)
|
280
|
+
@args = args
|
281
|
+
@insert_opts = insert_opts
|
282
|
+
end
|
283
|
+
end
|
284
|
+
|
285
|
+
# Result of a single insertion.
|
286
|
+
class InsertResult
|
287
|
+
# Inserted job row, or an existing job row if insert was skipped due to a
|
288
|
+
# previously existing unique job.
|
289
|
+
attr_reader :job
|
290
|
+
|
291
|
+
# True if for a unique job, the insertion was skipped due to an equivalent
|
292
|
+
# job matching unique property already being present.
|
293
|
+
attr_reader :unique_skipped_as_duplicated
|
294
|
+
|
295
|
+
def initialize(job, unique_skipped_as_duplicated:)
|
296
|
+
@job = job
|
297
|
+
@unique_skipped_as_duplicated = unique_skipped_as_duplicated
|
298
|
+
end
|
299
|
+
end
|
300
|
+
end
|
data/lib/driver.rb
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
module River
|
2
|
+
# Contains an interface used by the top-level River module to interface with
|
3
|
+
# its driver implementations. All types and methods in this module should be
|
4
|
+
# considered to be for internal use only and subject to change. API stability
|
5
|
+
# is not guaranteed.
|
6
|
+
module Driver
|
7
|
+
# Insert parameters for a job. This is sent to underlying drivers and is meant
|
8
|
+
# for internal use only. Its interface is subject to change.
|
9
|
+
class JobInsertParams
|
10
|
+
attr_accessor :encoded_args
|
11
|
+
attr_accessor :kind
|
12
|
+
attr_accessor :max_attempts
|
13
|
+
attr_accessor :priority
|
14
|
+
attr_accessor :queue
|
15
|
+
attr_accessor :scheduled_at
|
16
|
+
attr_accessor :state
|
17
|
+
attr_accessor :tags
|
18
|
+
attr_accessor :unique_key
|
19
|
+
attr_accessor :unique_states
|
20
|
+
|
21
|
+
def initialize(
|
22
|
+
encoded_args:,
|
23
|
+
kind:,
|
24
|
+
max_attempts:,
|
25
|
+
priority:,
|
26
|
+
queue:,
|
27
|
+
scheduled_at:,
|
28
|
+
state:,
|
29
|
+
tags:,
|
30
|
+
unique_key: nil,
|
31
|
+
unique_states: nil
|
32
|
+
)
|
33
|
+
self.encoded_args = encoded_args
|
34
|
+
self.kind = kind
|
35
|
+
self.max_attempts = max_attempts
|
36
|
+
self.priority = priority
|
37
|
+
self.queue = queue
|
38
|
+
self.scheduled_at = scheduled_at
|
39
|
+
self.state = state
|
40
|
+
self.tags = tags
|
41
|
+
self.unique_key = unique_key
|
42
|
+
self.unique_states = unique_states
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
data/lib/insert_opts.rb
ADDED
@@ -0,0 +1,146 @@
|
|
1
|
+
module River
|
2
|
+
# Options for job insertion, and which can be provided by implementing
|
3
|
+
# #insert_opts on job args, or specified as a parameter on #insert or
|
4
|
+
# #insert_many.
|
5
|
+
class InsertOpts
|
6
|
+
# The maximum number of total attempts (including both the original run and
|
7
|
+
# all retries) before a job is abandoned and set as discarded.
|
8
|
+
attr_accessor :max_attempts
|
9
|
+
|
10
|
+
# The priority of the job, with 1 being the highest priority and 4 being the
|
11
|
+
# lowest. When fetching available jobs to work, the highest priority jobs
|
12
|
+
# will always be fetched before any lower priority jobs are fetched. Note
|
13
|
+
# that if your workers are swamped with more high-priority jobs then they
|
14
|
+
# can handle, lower priority jobs may not be fetched.
|
15
|
+
#
|
16
|
+
# Defaults to PRIORITY_DEFAULT.
|
17
|
+
attr_accessor :priority
|
18
|
+
|
19
|
+
# The name of the job queue in which to insert the job.
|
20
|
+
#
|
21
|
+
# Defaults to QUEUE_DEFAULT.
|
22
|
+
attr_accessor :queue
|
23
|
+
|
24
|
+
# A time in future at which to schedule the job (i.e. in cases where it
|
25
|
+
# shouldn't be run immediately). The job is guaranteed not to run before
|
26
|
+
# this time, but may run slightly after depending on the number of other
|
27
|
+
# scheduled jobs and how busy the queue is.
|
28
|
+
#
|
29
|
+
# Use of this option generally only makes sense when passing options into
|
30
|
+
# Insert rather than when a job args is returning `#insert_opts`, however,
|
31
|
+
# it will work in both cases.
|
32
|
+
attr_accessor :scheduled_at
|
33
|
+
|
34
|
+
# An arbitrary list of keywords to add to the job. They have no functional
|
35
|
+
# behavior and are meant entirely as a user-specified construct to help
|
36
|
+
# group and categorize jobs.
|
37
|
+
#
|
38
|
+
# If tags are specified from both a job args override and from options on
|
39
|
+
# Insert, the latter takes precedence. Tags are not merged.
|
40
|
+
attr_accessor :tags
|
41
|
+
|
42
|
+
# Options relating to job uniqueness. No unique options means that the job
|
43
|
+
# is never treated as unique.
|
44
|
+
attr_accessor :unique_opts
|
45
|
+
|
46
|
+
def initialize(
|
47
|
+
max_attempts: nil,
|
48
|
+
priority: nil,
|
49
|
+
queue: nil,
|
50
|
+
scheduled_at: nil,
|
51
|
+
tags: nil,
|
52
|
+
unique_opts: nil
|
53
|
+
)
|
54
|
+
self.max_attempts = max_attempts
|
55
|
+
self.priority = priority
|
56
|
+
self.queue = queue
|
57
|
+
self.scheduled_at = scheduled_at
|
58
|
+
self.tags = tags
|
59
|
+
self.unique_opts = unique_opts
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
# Parameters for uniqueness for a job.
|
64
|
+
#
|
65
|
+
# If all properties are nil, no uniqueness at is enforced. As each property is
|
66
|
+
# initialized, it's added as a dimension on the uniqueness matrix, and with
|
67
|
+
# any property on, the job's kind always counts toward uniqueness.
|
68
|
+
#
|
69
|
+
# So for example, if only #by_queue is on, then for the given job kind, only a
|
70
|
+
# single instance is allowed in any given queue, regardless of other
|
71
|
+
# properties on the job. If both #by_args and #by_queue are on, then for the
|
72
|
+
# given job kind, a single instance is allowed for each combination of args
|
73
|
+
# and queues. If either args or queue is changed on a new job, it's allowed to
|
74
|
+
# be inserted as a new job.
|
75
|
+
#
|
76
|
+
# Uniquenes is checked at insert time by taking a Postgres advisory lock,
|
77
|
+
# doing a look up for an equivalent row, and inserting only if none was found.
|
78
|
+
# There's no database-level mechanism that guarantees jobs stay unique, so if
|
79
|
+
# an equivalent row is inserted out of band (or batch inserted, where a unique
|
80
|
+
# check doesn't occur), it's conceivable that duplicates could coexist.
|
81
|
+
class UniqueOpts
|
82
|
+
# Indicates that uniqueness should be enforced for any specific instance of
|
83
|
+
# encoded args for a job.
|
84
|
+
#
|
85
|
+
# Default is false, meaning that as long as any other unique property is
|
86
|
+
# enabled, uniqueness will be enforced for a kind regardless of input args.
|
87
|
+
attr_accessor :by_args
|
88
|
+
|
89
|
+
# Defines uniqueness within a given period. On an insert time is rounded
|
90
|
+
# down to the nearest multiple of the given period, and a job is only
|
91
|
+
# inserted if there isn't an existing job that will run between then and the
|
92
|
+
# next multiple of the period.
|
93
|
+
#
|
94
|
+
# The period should be specified in seconds. So a job that's unique every 15
|
95
|
+
# minute period would have a value of 900.
|
96
|
+
#
|
97
|
+
# Default is no unique period, meaning that as long as any other unique
|
98
|
+
# property is enabled, uniqueness will be enforced across all jobs of the
|
99
|
+
# kind in the database, regardless of when they were scheduled.
|
100
|
+
attr_accessor :by_period
|
101
|
+
|
102
|
+
# Indicates that uniqueness should be enforced within each queue.
|
103
|
+
#
|
104
|
+
# Default is false, meaning that as long as any other unique property is
|
105
|
+
# enabled, uniqueness will be enforced for a kind across all queues.
|
106
|
+
attr_accessor :by_queue
|
107
|
+
|
108
|
+
# Indicates that uniqueness should be enforced across any of the states in
|
109
|
+
# the given set. For example, if the given states were `(scheduled,
|
110
|
+
# running)` then a new job could be inserted even if one of the same kind
|
111
|
+
# was already being worked by the queue (new jobs are inserted as
|
112
|
+
# `available`).
|
113
|
+
#
|
114
|
+
# Unlike other unique options, ByState gets a default when it's not set for
|
115
|
+
# user convenience. The default is equivalent to:
|
116
|
+
#
|
117
|
+
# by_state: [River::JOB_STATE_AVAILABLE, River::JOB_STATE_COMPLETED, River::JOB_STATE_PENDING, River::JOB_STATE_RUNNING, River::JOB_STATE_RETRYABLE, River::JOB_STATE_SCHEDULED]
|
118
|
+
#
|
119
|
+
# With this setting, any jobs of the same kind that have been completed or
|
120
|
+
# discarded, but not yet cleaned out by the system, won't count towards the
|
121
|
+
# uniqueness of a new insert.
|
122
|
+
#
|
123
|
+
# The pending, scheduled, available, and running states are required when
|
124
|
+
# customizing this list.
|
125
|
+
attr_accessor :by_state
|
126
|
+
|
127
|
+
# Indicates that the job kind should not be considered for uniqueness. This
|
128
|
+
# is useful when you want to enforce uniqueness based on other properties
|
129
|
+
# across multiple worker types.
|
130
|
+
attr_accessor :exclude_kind
|
131
|
+
|
132
|
+
def initialize(
|
133
|
+
by_args: nil,
|
134
|
+
by_period: nil,
|
135
|
+
by_queue: nil,
|
136
|
+
by_state: nil,
|
137
|
+
exclude_kind: nil
|
138
|
+
)
|
139
|
+
self.by_args = by_args
|
140
|
+
self.by_period = by_period
|
141
|
+
self.by_queue = by_queue
|
142
|
+
self.by_state = by_state
|
143
|
+
self.exclude_kind = exclude_kind
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
data/lib/job.rb
ADDED
@@ -0,0 +1,193 @@
|
|
1
|
+
module River
|
2
|
+
JOB_STATE_AVAILABLE = "available"
|
3
|
+
JOB_STATE_CANCELLED = "cancelled"
|
4
|
+
JOB_STATE_COMPLETED = "completed"
|
5
|
+
JOB_STATE_DISCARDED = "discarded"
|
6
|
+
JOB_STATE_PENDING = "pending"
|
7
|
+
JOB_STATE_RETRYABLE = "retryable"
|
8
|
+
JOB_STATE_RUNNING = "running"
|
9
|
+
JOB_STATE_SCHEDULED = "scheduled"
|
10
|
+
|
11
|
+
# Provides a way of creating a job args from a simple Ruby hash for a quick
|
12
|
+
# way to insert a job without having to define a class. The first argument is
|
13
|
+
# a "kind" string for identifying the job in the database and the second is a
|
14
|
+
# hash that will be encoded to JSON.
|
15
|
+
#
|
16
|
+
# For example:
|
17
|
+
#
|
18
|
+
# insert_res = client.insert(River::JobArgsHash.new("job_kind", {
|
19
|
+
# job_num: 1
|
20
|
+
# }))
|
21
|
+
class JobArgsHash
|
22
|
+
def initialize(kind, hash)
|
23
|
+
raise "kind should be non-nil" if !kind
|
24
|
+
raise "hash should be non-nil" if !hash
|
25
|
+
|
26
|
+
@kind = kind
|
27
|
+
@hash = hash
|
28
|
+
end
|
29
|
+
|
30
|
+
attr_reader :kind
|
31
|
+
|
32
|
+
def to_json
|
33
|
+
JSON.dump(@hash)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
# JobRow contains the properties of a job that are persisted to the database.
|
38
|
+
class JobRow
|
39
|
+
# ID of the job. Generated as part of a Postgres sequence and generally
|
40
|
+
# ascending in nature, but there may be gaps in it as transactions roll
|
41
|
+
# back.
|
42
|
+
attr_accessor :id
|
43
|
+
|
44
|
+
# The job's args as a hash decoded from JSON.
|
45
|
+
attr_accessor :args
|
46
|
+
|
47
|
+
# The attempt number of the job. Jobs are inserted at 0, the number is
|
48
|
+
# incremented to 1 the first time work its worked, and may increment further
|
49
|
+
# if it's either snoozed or errors.
|
50
|
+
attr_accessor :attempt
|
51
|
+
|
52
|
+
# The time that the job was last worked. Starts out as `nil` on a new
|
53
|
+
# insert.
|
54
|
+
attr_accessor :attempted_at
|
55
|
+
|
56
|
+
# The set of worker IDs that have worked this job. A worker ID differs
|
57
|
+
# between different programs, but is shared by all executors within any
|
58
|
+
# given one. (i.e. Different Go processes have different IDs, but IDs are
|
59
|
+
# shared within any given process.) A process generates a new ID based on
|
60
|
+
# host and current time when it starts up.
|
61
|
+
attr_accessor :attempted_by
|
62
|
+
|
63
|
+
# When the job record was created.
|
64
|
+
attr_accessor :created_at
|
65
|
+
|
66
|
+
# A set of errors that occurred when the job was worked, one for each
|
67
|
+
# attempt. Ordered from earliest error to the latest error.
|
68
|
+
attr_accessor :errors
|
69
|
+
|
70
|
+
# The time at which the job was "finalized", meaning it was either completed
|
71
|
+
# successfully or errored for the last time such that it'll no longer be
|
72
|
+
# retried.
|
73
|
+
attr_accessor :finalized_at
|
74
|
+
|
75
|
+
# Kind uniquely identifies the type of job and instructs which worker
|
76
|
+
# should work it. It is set at insertion time via `#kind` on job args.
|
77
|
+
attr_accessor :kind
|
78
|
+
|
79
|
+
# The maximum number of attempts that the job will be tried before it errors
|
80
|
+
# for the last time and will no longer be worked.
|
81
|
+
attr_accessor :max_attempts
|
82
|
+
|
83
|
+
# Arbitrary metadata associated with the job.
|
84
|
+
attr_accessor :metadata
|
85
|
+
|
86
|
+
# The priority of the job, with 1 being the highest priority and 4 being the
|
87
|
+
# lowest. When fetching available jobs to work, the highest priority jobs
|
88
|
+
# will always be fetched before any lower priority jobs are fetched. Note
|
89
|
+
# that if your workers are swamped with more high-priority jobs then they
|
90
|
+
# can handle, lower priority jobs may not be fetched.
|
91
|
+
attr_accessor :priority
|
92
|
+
|
93
|
+
# The name of the queue where the job will be worked. Queues can be
|
94
|
+
# configured independently and be used to isolate jobs.
|
95
|
+
attr_accessor :queue
|
96
|
+
|
97
|
+
# When the job is scheduled to become available to be worked. Jobs default
|
98
|
+
# to running immediately, but may be scheduled for the future when they're
|
99
|
+
# inserted. They may also be scheduled for later because they were snoozed
|
100
|
+
# or because they errored and have additional retry attempts remaining.
|
101
|
+
attr_accessor :scheduled_at
|
102
|
+
|
103
|
+
# The state of job like `available` or `completed`. Jobs are `available`
|
104
|
+
# when they're first inserted.
|
105
|
+
attr_accessor :state
|
106
|
+
|
107
|
+
# Tags are an arbitrary list of keywords to add to the job. They have no
|
108
|
+
# functional behavior and are meant entirely as a user-specified construct
|
109
|
+
# to help group and categorize jobs.
|
110
|
+
attr_accessor :tags
|
111
|
+
|
112
|
+
# A unique key for the job within its kind that's used for unique job
|
113
|
+
# insertions. It's generated by hashing an inserted job's unique opts
|
114
|
+
# configuration.
|
115
|
+
attr_accessor :unique_key
|
116
|
+
|
117
|
+
# A list of states that the job must be in to be considered for uniqueness.
|
118
|
+
attr_accessor :unique_states
|
119
|
+
|
120
|
+
def initialize(
|
121
|
+
id:,
|
122
|
+
args:,
|
123
|
+
attempt:,
|
124
|
+
created_at:,
|
125
|
+
kind:,
|
126
|
+
max_attempts:,
|
127
|
+
metadata:,
|
128
|
+
priority:,
|
129
|
+
queue:,
|
130
|
+
scheduled_at:,
|
131
|
+
state:,
|
132
|
+
|
133
|
+
# nullable/optional
|
134
|
+
attempted_at: nil,
|
135
|
+
attempted_by: nil,
|
136
|
+
errors: nil,
|
137
|
+
finalized_at: nil,
|
138
|
+
tags: nil,
|
139
|
+
unique_key: nil,
|
140
|
+
unique_states: nil
|
141
|
+
)
|
142
|
+
self.id = id
|
143
|
+
self.args = args
|
144
|
+
self.attempt = attempt
|
145
|
+
self.attempted_at = attempted_at
|
146
|
+
self.attempted_by = attempted_by
|
147
|
+
self.created_at = created_at
|
148
|
+
self.errors = errors
|
149
|
+
self.finalized_at = finalized_at
|
150
|
+
self.kind = kind
|
151
|
+
self.max_attempts = max_attempts
|
152
|
+
self.metadata = metadata
|
153
|
+
self.priority = priority
|
154
|
+
self.queue = queue
|
155
|
+
self.scheduled_at = scheduled_at
|
156
|
+
self.state = state
|
157
|
+
self.tags = tags
|
158
|
+
self.unique_key = unique_key
|
159
|
+
self.unique_states = unique_states
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
# A failed job work attempt containing information about the error or panic
|
164
|
+
# that occurred.
|
165
|
+
class AttemptError
|
166
|
+
# The time at which the error occurred.
|
167
|
+
attr_accessor :at
|
168
|
+
|
169
|
+
# The attempt number on which the error occurred (maps to #attempt on a job
|
170
|
+
# row).
|
171
|
+
attr_accessor :attempt
|
172
|
+
|
173
|
+
# Contains the stringified error of an error returned from a job or a panic
|
174
|
+
# value in case of a panic.
|
175
|
+
attr_accessor :error
|
176
|
+
|
177
|
+
# Contains a stack trace from a job that panicked. The trace is produced by
|
178
|
+
# invoking `debug.Trace()` in Go.
|
179
|
+
attr_accessor :trace
|
180
|
+
|
181
|
+
def initialize(
|
182
|
+
at:,
|
183
|
+
attempt:,
|
184
|
+
error:,
|
185
|
+
trace:
|
186
|
+
)
|
187
|
+
self.at = at
|
188
|
+
self.attempt = attempt
|
189
|
+
self.error = error
|
190
|
+
self.trace = trace
|
191
|
+
end
|
192
|
+
end
|
193
|
+
end
|
data/lib/riverqueue.rb
CHANGED
@@ -0,0 +1,41 @@
|
|
1
|
+
module River
|
2
|
+
class UniqueBitmask
|
3
|
+
JOB_STATE_BIT_POSITIONS = {
|
4
|
+
::River::JOB_STATE_AVAILABLE => 7,
|
5
|
+
::River::JOB_STATE_CANCELLED => 6,
|
6
|
+
::River::JOB_STATE_COMPLETED => 5,
|
7
|
+
::River::JOB_STATE_DISCARDED => 4,
|
8
|
+
::River::JOB_STATE_PENDING => 3,
|
9
|
+
::River::JOB_STATE_RETRYABLE => 2,
|
10
|
+
::River::JOB_STATE_RUNNING => 1,
|
11
|
+
::River::JOB_STATE_SCHEDULED => 0
|
12
|
+
}.freeze
|
13
|
+
private_constant :JOB_STATE_BIT_POSITIONS
|
14
|
+
|
15
|
+
def self.from_states(states)
|
16
|
+
val = 0
|
17
|
+
|
18
|
+
states.each do |state|
|
19
|
+
bit_index = JOB_STATE_BIT_POSITIONS[state]
|
20
|
+
|
21
|
+
bit_position = 7 - (bit_index % 8)
|
22
|
+
val |= 1 << bit_position
|
23
|
+
end
|
24
|
+
|
25
|
+
format("%08b", val)
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.to_states(mask)
|
29
|
+
states = [] #: Array[jobStateAll] # rubocop:disable Layout/LeadingCommentSpace
|
30
|
+
|
31
|
+
JOB_STATE_BIT_POSITIONS.each do |state, bit_index|
|
32
|
+
bit_position = 7 - (bit_index % 8)
|
33
|
+
if (mask & (1 << bit_position)) != 0
|
34
|
+
states << state
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
states.sort
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: riverqueue
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Blake Gentry
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2024-
|
12
|
+
date: 2024-12-20 00:00:00.000000000 Z
|
13
13
|
dependencies: []
|
14
14
|
description: River is a fast job queue for Go. Use this gem in conjunction with gems
|
15
15
|
riverqueue-activerecord or riverqueue-sequel to insert jobs in Ruby which will be
|
@@ -19,7 +19,12 @@ executables: []
|
|
19
19
|
extensions: []
|
20
20
|
extra_rdoc_files: []
|
21
21
|
files:
|
22
|
+
- lib/client.rb
|
23
|
+
- lib/driver.rb
|
24
|
+
- lib/insert_opts.rb
|
25
|
+
- lib/job.rb
|
22
26
|
- lib/riverqueue.rb
|
27
|
+
- lib/unique_bitmask.rb
|
23
28
|
homepage: https://riverqueue.com
|
24
29
|
licenses:
|
25
30
|
- LGPL-3.0-or-later
|
@@ -43,7 +48,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
43
48
|
- !ruby/object:Gem::Version
|
44
49
|
version: '0'
|
45
50
|
requirements: []
|
46
|
-
rubygems_version: 3.
|
51
|
+
rubygems_version: 3.5.16
|
47
52
|
signing_key:
|
48
53
|
specification_version: 4
|
49
54
|
summary: River is a fast job queue for Go.
|