riverqueue 0.6.1 → 0.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/client.rb +330 -0
- data/lib/driver.rb +63 -0
- data/lib/fnv.rb +35 -0
- data/lib/insert_opts.rb +136 -0
- data/lib/job.rb +187 -0
- metadata +7 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8aa1b2a14e085df2b2a7e79fa6c6cd5b19aeb2c893a4fd369b99e469dd5916b9
|
4
|
+
data.tar.gz: 3cebd975dcadb223ecc163918b2e8bb0ce67bb671be99c6f706d3d9ec2da6507
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 868459a39a19fe9aacec7552b03148d97228ff58bdfb6b79a6e724c800cc97ef10017097a46f5132684d5d4e97ea9c788b83c813dcb6dc812835e9c0c9ec9645
|
7
|
+
data.tar.gz: '05538541ee6466adc722af9d6ab4cbf4c1a081a59c9eb887089b8abea8474415d61dfa9fdf827e4863ee393874258f09e6c54b06ffac87ddc90931f346f12f0b'
|
data/lib/client.rb
ADDED
@@ -0,0 +1,330 @@
|
|
1
|
+
require "digest"
|
2
|
+
require "fnv"
|
3
|
+
require "time"
|
4
|
+
|
5
|
+
module River
|
6
|
+
# Default number of maximum attempts for a job.
|
7
|
+
MAX_ATTEMPTS_DEFAULT = 25
|
8
|
+
|
9
|
+
# Default priority for a job.
|
10
|
+
PRIORITY_DEFAULT = 1
|
11
|
+
|
12
|
+
# Default queue for a job.
|
13
|
+
QUEUE_DEFAULT = "default"
|
14
|
+
|
15
|
+
# Provides a client for River that inserts jobs. Unlike the Go version of the
|
16
|
+
# River client, this one can insert jobs only. Jobs can only be worked from Go
|
17
|
+
# code, so job arg kinds and JSON encoding details must be shared between Ruby
|
18
|
+
# and Go code.
|
19
|
+
#
|
20
|
+
# Used in conjunction with a River driver like:
|
21
|
+
#
|
22
|
+
# DB = Sequel.connect(...)
|
23
|
+
# client = River::Client.new(River::Driver::Sequel.new(DB))
|
24
|
+
#
|
25
|
+
# River drivers are found in separate gems like `riverqueue-sequel` to help
|
26
|
+
# minimize transient dependencies.
|
27
|
+
class Client
|
28
|
+
def initialize(driver, advisory_lock_prefix: nil)
|
29
|
+
@driver = driver
|
30
|
+
@advisory_lock_prefix = check_advisory_lock_prefix_bounds(advisory_lock_prefix)
|
31
|
+
@time_now_utc = -> { Time.now.utc } # for test time stubbing
|
32
|
+
end
|
33
|
+
|
34
|
+
# Inserts a new job for work given a job args implementation and insertion
|
35
|
+
# options (which may be omitted).
|
36
|
+
#
|
37
|
+
# With job args only:
|
38
|
+
#
|
39
|
+
# insert_res = client.insert(SimpleArgs.new(job_num: 1))
|
40
|
+
# insert_res.job # inserted job row
|
41
|
+
#
|
42
|
+
# With insert opts:
|
43
|
+
#
|
44
|
+
# insert_res = client.insert(SimpleArgs.new(job_num: 1), insert_opts: InsertOpts.new(queue: "high_priority"))
|
45
|
+
# insert_res.job # inserted job row
|
46
|
+
#
|
47
|
+
# Job arg implementations are expected to respond to:
|
48
|
+
#
|
49
|
+
# * `#kind`: A string that uniquely identifies the job in the database.
|
50
|
+
# * `#to_json`: Encodes the args to JSON for persistence in the database.
|
51
|
+
# Must match encoding an args struct on the Go side to be workable.
|
52
|
+
#
|
53
|
+
# They may also respond to `#insert_opts` which is expected to return an
|
54
|
+
# `InsertOpts` that contains options that will apply to all jobs of this
|
55
|
+
# kind. Insertion options provided as an argument to `#insert` override
|
56
|
+
# those returned by job args.
|
57
|
+
#
|
58
|
+
# For example:
|
59
|
+
#
|
60
|
+
# class SimpleArgs
|
61
|
+
# attr_accessor :job_num
|
62
|
+
#
|
63
|
+
# def initialize(job_num:)
|
64
|
+
# self.job_num = job_num
|
65
|
+
# end
|
66
|
+
#
|
67
|
+
# def kind = "simple"
|
68
|
+
#
|
69
|
+
# def to_json = JSON.dump({job_num: job_num})
|
70
|
+
# end
|
71
|
+
#
|
72
|
+
# See also JobArgsHash for an easy way to insert a job from a hash.
|
73
|
+
#
|
74
|
+
# Returns an instance of InsertResult.
|
75
|
+
def insert(args, insert_opts: EMPTY_INSERT_OPTS)
|
76
|
+
insert_params, unique_opts = make_insert_params(args, insert_opts)
|
77
|
+
check_unique_job(insert_params, unique_opts) do
|
78
|
+
job = @driver.job_insert(insert_params)
|
79
|
+
InsertResult.new(job)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
# Inserts many new jobs as part of a single batch operation for improved
|
84
|
+
# efficiency.
|
85
|
+
#
|
86
|
+
# Takes an array of job args or InsertManyParams which encapsulate job args
|
87
|
+
# and a paired InsertOpts.
|
88
|
+
#
|
89
|
+
# With job args:
|
90
|
+
#
|
91
|
+
# num_inserted = client.insert_many([
|
92
|
+
# SimpleArgs.new(job_num: 1),
|
93
|
+
# SimpleArgs.new(job_num: 2)
|
94
|
+
# ])
|
95
|
+
#
|
96
|
+
# With InsertManyParams:
|
97
|
+
#
|
98
|
+
# num_inserted = client.insert_many([
|
99
|
+
# River::InsertManyParams.new(SimpleArgs.new(job_num: 1), insert_opts: InsertOpts.new(max_attempts: 5)),
|
100
|
+
# River::InsertManyParams.new(SimpleArgs.new(job_num: 2), insert_opts: InsertOpts.new(queue: "high_priority"))
|
101
|
+
# ])
|
102
|
+
#
|
103
|
+
# Job arg implementations are expected to respond to:
|
104
|
+
#
|
105
|
+
# * `#kind`: A string that uniquely identifies the job in the database.
|
106
|
+
# * `#to_json`: Encodes the args to JSON for persistence in the database.
|
107
|
+
# Must match encoding an args struct on the Go side to be workable.
|
108
|
+
#
|
109
|
+
# For example:
|
110
|
+
#
|
111
|
+
# class SimpleArgs
|
112
|
+
# attr_accessor :job_num
|
113
|
+
#
|
114
|
+
# def initialize(job_num:)
|
115
|
+
# self.job_num = job_num
|
116
|
+
# end
|
117
|
+
#
|
118
|
+
# def kind = "simple"
|
119
|
+
#
|
120
|
+
# def to_json = JSON.dump({job_num: job_num})
|
121
|
+
# end
|
122
|
+
#
|
123
|
+
# See also JobArgsHash for an easy way to insert a job from a hash.
|
124
|
+
#
|
125
|
+
# Unique job insertion isn't supported with bulk insertion because it'd run
|
126
|
+
# the risk of major lock contention.
|
127
|
+
#
|
128
|
+
# Returns the number of jobs inserted.
|
129
|
+
def insert_many(args)
|
130
|
+
all_params = args.map do |arg|
|
131
|
+
if arg.is_a?(InsertManyParams)
|
132
|
+
make_insert_params(arg.args, arg.insert_opts || EMPTY_INSERT_OPTS, is_insert_many: true).first # unique opts ignored on batch insert
|
133
|
+
else # jobArgs
|
134
|
+
make_insert_params(arg, EMPTY_INSERT_OPTS, is_insert_many: true).first # unique opts ignored on batch insert
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
@driver.job_insert_many(all_params)
|
139
|
+
end
|
140
|
+
|
141
|
+
private def check_advisory_lock_prefix_bounds(advisory_lock_prefix)
|
142
|
+
return nil if advisory_lock_prefix.nil?
|
143
|
+
|
144
|
+
# 2**32-1 is 0xffffffff (the largest number that's four bytes)
|
145
|
+
if advisory_lock_prefix < 0 || advisory_lock_prefix > 2**32 - 1
|
146
|
+
raise ArgumentError, "advisory lock prefix must fit inside four bytes"
|
147
|
+
end
|
148
|
+
advisory_lock_prefix
|
149
|
+
end
|
150
|
+
|
151
|
+
# Default states that are used during a unique insert. Can be overridden by
|
152
|
+
# setting UniqueOpts#by_state.
|
153
|
+
DEFAULT_UNIQUE_STATES = [
|
154
|
+
JOB_STATE_AVAILABLE,
|
155
|
+
JOB_STATE_COMPLETED,
|
156
|
+
JOB_STATE_RETRYABLE,
|
157
|
+
JOB_STATE_RUNNING,
|
158
|
+
JOB_STATE_SCHEDULED
|
159
|
+
].freeze
|
160
|
+
private_constant :DEFAULT_UNIQUE_STATES
|
161
|
+
|
162
|
+
EMPTY_INSERT_OPTS = InsertOpts.new.freeze
|
163
|
+
private_constant :EMPTY_INSERT_OPTS
|
164
|
+
|
165
|
+
private def check_unique_job(insert_params, unique_opts, &block)
|
166
|
+
return block.call if unique_opts.nil?
|
167
|
+
|
168
|
+
any_unique_opts = false
|
169
|
+
get_params = Driver::JobGetByKindAndUniquePropertiesParam.new(kind: insert_params.kind)
|
170
|
+
unique_key = ""
|
171
|
+
|
172
|
+
# It's extremely important here that this lock string format and algorithm
|
173
|
+
# match the one in the main River library _exactly_. Don't change them
|
174
|
+
# unless they're updated everywhere.
|
175
|
+
if unique_opts.by_args
|
176
|
+
any_unique_opts = true
|
177
|
+
get_params.encoded_args = insert_params.encoded_args
|
178
|
+
unique_key += "&args=#{insert_params.encoded_args}"
|
179
|
+
end
|
180
|
+
|
181
|
+
if unique_opts.by_period
|
182
|
+
lower_period_bound = truncate_time(@time_now_utc.call, unique_opts.by_period).utc
|
183
|
+
|
184
|
+
any_unique_opts = true
|
185
|
+
get_params.created_at = [lower_period_bound, lower_period_bound + unique_opts.by_period]
|
186
|
+
unique_key += "&period=#{lower_period_bound.strftime("%FT%TZ")}"
|
187
|
+
end
|
188
|
+
|
189
|
+
if unique_opts.by_queue
|
190
|
+
any_unique_opts = true
|
191
|
+
get_params.queue = insert_params.queue
|
192
|
+
unique_key += "&queue=#{insert_params.queue}"
|
193
|
+
end
|
194
|
+
|
195
|
+
if unique_opts.by_state
|
196
|
+
any_unique_opts = true
|
197
|
+
get_params.state = unique_opts.by_state
|
198
|
+
unique_key += "&state=#{unique_opts.by_state.join(",")}"
|
199
|
+
else
|
200
|
+
get_params.state = DEFAULT_UNIQUE_STATES
|
201
|
+
unique_key += "&state=#{DEFAULT_UNIQUE_STATES.join(",")}"
|
202
|
+
end
|
203
|
+
|
204
|
+
return block.call unless any_unique_opts
|
205
|
+
|
206
|
+
# fast path
|
207
|
+
if !unique_opts.by_state || unique_opts.by_state.sort == DEFAULT_UNIQUE_STATES
|
208
|
+
unique_key_hash = Digest::SHA256.digest(unique_key)
|
209
|
+
job, unique_skipped_as_duplicate = @driver.job_insert_unique(insert_params, unique_key_hash)
|
210
|
+
return InsertResult.new(job, unique_skipped_as_duplicated: unique_skipped_as_duplicate)
|
211
|
+
end
|
212
|
+
|
213
|
+
@driver.transaction do
|
214
|
+
lock_str = "unique_key"
|
215
|
+
lock_str += "kind=#{insert_params.kind}"
|
216
|
+
lock_str += unique_key
|
217
|
+
|
218
|
+
lock_key = if @advisory_lock_prefix.nil?
|
219
|
+
FNV.fnv1_hash(lock_str, size: 64)
|
220
|
+
else
|
221
|
+
# Steep should be able to tell that this is not nil, but it can't.
|
222
|
+
prefix = @advisory_lock_prefix #: Integer # rubocop:disable Layout/LeadingCommentSpace
|
223
|
+
prefix << 32 | FNV.fnv1_hash(lock_str, size: 32)
|
224
|
+
end
|
225
|
+
|
226
|
+
# Packs a uint64 then unpacks to int64, which we need to do to keep the
|
227
|
+
# value within the bounds of Postgres' bigint. Overflow is okay because
|
228
|
+
# we can use the full bigint space (including negative numbers) for the
|
229
|
+
# advisory lock.
|
230
|
+
lock_key = uint64_to_int64(lock_key)
|
231
|
+
|
232
|
+
@driver.advisory_lock(lock_key)
|
233
|
+
|
234
|
+
existing_job = @driver.job_get_by_kind_and_unique_properties(get_params)
|
235
|
+
return InsertResult.new(existing_job, unique_skipped_as_duplicated: true) if existing_job
|
236
|
+
|
237
|
+
block.call
|
238
|
+
end
|
239
|
+
end
|
240
|
+
|
241
|
+
private def make_insert_params(args, insert_opts, is_insert_many: false)
|
242
|
+
raise "args should respond to `#kind`" if !args.respond_to?(:kind)
|
243
|
+
|
244
|
+
# ~all objects in Ruby respond to `#to_json`, so check non-nil instead.
|
245
|
+
args_json = args.to_json
|
246
|
+
raise "args should return non-nil from `#to_json`" if !args_json
|
247
|
+
|
248
|
+
args_insert_opts = if args.respond_to?(:insert_opts)
|
249
|
+
args_with_insert_opts = args #: _JobArgsWithInsertOpts # rubocop:disable Layout/LeadingCommentSpace
|
250
|
+
args_with_insert_opts.insert_opts || EMPTY_INSERT_OPTS
|
251
|
+
else
|
252
|
+
EMPTY_INSERT_OPTS
|
253
|
+
end
|
254
|
+
|
255
|
+
scheduled_at = insert_opts.scheduled_at || args_insert_opts.scheduled_at
|
256
|
+
unique_opts = insert_opts.unique_opts || args_insert_opts.unique_opts
|
257
|
+
|
258
|
+
raise ArgumentError, "unique opts can't be used with `#insert_many`" if is_insert_many && unique_opts
|
259
|
+
|
260
|
+
[
|
261
|
+
Driver::JobInsertParams.new(
|
262
|
+
encoded_args: args_json,
|
263
|
+
kind: args.kind,
|
264
|
+
max_attempts: insert_opts.max_attempts || args_insert_opts.max_attempts || MAX_ATTEMPTS_DEFAULT,
|
265
|
+
priority: insert_opts.priority || args_insert_opts.priority || PRIORITY_DEFAULT,
|
266
|
+
queue: insert_opts.queue || args_insert_opts.queue || QUEUE_DEFAULT,
|
267
|
+
scheduled_at: scheduled_at&.utc, # database defaults to now
|
268
|
+
state: scheduled_at ? JOB_STATE_SCHEDULED : JOB_STATE_AVAILABLE,
|
269
|
+
tags: validate_tags(insert_opts.tags || args_insert_opts.tags)
|
270
|
+
),
|
271
|
+
unique_opts
|
272
|
+
]
|
273
|
+
end
|
274
|
+
|
275
|
+
# Truncates the given time down to the interval. For example:
|
276
|
+
#
|
277
|
+
# Thu Jan 15 21:26:36 UTC 2024 @ 15 minutes ->
|
278
|
+
# Thu Jan 15 21:15:00 UTC 2024
|
279
|
+
private def truncate_time(time, interval_seconds)
|
280
|
+
Time.at((time.to_f / interval_seconds).floor * interval_seconds)
|
281
|
+
end
|
282
|
+
|
283
|
+
# Moves an integer that may occupy the entire uint64 space to one that's
|
284
|
+
# bounded within int64. Allows overflow.
|
285
|
+
private def uint64_to_int64(int)
|
286
|
+
[int].pack("Q").unpack1("q") #: Integer # rubocop:disable Layout/LeadingCommentSpace
|
287
|
+
end
|
288
|
+
|
289
|
+
TAG_RE = /\A[\w][\w\-]+[\w]\z/
|
290
|
+
private_constant :TAG_RE
|
291
|
+
|
292
|
+
private def validate_tags(tags)
|
293
|
+
tags&.each do |tag|
|
294
|
+
raise ArgumentError, "tags should be 255 characters or less" if tag.length > 255
|
295
|
+
raise ArgumentError, "tag should match regex #{TAG_RE.inspect}" unless TAG_RE.match(tag)
|
296
|
+
end
|
297
|
+
end
|
298
|
+
end
|
299
|
+
|
300
|
+
# A single job to insert that's part of an #insert_many batch insert. Unlike
|
301
|
+
# sending raw job args, supports an InsertOpts to pair with the job.
|
302
|
+
class InsertManyParams
|
303
|
+
# Job args to insert.
|
304
|
+
attr_reader :args
|
305
|
+
|
306
|
+
# Insertion options to use with the insert.
|
307
|
+
attr_reader :insert_opts
|
308
|
+
|
309
|
+
def initialize(args, insert_opts: nil)
|
310
|
+
@args = args
|
311
|
+
@insert_opts = insert_opts
|
312
|
+
end
|
313
|
+
end
|
314
|
+
|
315
|
+
# Result of a single insertion.
|
316
|
+
class InsertResult
|
317
|
+
# Inserted job row, or an existing job row if insert was skipped due to a
|
318
|
+
# previously existing unique job.
|
319
|
+
attr_reader :job
|
320
|
+
|
321
|
+
# True if for a unique job, the insertion was skipped due to an equivalent
|
322
|
+
# job matching unique property already being present.
|
323
|
+
attr_reader :unique_skipped_as_duplicated
|
324
|
+
|
325
|
+
def initialize(job, unique_skipped_as_duplicated: false)
|
326
|
+
@job = job
|
327
|
+
@unique_skipped_as_duplicated = unique_skipped_as_duplicated
|
328
|
+
end
|
329
|
+
end
|
330
|
+
end
|
data/lib/driver.rb
ADDED
@@ -0,0 +1,63 @@
|
|
1
|
+
module River
|
2
|
+
# Contains an interface used by the top-level River module to interface with
|
3
|
+
# its driver implementations. All types and methods in this module should be
|
4
|
+
# considered to be for internal use only and subject to change. API stability
|
5
|
+
# is not guaranteed.
|
6
|
+
module Driver
|
7
|
+
# Parameters for looking up a job by kind and unique properties.
|
8
|
+
class JobGetByKindAndUniquePropertiesParam
|
9
|
+
attr_accessor :created_at
|
10
|
+
attr_accessor :encoded_args
|
11
|
+
attr_accessor :kind
|
12
|
+
attr_accessor :queue
|
13
|
+
attr_accessor :state
|
14
|
+
|
15
|
+
def initialize(
|
16
|
+
kind:,
|
17
|
+
created_at: nil,
|
18
|
+
encoded_args: nil,
|
19
|
+
queue: nil,
|
20
|
+
state: nil
|
21
|
+
)
|
22
|
+
self.kind = kind
|
23
|
+
self.created_at = created_at
|
24
|
+
self.encoded_args = encoded_args
|
25
|
+
self.queue = queue
|
26
|
+
self.state = state
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
# Insert parameters for a job. This is sent to underlying drivers and is meant
|
31
|
+
# for internal use only. Its interface is subject to change.
|
32
|
+
class JobInsertParams
|
33
|
+
attr_accessor :encoded_args
|
34
|
+
attr_accessor :kind
|
35
|
+
attr_accessor :max_attempts
|
36
|
+
attr_accessor :priority
|
37
|
+
attr_accessor :queue
|
38
|
+
attr_accessor :scheduled_at
|
39
|
+
attr_accessor :state
|
40
|
+
attr_accessor :tags
|
41
|
+
|
42
|
+
def initialize(
|
43
|
+
encoded_args:,
|
44
|
+
kind:,
|
45
|
+
max_attempts:,
|
46
|
+
priority:,
|
47
|
+
queue:,
|
48
|
+
scheduled_at:,
|
49
|
+
state:,
|
50
|
+
tags:
|
51
|
+
)
|
52
|
+
self.encoded_args = encoded_args
|
53
|
+
self.kind = kind
|
54
|
+
self.max_attempts = max_attempts
|
55
|
+
self.priority = priority
|
56
|
+
self.queue = queue
|
57
|
+
self.scheduled_at = scheduled_at
|
58
|
+
self.state = state
|
59
|
+
self.tags = tags
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
data/lib/fnv.rb
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
module River
|
2
|
+
# FNV is the Fowler–Noll–Vo hash function, a simple hash that's very easy to
|
3
|
+
# implement, and hash the perfect characteristics for use with the 64 bits of
|
4
|
+
# available space in a PG advisory lock.
|
5
|
+
#
|
6
|
+
# I'm implemented it myself so that the River gem can stay dependency free
|
7
|
+
# (and because it's quite easy to do).
|
8
|
+
module FNV
|
9
|
+
def self.fnv1_hash(str, size:)
|
10
|
+
hash = OFFSET_BASIS.fetch(size)
|
11
|
+
mask = (2**size - 1).to_int # creates a mask of 1s of `size` bits long like 0xffffffff
|
12
|
+
prime = PRIME.fetch(size)
|
13
|
+
|
14
|
+
str.each_byte do |byte|
|
15
|
+
hash *= prime
|
16
|
+
hash &= mask
|
17
|
+
hash ^= byte
|
18
|
+
end
|
19
|
+
|
20
|
+
hash
|
21
|
+
end
|
22
|
+
|
23
|
+
OFFSET_BASIS = {
|
24
|
+
32 => 0x811c9dc5,
|
25
|
+
64 => 0xcbf29ce484222325
|
26
|
+
}.freeze
|
27
|
+
private_constant :OFFSET_BASIS
|
28
|
+
|
29
|
+
PRIME = {
|
30
|
+
32 => 0x01000193,
|
31
|
+
64 => 0x00000100000001B3
|
32
|
+
}.freeze
|
33
|
+
private_constant :PRIME
|
34
|
+
end
|
35
|
+
end
|
data/lib/insert_opts.rb
ADDED
@@ -0,0 +1,136 @@
|
|
1
|
+
module River
|
2
|
+
# Options for job insertion, and which can be provided by implementing
|
3
|
+
# #insert_opts on job args, or specified as a parameter on #insert or
|
4
|
+
# #insert_many.
|
5
|
+
class InsertOpts
|
6
|
+
# The maximum number of total attempts (including both the original run and
|
7
|
+
# all retries) before a job is abandoned and set as discarded.
|
8
|
+
attr_accessor :max_attempts
|
9
|
+
|
10
|
+
# The priority of the job, with 1 being the highest priority and 4 being the
|
11
|
+
# lowest. When fetching available jobs to work, the highest priority jobs
|
12
|
+
# will always be fetched before any lower priority jobs are fetched. Note
|
13
|
+
# that if your workers are swamped with more high-priority jobs then they
|
14
|
+
# can handle, lower priority jobs may not be fetched.
|
15
|
+
#
|
16
|
+
# Defaults to PRIORITY_DEFAULT.
|
17
|
+
attr_accessor :priority
|
18
|
+
|
19
|
+
# The name of the job queue in which to insert the job.
|
20
|
+
#
|
21
|
+
# Defaults to QUEUE_DEFAULT.
|
22
|
+
attr_accessor :queue
|
23
|
+
|
24
|
+
# A time in future at which to schedule the job (i.e. in cases where it
|
25
|
+
# shouldn't be run immediately). The job is guaranteed not to run before
|
26
|
+
# this time, but may run slightly after depending on the number of other
|
27
|
+
# scheduled jobs and how busy the queue is.
|
28
|
+
#
|
29
|
+
# Use of this option generally only makes sense when passing options into
|
30
|
+
# Insert rather than when a job args is returning `#insert_opts`, however,
|
31
|
+
# it will work in both cases.
|
32
|
+
attr_accessor :scheduled_at
|
33
|
+
|
34
|
+
# An arbitrary list of keywords to add to the job. They have no functional
|
35
|
+
# behavior and are meant entirely as a user-specified construct to help
|
36
|
+
# group and categorize jobs.
|
37
|
+
#
|
38
|
+
# If tags are specified from both a job args override and from options on
|
39
|
+
# Insert, the latter takes precedence. Tags are not merged.
|
40
|
+
attr_accessor :tags
|
41
|
+
|
42
|
+
# Options relating to job uniqueness. No unique options means that the job
|
43
|
+
# is never treated as unique.
|
44
|
+
attr_accessor :unique_opts
|
45
|
+
|
46
|
+
def initialize(
|
47
|
+
max_attempts: nil,
|
48
|
+
priority: nil,
|
49
|
+
queue: nil,
|
50
|
+
scheduled_at: nil,
|
51
|
+
tags: nil,
|
52
|
+
unique_opts: nil
|
53
|
+
)
|
54
|
+
self.max_attempts = max_attempts
|
55
|
+
self.priority = priority
|
56
|
+
self.queue = queue
|
57
|
+
self.scheduled_at = scheduled_at
|
58
|
+
self.tags = tags
|
59
|
+
self.unique_opts = unique_opts
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
# Parameters for uniqueness for a job.
|
64
|
+
#
|
65
|
+
# If all properties are nil, no uniqueness at is enforced. As each property is
|
66
|
+
# initialized, it's added as a dimension on the uniqueness matrix, and with
|
67
|
+
# any property on, the job's kind always counts toward uniqueness.
|
68
|
+
#
|
69
|
+
# So for example, if only #by_queue is on, then for the given job kind, only a
|
70
|
+
# single instance is allowed in any given queue, regardless of other
|
71
|
+
# properties on the job. If both #by_args and #by_queue are on, then for the
|
72
|
+
# given job kind, a single instance is allowed for each combination of args
|
73
|
+
# and queues. If either args or queue is changed on a new job, it's allowed to
|
74
|
+
# be inserted as a new job.
|
75
|
+
#
|
76
|
+
# Uniquenes is checked at insert time by taking a Postgres advisory lock,
|
77
|
+
# doing a look up for an equivalent row, and inserting only if none was found.
|
78
|
+
# There's no database-level mechanism that guarantees jobs stay unique, so if
|
79
|
+
# an equivalent row is inserted out of band (or batch inserted, where a unique
|
80
|
+
# check doesn't occur), it's conceivable that duplicates could coexist.
|
81
|
+
class UniqueOpts
|
82
|
+
# Indicates that uniqueness should be enforced for any specific instance of
|
83
|
+
# encoded args for a job.
|
84
|
+
#
|
85
|
+
# Default is false, meaning that as long as any other unique property is
|
86
|
+
# enabled, uniqueness will be enforced for a kind regardless of input args.
|
87
|
+
attr_accessor :by_args
|
88
|
+
|
89
|
+
# Defines uniqueness within a given period. On an insert time is rounded
|
90
|
+
# down to the nearest multiple of the given period, and a job is only
|
91
|
+
# inserted if there isn't an existing job that will run between then and the
|
92
|
+
# next multiple of the period.
|
93
|
+
#
|
94
|
+
# The period should be specified in seconds. So a job that's unique every 15
|
95
|
+
# minute period would have a value of 900.
|
96
|
+
#
|
97
|
+
# Default is no unique period, meaning that as long as any other unique
|
98
|
+
# property is enabled, uniqueness will be enforced across all jobs of the
|
99
|
+
# kind in the database, regardless of when they were scheduled.
|
100
|
+
attr_accessor :by_period
|
101
|
+
|
102
|
+
# Indicates that uniqueness should be enforced within each queue.
|
103
|
+
#
|
104
|
+
# Default is false, meaning that as long as any other unique property is
|
105
|
+
# enabled, uniqueness will be enforced for a kind across all queues.
|
106
|
+
attr_accessor :by_queue
|
107
|
+
|
108
|
+
# Indicates that uniqueness should be enforced across any of the states in
|
109
|
+
# the given set. For example, if the given states were `(scheduled,
|
110
|
+
# running)` then a new job could be inserted even if one of the same kind
|
111
|
+
# was already being worked by the queue (new jobs are inserted as
|
112
|
+
# `available`).
|
113
|
+
#
|
114
|
+
# Unlike other unique options, ByState gets a default when it's not set for
|
115
|
+
# user convenience. The default is equivalent to:
|
116
|
+
#
|
117
|
+
# by_state: [River::JOB_STATE_AVAILABLE, River::JOB_STATE_COMPLETED, River::JOB_STATE_RUNNING, River::JOB_STATE_RETRYABLE, River::JOB_STATE_SCHEDULED]
|
118
|
+
#
|
119
|
+
# With this setting, any jobs of the same kind that have been completed or
|
120
|
+
# discarded, but not yet cleaned out by the system, won't count towards the
|
121
|
+
# uniqueness of a new insert.
|
122
|
+
attr_accessor :by_state
|
123
|
+
|
124
|
+
def initialize(
|
125
|
+
by_args: nil,
|
126
|
+
by_period: nil,
|
127
|
+
by_queue: nil,
|
128
|
+
by_state: nil
|
129
|
+
)
|
130
|
+
self.by_args = by_args
|
131
|
+
self.by_period = by_period
|
132
|
+
self.by_queue = by_queue
|
133
|
+
self.by_state = by_state
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
data/lib/job.rb
ADDED
@@ -0,0 +1,187 @@
|
|
1
|
+
module River
|
2
|
+
JOB_STATE_AVAILABLE = "available"
|
3
|
+
JOB_STATE_CANCELLED = "cancelled"
|
4
|
+
JOB_STATE_COMPLETED = "completed"
|
5
|
+
JOB_STATE_DISCARDED = "discarded"
|
6
|
+
JOB_STATE_RETRYABLE = "retryable"
|
7
|
+
JOB_STATE_RUNNING = "running"
|
8
|
+
JOB_STATE_SCHEDULED = "scheduled"
|
9
|
+
|
10
|
+
# Provides a way of creating a job args from a simple Ruby hash for a quick
|
11
|
+
# way to insert a job without having to define a class. The first argument is
|
12
|
+
# a "kind" string for identifying the job in the database and the second is a
|
13
|
+
# hash that will be encoded to JSON.
|
14
|
+
#
|
15
|
+
# For example:
|
16
|
+
#
|
17
|
+
# insert_res = client.insert(River::JobArgsHash.new("job_kind", {
|
18
|
+
# job_num: 1
|
19
|
+
# }))
|
20
|
+
class JobArgsHash
|
21
|
+
def initialize(kind, hash)
|
22
|
+
raise "kind should be non-nil" if !kind
|
23
|
+
raise "hash should be non-nil" if !hash
|
24
|
+
|
25
|
+
@kind = kind
|
26
|
+
@hash = hash
|
27
|
+
end
|
28
|
+
|
29
|
+
attr_reader :kind
|
30
|
+
|
31
|
+
def to_json
|
32
|
+
JSON.dump(@hash)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
# JobRow contains the properties of a job that are persisted to the database.
|
37
|
+
class JobRow
|
38
|
+
# ID of the job. Generated as part of a Postgres sequence and generally
|
39
|
+
# ascending in nature, but there may be gaps in it as transactions roll
|
40
|
+
# back.
|
41
|
+
attr_accessor :id
|
42
|
+
|
43
|
+
# The job's args as a hash decoded from JSON.
|
44
|
+
attr_accessor :args
|
45
|
+
|
46
|
+
# The attempt number of the job. Jobs are inserted at 0, the number is
|
47
|
+
# incremented to 1 the first time work its worked, and may increment further
|
48
|
+
# if it's either snoozed or errors.
|
49
|
+
attr_accessor :attempt
|
50
|
+
|
51
|
+
# The time that the job was last worked. Starts out as `nil` on a new
|
52
|
+
# insert.
|
53
|
+
attr_accessor :attempted_at
|
54
|
+
|
55
|
+
# The set of worker IDs that have worked this job. A worker ID differs
|
56
|
+
# between different programs, but is shared by all executors within any
|
57
|
+
# given one. (i.e. Different Go processes have different IDs, but IDs are
|
58
|
+
# shared within any given process.) A process generates a new ID based on
|
59
|
+
# host and current time when it starts up.
|
60
|
+
attr_accessor :attempted_by
|
61
|
+
|
62
|
+
# When the job record was created.
|
63
|
+
attr_accessor :created_at
|
64
|
+
|
65
|
+
# A set of errors that occurred when the job was worked, one for each
|
66
|
+
# attempt. Ordered from earliest error to the latest error.
|
67
|
+
attr_accessor :errors
|
68
|
+
|
69
|
+
# The time at which the job was "finalized", meaning it was either completed
|
70
|
+
# successfully or errored for the last time such that it'll no longer be
|
71
|
+
# retried.
|
72
|
+
attr_accessor :finalized_at
|
73
|
+
|
74
|
+
# Kind uniquely identifies the type of job and instructs which worker
|
75
|
+
# should work it. It is set at insertion time via `#kind` on job args.
|
76
|
+
attr_accessor :kind
|
77
|
+
|
78
|
+
# The maximum number of attempts that the job will be tried before it errors
|
79
|
+
# for the last time and will no longer be worked.
|
80
|
+
attr_accessor :max_attempts
|
81
|
+
|
82
|
+
# Arbitrary metadata associated with the job.
|
83
|
+
attr_accessor :metadata
|
84
|
+
|
85
|
+
# The priority of the job, with 1 being the highest priority and 4 being the
|
86
|
+
# lowest. When fetching available jobs to work, the highest priority jobs
|
87
|
+
# will always be fetched before any lower priority jobs are fetched. Note
|
88
|
+
# that if your workers are swamped with more high-priority jobs then they
|
89
|
+
# can handle, lower priority jobs may not be fetched.
|
90
|
+
attr_accessor :priority
|
91
|
+
|
92
|
+
# The name of the queue where the job will be worked. Queues can be
|
93
|
+
# configured independently and be used to isolate jobs.
|
94
|
+
attr_accessor :queue
|
95
|
+
|
96
|
+
# When the job is scheduled to become available to be worked. Jobs default
|
97
|
+
# to running immediately, but may be scheduled for the future when they're
|
98
|
+
# inserted. They may also be scheduled for later because they were snoozed
|
99
|
+
# or because they errored and have additional retry attempts remaining.
|
100
|
+
attr_accessor :scheduled_at
|
101
|
+
|
102
|
+
# The state of job like `available` or `completed`. Jobs are `available`
|
103
|
+
# when they're first inserted.
|
104
|
+
attr_accessor :state
|
105
|
+
|
106
|
+
# Tags are an arbitrary list of keywords to add to the job. They have no
|
107
|
+
# functional behavior and are meant entirely as a user-specified construct
|
108
|
+
# to help group and categorize jobs.
|
109
|
+
attr_accessor :tags
|
110
|
+
|
111
|
+
# A unique key for the job within its kind that's used for unique job
|
112
|
+
# insertions. It's generated by hashing an inserted job's unique opts
|
113
|
+
# configuration.
|
114
|
+
attr_accessor :unique_key
|
115
|
+
|
116
|
+
def initialize(
|
117
|
+
id:,
|
118
|
+
args:,
|
119
|
+
attempt:,
|
120
|
+
created_at:,
|
121
|
+
kind:,
|
122
|
+
max_attempts:,
|
123
|
+
metadata:,
|
124
|
+
priority:,
|
125
|
+
queue:,
|
126
|
+
scheduled_at:,
|
127
|
+
state:,
|
128
|
+
|
129
|
+
# nullable/optional
|
130
|
+
attempted_at: nil,
|
131
|
+
attempted_by: nil,
|
132
|
+
errors: nil,
|
133
|
+
finalized_at: nil,
|
134
|
+
tags: nil,
|
135
|
+
unique_key: nil
|
136
|
+
)
|
137
|
+
self.id = id
|
138
|
+
self.args = args
|
139
|
+
self.attempt = attempt
|
140
|
+
self.attempted_at = attempted_at
|
141
|
+
self.attempted_by = attempted_by
|
142
|
+
self.created_at = created_at
|
143
|
+
self.errors = errors
|
144
|
+
self.finalized_at = finalized_at
|
145
|
+
self.kind = kind
|
146
|
+
self.max_attempts = max_attempts
|
147
|
+
self.metadata = metadata
|
148
|
+
self.priority = priority
|
149
|
+
self.queue = queue
|
150
|
+
self.scheduled_at = scheduled_at
|
151
|
+
self.state = state
|
152
|
+
self.tags = tags
|
153
|
+
self.unique_key = unique_key
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
# A failed job work attempt containing information about the error or panic
|
158
|
+
# that occurred.
|
159
|
+
class AttemptError
|
160
|
+
# The time at which the error occurred.
|
161
|
+
attr_accessor :at
|
162
|
+
|
163
|
+
# The attempt number on which the error occurred (maps to #attempt on a job
|
164
|
+
# row).
|
165
|
+
attr_accessor :attempt
|
166
|
+
|
167
|
+
# Contains the stringified error of an error returned from a job or a panic
|
168
|
+
# value in case of a panic.
|
169
|
+
attr_accessor :error
|
170
|
+
|
171
|
+
# Contains a stack trace from a job that panicked. The trace is produced by
|
172
|
+
# invoking `debug.Trace()` in Go.
|
173
|
+
attr_accessor :trace
|
174
|
+
|
175
|
+
def initialize(
|
176
|
+
at:,
|
177
|
+
attempt:,
|
178
|
+
error:,
|
179
|
+
trace:
|
180
|
+
)
|
181
|
+
self.at = at
|
182
|
+
self.attempt = attempt
|
183
|
+
self.error = error
|
184
|
+
self.trace = trace
|
185
|
+
end
|
186
|
+
end
|
187
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: riverqueue
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.7.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Blake Gentry
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2024-08-
|
12
|
+
date: 2024-08-31 00:00:00.000000000 Z
|
13
13
|
dependencies: []
|
14
14
|
description: River is a fast job queue for Go. Use this gem in conjunction with gems
|
15
15
|
riverqueue-activerecord or riverqueue-sequel to insert jobs in Ruby which will be
|
@@ -19,6 +19,11 @@ executables: []
|
|
19
19
|
extensions: []
|
20
20
|
extra_rdoc_files: []
|
21
21
|
files:
|
22
|
+
- lib/client.rb
|
23
|
+
- lib/driver.rb
|
24
|
+
- lib/fnv.rb
|
25
|
+
- lib/insert_opts.rb
|
26
|
+
- lib/job.rb
|
22
27
|
- lib/riverqueue.rb
|
23
28
|
homepage: https://riverqueue.com
|
24
29
|
licenses:
|