riverqueue 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +182 -0
- data/LICENSE +374 -0
- data/README.md +22 -0
- data/docs/README.md +812 -0
- data/docs/conformance.md +72 -0
- data/docs/migrations.md +118 -0
- data/docs/testing.md +173 -0
- data/docs/workers.md +126 -0
- data/exe/river +7 -0
- data/lib/cli.rb +55 -0
- data/lib/client.rb +347 -54
- data/lib/client_runtime.rb +721 -0
- data/lib/config.rb +154 -0
- data/lib/driver/job_row_decoder.rb +85 -0
- data/lib/driver/postgres_capabilities.rb +41 -0
- data/lib/driver/runtime.rb +772 -0
- data/lib/driver.rb +13 -6
- data/lib/errors.rb +83 -0
- data/lib/event.rb +85 -0
- data/lib/insert_opts.rb +46 -12
- data/lib/job.rb +28 -13
- data/lib/job_persisted_logging/plugin.rb +108 -0
- data/lib/migration_cli.rb +90 -0
- data/lib/migrator.rb +189 -0
- data/lib/params.rb +96 -0
- data/lib/periodic_cron.rb +59 -0
- data/lib/periodic_job.rb +143 -0
- data/lib/resumable.rb +148 -0
- data/lib/riverqueue/testing/minitest.rb +18 -0
- data/lib/riverqueue/testing/rspec.rb +109 -0
- data/lib/riverqueue/testing.rb +134 -0
- data/lib/riverqueue.rb +16 -0
- data/lib/unique_args.rb +97 -0
- data/lib/unique_bitmask.rb +14 -12
- data/lib/worker.rb +152 -0
- data/lib/worker_runner.rb +106 -0
- data/migration/LICENSE +374 -0
- data/migration/README.md +13 -0
- data/migration/manifest.json +37 -0
- data/migration/postgresql/main/001_create_river_migration.down.sql +1 -0
- data/migration/postgresql/main/001_create_river_migration.up.sql +8 -0
- data/migration/postgresql/main/002_initial_schema.down.sql +5 -0
- data/migration/postgresql/main/002_initial_schema.up.sql +96 -0
- data/migration/postgresql/main/003_river_job_tags_non_null.down.sql +3 -0
- data/migration/postgresql/main/003_river_job_tags_non_null.up.sql +3 -0
- data/migration/postgresql/main/004_pending_and_more.down.sql +42 -0
- data/migration/postgresql/main/004_pending_and_more.up.sql +45 -0
- data/migration/postgresql/main/005_migration_unique_client.down.sql +57 -0
- data/migration/postgresql/main/005_migration_unique_client.up.sql +79 -0
- data/migration/postgresql/main/006_bulk_unique.down.sql +16 -0
- data/migration/postgresql/main/006_bulk_unique.up.sql +40 -0
- data/migration/postgresql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql +56 -0
- data/migration/postgresql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql +44 -0
- data/migration/postgresql/main/008_job_id_autoincrement.down.sql +3 -0
- data/migration/postgresql/main/008_job_id_autoincrement.up.sql +3 -0
- data/migration/sqlite/main/001_create_river_migration.down.sql +1 -0
- data/migration/sqlite/main/001_create_river_migration.up.sql +8 -0
- data/migration/sqlite/main/002_initial_schema.down.sql +8 -0
- data/migration/sqlite/main/002_initial_schema.up.sql +19 -0
- data/migration/sqlite/main/003_river_job_tags_non_null.down.sql +6 -0
- data/migration/sqlite/main/003_river_job_tags_non_null.up.sql +6 -0
- data/migration/sqlite/main/004_pending_and_more.down.sql +26 -0
- data/migration/sqlite/main/004_pending_and_more.up.sql +33 -0
- data/migration/sqlite/main/005_migration_unique_client.down.sql +37 -0
- data/migration/sqlite/main/005_migration_unique_client.up.sql +64 -0
- data/migration/sqlite/main/006_bulk_unique.down.sql +7 -0
- data/migration/sqlite/main/006_bulk_unique.up.sql +63 -0
- data/migration/sqlite/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql +255 -0
- data/migration/sqlite/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql +261 -0
- data/migration/sqlite/main/008_job_id_autoincrement.down.sql +121 -0
- data/migration/sqlite/main/008_job_id_autoincrement.up.sql +123 -0
- data/sig/client.rbs +98 -0
- data/sig/driver.rbs +172 -0
- data/sig/insert_opts.rbs +25 -0
- data/sig/job.rbs +74 -0
- data/sig/job_persisted_logging/plugin.rbs +29 -0
- data/sig/migrator.rbs +44 -0
- data/sig/riverqueue.rbs +2 -0
- data/sig/runtime.rbs +402 -0
- data/sig/testing.rbs +95 -0
- data/sig/unique_args.rbs +8 -0
- data/sig/unique_bitmask.rbs +8 -0
- data/sig/worker_runner.rbs +22 -0
- metadata +170 -13
data/lib/client.rb
CHANGED
|
@@ -1,20 +1,22 @@
|
|
|
1
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest/sha2"
|
|
2
4
|
require "time"
|
|
3
5
|
|
|
4
6
|
module River
|
|
5
7
|
# Default number of maximum attempts for a job.
|
|
6
8
|
MAX_ATTEMPTS_DEFAULT = 25
|
|
7
9
|
|
|
10
|
+
# Largest attempt limit supported by all River databases.
|
|
11
|
+
MAX_ATTEMPTS_LIMIT = 32_767
|
|
12
|
+
|
|
8
13
|
# Default priority for a job.
|
|
9
14
|
PRIORITY_DEFAULT = 1
|
|
10
15
|
|
|
11
16
|
# Default queue for a job.
|
|
12
17
|
QUEUE_DEFAULT = "default"
|
|
13
18
|
|
|
14
|
-
# Provides a client for
|
|
15
|
-
# River client, this one can insert jobs only. Jobs can only be worked from Go
|
|
16
|
-
# code, so job arg kinds and JSON encoding details must be shared between Ruby
|
|
17
|
-
# and Go code.
|
|
19
|
+
# Provides a River client for inserting and working jobs.
|
|
18
20
|
#
|
|
19
21
|
# Used in conjunction with a River driver like:
|
|
20
22
|
#
|
|
@@ -24,11 +26,40 @@ module River
|
|
|
24
26
|
# River drivers are found in separate gems like `riverqueue-sequel` to help
|
|
25
27
|
# minimize transient dependencies.
|
|
26
28
|
class Client
|
|
27
|
-
|
|
29
|
+
# Configuration used by this client.
|
|
30
|
+
attr_reader :config
|
|
31
|
+
|
|
32
|
+
# Database driver used by this client.
|
|
33
|
+
attr_reader :driver
|
|
34
|
+
|
|
35
|
+
# Creates a client backed by the given database driver. Pass a Config to
|
|
36
|
+
# customize workers, queues, plugins, and runtime behavior.
|
|
37
|
+
def initialize(driver, config: nil)
|
|
38
|
+
@config = config || Config.new
|
|
28
39
|
@driver = driver
|
|
40
|
+
@runtime = ClientRuntime.new(self, driver, @config)
|
|
29
41
|
@time_now_utc = -> { Time.now.utc } # for test time stubbing
|
|
30
42
|
end
|
|
31
43
|
|
|
44
|
+
# Internal extension point used by separately packaged batch workers after
|
|
45
|
+
# they atomically claim additional jobs alongside the batch leader.
|
|
46
|
+
# Errors honor the registered worker's retry hooks and River's cancellation,
|
|
47
|
+
# snooze, and interruption signals. Timings use the row's claim timestamp.
|
|
48
|
+
def __finish_claimed_job(row, error = nil)
|
|
49
|
+
@runtime.finish_claimed(row, error)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def __perform_job(id, allow_scheduled: false)
|
|
53
|
+
@runtime.perform_job(id, allow_scheduled: allow_scheduled)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def __interrupt_workers = @runtime.interrupt_workers
|
|
57
|
+
|
|
58
|
+
def __runtime_healthy? = @runtime.healthy?
|
|
59
|
+
|
|
60
|
+
# Returns the client ID recorded on jobs while this client works them.
|
|
61
|
+
def id = config.id
|
|
62
|
+
|
|
32
63
|
# Inserts a new job for work given a job args implementation and insertion
|
|
33
64
|
# options (which may be omitted).
|
|
34
65
|
#
|
|
@@ -37,14 +68,14 @@ module River
|
|
|
37
68
|
# insert_res = client.insert(SimpleArgs.new(job_num: 1))
|
|
38
69
|
# insert_res.job # inserted job row
|
|
39
70
|
#
|
|
40
|
-
# With
|
|
71
|
+
# With keyword options:
|
|
41
72
|
#
|
|
42
|
-
# insert_res = client.insert(SimpleArgs.new(job_num: 1),
|
|
73
|
+
# insert_res = client.insert(SimpleArgs.new(job_num: 1), queue: :high_priority)
|
|
43
74
|
# insert_res.job # inserted job row
|
|
44
75
|
#
|
|
45
76
|
# Job arg implementations are expected to respond to:
|
|
46
77
|
#
|
|
47
|
-
# * `#kind`: A
|
|
78
|
+
# * `#kind`: A symbol or string identifying the job kind in the database.
|
|
48
79
|
# * `#to_json`: Encodes the args to JSON for persistence in the database.
|
|
49
80
|
# Must match encoding an args struct on the Go side to be workable.
|
|
50
81
|
#
|
|
@@ -64,15 +95,19 @@ module River
|
|
|
64
95
|
#
|
|
65
96
|
# def kind = "simple"
|
|
66
97
|
#
|
|
67
|
-
# def to_json = JSON.
|
|
98
|
+
# def to_json = JSON.generate({job_num: job_num})
|
|
68
99
|
# end
|
|
69
100
|
#
|
|
70
101
|
# See also JobArgsHash for an easy way to insert a job from a hash.
|
|
71
102
|
#
|
|
72
|
-
# Returns an instance of
|
|
73
|
-
|
|
103
|
+
# Returns an instance of JobInsertResult. A reusable InsertOpts may instead
|
|
104
|
+
# be supplied as +insert_opts:+; do not combine it with keyword options.
|
|
105
|
+
# @type method insert: (jobArgs, ?insert_opts: InsertOpts?, **untyped) -> JobInsertResult
|
|
106
|
+
def insert(args, insert_opts: nil, **options)
|
|
107
|
+
raise ArgumentError, "use insert_opts or keyword options, not both" if insert_opts && !options.empty?
|
|
108
|
+
insert_opts ||= options.empty? ? EMPTY_INSERT_OPTS : InsertOpts.new(**options)
|
|
74
109
|
insert_params = make_insert_params(args, insert_opts)
|
|
75
|
-
insert_and_check_unique_job(insert_params)
|
|
110
|
+
run_insert_plugins([insert_params]) { [insert_and_check_unique_job(insert_params)] }.first
|
|
76
111
|
end
|
|
77
112
|
|
|
78
113
|
# Inserts many new jobs as part of a single batch operation for improved
|
|
@@ -83,21 +118,21 @@ module River
|
|
|
83
118
|
#
|
|
84
119
|
# With job args:
|
|
85
120
|
#
|
|
86
|
-
#
|
|
121
|
+
# insert_results = client.insert_many([
|
|
87
122
|
# SimpleArgs.new(job_num: 1),
|
|
88
123
|
# SimpleArgs.new(job_num: 2)
|
|
89
124
|
# ])
|
|
90
125
|
#
|
|
91
126
|
# With InsertManyParams:
|
|
92
127
|
#
|
|
93
|
-
#
|
|
94
|
-
# River::InsertManyParams.new(SimpleArgs.new(job_num: 1),
|
|
95
|
-
# River::InsertManyParams.new(SimpleArgs.new(job_num: 2),
|
|
128
|
+
# insert_results = client.insert_many([
|
|
129
|
+
# River::InsertManyParams.new(SimpleArgs.new(job_num: 1), max_attempts: 5),
|
|
130
|
+
# River::InsertManyParams.new(SimpleArgs.new(job_num: 2), queue: :high_priority)
|
|
96
131
|
# ])
|
|
97
132
|
#
|
|
98
133
|
# Job arg implementations are expected to respond to:
|
|
99
134
|
#
|
|
100
|
-
# * `#kind`: A
|
|
135
|
+
# * `#kind`: A symbol or string identifying the job kind in the database.
|
|
101
136
|
# * `#to_json`: Encodes the args to JSON for persistence in the database.
|
|
102
137
|
# Must match encoding an args struct on the Go side to be workable.
|
|
103
138
|
#
|
|
@@ -112,13 +147,18 @@ module River
|
|
|
112
147
|
#
|
|
113
148
|
# def kind = "simple"
|
|
114
149
|
#
|
|
115
|
-
# def to_json = JSON.
|
|
150
|
+
# def to_json = JSON.generate({job_num: job_num})
|
|
116
151
|
# end
|
|
117
152
|
#
|
|
118
153
|
# See also JobArgsHash for an easy way to insert a job from a hash.
|
|
119
154
|
#
|
|
120
|
-
# Returns
|
|
155
|
+
# Returns one JobInsertResult for each argument, in input order.
|
|
156
|
+
# Raises ArgumentError if the batch is empty.
|
|
157
|
+
# A batch cannot contain repeated unique keys in their enforced states;
|
|
158
|
+
# the whole batch fails instead of partially inserting it.
|
|
121
159
|
def insert_many(args)
|
|
160
|
+
raise ArgumentError, "no jobs to insert" if args.empty?
|
|
161
|
+
|
|
122
162
|
all_params = args.map do |arg|
|
|
123
163
|
if arg.is_a?(InsertManyParams)
|
|
124
164
|
make_insert_params(arg.args, arg.insert_opts || EMPTY_INSERT_OPTS)
|
|
@@ -127,10 +167,185 @@ module River
|
|
|
127
167
|
end
|
|
128
168
|
end
|
|
129
169
|
|
|
130
|
-
|
|
131
|
-
.
|
|
132
|
-
|
|
170
|
+
run_insert_plugins(all_params) do
|
|
171
|
+
@driver.job_insert_many(all_params)
|
|
172
|
+
.map do |job, unique_skipped_as_duplicate|
|
|
173
|
+
JobInsertResult.new(job, unique_skipped_as_duplicated: unique_skipped_as_duplicate)
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
# Cancels a job by ID and returns its updated JobRow.
|
|
179
|
+
#
|
|
180
|
+
# Raises NotFoundError if the job does not exist.
|
|
181
|
+
def job_cancel(id)
|
|
182
|
+
@driver.job_cancel(id) || raise(NotFoundError, "job not found: #{id}")
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
# Deletes a job by ID and returns its former JobRow.
|
|
186
|
+
#
|
|
187
|
+
# Raises NotFoundError if the job does not exist and JobRunningError if it
|
|
188
|
+
# is currently running.
|
|
189
|
+
def job_delete(id)
|
|
190
|
+
job = @driver.job_delete(id) || raise(NotFoundError, "job not found: #{id}")
|
|
191
|
+
raise JobRunningError, "running jobs cannot be deleted" if job.state == JOB_STATE_RUNNING
|
|
192
|
+
|
|
193
|
+
job
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
# Deletes jobs matching the supplied filters and returns a
|
|
197
|
+
# JobDeleteManyResult. At least one filter is required. Accepts keyword
|
|
198
|
+
# filters or a reusable JobListParams, but not both.
|
|
199
|
+
# @type method job_delete_many: (?JobListParams?, **untyped) -> untyped
|
|
200
|
+
def job_delete_many(params = nil, **filters)
|
|
201
|
+
raise ArgumentError, "use params or keyword filters, not both" if params && !filters.empty?
|
|
202
|
+
params ||= JobListParams.new(**filters)
|
|
203
|
+
raise ArgumentError, "delete with no filters is not allowed" unless params.filters?
|
|
204
|
+
|
|
205
|
+
JobDeleteManyResult.new(@driver.job_delete_many(params))
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
# Fetches a job by ID.
|
|
209
|
+
#
|
|
210
|
+
# Raises NotFoundError if the job does not exist.
|
|
211
|
+
def job_get(id)
|
|
212
|
+
@driver.job_get_by_id(id) || raise(NotFoundError, "job not found: #{id}")
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
# Lists jobs matching the supplied filters and ordering.
|
|
216
|
+
#
|
|
217
|
+
# The returned JobListResult includes a cursor suitable for the next page.
|
|
218
|
+
# Accepts keyword filters or a reusable JobListParams, but not both.
|
|
219
|
+
# @type method job_list: (?JobListParams?, **untyped) -> untyped
|
|
220
|
+
def job_list(params = nil, **filters)
|
|
221
|
+
raise ArgumentError, "use params or keyword filters, not both" if params && !filters.empty?
|
|
222
|
+
params ||= JobListParams.new(**filters)
|
|
223
|
+
jobs = @driver.job_list(params)
|
|
224
|
+
last = jobs.last
|
|
225
|
+
cursor = last && JobListCursor.new(id: last.id, sort_by: params.sort_by, sort_order: params.sort_order, value: last.public_send(params.sort_by))
|
|
226
|
+
JobListResult.new(jobs, cursor)
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
# Makes a non-running job immediately available for another attempt and
|
|
230
|
+
# returns its updated JobRow. Raises NotFoundError if it does not exist.
|
|
231
|
+
# Clears any cancellation request from its previous attempt. Raises
|
|
232
|
+
# ArgumentError if another attempt would exceed MAX_ATTEMPTS_LIMIT.
|
|
233
|
+
def job_retry(id)
|
|
234
|
+
@driver.job_retry(id) || raise(NotFoundError, "job not found: #{id}")
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
# Applies keyword attributes or a JobUpdateParams to a job and returns its
|
|
238
|
+
# updated JobRow. Explicit nil clears nullable fields; omitted fields are
|
|
239
|
+
# unchanged. Metadata must be a Hash; use an empty Hash to clear it.
|
|
240
|
+
# attempt must be between 0 and MAX_ATTEMPTS_LIMIT (32,767);
|
|
241
|
+
# max_attempts must be between 1 and MAX_ATTEMPTS_LIMIT.
|
|
242
|
+
# Raises NotFoundError if the job does not exist.
|
|
243
|
+
# @type method job_update: (Integer, ?JobUpdateParams?, **untyped) -> JobRow
|
|
244
|
+
def job_update(id, params = nil, **attributes)
|
|
245
|
+
raise ArgumentError, "use params or keyword attributes, not both" if params && !attributes.empty?
|
|
246
|
+
params ||= JobUpdateParams.new(**attributes)
|
|
247
|
+
@driver.job_update(id, params) || raise(NotFoundError, "job not found: #{id}")
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
# Returns the live PeriodicJobBundle used to add and remove periodic jobs.
|
|
251
|
+
def periodic_jobs
|
|
252
|
+
raise ArgumentError, "cannot modify periodic jobs when leader_election_disabled is true" if @config.leader_election_disabled
|
|
253
|
+
|
|
254
|
+
@runtime.periodic_jobs
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
# Adds a queue to the client and returns self. If the client is running, it
|
|
258
|
+
# begins working the queue immediately.
|
|
259
|
+
def queue_add(name, queue_config)
|
|
260
|
+
@runtime.queue_add(name.to_s, queue_config)
|
|
261
|
+
self
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
# Fetches a queue by name.
|
|
265
|
+
#
|
|
266
|
+
# Raises NotFoundError if the queue does not exist.
|
|
267
|
+
def queue_get(name)
|
|
268
|
+
@driver.queue_get(name.to_s) || raise(NotFoundError, "queue not found: #{name}")
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
# Lists up to +max+ known queues.
|
|
272
|
+
def queue_list(max: 100)
|
|
273
|
+
QueueListResult.new(@driver.queue_list(max: max))
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
# Pauses a named queue, or all queues when +name+ is +"*"+.
|
|
277
|
+
def queue_pause(name)
|
|
278
|
+
name = name.to_s
|
|
279
|
+
queues = @driver.queue_pause(name)
|
|
280
|
+
@runtime.wake
|
|
281
|
+
queues.each do |queue|
|
|
282
|
+
@runtime.publish_queue(EVENT_QUEUE_PAUSED, queue)
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
true
|
|
286
|
+
end
|
|
287
|
+
|
|
288
|
+
# Removes a configured queue, waits for active jobs in it to finish, and
|
|
289
|
+
# returns self.
|
|
290
|
+
# Raises ThreadError if called from that queue's own worker or producer.
|
|
291
|
+
def queue_remove(name)
|
|
292
|
+
@runtime.queue_remove(name.to_s)
|
|
293
|
+
self
|
|
294
|
+
end
|
|
295
|
+
|
|
296
|
+
# Resumes a named queue, or all queues when +name+ is +"*"+.
|
|
297
|
+
def queue_resume(name)
|
|
298
|
+
name = name.to_s
|
|
299
|
+
queues = @driver.queue_resume(name)
|
|
300
|
+
@runtime.wake
|
|
301
|
+
queues.each do |queue|
|
|
302
|
+
@runtime.publish_queue(EVENT_QUEUE_RESUMED, queue)
|
|
133
303
|
end
|
|
304
|
+
|
|
305
|
+
true
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
# Replaces a queue's metadata and returns the updated Queue.
|
|
309
|
+
def queue_update(name, metadata:)
|
|
310
|
+
@driver.queue_update(name.to_s, metadata: metadata) || raise(NotFoundError, "queue not found: #{name}")
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
# Starts polling configured queues and working jobs in background threads.
|
|
314
|
+
# Returns self.
|
|
315
|
+
def start
|
|
316
|
+
@runtime.start
|
|
317
|
+
self
|
|
318
|
+
end
|
|
319
|
+
|
|
320
|
+
# Returns true while the client runtime is started.
|
|
321
|
+
def started? = @runtime.started?
|
|
322
|
+
|
|
323
|
+
# Stops fetching and producing periodic jobs, waits for active jobs to
|
|
324
|
+
# finish, and returns self. Pass wait: false to request stop without
|
|
325
|
+
# waiting; call stop again to finish draining and release resources.
|
|
326
|
+
# In-flight fetches or maintenance operations may finish. Until a waiting
|
|
327
|
+
# stop completes, started? remains true and stopped? remains false.
|
|
328
|
+
# Raises ThreadError for a waiting call from this client's runtime threads.
|
|
329
|
+
def stop(wait: true)
|
|
330
|
+
@runtime.stop(wait: wait)
|
|
331
|
+
self
|
|
332
|
+
end
|
|
333
|
+
|
|
334
|
+
# Stops fetching new jobs, interrupts active work, and returns self.
|
|
335
|
+
# Raises ThreadError if called from this client's own runtime threads.
|
|
336
|
+
def stop_and_cancel
|
|
337
|
+
@runtime.stop(cancel: true)
|
|
338
|
+
self
|
|
339
|
+
end
|
|
340
|
+
|
|
341
|
+
# Returns true when the client runtime is fully stopped.
|
|
342
|
+
def stopped? = @runtime.stopped?
|
|
343
|
+
|
|
344
|
+
# Subscribes to event kinds and returns a Subscription.
|
|
345
|
+
#
|
|
346
|
+
# Call Subscription#close when the subscription is no longer needed.
|
|
347
|
+
def subscribe(*kinds, buffer_size: 100)
|
|
348
|
+
@runtime.subscribe(kinds, buffer_size: buffer_size)
|
|
134
349
|
end
|
|
135
350
|
|
|
136
351
|
# Default states that are used during a unique insert. Can be overridden by
|
|
@@ -143,7 +358,10 @@ module River
|
|
|
143
358
|
JOB_STATE_RUNNING,
|
|
144
359
|
JOB_STATE_SCHEDULED
|
|
145
360
|
].freeze
|
|
146
|
-
|
|
361
|
+
|
|
362
|
+
EMPTY_INSERT_OPTS = InsertOpts.new.freeze
|
|
363
|
+
|
|
364
|
+
INITIAL_STATES = [JOB_STATE_AVAILABLE, JOB_STATE_PENDING, JOB_STATE_SCHEDULED].freeze
|
|
147
365
|
|
|
148
366
|
REQUIRED_UNIQUE_STATES = [
|
|
149
367
|
JOB_STATE_AVAILABLE,
|
|
@@ -151,22 +369,22 @@ module River
|
|
|
151
369
|
JOB_STATE_RUNNING,
|
|
152
370
|
JOB_STATE_SCHEDULED
|
|
153
371
|
].freeze
|
|
154
|
-
private_constant :REQUIRED_UNIQUE_STATES
|
|
155
372
|
|
|
156
|
-
|
|
157
|
-
|
|
373
|
+
TAG_RE = /\A\w[\w-]+\w\z/
|
|
374
|
+
|
|
375
|
+
private_constant :DEFAULT_UNIQUE_STATES, :EMPTY_INSERT_OPTS, :INITIAL_STATES, :REQUIRED_UNIQUE_STATES, :TAG_RE
|
|
158
376
|
|
|
159
377
|
private def insert_and_check_unique_job(insert_params)
|
|
160
378
|
job, unique_skipped_as_duplicate = @driver.job_insert(insert_params)
|
|
161
|
-
|
|
379
|
+
JobInsertResult.new(job, unique_skipped_as_duplicated: unique_skipped_as_duplicate)
|
|
162
380
|
end
|
|
163
381
|
|
|
164
382
|
private def make_insert_params(args, insert_opts)
|
|
165
|
-
raise "args should respond to `#kind`" if !args.respond_to?(:kind)
|
|
383
|
+
raise ArgumentError, "args should respond to `#kind`" if !args.respond_to?(:kind)
|
|
166
384
|
|
|
167
385
|
# ~all objects in Ruby respond to `#to_json`, so check non-nil instead.
|
|
168
386
|
args_json = args.to_json
|
|
169
|
-
raise "args should return non-nil from `#to_json`" if !args_json
|
|
387
|
+
raise ArgumentError, "args should return non-nil from `#to_json`" if !args_json
|
|
170
388
|
|
|
171
389
|
args_insert_opts = if args.respond_to?(:insert_opts)
|
|
172
390
|
args_with_insert_opts = args #: _JobArgsWithInsertOpts # rubocop:disable Layout/LeadingCommentSpace
|
|
@@ -175,25 +393,37 @@ module River
|
|
|
175
393
|
EMPTY_INSERT_OPTS
|
|
176
394
|
end
|
|
177
395
|
|
|
396
|
+
max_attempts = insert_opts.max_attempts || args_insert_opts.max_attempts || MAX_ATTEMPTS_DEFAULT
|
|
397
|
+
raise ArgumentError, "max_attempts must be greater than zero" unless max_attempts > 0
|
|
398
|
+
raise ArgumentError, "max_attempts must not exceed #{MAX_ATTEMPTS_LIMIT}" if max_attempts > MAX_ATTEMPTS_LIMIT
|
|
399
|
+
|
|
400
|
+
queue = (insert_opts.queue || args_insert_opts.queue || QUEUE_DEFAULT).to_s
|
|
401
|
+
raise ArgumentError, "invalid queue name: #{queue.inspect}" unless queue.match?(QUEUE_NAME_REGEX) && queue.length < 128
|
|
402
|
+
|
|
178
403
|
scheduled_at = insert_opts.scheduled_at || args_insert_opts.scheduled_at
|
|
404
|
+
now = @time_now_utc.call
|
|
405
|
+
state = validate_insert_state(insert_opts.state || args_insert_opts.state || ((scheduled_at && scheduled_at > now) ? JOB_STATE_SCHEDULED : JOB_STATE_AVAILABLE))
|
|
179
406
|
|
|
180
407
|
insert_params = Driver::JobInsertParams.new(
|
|
408
|
+
args: args,
|
|
181
409
|
encoded_args: args_json,
|
|
182
|
-
kind: args.kind,
|
|
183
|
-
max_attempts:
|
|
410
|
+
kind: args.kind.to_s,
|
|
411
|
+
max_attempts: max_attempts,
|
|
412
|
+
metadata: (args_insert_opts.metadata || {}).merge(insert_opts.metadata || {}),
|
|
184
413
|
priority: insert_opts.priority || args_insert_opts.priority || PRIORITY_DEFAULT,
|
|
185
|
-
queue:
|
|
186
|
-
scheduled_at: scheduled_at&.
|
|
187
|
-
state:
|
|
414
|
+
queue: queue,
|
|
415
|
+
scheduled_at: scheduled_at&.getutc || now,
|
|
416
|
+
state: state,
|
|
188
417
|
tags: validate_tags(insert_opts.tags || args_insert_opts.tags || [])
|
|
189
418
|
)
|
|
190
419
|
|
|
191
420
|
unique_opts = insert_opts.unique_opts || args_insert_opts.unique_opts
|
|
192
|
-
if unique_opts
|
|
421
|
+
if unique_opts && !unique_opts.empty?
|
|
193
422
|
unique_key, unique_states = make_unique_key_and_bitmask(insert_params, unique_opts)
|
|
194
423
|
insert_params.unique_key = unique_key
|
|
195
424
|
insert_params.unique_states = unique_states
|
|
196
425
|
end
|
|
426
|
+
|
|
197
427
|
insert_params
|
|
198
428
|
end
|
|
199
429
|
|
|
@@ -208,18 +438,13 @@ module River
|
|
|
208
438
|
end
|
|
209
439
|
|
|
210
440
|
if unique_opts.by_args
|
|
211
|
-
|
|
212
|
-
filtered_args = if unique_opts.by_args.is_a?(Array)
|
|
213
|
-
parsed_args.slice(*unique_opts.by_args)
|
|
214
|
-
else
|
|
215
|
-
parsed_args
|
|
216
|
-
end
|
|
217
|
-
|
|
218
|
-
encoded_args = JSON.generate(filtered_args.sort.to_h)
|
|
441
|
+
encoded_args = UniqueArgs.encode(insert_params.encoded_args, unique_opts.by_args)
|
|
219
442
|
unique_key += "&args=#{encoded_args}"
|
|
220
443
|
end
|
|
221
444
|
|
|
222
|
-
if unique_opts.by_period
|
|
445
|
+
if unique_opts.by_period && unique_opts.by_period != 0
|
|
446
|
+
raise ArgumentError, "by_period should not be less than 1 second" if unique_opts.by_period < 1
|
|
447
|
+
|
|
223
448
|
lower_period_bound = truncate_time(insert_params.scheduled_at || @time_now_utc.call, unique_opts.by_period).utc
|
|
224
449
|
|
|
225
450
|
unique_key += "&period=#{lower_period_bound.strftime("%FT%TZ")}"
|
|
@@ -229,18 +454,66 @@ module River
|
|
|
229
454
|
unique_key += "&queue=#{insert_params.queue}"
|
|
230
455
|
end
|
|
231
456
|
|
|
457
|
+
# Without kind or another dimension, every job would share the same key.
|
|
458
|
+
raise ArgumentError, "exclude_kind requires by_args, by_queue, or by_period" if unique_key.empty?
|
|
459
|
+
|
|
232
460
|
unique_key_hash = Digest::SHA256.digest(unique_key)
|
|
233
|
-
|
|
461
|
+
states = unique_opts.by_state
|
|
462
|
+
unique_states = validate_unique_states((states && !states.empty?) ? states : DEFAULT_UNIQUE_STATES)
|
|
234
463
|
|
|
235
464
|
[unique_key_hash, UniqueBitmask.from_states(unique_states)]
|
|
236
465
|
end
|
|
237
466
|
|
|
467
|
+
private def run_insert_plugins(all_params, &insert_operation)
|
|
468
|
+
unless config.plugins.any? { |plugin| plugin.respond_to?(:insert_begin) || plugin.respond_to?(:insert_end) || plugin.respond_to?(:insert_many) }
|
|
469
|
+
results = insert_operation.call
|
|
470
|
+
@runtime.wake
|
|
471
|
+
return results
|
|
472
|
+
end
|
|
473
|
+
|
|
474
|
+
operation = -> do
|
|
475
|
+
all_params.each do |insert_params|
|
|
476
|
+
config.plugins.each do |plugin|
|
|
477
|
+
plugin.insert_begin(insert_params) if plugin.respond_to?(:insert_begin)
|
|
478
|
+
end
|
|
479
|
+
end
|
|
480
|
+
|
|
481
|
+
# Hooks may change any job in the batch, including an earlier entry.
|
|
482
|
+
# Validate the final states only after all insertion hooks have run.
|
|
483
|
+
all_params.each { |params| params.state = validate_insert_state(params.state) }
|
|
484
|
+
|
|
485
|
+
results = insert_operation.call
|
|
486
|
+
results.each do |result|
|
|
487
|
+
config.plugins.reverse_each do |plugin|
|
|
488
|
+
plugin.insert_end(result) if plugin.respond_to?(:insert_end)
|
|
489
|
+
end
|
|
490
|
+
end
|
|
491
|
+
|
|
492
|
+
results
|
|
493
|
+
end
|
|
494
|
+
|
|
495
|
+
config.plugins.reverse_each do |plugin|
|
|
496
|
+
next unless plugin.respond_to?(:insert_many)
|
|
497
|
+
|
|
498
|
+
next_operation = operation
|
|
499
|
+
operation = -> { plugin.insert_many(all_params, next_operation) }
|
|
500
|
+
end
|
|
501
|
+
|
|
502
|
+
results = @driver.transaction { operation.call }
|
|
503
|
+
@runtime.wake
|
|
504
|
+
results
|
|
505
|
+
end
|
|
506
|
+
|
|
238
507
|
# Truncates the given time down to the interval. For example:
|
|
239
508
|
#
|
|
240
509
|
# Thu Jan 15 21:26:36 UTC 2024 @ 15 minutes ->
|
|
241
510
|
# Thu Jan 15 21:15:00 UTC 2024
|
|
242
511
|
private def truncate_time(time, interval_seconds)
|
|
243
|
-
|
|
512
|
+
# Go truncates from year 1, not the Unix epoch. Rational arithmetic also
|
|
513
|
+
# avoids floating-point rounding across a period boundary.
|
|
514
|
+
epoch_offset = 62_135_596_800
|
|
515
|
+
interval = interval_seconds.to_r
|
|
516
|
+
Time.at(((time.to_r + epoch_offset) / interval).floor * interval - epoch_offset)
|
|
244
517
|
end
|
|
245
518
|
|
|
246
519
|
# Moves an integer that may occupy the entire uint64 space to one that's
|
|
@@ -249,8 +522,12 @@ module River
|
|
|
249
522
|
[int].pack("Q").unpack1("q") #: Integer # rubocop:disable Layout/LeadingCommentSpace
|
|
250
523
|
end
|
|
251
524
|
|
|
252
|
-
|
|
253
|
-
|
|
525
|
+
private def validate_insert_state(state)
|
|
526
|
+
state = state.to_s #: jobStateInitial # rubocop:disable Layout/LeadingCommentSpace
|
|
527
|
+
raise ArgumentError, "invalid insertion state: #{state.inspect}; must be available, pending, or scheduled" unless INITIAL_STATES.include?(state)
|
|
528
|
+
|
|
529
|
+
state
|
|
530
|
+
end
|
|
254
531
|
|
|
255
532
|
private def validate_tags(tags)
|
|
256
533
|
tags.each do |tag|
|
|
@@ -260,9 +537,11 @@ module River
|
|
|
260
537
|
end
|
|
261
538
|
|
|
262
539
|
private def validate_unique_states(states)
|
|
540
|
+
states = states.map(&:to_s) #: Array[jobStateAll] # rubocop:disable Layout/LeadingCommentSpace
|
|
263
541
|
REQUIRED_UNIQUE_STATES.each do |required_state|
|
|
264
542
|
raise ArgumentError, "by_state should include required state #{required_state}" unless states.include?(required_state)
|
|
265
543
|
end
|
|
544
|
+
|
|
266
545
|
states
|
|
267
546
|
end
|
|
268
547
|
end
|
|
@@ -276,25 +555,39 @@ module River
|
|
|
276
555
|
# Insertion options to use with the insert.
|
|
277
556
|
attr_reader :insert_opts
|
|
278
557
|
|
|
279
|
-
|
|
558
|
+
# Pairs job arguments with per-job keyword options for Client#insert_many.
|
|
559
|
+
# A reusable InsertOpts may be supplied instead as +insert_opts:+.
|
|
560
|
+
# @type method initialize: (jobArgs, ?insert_opts: InsertOpts?, **untyped) -> void
|
|
561
|
+
def initialize(args, insert_opts: nil, **options)
|
|
562
|
+
raise ArgumentError, "use insert_opts or keyword options, not both" if insert_opts && !options.empty?
|
|
563
|
+
|
|
280
564
|
@args = args
|
|
281
|
-
@insert_opts = insert_opts
|
|
565
|
+
@insert_opts = insert_opts || (InsertOpts.new(**options) unless options.empty?)
|
|
282
566
|
end
|
|
283
567
|
end
|
|
284
568
|
|
|
285
569
|
# Result of a single insertion.
|
|
286
|
-
class
|
|
570
|
+
class JobInsertResult
|
|
287
571
|
# Inserted job row, or an existing job row if insert was skipped due to a
|
|
288
572
|
# previously existing unique job.
|
|
289
573
|
attr_reader :job
|
|
290
574
|
|
|
291
575
|
# True if for a unique job, the insertion was skipped due to an equivalent
|
|
292
576
|
# job matching unique property already being present.
|
|
293
|
-
|
|
577
|
+
def unique_skipped_as_duplicate? = @unique_skipped_as_duplicate
|
|
578
|
+
|
|
579
|
+
# Compatibility with the spelling used in earlier releases.
|
|
580
|
+
alias_method :unique_skipped_as_duplicated, :unique_skipped_as_duplicate?
|
|
294
581
|
|
|
582
|
+
# Creates an insertion result. Applications normally receive instances from
|
|
583
|
+
# Client#insert or Client#insert_many. The keyword retains its original
|
|
584
|
+
# spelling for compatibility with earlier releases.
|
|
295
585
|
def initialize(job, unique_skipped_as_duplicated:)
|
|
296
586
|
@job = job
|
|
297
|
-
@
|
|
587
|
+
@unique_skipped_as_duplicate = unique_skipped_as_duplicated
|
|
298
588
|
end
|
|
299
589
|
end
|
|
590
|
+
|
|
591
|
+
# Compatibility with the insertion result class name used in earlier releases.
|
|
592
|
+
InsertResult = JobInsertResult
|
|
300
593
|
end
|