riverqueue 0.11.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +132 -0
  3. data/LICENSE +374 -0
  4. data/docs/README.md +765 -0
  5. data/docs/migrations.md +121 -0
  6. data/docs/testing.md +173 -0
  7. data/docs/workers.md +126 -0
  8. data/exe/river +7 -0
  9. data/lib/cli.rb +55 -0
  10. data/lib/client.rb +314 -54
  11. data/lib/client_runtime.rb +612 -0
  12. data/lib/config.rb +154 -0
  13. data/lib/driver/job_row_decoder.rb +85 -0
  14. data/lib/driver/postgres_capabilities.rb +41 -0
  15. data/lib/driver/runtime.rb +661 -0
  16. data/lib/driver.rb +13 -6
  17. data/lib/errors.rb +83 -0
  18. data/lib/event.rb +78 -0
  19. data/lib/insert_opts.rb +40 -11
  20. data/lib/job.rb +28 -13
  21. data/lib/job_persisted_logging/plugin.rb +108 -0
  22. data/lib/migration_cli.rb +90 -0
  23. data/lib/migrator.rb +189 -0
  24. data/lib/params.rb +96 -0
  25. data/lib/periodic_cron.rb +24 -0
  26. data/lib/periodic_job.rb +135 -0
  27. data/lib/resumable.rb +148 -0
  28. data/lib/riverqueue/testing/minitest.rb +18 -0
  29. data/lib/riverqueue/testing/rspec.rb +109 -0
  30. data/lib/riverqueue/testing.rb +134 -0
  31. data/lib/riverqueue.rb +16 -0
  32. data/lib/unique_args.rb +97 -0
  33. data/lib/unique_bitmask.rb +14 -12
  34. data/lib/worker.rb +152 -0
  35. data/lib/worker_runner.rb +106 -0
  36. data/migration/LICENSE +374 -0
  37. data/migration/README.md +11 -0
  38. data/migration/manifest.json +38 -0
  39. data/migration/postgresql/main/001_create_river_migration.down.sql +1 -0
  40. data/migration/postgresql/main/001_create_river_migration.up.sql +8 -0
  41. data/migration/postgresql/main/002_initial_schema.down.sql +5 -0
  42. data/migration/postgresql/main/002_initial_schema.up.sql +96 -0
  43. data/migration/postgresql/main/003_river_job_tags_non_null.down.sql +3 -0
  44. data/migration/postgresql/main/003_river_job_tags_non_null.up.sql +3 -0
  45. data/migration/postgresql/main/004_pending_and_more.down.sql +42 -0
  46. data/migration/postgresql/main/004_pending_and_more.up.sql +45 -0
  47. data/migration/postgresql/main/005_migration_unique_client.down.sql +57 -0
  48. data/migration/postgresql/main/005_migration_unique_client.up.sql +79 -0
  49. data/migration/postgresql/main/006_bulk_unique.down.sql +16 -0
  50. data/migration/postgresql/main/006_bulk_unique.up.sql +40 -0
  51. data/migration/postgresql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql +56 -0
  52. data/migration/postgresql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql +44 -0
  53. data/migration/postgresql/main/008_job_id_autoincrement.down.sql +3 -0
  54. data/migration/postgresql/main/008_job_id_autoincrement.up.sql +3 -0
  55. data/migration/sqlite/main/001_create_river_migration.down.sql +1 -0
  56. data/migration/sqlite/main/001_create_river_migration.up.sql +8 -0
  57. data/migration/sqlite/main/002_initial_schema.down.sql +8 -0
  58. data/migration/sqlite/main/002_initial_schema.up.sql +19 -0
  59. data/migration/sqlite/main/003_river_job_tags_non_null.down.sql +6 -0
  60. data/migration/sqlite/main/003_river_job_tags_non_null.up.sql +6 -0
  61. data/migration/sqlite/main/004_pending_and_more.down.sql +26 -0
  62. data/migration/sqlite/main/004_pending_and_more.up.sql +33 -0
  63. data/migration/sqlite/main/005_migration_unique_client.down.sql +37 -0
  64. data/migration/sqlite/main/005_migration_unique_client.up.sql +64 -0
  65. data/migration/sqlite/main/006_bulk_unique.down.sql +7 -0
  66. data/migration/sqlite/main/006_bulk_unique.up.sql +63 -0
  67. data/migration/sqlite/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql +255 -0
  68. data/migration/sqlite/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql +261 -0
  69. data/migration/sqlite/main/008_job_id_autoincrement.down.sql +121 -0
  70. data/migration/sqlite/main/008_job_id_autoincrement.up.sql +123 -0
  71. data/sig/client.rbs +95 -0
  72. data/sig/driver.rbs +170 -0
  73. data/sig/insert_opts.rbs +25 -0
  74. data/sig/job.rbs +72 -0
  75. data/sig/job_persisted_logging/plugin.rbs +29 -0
  76. data/sig/migrator.rbs +44 -0
  77. data/sig/riverqueue.rbs +2 -0
  78. data/sig/runtime.rbs +382 -0
  79. data/sig/testing.rbs +95 -0
  80. data/sig/unique_args.rbs +8 -0
  81. data/sig/unique_bitmask.rbs +8 -0
  82. data/sig/worker_runner.rbs +22 -0
  83. metadata +165 -10
@@ -0,0 +1,661 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "time"
4
+
5
+ module River::Driver
6
+ # Database operations used by River's worker runtime. Driver gems implement a
7
+ # small set of raw SQL primitives and include this module so the state machine
8
+ # stays identical across ActiveRecord and Sequel.
9
+ module Runtime
10
+ def init_driver
11
+ postgres_capabilities if runtime_postgres?
12
+ end
13
+
14
+ def postgres_capabilities
15
+ @postgres_capabilities_cache.fetch(runtime_connection_pool) do
16
+ row = runtime_query_rows(<<~SQL).first
17
+ SELECT version()::text AS product,
18
+ current_setting('server_version_num')::int AS version_num,
19
+ coalesce(current_setting('yb_enable_listen_notify', true), 'off')::boolean AS yb_listen_notify_enabled
20
+ SQL
21
+ PostgresCapabilities.new(
22
+ product: runtime_value(row, :product),
23
+ version_num: runtime_value(row, :version_num),
24
+ yb_listen_notify_enabled: runtime_value(row, :yb_listen_notify_enabled) == true
25
+ )
26
+ end
27
+ end
28
+
29
+ # Notifications and rows must commit together. The insertion drivers call
30
+ # this inside their transaction; PostgreSQL defers delivery until commit.
31
+ private def postgres_notify_insert(params)
32
+ queues = params.select { |param| param.state == River::JOB_STATE_AVAILABLE }.map(&:queue).uniq
33
+ return if queues.empty?
34
+ return unless postgres_capabilities.supports_listen_notify
35
+
36
+ payloads = queues.map { |queue| "(#{runtime_quote(JSON.generate(queue: queue))})" }.join(",")
37
+ runtime_execute("SELECT pg_notify(current_schema() || '.river_insert', payload) FROM (VALUES #{payloads}) AS notifications(payload)")
38
+ end
39
+
40
+ def job_cancel(id, now: Time.now.utc)
41
+ original_id = Integer(id)
42
+ transaction do
43
+ updated_id = runtime_returning_ids(<<~SQL).first
44
+ UPDATE river_job
45
+ SET state = CASE WHEN state = 'running' THEN state ELSE 'cancelled' END,
46
+ finalized_at = CASE WHEN state = 'running' THEN finalized_at ELSE #{runtime_time(now)} END,
47
+ metadata = #{runtime_merge_metadata("cancel_attempted_at" => now.iso8601(6))}
48
+ WHERE id = #{original_id}
49
+ AND state NOT IN ('cancelled', 'completed', 'discarded')
50
+ AND finalized_at IS NULL
51
+ RETURNING id
52
+ SQL
53
+ row = job_get_by_id(updated_id || original_id)
54
+ # Other engines interrupt running workers through this notification;
55
+ # the durable marker remains Ruby's polling fallback.
56
+ runtime_notify("river_control", action: "cancel", job_id: row.id, queue: row.queue) if updated_id
57
+ row
58
+ end
59
+ end
60
+
61
+ def job_claim(id:, attempted_by:, allow_scheduled: false, now: Time.now.utc)
62
+ predicate = "id = #{Integer(id)} AND state IN ('available', 'retryable', 'scheduled')"
63
+ predicate += " AND scheduled_at <= #{runtime_time(now)}" unless allow_scheduled
64
+
65
+ runtime_claim_jobs(predicate, 1, attempted_by, now).first
66
+ end
67
+
68
+ # Drivers may combine the cancellation probe and completion atomically.
69
+ # :cancelled asks the runtime to apply its normal cancellation/error hooks.
70
+ def job_complete(id:, finalized_at:, metadata: nil, now: Time.now.utc)
71
+ id = Integer(id)
72
+ return :cancelled if job_get_cancelled_ids([id]).include?(id)
73
+
74
+ job_set_state_if_running(id: id, finalized_at: finalized_at, metadata: metadata, now: now, state: "completed")
75
+ end
76
+
77
+ def job_delete(id)
78
+ existing = job_get_by_id(id)
79
+ return nil unless existing
80
+ return existing if existing.state == River::JOB_STATE_RUNNING
81
+
82
+ deleted_id = runtime_returning_ids("DELETE FROM river_job WHERE id = #{Integer(id)} AND state != 'running' RETURNING id").first
83
+ deleted_id ? existing : job_get_by_id(id)
84
+ end
85
+
86
+ def job_delete_finalized(retention:, now: Time.now.utc, max: 1_000)
87
+ clauses = retention.filter_map do |state, seconds|
88
+ next unless seconds
89
+
90
+ "(state = #{runtime_quote(state.to_s)} AND finalized_at < #{runtime_time(now - seconds)})"
91
+ end
92
+
93
+ return 0 if clauses.empty?
94
+
95
+ ids = runtime_query_rows(<<~SQL).map { |row| runtime_value(row, :id).to_i }
96
+ SELECT id FROM river_job WHERE #{clauses.join(" OR ")} ORDER BY id LIMIT #{Integer(max)}
97
+ SQL
98
+ return 0 if ids.empty?
99
+
100
+ runtime_returning_ids(<<~SQL).length
101
+ DELETE FROM river_job
102
+ WHERE id IN (#{ids.join(",")}) AND (#{clauses.join(" OR ")})
103
+ RETURNING id
104
+ SQL
105
+ end
106
+
107
+ def job_delete_if_running(id)
108
+ runtime_returning_ids("DELETE FROM river_job WHERE id = #{Integer(id)} AND state = 'running' RETURNING id").any?
109
+ end
110
+
111
+ def job_delete_many(params)
112
+ transaction do
113
+ jobs = job_list(params).reject { |job| job.state == River::JOB_STATE_RUNNING }
114
+ next [] if jobs.empty?
115
+
116
+ ids = jobs.map(&:id)
117
+ deleted = runtime_returning_ids(<<~SQL)
118
+ DELETE FROM river_job
119
+ WHERE id IN (#{ids.join(",")}) AND state != 'running'
120
+ RETURNING id
121
+ SQL
122
+ jobs.select { |job| deleted.include?(job.id) }
123
+ end
124
+ end
125
+
126
+ def job_get_available(queue:, max:, attempted_by:, kinds: nil, now: Time.now.utc)
127
+ return [] if kinds&.empty?
128
+
129
+ predicate = "state = 'available' AND queue = #{runtime_quote(queue)} AND scheduled_at <= #{runtime_time(now)}"
130
+ predicate += " AND #{runtime_in_clause("kind", kinds)}" if kinds
131
+ runtime_claim_jobs(predicate, max, attempted_by, now)
132
+ end
133
+
134
+ def job_get_cancelled_ids(ids)
135
+ return [] if ids.empty?
136
+
137
+ runtime_query_rows("SELECT id FROM river_job WHERE id IN (#{ids.map { |id| Integer(id) }.join(",")}) AND (#{runtime_cancel_attempted}) ORDER BY id")
138
+ .map { |row| runtime_value(row, :id).to_i }
139
+ end
140
+
141
+ def job_list(params = nil)
142
+ params ||= River::JobListParams.new
143
+ return runtime_job_list_without_params if params == :all
144
+
145
+ clauses = [] #: Array[String]
146
+ clauses << runtime_cursor_clause(params) if params.after
147
+ clauses << "id #{(params.sort_order == :asc) ? ">" : "<"} #{Integer(params.after_id)}" if params.after_id
148
+ clauses << runtime_in_clause("id", params.ids.map { |value| Integer(value) }) if params.ids&.any?
149
+ clauses << runtime_in_clause("kind", params.kinds) if params.kinds&.any?
150
+ clauses << runtime_in_clause("priority", params.priorities.map { |value| Integer(value) }) if params.priorities&.any?
151
+ clauses << runtime_in_clause("queue", params.queues) if params.queues&.any?
152
+ clauses << runtime_in_clause("state", params.states) if params.states&.any?
153
+
154
+ finalized = params.sort_by == :finalized_at && params.states&.length == 1 &&
155
+ %w[cancelled completed discarded].include?(params.states.first)
156
+ # Schemas require finalized timestamps for terminal states. Spell this out
157
+ # so PostgreSQL can use the partial (state, finalized_at) index.
158
+ clauses << "finalized_at IS NOT NULL" if finalized
159
+ # Explicit NULLS LAST prevents a backward index scan, even when there are
160
+ # no nulls. Only request it for timestamps that can actually be null.
161
+ null_order = (params.sort_by == :finalized_at && !finalized) ? " NULLS LAST" : ""
162
+
163
+ params.metadata&.each { |key, value| clauses << runtime_metadata_equals(key, value) }
164
+ Array(params.tags_all).each { |tag| clauses << runtime_tag_contains(tag) }
165
+ if params.tags_any&.any?
166
+ clauses << "(" + params.tags_any.map { |tag| runtime_tag_contains(tag) }.join(" OR ") + ")"
167
+ end
168
+
169
+ where = clauses.empty? ? "" : "WHERE #{clauses.join(" AND ")}"
170
+ runtime_job_rows(<<~SQL)
171
+ #{where}
172
+ ORDER BY #{params.sort_by} #{params.sort_order.to_s.upcase}#{null_order}, id #{params.sort_order.to_s.upcase}
173
+ LIMIT #{params.limit}
174
+ SQL
175
+ end
176
+
177
+ def job_metadata_merge(id, metadata)
178
+ updated_id = runtime_returning_ids(<<~SQL).first
179
+ UPDATE river_job
180
+ SET metadata = #{runtime_merge_metadata(metadata)}
181
+ WHERE id = #{Integer(id)}
182
+ RETURNING id
183
+ SQL
184
+ updated_id ? job_get_by_id(updated_id) : nil
185
+ end
186
+
187
+ def job_rescue_stuck(horizon:, retry_policy:, now: Time.now.utc, max: 1_000)
188
+ transaction do
189
+ # Select only stuck jobs before applying the limit, and hold their locks
190
+ # until rescue finishes so a newer attempt cannot be rescued by mistake.
191
+ lock = runtime_postgres? ? "FOR UPDATE SKIP LOCKED" : ""
192
+ ids = runtime_returning_ids(<<~SQL)
193
+ SELECT id FROM river_job
194
+ WHERE state = 'running' AND attempted_at < #{runtime_time(horizon)}
195
+ ORDER BY id LIMIT #{Integer(max)} #{lock}
196
+ SQL
197
+ jobs = ids.map { |id| runtime_read_job(id) }
198
+ jobs.each do |job|
199
+ cancelled = job.metadata.key?("cancel_attempted_at")
200
+ final = cancelled || job.attempt >= job.max_attempts
201
+ state = if cancelled
202
+ River::JOB_STATE_CANCELLED
203
+ elsif final
204
+ River::JOB_STATE_DISCARDED
205
+ else
206
+ River::JOB_STATE_RETRYABLE
207
+ end
208
+ error = River::AttemptError.new(at: now, attempt: job.attempt, error: "Stuck job rescued by River", trace: "")
209
+ job_set_state_if_running(
210
+ id: job.id,
211
+ error: error,
212
+ finalized_at: final ? now : nil,
213
+ metadata: {"river:rescue_count" => job.metadata.fetch("river:rescue_count", 0).to_i + 1},
214
+ now: now,
215
+ scheduled_at: final ? nil : retry_policy.next_retry(job, error, now: now),
216
+ state: state
217
+ )
218
+ end
219
+
220
+ jobs.length
221
+ end
222
+ end
223
+
224
+ def job_retry(id, now: Time.now.utc)
225
+ updated_id = runtime_returning_ids(<<~SQL).first
226
+ UPDATE river_job
227
+ SET state = 'available',
228
+ max_attempts = CASE WHEN attempt = max_attempts THEN max_attempts + 1 ELSE max_attempts END,
229
+ finalized_at = NULL,
230
+ scheduled_at = #{runtime_time(now)}
231
+ WHERE id = #{Integer(id)}
232
+ AND state != 'running'
233
+ AND (state != 'available' OR scheduled_at > #{runtime_time(now)})
234
+ RETURNING id
235
+ SQL
236
+ job_get_by_id(updated_id || id)
237
+ end
238
+
239
+ def job_schedule(now: Time.now.utc, max: 1_000)
240
+ transaction do
241
+ # Hold each selected row until its transition (including uniqueness
242
+ # conflict handling) finishes. A concurrent retry may change its due time.
243
+ lock = runtime_postgres? ? "FOR UPDATE SKIP LOCKED" : ""
244
+ ids = runtime_query_rows(<<~SQL).map { |row| runtime_value(row, :id).to_i }
245
+ SELECT id FROM river_job
246
+ WHERE state IN ('retryable', 'scheduled') AND scheduled_at <= #{runtime_time(now)}
247
+ ORDER BY priority, scheduled_at, id
248
+ LIMIT #{Integer(max)} #{lock}
249
+ SQL
250
+ ids.each do |id|
251
+ transaction do
252
+ runtime_execute("UPDATE river_job SET state = 'available' WHERE id = #{id} AND state IN ('retryable', 'scheduled')")
253
+ end
254
+ rescue runtime_unique_violation_class
255
+ runtime_execute(<<~SQL)
256
+ UPDATE river_job
257
+ SET state = 'discarded', finalized_at = #{runtime_time(now)},
258
+ metadata = #{runtime_merge_metadata("unique_key_conflict" => "scheduler_discarded")}
259
+ WHERE id = #{id}
260
+ SQL
261
+ end
262
+
263
+ ids.length
264
+ end
265
+ end
266
+
267
+ def job_set_state_if_running(id:, state:, now: Time.now.utc, attempt: nil,
268
+ error: nil, finalized_at: nil, metadata: nil, scheduled_at: nil)
269
+ state = state.to_s
270
+ retrying = [River::JOB_STATE_AVAILABLE, River::JOB_STATE_RETRYABLE, River::JOB_STATE_SCHEDULED].include?(state)
271
+ cancel_path = retrying ? runtime_cancel_attempted : "false"
272
+
273
+ assignments = [] #: Array[String]
274
+ assignments << "attempt = CASE WHEN NOT (#{cancel_path}) THEN #{Integer(attempt)} ELSE attempt END" unless attempt.nil?
275
+ assignments << "errors = #{runtime_append_error(error)}" if error
276
+
277
+ assignments << "finalized_at = CASE WHEN #{cancel_path} THEN #{runtime_time(now)} ELSE #{runtime_nullable_time(finalized_at)} END"
278
+ assignments << "metadata = #{runtime_merge_metadata(metadata)}" unless metadata.nil? || metadata.empty?
279
+ assignments << "scheduled_at = CASE WHEN NOT (#{cancel_path}) THEN #{runtime_time(scheduled_at)} ELSE scheduled_at END" if scheduled_at
280
+
281
+ assignments << "state = CASE WHEN #{cancel_path} THEN 'cancelled' ELSE #{runtime_state(state)} END"
282
+
283
+ id = runtime_returning_ids(<<~SQL).first
284
+ UPDATE river_job
285
+ SET #{assignments.join(",\n ")}
286
+ WHERE id = #{Integer(id)} AND state = 'running'
287
+ RETURNING id
288
+ SQL
289
+ id ? runtime_read_job(id) : nil
290
+ end
291
+
292
+ def job_update(id, params)
293
+ assignments = params.each.map do |field, value|
294
+ raise ArgumentError, "unknown update field: #{field}" unless River::JobUpdateParams.method_defined?(field)
295
+
296
+ "#{field} = #{runtime_update_value(field, value)}"
297
+ end
298
+
299
+ return job_get_by_id(id) if assignments.empty?
300
+
301
+ updated_id = runtime_returning_ids(<<~SQL).first
302
+ UPDATE river_job SET #{assignments.join(", ")}
303
+ WHERE id = #{Integer(id)}
304
+ RETURNING id
305
+ SQL
306
+ updated_id ? job_get_by_id(updated_id) : nil
307
+ end
308
+
309
+ def leader_acquire(id, ttl: 30, now: Time.now.utc)
310
+ transaction do
311
+ runtime_execute("DELETE FROM river_leader WHERE expires_at < #{runtime_time(now)}")
312
+ runtime_execute(<<~SQL)
313
+ INSERT INTO river_leader (leader_id, elected_at, expires_at)
314
+ VALUES (#{runtime_quote(id)}, #{runtime_time(now)}, #{runtime_time(now + ttl)})
315
+ ON CONFLICT (name) DO NOTHING
316
+ SQL
317
+ runtime_query_rows("SELECT leader_id FROM river_leader WHERE leader_id = #{runtime_quote(id)}").any?
318
+ end
319
+ end
320
+
321
+ def leader_release(id)
322
+ runtime_execute("DELETE FROM river_leader WHERE leader_id = #{runtime_quote(id)}")
323
+ end
324
+
325
+ def leader_renew(id, ttl: 30, now: Time.now.utc)
326
+ runtime_query_rows(<<~SQL).any?
327
+ UPDATE river_leader SET expires_at = #{runtime_time(now + ttl)}
328
+ WHERE leader_id = #{runtime_quote(id)} AND expires_at >= #{runtime_time(now)}
329
+ RETURNING leader_id
330
+ SQL
331
+ end
332
+
333
+ # Bounded batches release SQLite's writer lock between maintenance passes.
334
+ def notification_delete_before(horizon:, max: 10_000)
335
+ return 0 if runtime_postgres?
336
+
337
+ runtime_returning_ids(<<~SQL).length
338
+ DELETE FROM river_notification
339
+ WHERE id IN (
340
+ SELECT id FROM river_notification
341
+ WHERE created_at < #{runtime_time(horizon)}
342
+ ORDER BY created_at, id LIMIT #{Integer(max)}
343
+ )
344
+ RETURNING id
345
+ SQL
346
+ end
347
+
348
+ def queue_get(name)
349
+ row = runtime_query_rows("SELECT #{runtime_queue_columns} FROM river_queue WHERE name = #{runtime_quote(name)}").first
350
+ runtime_queue_from_row(row)
351
+ end
352
+
353
+ def queue_list(max: 100)
354
+ runtime_query_rows("SELECT #{runtime_queue_columns} FROM river_queue ORDER BY name LIMIT #{Integer(max)}")
355
+ .map { |row| runtime_queue_from_row(row) }
356
+ end
357
+
358
+ def queue_pause(name, now: Time.now.utc)
359
+ filter = (name == "*") ? "true" : "name = #{runtime_quote(name)}"
360
+ runtime_execute(<<~SQL)
361
+ UPDATE river_queue
362
+ SET paused_at = CASE WHEN paused_at IS NULL THEN #{runtime_time(now)} ELSE paused_at END,
363
+ updated_at = CASE WHEN paused_at IS NULL THEN #{runtime_time(now)} ELSE updated_at END
364
+ WHERE #{filter}
365
+ SQL
366
+ end
367
+
368
+ def queue_resume(name, now: Time.now.utc)
369
+ filter = (name == "*") ? "true" : "name = #{runtime_quote(name)}"
370
+ runtime_execute(<<~SQL)
371
+ UPDATE river_queue
372
+ SET updated_at = CASE WHEN paused_at IS NOT NULL THEN #{runtime_time(now)} ELSE updated_at END,
373
+ paused_at = NULL
374
+ WHERE #{filter}
375
+ SQL
376
+ end
377
+
378
+ def queue_update(name, metadata:, now: Time.now.utc)
379
+ id = runtime_query_rows(<<~SQL).first
380
+ UPDATE river_queue SET metadata = #{runtime_json(metadata)}, updated_at = #{runtime_time(now)}
381
+ WHERE name = #{runtime_quote(name)} RETURNING name
382
+ SQL
383
+ id ? queue_get(name) : nil
384
+ end
385
+
386
+ def queue_upsert(name, metadata: {}, now: Time.now.utc)
387
+ runtime_execute(<<~SQL)
388
+ INSERT INTO river_queue (name, created_at, metadata, updated_at)
389
+ VALUES (#{runtime_quote(name)}, #{runtime_time(now)}, #{runtime_json(metadata)}, #{runtime_time(now)})
390
+ ON CONFLICT (name) DO UPDATE SET updated_at = excluded.updated_at
391
+ SQL
392
+ queue_get(name)
393
+ end
394
+
395
+ private def runtime_append_error(error)
396
+ value = error.respond_to?(:to_h) ? error.to_h : error
397
+ if runtime_postgres?
398
+ "array_append(errors, #{runtime_json(value)})"
399
+ else
400
+ encoded = "json(#{runtime_quote(JSON.generate(value))})"
401
+ # Damaged JSONB may contain invalid UTF-8. Preserve its bytes as hex
402
+ # rather than creating an unreadable JSON string by casting it to text.
403
+ <<~SQL
404
+ CASE WHEN NOT json_valid(errors, 10) AND errors IS NOT NULL THEN
405
+ jsonb(json_array(CASE WHEN typeof(errors) = 'blob' THEN 'invalid JSONB: ' || hex(errors) ELSE errors END, #{encoded}))
406
+ WHEN coalesce(json_type(errors), 'array') <> 'array' THEN jsonb(json_array(json(errors), #{encoded}))
407
+ ELSE jsonb(json_insert(json(coalesce(errors, jsonb('[]'))), '$[#]', #{encoded})) END
408
+ SQL
409
+ end
410
+ end
411
+
412
+ private def runtime_cancel_attempted
413
+ runtime_postgres? ? "metadata ? 'cancel_attempted_at'" : "(CASE WHEN json_valid(metadata, 10) THEN metadata -> 'cancel_attempted_at' END) IS NOT NULL"
414
+ end
415
+
416
+ private def runtime_claim_jobs(predicate, max, attempted_by, now)
417
+ transaction do
418
+ attempted_by_sql = if runtime_postgres?
419
+ "array_append(CASE WHEN cardinality(attempted_by) >= 100 THEN attempted_by[(cardinality(attempted_by) - 98):] ELSE attempted_by END, #{runtime_quote(attempted_by)})"
420
+ else
421
+ # Preserve corrupt history so decoding can fail this attempt instead
422
+ # of silently repairing it or aborting the entire claim batch.
423
+ <<~SQL
424
+ CASE WHEN NOT json_valid(attempted_by, 10) AND attempted_by IS NOT NULL THEN attempted_by
425
+ ELSE jsonb(json_insert(json(coalesce(attempted_by, jsonb('[]'))), '$[#]', #{runtime_quote(attempted_by)})) END
426
+ SQL
427
+ end
428
+
429
+ lock_clause = runtime_postgres? ? "FOR UPDATE SKIP LOCKED" : ""
430
+ ids = runtime_returning_ids(<<~SQL)
431
+ UPDATE river_job
432
+ SET attempt = attempt + 1,
433
+ attempted_at = #{runtime_time(now)},
434
+ attempted_by = #{attempted_by_sql},
435
+ state = 'running'
436
+ WHERE id IN (
437
+ SELECT id FROM river_job
438
+ WHERE #{predicate}
439
+ ORDER BY priority ASC, scheduled_at ASC, id ASC
440
+ LIMIT #{Integer(max)}
441
+ #{lock_clause}
442
+ )
443
+ RETURNING id
444
+ SQL
445
+ ids.map { |id| runtime_read_job(id) }
446
+ end
447
+ end
448
+
449
+ private def runtime_read_job(id)
450
+ job_get_by_id(id)
451
+ rescue River::JobRowDecodeError => error
452
+ error.job
453
+ end
454
+
455
+ private def runtime_decode_postgres_job(row)
456
+ row = row.transform_keys(&:to_sym)
457
+ decoder = JobRowDecoder.new
458
+ decoder.finish(River::JobRow.new(
459
+ id: row[:id], args: decoder.decoded(:args, row[:args], default: {}),
460
+ attempt: row[:attempt], attempted_at: row[:attempted_at]&.getutc,
461
+ attempted_by: decoder.decoded(:attempted_by, row[:attempted_by], type: Array, strings: true),
462
+ created_at: row[:created_at].getutc,
463
+ errors: row[:errors]&.map { |error| JobRowDecoder.attempt_error(error) },
464
+ finalized_at: row[:finalized_at]&.getutc, kind: row[:kind], max_attempts: row[:max_attempts],
465
+ metadata: decoder.decoded(:metadata, row[:metadata], type: Hash, default: {}),
466
+ priority: row[:priority], queue: row[:queue], scheduled_at: row[:scheduled_at].getutc,
467
+ state: row[:state], tags: decoder.decoded(:tags, row[:tags], type: Array, strings: true),
468
+ unique_key: row[:unique_key]&.to_s,
469
+ unique_states: row[:unique_states] ? River::UniqueBitmask.to_states(row[:unique_states].to_i(2)) : nil
470
+ ))
471
+ end
472
+
473
+ private def runtime_decode_sqlite_job(row)
474
+ row = row.transform_keys(&:to_sym)
475
+ decoder = JobRowDecoder.new
476
+ errors = decoder.json(:errors, row[:errors], type: Array, default: [])
477
+ decoder.finish(River::JobRow.new(
478
+ id: row[:id], args: decoder.json(:args, row[:args], default: {}),
479
+ attempt: row[:attempt], attempted_at: runtime_parse_time(row[:attempted_at]),
480
+ attempted_by: decoder.json(:attempted_by, row[:attempted_by], type: Array, strings: true),
481
+ created_at: runtime_parse_time(row[:created_at]),
482
+ errors: errors.map { |error| JobRowDecoder.attempt_error(error) },
483
+ finalized_at: runtime_parse_time(row[:finalized_at]), kind: row[:kind], max_attempts: row[:max_attempts],
484
+ metadata: decoder.json(:metadata, row[:metadata], type: Hash, default: {}),
485
+ priority: row[:priority], queue: row[:queue], scheduled_at: runtime_parse_time(row[:scheduled_at]),
486
+ state: row[:state], tags: decoder.json(:tags, row[:tags], type: Array, strings: true),
487
+ unique_key: row[:unique_key]&.to_s,
488
+ unique_states: row[:unique_states] ? River::UniqueBitmask.to_states(row[:unique_states]) : nil
489
+ ))
490
+ end
491
+
492
+ private def runtime_in_clause(column, values)
493
+ "#{column} IN (#{values.map { |value| runtime_quote(value) }.join(",")})"
494
+ end
495
+
496
+ private def runtime_notify(topic, payload)
497
+ encoded = runtime_quote(JSON.generate(payload))
498
+ if runtime_postgres?
499
+ return unless postgres_capabilities.supports_listen_notify
500
+
501
+ runtime_execute("SELECT pg_notify(current_schema() || '.' || #{runtime_quote(topic)}, #{encoded})")
502
+ else
503
+ runtime_execute("INSERT INTO river_notification (topic, payload) VALUES (#{runtime_quote(topic)}, #{encoded})")
504
+ end
505
+ end
506
+
507
+ private def sqlite_insert_nonces(params)
508
+ keys = {} #: Hash[String, bool]
509
+ params.map do |param|
510
+ key = param.unique_key
511
+ if key && !key.empty? && River::UniqueBitmask.to_states((param.unique_states || "0").to_i(2)).include?(param.state)
512
+ # PostgreSQL rejects batches that upsert the same indexed row twice.
513
+ raise ArgumentError, "unique key appears more than once in batch" if keys[key]
514
+
515
+ keys[key] = true
516
+ end
517
+ SecureRandom.hex(8)
518
+ end
519
+ end
520
+
521
+ private def runtime_cursor_clause(params)
522
+ cursor = params.after
523
+ comparison = (params.sort_order == :asc) ? ">" : "<"
524
+ id_clause = "id #{comparison} #{Integer(cursor.id)}"
525
+ return id_clause if params.sort_by == :id
526
+
527
+ column = params.sort_by
528
+ return "(#{column} IS NULL AND #{id_clause})" if cursor.value.nil?
529
+
530
+ value = runtime_time(cursor.value)
531
+ "(#{column} IS NULL OR #{column} #{comparison} #{value} OR (#{column} = #{value} AND #{id_clause}))"
532
+ end
533
+
534
+ private def runtime_json(value)
535
+ encoded = value.is_a?(String) ? value : JSON.generate(value)
536
+ runtime_postgres? ? "#{runtime_quote(encoded)}::jsonb" : "jsonb(#{runtime_quote(encoded)})"
537
+ end
538
+
539
+ private def runtime_merge_metadata(metadata)
540
+ if runtime_postgres?
541
+ "metadata || #{runtime_json(metadata)}"
542
+ else
543
+ # PostgreSQL's || replaces top-level values, including JSON null.
544
+ # JSON Merge Patch would recursively merge objects and delete nulls.
545
+ merged = metadata.reduce("metadata") do |expression, (key, value)|
546
+ path = runtime_quote("$.#{JSON.generate(key.to_s)}")
547
+ "jsonb_set(#{expression}, #{path}, jsonb(#{runtime_quote(JSON.generate(value))}))"
548
+ end
549
+ "CASE WHEN json_valid(metadata, 10) THEN #{merged} ELSE metadata END"
550
+ end
551
+ end
552
+
553
+ private def runtime_metadata_equals(key, value)
554
+ encoded = runtime_quote(JSON.generate(value))
555
+ if runtime_postgres?
556
+ "metadata -> #{runtime_quote(key.to_s)} = #{encoded}::jsonb"
557
+ else
558
+ # Compare JSON trees so object key order is immaterial, while strings,
559
+ # booleans, null, and missing keys remain distinct. json_each also treats
560
+ # the requested metadata key literally instead of as a JSON path.
561
+ columns = "fullkey, CASE WHEN type IN ('integer', 'real') THEN 'number' ELSE type END, atom"
562
+ actual = <<~SQL
563
+ SELECT #{columns} FROM json_tree(CASE entry.type
564
+ WHEN 'text' THEN json_quote(entry.value)
565
+ WHEN 'null' THEN 'null'
566
+ WHEN 'true' THEN 'true'
567
+ WHEN 'false' THEN 'false'
568
+ ELSE entry.value END)
569
+ SQL
570
+ expected = "SELECT #{columns} FROM json_tree(#{encoded})"
571
+ <<~SQL
572
+ EXISTS (SELECT 1 FROM json_each(metadata) AS entry
573
+ WHERE entry.key = #{runtime_quote(key.to_s)}
574
+ AND NOT EXISTS (#{actual} EXCEPT #{expected})
575
+ AND NOT EXISTS (#{expected} EXCEPT #{actual}))
576
+ SQL
577
+ end
578
+ end
579
+
580
+ private def runtime_nullable_time(value)
581
+ value ? runtime_time(value) : "NULL"
582
+ end
583
+
584
+ private def runtime_parse_json(value)
585
+ value.is_a?(String) ? JSON.parse(value) : value.to_h
586
+ end
587
+
588
+ private def runtime_parse_time(value)
589
+ return nil unless value
590
+
591
+ if value.respond_to?(:getutc)
592
+ value.getutc
593
+ else
594
+ Time.parse(value.to_s + (value.to_s.match?(/(?:Z|[+-]\d{2}:?\d{2})\z/) ? "" : " UTC")).utc
595
+ end
596
+ end
597
+
598
+ private def runtime_queue_columns
599
+ runtime_postgres? ? "name, created_at, metadata, paused_at, updated_at" : "name, CAST(created_at AS text) AS created_at, json(metadata) AS metadata, CAST(paused_at AS text) AS paused_at, CAST(updated_at AS text) AS updated_at"
600
+ end
601
+
602
+ private def runtime_queue_from_row(row)
603
+ return nil unless row
604
+
605
+ River::Queue.new(
606
+ runtime_value(row, :name),
607
+ runtime_parse_time(runtime_value(row, :created_at)),
608
+ runtime_parse_json(runtime_value(row, :metadata)),
609
+ runtime_parse_time(runtime_value(row, :paused_at)),
610
+ runtime_parse_time(runtime_value(row, :updated_at))
611
+ )
612
+ end
613
+
614
+ private def runtime_returning_ids(sql)
615
+ runtime_query_rows(sql).map { |row| runtime_value(row, :id).to_i }
616
+ end
617
+
618
+ private def runtime_state(value)
619
+ runtime_postgres? ? "#{runtime_quote(value)}::river_job_state" : runtime_quote(value)
620
+ end
621
+
622
+ private def runtime_tag_contains(tag)
623
+ if runtime_postgres?
624
+ "tags @> ARRAY[#{runtime_quote(tag)}]::varchar[]"
625
+ else
626
+ "EXISTS (SELECT 1 FROM json_each(json(tags)) WHERE value = #{runtime_quote(tag)})"
627
+ end
628
+ end
629
+
630
+ private def runtime_time(value)
631
+ raise ArgumentError, "time cannot be nil" unless value
632
+
633
+ cast = runtime_postgres? ? "::timestamptz" : ""
634
+ encoded = if runtime_postgres?
635
+ value.getutc.iso8601(6)
636
+ else
637
+ value.getutc.round(3).strftime("%Y-%m-%d %H:%M:%S.%3N")
638
+ end
639
+ "#{runtime_quote(encoded)}#{cast}"
640
+ end
641
+
642
+ private def runtime_update_value(field, value)
643
+ case field
644
+ when :attempt, :max_attempts
645
+ Integer(value).to_s
646
+ when :attempted_at, :finalized_at
647
+ value ? runtime_time(value) : "NULL"
648
+ when :attempted_by
649
+ runtime_postgres? ? "ARRAY[#{Array(value).map { |item| runtime_quote(item) }.join(",")}]::text[]" : runtime_json(Array(value))
650
+ when :errors
651
+ input_values = Array(value) #: Array[untyped]
652
+ values = input_values.map { |error| error.respond_to?(:to_h) ? error.to_h : error }
653
+ runtime_postgres? ? "ARRAY[#{values.map { |item| runtime_json(item) }.join(",")}]::jsonb[]" : runtime_json(values)
654
+ when :metadata
655
+ runtime_json(value)
656
+ when :state
657
+ runtime_state(value)
658
+ end
659
+ end
660
+ end
661
+ end