@pgpm/database-jobs 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/LICENSE +22 -0
  2. package/Makefile +6 -0
  3. package/README.md +5 -0
  4. package/__tests__/__snapshots__/jobs.test.ts.snap +19 -0
  5. package/__tests__/jobs.test.ts +138 -0
  6. package/deploy/schemas/app_jobs/helpers/json_build_object_apply.sql +28 -0
  7. package/deploy/schemas/app_jobs/procedures/add_job.sql +103 -0
  8. package/deploy/schemas/app_jobs/procedures/add_scheduled_job.sql +97 -0
  9. package/deploy/schemas/app_jobs/procedures/complete_job.sql +32 -0
  10. package/deploy/schemas/app_jobs/procedures/complete_jobs.sql +19 -0
  11. package/deploy/schemas/app_jobs/procedures/do_notify.sql +16 -0
  12. package/deploy/schemas/app_jobs/procedures/fail_job.sql +41 -0
  13. package/deploy/schemas/app_jobs/procedures/get_job.sql +92 -0
  14. package/deploy/schemas/app_jobs/procedures/get_scheduled_job.sql +61 -0
  15. package/deploy/schemas/app_jobs/procedures/permanently_fail_jobs.sql +24 -0
  16. package/deploy/schemas/app_jobs/procedures/release_jobs.sql +34 -0
  17. package/deploy/schemas/app_jobs/procedures/release_scheduled_jobs.sql +26 -0
  18. package/deploy/schemas/app_jobs/procedures/reschedule_jobs.sql +26 -0
  19. package/deploy/schemas/app_jobs/procedures/run_scheduled_job.sql +78 -0
  20. package/deploy/schemas/app_jobs/schema.sql +7 -0
  21. package/deploy/schemas/app_jobs/tables/job_queues/grants/grant_select_insert_update_delete_to_administrator.sql +12 -0
  22. package/deploy/schemas/app_jobs/tables/job_queues/indexes/job_queues_locked_by_idx.sql +8 -0
  23. package/deploy/schemas/app_jobs/tables/job_queues/table.sql +12 -0
  24. package/deploy/schemas/app_jobs/tables/jobs/grants/grant_select_insert_update_delete_to_administrator.sql +12 -0
  25. package/deploy/schemas/app_jobs/tables/jobs/indexes/jobs_locked_by_idx.sql +8 -0
  26. package/deploy/schemas/app_jobs/tables/jobs/indexes/priority_run_at_id_idx.sql +8 -0
  27. package/deploy/schemas/app_jobs/tables/jobs/table.sql +27 -0
  28. package/deploy/schemas/app_jobs/tables/jobs/triggers/decrease_job_queue_count.sql +45 -0
  29. package/deploy/schemas/app_jobs/tables/jobs/triggers/increase_job_queue_count.sql +32 -0
  30. package/deploy/schemas/app_jobs/tables/jobs/triggers/notify_worker.sql +13 -0
  31. package/deploy/schemas/app_jobs/tables/jobs/triggers/timestamps.sql +20 -0
  32. package/deploy/schemas/app_jobs/tables/scheduled_jobs/grants/grant_select_insert_update_delete_to_administrator.sql +12 -0
  33. package/deploy/schemas/app_jobs/tables/scheduled_jobs/indexes/scheduled_jobs_locked_by_idx.sql +8 -0
  34. package/deploy/schemas/app_jobs/tables/scheduled_jobs/indexes/scheduled_jobs_priority_id_idx.sql +8 -0
  35. package/deploy/schemas/app_jobs/tables/scheduled_jobs/table.sql +27 -0
  36. package/deploy/schemas/app_jobs/tables/scheduled_jobs/triggers/notify_scheduled_job.sql +12 -0
  37. package/deploy/schemas/app_jobs/triggers/tg_add_job_with_fields.sql +50 -0
  38. package/deploy/schemas/app_jobs/triggers/tg_add_job_with_row.sql +26 -0
  39. package/deploy/schemas/app_jobs/triggers/tg_add_job_with_row_id.sql +27 -0
  40. package/deploy/schemas/app_jobs/triggers/tg_update_timestamps.sql +21 -0
  41. package/jest.config.js +15 -0
  42. package/launchql-database-jobs.control +8 -0
  43. package/launchql.plan +38 -0
  44. package/package.json +29 -0
  45. package/revert/schemas/app_jobs/helpers/json_build_object_apply.sql +7 -0
  46. package/revert/schemas/app_jobs/procedures/add_job.sql +7 -0
  47. package/revert/schemas/app_jobs/procedures/add_scheduled_job.sql +7 -0
  48. package/revert/schemas/app_jobs/procedures/complete_job.sql +7 -0
  49. package/revert/schemas/app_jobs/procedures/complete_jobs.sql +7 -0
  50. package/revert/schemas/app_jobs/procedures/do_notify.sql +7 -0
  51. package/revert/schemas/app_jobs/procedures/fail_job.sql +7 -0
  52. package/revert/schemas/app_jobs/procedures/get_job.sql +7 -0
  53. package/revert/schemas/app_jobs/procedures/get_scheduled_job.sql +7 -0
  54. package/revert/schemas/app_jobs/procedures/permanently_fail_jobs.sql +7 -0
  55. package/revert/schemas/app_jobs/procedures/release_jobs.sql +7 -0
  56. package/revert/schemas/app_jobs/procedures/release_scheduled_jobs.sql +7 -0
  57. package/revert/schemas/app_jobs/procedures/reschedule_jobs.sql +7 -0
  58. package/revert/schemas/app_jobs/procedures/run_scheduled_job.sql +7 -0
  59. package/revert/schemas/app_jobs/schema.sql +7 -0
  60. package/revert/schemas/app_jobs/tables/job_queues/grants/grant_select_insert_update_delete_to_administrator.sql +7 -0
  61. package/revert/schemas/app_jobs/tables/job_queues/indexes/job_queues_locked_by_idx.sql +7 -0
  62. package/revert/schemas/app_jobs/tables/job_queues/table.sql +7 -0
  63. package/revert/schemas/app_jobs/tables/jobs/grants/grant_select_insert_update_delete_to_administrator.sql +7 -0
  64. package/revert/schemas/app_jobs/tables/jobs/indexes/jobs_locked_by_idx.sql +7 -0
  65. package/revert/schemas/app_jobs/tables/jobs/indexes/priority_run_at_id_idx.sql +7 -0
  66. package/revert/schemas/app_jobs/tables/jobs/table.sql +7 -0
  67. package/revert/schemas/app_jobs/tables/jobs/triggers/decrease_job_queue_count.sql +7 -0
  68. package/revert/schemas/app_jobs/tables/jobs/triggers/increase_job_queue_count.sql +7 -0
  69. package/revert/schemas/app_jobs/tables/jobs/triggers/notify_worker.sql +5 -0
  70. package/revert/schemas/app_jobs/tables/jobs/triggers/timestamps.sql +9 -0
  71. package/revert/schemas/app_jobs/tables/scheduled_jobs/grants/grant_select_insert_update_delete_to_administrator.sql +7 -0
  72. package/revert/schemas/app_jobs/tables/scheduled_jobs/indexes/scheduled_jobs_locked_by_idx.sql +7 -0
  73. package/revert/schemas/app_jobs/tables/scheduled_jobs/indexes/scheduled_jobs_priority_id_idx.sql +7 -0
  74. package/revert/schemas/app_jobs/tables/scheduled_jobs/table.sql +7 -0
  75. package/revert/schemas/app_jobs/tables/scheduled_jobs/triggers/notify_scheduled_job.sql +8 -0
  76. package/revert/schemas/app_jobs/triggers/tg_add_job_with_fields.sql +7 -0
  77. package/revert/schemas/app_jobs/triggers/tg_add_job_with_row.sql +7 -0
  78. package/revert/schemas/app_jobs/triggers/tg_add_job_with_row_id.sql +5 -0
  79. package/revert/schemas/app_jobs/triggers/tg_update_timestamps.sql +7 -0
  80. package/sql/launchql-database-jobs--0.4.6.sql +769 -0
  81. package/verify/schemas/app_jobs/helpers/json_build_object_apply.sql +7 -0
  82. package/verify/schemas/app_jobs/procedures/add_job.sql +7 -0
  83. package/verify/schemas/app_jobs/procedures/add_scheduled_job.sql +7 -0
  84. package/verify/schemas/app_jobs/procedures/complete_job.sql +7 -0
  85. package/verify/schemas/app_jobs/procedures/complete_jobs.sql +7 -0
  86. package/verify/schemas/app_jobs/procedures/do_notify.sql +7 -0
  87. package/verify/schemas/app_jobs/procedures/fail_job.sql +7 -0
  88. package/verify/schemas/app_jobs/procedures/get_job.sql +7 -0
  89. package/verify/schemas/app_jobs/procedures/get_scheduled_job.sql +7 -0
  90. package/verify/schemas/app_jobs/procedures/permanently_fail_jobs.sql +7 -0
  91. package/verify/schemas/app_jobs/procedures/release_jobs.sql +7 -0
  92. package/verify/schemas/app_jobs/procedures/release_scheduled_jobs.sql +7 -0
  93. package/verify/schemas/app_jobs/procedures/reschedule_jobs.sql +7 -0
  94. package/verify/schemas/app_jobs/procedures/run_scheduled_job.sql +7 -0
  95. package/verify/schemas/app_jobs/schema.sql +7 -0
  96. package/verify/schemas/app_jobs/tables/job_queues/grants/grant_select_insert_update_delete_to_administrator.sql +10 -0
  97. package/verify/schemas/app_jobs/tables/job_queues/indexes/job_queues_locked_by_idx.sql +7 -0
  98. package/verify/schemas/app_jobs/tables/job_queues/table.sql +7 -0
  99. package/verify/schemas/app_jobs/tables/jobs/grants/grant_select_insert_update_delete_to_administrator.sql +10 -0
  100. package/verify/schemas/app_jobs/tables/jobs/indexes/jobs_locked_by_idx.sql +7 -0
  101. package/verify/schemas/app_jobs/tables/jobs/indexes/priority_run_at_id_idx.sql +7 -0
  102. package/verify/schemas/app_jobs/tables/jobs/table.sql +7 -0
  103. package/verify/schemas/app_jobs/tables/jobs/triggers/decrease_job_queue_count.sql +10 -0
  104. package/verify/schemas/app_jobs/tables/jobs/triggers/increase_job_queue_count.sql +10 -0
  105. package/verify/schemas/app_jobs/tables/jobs/triggers/notify_worker.sql +6 -0
  106. package/verify/schemas/app_jobs/tables/jobs/triggers/timestamps.sql +16 -0
  107. package/verify/schemas/app_jobs/tables/scheduled_jobs/grants/grant_select_insert_update_delete_to_administrator.sql +10 -0
  108. package/verify/schemas/app_jobs/tables/scheduled_jobs/indexes/scheduled_jobs_locked_by_idx.sql +7 -0
  109. package/verify/schemas/app_jobs/tables/scheduled_jobs/indexes/scheduled_jobs_priority_id_idx.sql +7 -0
  110. package/verify/schemas/app_jobs/tables/scheduled_jobs/table.sql +7 -0
  111. package/verify/schemas/app_jobs/tables/scheduled_jobs/triggers/notify_scheduled_job.sql +8 -0
  112. package/verify/schemas/app_jobs/triggers/tg_add_job_with_fields.sql +7 -0
  113. package/verify/schemas/app_jobs/triggers/tg_add_job_with_row.sql +7 -0
  114. package/verify/schemas/app_jobs/triggers/tg_add_job_with_row_id.sql +6 -0
  115. package/verify/schemas/app_jobs/triggers/tg_update_timestamps.sql +7 -0
@@ -0,0 +1,769 @@
1
+ \echo Use "CREATE EXTENSION launchql-database-jobs" to load this file. \quit
2
+ CREATE SCHEMA IF NOT EXISTS app_jobs;
3
+
4
+ GRANT USAGE ON SCHEMA app_jobs TO administrator;
5
+
6
+ ALTER DEFAULT PRIVILEGES IN SCHEMA app_jobs
7
+ GRANT EXECUTE ON FUNCTIONS TO administrator;
8
+
9
+ CREATE FUNCTION app_jobs.json_build_object_apply(arguments text[]) RETURNS pg_catalog.json AS $EOFCODE$
10
+ DECLARE
11
+ arg text;
12
+ _sql text;
13
+ _res json;
14
+ args text[];
15
+ BEGIN
16
+ _sql = 'SELECT json_build_object(';
17
+ FOR arg IN
18
+ SELECT
19
+ unnest(arguments)
20
+ LOOP
21
+ args = array_append(args, format('''%s''', arg));
22
+ END LOOP;
23
+ _sql = _sql || format('%s);', array_to_string(args, ','));
24
+ EXECUTE _sql INTO _res;
25
+ RETURN _res;
26
+ END;
27
+ $EOFCODE$ LANGUAGE plpgsql;
28
+
29
+ CREATE TABLE app_jobs.jobs (
30
+ id bigserial PRIMARY KEY,
31
+ database_id uuid NOT NULL,
32
+ queue_name text DEFAULT public.gen_random_uuid()::text,
33
+ task_identifier text NOT NULL,
34
+ payload pg_catalog.json DEFAULT '{}'::json NOT NULL,
35
+ priority int DEFAULT 0 NOT NULL,
36
+ run_at timestamptz DEFAULT now() NOT NULL,
37
+ attempts int DEFAULT 0 NOT NULL,
38
+ max_attempts int DEFAULT 25 NOT NULL,
39
+ key text,
40
+ last_error text,
41
+ locked_at timestamptz,
42
+ locked_by text,
43
+ CHECK (length(key) < 513),
44
+ CHECK (length(task_identifier) < 127),
45
+ CHECK (max_attempts > 0),
46
+ CHECK (length(queue_name) < 127),
47
+ CHECK (length(locked_by) > 3),
48
+ UNIQUE (key)
49
+ );
50
+
51
+ CREATE TABLE app_jobs.job_queues (
52
+ queue_name text NOT NULL PRIMARY KEY,
53
+ job_count int DEFAULT 0 NOT NULL,
54
+ locked_at timestamptz,
55
+ locked_by text
56
+ );
57
+
58
+ CREATE FUNCTION app_jobs.add_job(db_id uuid, identifier text, payload pg_catalog.json DEFAULT '{}'::json, job_key text DEFAULT NULL, queue_name text DEFAULT NULL, run_at timestamptz DEFAULT now(), max_attempts int DEFAULT 25, priority int DEFAULT 0) RETURNS app_jobs.jobs AS $EOFCODE$
59
+ DECLARE
60
+ v_job app_jobs.jobs;
61
+ BEGIN
62
+ IF job_key IS NOT NULL THEN
63
+ -- Upsert job
64
+ INSERT INTO app_jobs.jobs (
65
+ database_id,
66
+ task_identifier,
67
+ payload,
68
+ queue_name,
69
+ run_at,
70
+ max_attempts,
71
+ key,
72
+ priority
73
+ ) VALUES (
74
+ db_id,
75
+ identifier,
76
+ coalesce(payload,
77
+ '{}'::json),
78
+ queue_name,
79
+ coalesce(run_at, now()),
80
+ coalesce(max_attempts, 25),
81
+ job_key,
82
+ coalesce(priority, 0)
83
+ )
84
+ ON CONFLICT (key)
85
+ DO UPDATE SET
86
+ task_identifier = EXCLUDED.task_identifier,
87
+ payload = EXCLUDED.payload,
88
+ queue_name = EXCLUDED.queue_name,
89
+ max_attempts = EXCLUDED.max_attempts,
90
+ run_at = EXCLUDED.run_at,
91
+ priority = EXCLUDED.priority,
92
+ -- always reset error/retry state
93
+ attempts = 0, last_error = NULL
94
+ WHERE
95
+ jobs.locked_at IS NULL
96
+ RETURNING
97
+ * INTO v_job;
98
+
99
+ -- If upsert succeeded (insert or update), return early
100
+
101
+ IF NOT (v_job IS NULL) THEN
102
+ RETURN v_job;
103
+ END IF;
104
+
105
+ -- Upsert failed -> there must be an existing job that is locked. Remove
106
+ -- existing key to allow a new one to be inserted, and prevent any
107
+ -- subsequent retries by bumping attempts to the max allowed.
108
+
109
+ UPDATE
110
+ app_jobs.jobs
111
+ SET
112
+ KEY = NULL,
113
+ attempts = jobs.max_attempts
114
+ WHERE
115
+ KEY = job_key;
116
+ END IF;
117
+
118
+ INSERT INTO app_jobs.jobs (
119
+ database_id,
120
+ task_identifier,
121
+ payload,
122
+ queue_name,
123
+ run_at,
124
+ max_attempts,
125
+ priority
126
+ ) VALUES (
127
+ db_id,
128
+ identifier,
129
+ payload,
130
+ queue_name,
131
+ run_at,
132
+ max_attempts,
133
+ priority
134
+ )
135
+ RETURNING * INTO v_job;
136
+
137
+ RETURN v_job;
138
+ END;
139
+ $EOFCODE$ LANGUAGE plpgsql VOLATILE SECURITY DEFINER;
140
+
141
+ CREATE TABLE app_jobs.scheduled_jobs (
142
+ id bigserial PRIMARY KEY,
143
+ database_id uuid NOT NULL,
144
+ queue_name text DEFAULT public.gen_random_uuid()::text,
145
+ task_identifier text NOT NULL,
146
+ payload pg_catalog.json DEFAULT '{}'::json NOT NULL,
147
+ priority int DEFAULT 0 NOT NULL,
148
+ max_attempts int DEFAULT 25 NOT NULL,
149
+ key text,
150
+ locked_at timestamptz,
151
+ locked_by text,
152
+ schedule_info pg_catalog.json NOT NULL,
153
+ last_scheduled timestamptz,
154
+ last_scheduled_id bigint,
155
+ CHECK (length(key) < 513),
156
+ CHECK (length(task_identifier) < 127),
157
+ CHECK (max_attempts > 0),
158
+ CHECK (length(queue_name) < 127),
159
+ CHECK (length(locked_by) > 3),
160
+ UNIQUE (key)
161
+ );
162
+
163
+ CREATE FUNCTION app_jobs.add_scheduled_job(db_id uuid, identifier text, payload pg_catalog.json DEFAULT '{}'::json, schedule_info pg_catalog.json DEFAULT '{}'::json, job_key text DEFAULT NULL, queue_name text DEFAULT NULL, max_attempts int DEFAULT 25, priority int DEFAULT 0) RETURNS app_jobs.scheduled_jobs AS $EOFCODE$
164
+ DECLARE
165
+ v_job app_jobs.scheduled_jobs;
166
+ BEGIN
167
+ IF job_key IS NOT NULL THEN
168
+
169
+ -- Upsert job
170
+ INSERT INTO app_jobs.scheduled_jobs (
171
+ database_id,
172
+ task_identifier,
173
+ payload,
174
+ queue_name,
175
+ schedule_info,
176
+ max_attempts,
177
+ key,
178
+ priority
179
+ ) VALUES (
180
+ db_id,
181
+ identifier,
182
+ coalesce(payload, '{}'::json),
183
+ queue_name,
184
+ schedule_info,
185
+ coalesce(max_attempts, 25),
186
+ job_key,
187
+ coalesce(priority, 0)
188
+ )
189
+ ON CONFLICT (key)
190
+ DO UPDATE SET
191
+ task_identifier = EXCLUDED.task_identifier,
192
+ payload = EXCLUDED.payload,
193
+ queue_name = EXCLUDED.queue_name,
194
+ max_attempts = EXCLUDED.max_attempts,
195
+ schedule_info = EXCLUDED.schedule_info,
196
+ priority = EXCLUDED.priority
197
+ WHERE
198
+ scheduled_jobs.locked_at IS NULL
199
+ RETURNING
200
+ * INTO v_job;
201
+
202
+ -- If upsert succeeded (insert or update), return early
203
+
204
+ IF NOT (v_job IS NULL) THEN
205
+ RETURN v_job;
206
+ END IF;
207
+
208
+ -- Upsert failed -> there must be an existing scheduled job that is locked. Remove
209
+ -- and allow a new one to be inserted
210
+
211
+ DELETE FROM
212
+ app_jobs.scheduled_jobs
213
+ WHERE
214
+ KEY = job_key;
215
+ END IF;
216
+
217
+ INSERT INTO app_jobs.scheduled_jobs (
218
+ database_id,
219
+ task_identifier,
220
+ payload,
221
+ queue_name,
222
+ schedule_info,
223
+ max_attempts,
224
+ priority
225
+ ) VALUES (
226
+ db_id,
227
+ identifier,
228
+ payload,
229
+ queue_name,
230
+ schedule_info,
231
+ max_attempts,
232
+ priority
233
+ ) RETURNING * INTO v_job;
234
+ RETURN v_job;
235
+ END;
236
+ $EOFCODE$ LANGUAGE plpgsql VOLATILE SECURITY DEFINER;
237
+
238
+ CREATE FUNCTION app_jobs.complete_job(worker_id text, job_id bigint) RETURNS app_jobs.jobs LANGUAGE plpgsql AS $EOFCODE$
239
+ DECLARE
240
+ v_row app_jobs.jobs;
241
+ BEGIN
242
+ DELETE FROM app_jobs.jobs
243
+ WHERE id = job_id
244
+ RETURNING
245
+ * INTO v_row;
246
+ IF v_row.queue_name IS NOT NULL THEN
247
+ UPDATE
248
+ app_jobs.job_queues
249
+ SET
250
+ locked_by = NULL,
251
+ locked_at = NULL
252
+ WHERE
253
+ queue_name = v_row.queue_name
254
+ AND locked_by = worker_id;
255
+ END IF;
256
+ RETURN v_row;
257
+ END;
258
+ $EOFCODE$;
259
+
260
+ CREATE FUNCTION app_jobs.complete_jobs(job_ids bigint[]) RETURNS SETOF app_jobs.jobs LANGUAGE sql AS $EOFCODE$
261
+ DELETE FROM app_jobs.jobs
262
+ WHERE id = ANY (job_ids)
263
+ AND (locked_by IS NULL
264
+ OR locked_at < NOW() - interval '4 hours')
265
+ RETURNING
266
+ *;
267
+ $EOFCODE$;
268
+
269
+ CREATE FUNCTION app_jobs.do_notify() RETURNS trigger AS $EOFCODE$
270
+ BEGIN
271
+ PERFORM
272
+ pg_notify(TG_ARGV[0], '');
273
+ RETURN NEW;
274
+ END;
275
+ $EOFCODE$ LANGUAGE plpgsql;
276
+
277
+ CREATE FUNCTION app_jobs.fail_job(worker_id text, job_id bigint, error_message text) RETURNS app_jobs.jobs LANGUAGE plpgsql STRICT AS $EOFCODE$
278
+ DECLARE
279
+ v_row app_jobs.jobs;
280
+ BEGIN
281
+ UPDATE
282
+ app_jobs.jobs
283
+ SET
284
+ last_error = error_message,
285
+ run_at = greatest (now(), run_at) + (exp(least (attempts, 10))::text || ' seconds')::interval,
286
+ locked_by = NULL,
287
+ locked_at = NULL
288
+ WHERE
289
+ id = job_id
290
+ AND locked_by = worker_id
291
+ RETURNING
292
+ * INTO v_row;
293
+ IF v_row.queue_name IS NOT NULL THEN
294
+ UPDATE
295
+ app_jobs.job_queues
296
+ SET
297
+ locked_by = NULL,
298
+ locked_at = NULL
299
+ WHERE
300
+ queue_name = v_row.queue_name
301
+ AND locked_by = worker_id;
302
+ END IF;
303
+ RETURN v_row;
304
+ END;
305
+ $EOFCODE$;
306
+
307
+ CREATE FUNCTION app_jobs.get_job(worker_id text, task_identifiers text[] DEFAULT NULL, job_expiry interval DEFAULT '4 hours') RETURNS app_jobs.jobs LANGUAGE plpgsql AS $EOFCODE$
308
+ DECLARE
309
+ v_job_id bigint;
310
+ v_queue_name text;
311
+ v_row app_jobs.jobs;
312
+ v_now timestamptz = now();
313
+ BEGIN
314
+
315
+ IF worker_id IS NULL THEN
316
+ RAISE exception 'INVALID_WORKER_ID';
317
+ END IF;
318
+
319
+ --
320
+
321
+ SELECT
322
+ jobs.queue_name,
323
+ jobs.id INTO v_queue_name,
324
+ v_job_id
325
+ FROM
326
+ app_jobs.jobs
327
+ WHERE (jobs.locked_at IS NULL
328
+ OR jobs.locked_at < (v_now - job_expiry))
329
+ AND (jobs.queue_name IS NULL
330
+ OR EXISTS (
331
+ SELECT
332
+ 1
333
+ FROM
334
+ app_jobs.job_queues
335
+ WHERE
336
+ job_queues.queue_name = jobs.queue_name
337
+ AND (job_queues.locked_at IS NULL
338
+ OR job_queues.locked_at < (v_now - job_expiry))
339
+ FOR UPDATE
340
+ SKIP LOCKED))
341
+ AND run_at <= v_now
342
+ AND attempts < max_attempts
343
+ AND (task_identifiers IS NULL
344
+ OR task_identifier = ANY (task_identifiers))
345
+ ORDER BY
346
+ priority ASC,
347
+ run_at ASC,
348
+ id ASC
349
+ LIMIT 1
350
+ FOR UPDATE
351
+ SKIP LOCKED;
352
+
353
+ --
354
+
355
+ IF v_job_id IS NULL THEN
356
+ RETURN NULL;
357
+ END IF;
358
+
359
+ --
360
+
361
+ IF v_queue_name IS NOT NULL THEN
362
+ UPDATE
363
+ app_jobs.job_queues
364
+ SET
365
+ locked_by = worker_id,
366
+ locked_at = v_now
367
+ WHERE
368
+ job_queues.queue_name = v_queue_name;
369
+ END IF;
370
+
371
+ --
372
+
373
+ UPDATE
374
+ app_jobs.jobs
375
+ SET
376
+ attempts = attempts + 1,
377
+ locked_by = worker_id,
378
+ locked_at = v_now
379
+ WHERE
380
+ id = v_job_id
381
+ RETURNING
382
+ * INTO v_row;
383
+
384
+ --
385
+ RETURN v_row;
386
+ END;
387
+ $EOFCODE$;
388
+
389
+ CREATE FUNCTION app_jobs.get_scheduled_job(worker_id text, task_identifiers text[] DEFAULT NULL) RETURNS app_jobs.scheduled_jobs LANGUAGE plpgsql AS $EOFCODE$
390
+ DECLARE
391
+ v_job_id bigint;
392
+ v_row app_jobs.scheduled_jobs;
393
+ BEGIN
394
+
395
+ --
396
+
397
+ IF worker_id IS NULL THEN
398
+ RAISE exception 'INVALID_WORKER_ID';
399
+ END IF;
400
+
401
+ --
402
+
403
+ SELECT
404
+ scheduled_jobs.id INTO v_job_id
405
+ FROM
406
+ app_jobs.scheduled_jobs
407
+ WHERE (scheduled_jobs.locked_at IS NULL)
408
+ AND (task_identifiers IS NULL
409
+ OR task_identifier = ANY (task_identifiers))
410
+ ORDER BY
411
+ priority ASC,
412
+ id ASC
413
+ LIMIT 1
414
+ FOR UPDATE
415
+ SKIP LOCKED;
416
+
417
+ --
418
+
419
+ IF v_job_id IS NULL THEN
420
+ RETURN NULL;
421
+ END IF;
422
+
423
+ --
424
+
425
+ UPDATE
426
+ app_jobs.scheduled_jobs
427
+ SET
428
+ locked_by = worker_id,
429
+ locked_at = NOW()
430
+ WHERE
431
+ id = v_job_id
432
+ RETURNING
433
+ * INTO v_row;
434
+
435
+ --
436
+
437
+ RETURN v_row;
438
+ END;
439
+ $EOFCODE$;
440
+
441
+ CREATE FUNCTION app_jobs.permanently_fail_jobs(job_ids bigint[], error_message text DEFAULT NULL) RETURNS SETOF app_jobs.jobs LANGUAGE sql AS $EOFCODE$
442
+ UPDATE
443
+ app_jobs.jobs
444
+ SET
445
+ last_error = coalesce(error_message, 'Manually marked as failed'),
446
+ attempts = max_attempts
447
+ WHERE
448
+ id = ANY (job_ids)
449
+ AND (locked_by IS NULL
450
+ OR locked_at < NOW() - interval '4 hours')
451
+ RETURNING
452
+ *;
453
+ $EOFCODE$;
454
+
455
+ CREATE FUNCTION app_jobs.release_jobs(worker_id text) RETURNS void AS $EOFCODE$
456
+ DECLARE
457
+ BEGIN
458
+ -- clear the job
459
+ UPDATE
460
+ app_jobs.jobs
461
+ SET
462
+ locked_at = NULL,
463
+ locked_by = NULL,
464
+ attempts = GREATEST (attempts - 1, 0)
465
+ WHERE
466
+ locked_by = worker_id;
467
+ -- clear the queue
468
+ UPDATE
469
+ app_jobs.job_queues
470
+ SET
471
+ locked_at = NULL,
472
+ locked_by = NULL
473
+ WHERE
474
+ locked_by = worker_id;
475
+ END;
476
+ $EOFCODE$ LANGUAGE plpgsql VOLATILE;
477
+
478
+ CREATE FUNCTION app_jobs.release_scheduled_jobs(worker_id text, ids bigint[] DEFAULT NULL) RETURNS void AS $EOFCODE$
479
+ DECLARE
480
+ BEGIN
481
+ -- clear the scheduled job
482
+ UPDATE
483
+ app_jobs.scheduled_jobs s
484
+ SET
485
+ locked_at = NULL,
486
+ locked_by = NULL
487
+ WHERE
488
+ locked_by = worker_id
489
+ AND (ids IS NULL
490
+ OR s.id = ANY (ids));
491
+ END;
492
+ $EOFCODE$ LANGUAGE plpgsql VOLATILE;
493
+
494
+ CREATE FUNCTION app_jobs.reschedule_jobs(job_ids bigint[], run_at timestamptz DEFAULT NULL, priority int DEFAULT NULL, attempts int DEFAULT NULL, max_attempts int DEFAULT NULL) RETURNS SETOF app_jobs.jobs LANGUAGE sql AS $EOFCODE$
495
+ UPDATE
496
+ app_jobs.jobs
497
+ SET
498
+ run_at = coalesce(reschedule_jobs.run_at, jobs.run_at),
499
+ priority = coalesce(reschedule_jobs.priority, jobs.priority),
500
+ attempts = coalesce(reschedule_jobs.attempts, jobs.attempts),
501
+ max_attempts = coalesce(reschedule_jobs.max_attempts, jobs.max_attempts)
502
+ WHERE
503
+ id = ANY (job_ids)
504
+ AND (locked_by IS NULL
505
+ OR locked_at < NOW() - interval '4 hours')
506
+ RETURNING
507
+ *;
508
+ $EOFCODE$;
509
+
510
+ CREATE FUNCTION app_jobs.run_scheduled_job(id bigint, job_expiry interval DEFAULT '1 hours') RETURNS app_jobs.jobs AS $EOFCODE$
511
+ DECLARE
512
+ j app_jobs.jobs;
513
+ last_id bigint;
514
+ lkd_by text;
515
+ BEGIN
516
+ -- check last scheduled
517
+ SELECT
518
+ last_scheduled_id
519
+ FROM
520
+ app_jobs.scheduled_jobs s
521
+ WHERE
522
+ s.id = run_scheduled_job.id INTO last_id;
523
+
524
+ -- if it's been scheduled check if it's been run
525
+
526
+ IF (last_id IS NOT NULL) THEN
527
+ SELECT
528
+ locked_by
529
+ FROM
530
+ app_jobs.jobs js
531
+ WHERE
532
+ js.id = last_id
533
+ AND (js.locked_at IS NULL -- never been run
534
+ OR js.locked_at >= (NOW() - job_expiry)
535
+ -- still running within a safe interval
536
+ ) INTO lkd_by;
537
+ IF (FOUND) THEN
538
+ RAISE EXCEPTION 'ALREADY_SCHEDULED';
539
+ END IF;
540
+ END IF;
541
+
542
+ -- insert new job
543
+ INSERT INTO app_jobs.jobs (
544
+ database_id,
545
+ queue_name,
546
+ task_identifier,
547
+ payload,
548
+ priority,
549
+ max_attempts,
550
+ key
551
+ ) SELECT
552
+ database_id,
553
+ queue_name,
554
+ task_identifier,
555
+ payload,
556
+ priority,
557
+ max_attempts,
558
+ key
559
+ FROM
560
+ app_jobs.scheduled_jobs s
561
+ WHERE
562
+ s.id = run_scheduled_job.id
563
+ RETURNING
564
+ * INTO j;
565
+ -- update the scheduled job
566
+ UPDATE
567
+ app_jobs.scheduled_jobs s
568
+ SET
569
+ last_scheduled = NOW(),
570
+ last_scheduled_id = j.id
571
+ WHERE
572
+ s.id = run_scheduled_job.id;
573
+ RETURN j;
574
+ END;
575
+ $EOFCODE$ LANGUAGE plpgsql VOLATILE;
576
+
577
+ GRANT SELECT, INSERT, UPDATE, DELETE ON app_jobs.job_queues TO administrator;
578
+
579
+ CREATE INDEX job_queues_locked_by_idx ON app_jobs.job_queues (locked_by);
580
+
581
+ GRANT SELECT, INSERT, UPDATE, DELETE ON app_jobs.jobs TO administrator;
582
+
583
+ CREATE INDEX jobs_locked_by_idx ON app_jobs.jobs (locked_by);
584
+
585
+ CREATE INDEX priority_run_at_id_idx ON app_jobs.jobs (priority, run_at, id);
586
+
587
+ CREATE FUNCTION app_jobs.tg_decrease_job_queue_count() RETURNS trigger AS $EOFCODE$
588
+ DECLARE
589
+ v_new_job_count int;
590
+ BEGIN
591
+ UPDATE
592
+ app_jobs.job_queues
593
+ SET
594
+ job_count = job_queues.job_count - 1
595
+ WHERE
596
+ queue_name = OLD.queue_name
597
+ RETURNING
598
+ job_count INTO v_new_job_count;
599
+ IF v_new_job_count <= 0 THEN
600
+ DELETE FROM app_jobs.job_queues
601
+ WHERE queue_name = OLD.queue_name
602
+ AND job_count <= 0;
603
+ END IF;
604
+ RETURN OLD;
605
+ END;
606
+ $EOFCODE$ LANGUAGE plpgsql VOLATILE;
607
+
608
+ CREATE TRIGGER decrease_job_queue_count_on_delete
609
+ AFTER DELETE
610
+ ON app_jobs.jobs
611
+ FOR EACH ROW
612
+ WHEN (old.queue_name IS NOT NULL)
613
+ EXECUTE PROCEDURE app_jobs.tg_decrease_job_queue_count();
614
+
615
+ CREATE TRIGGER decrease_job_queue_count_on_update
616
+ AFTER UPDATE OF queue_name
617
+ ON app_jobs.jobs
618
+ FOR EACH ROW
619
+ WHEN (new.queue_name IS DISTINCT FROM old.queue_name
620
+ AND old.queue_name IS NOT NULL)
621
+ EXECUTE PROCEDURE app_jobs.tg_decrease_job_queue_count();
622
+
623
+ CREATE FUNCTION app_jobs.tg_increase_job_queue_count() RETURNS trigger AS $EOFCODE$
624
+ BEGIN
625
+ INSERT INTO app_jobs.job_queues (queue_name, job_count)
626
+ VALUES (NEW.queue_name, 1)
627
+ ON CONFLICT (queue_name)
628
+ DO UPDATE SET
629
+ job_count = job_queues.job_count + 1;
630
+ RETURN NEW;
631
+ END;
632
+ $EOFCODE$ LANGUAGE plpgsql VOLATILE;
633
+
634
+ CREATE TRIGGER _500_increase_job_queue_count_on_insert
635
+ AFTER INSERT
636
+ ON app_jobs.jobs
637
+ FOR EACH ROW
638
+ WHEN (new.queue_name IS NOT NULL)
639
+ EXECUTE PROCEDURE app_jobs.tg_increase_job_queue_count();
640
+
641
+ CREATE TRIGGER _500_increase_job_queue_count_on_update
642
+ AFTER UPDATE OF queue_name
643
+ ON app_jobs.jobs
644
+ FOR EACH ROW
645
+ WHEN (new.queue_name IS DISTINCT FROM old.queue_name
646
+ AND new.queue_name IS NOT NULL)
647
+ EXECUTE PROCEDURE app_jobs.tg_increase_job_queue_count();
648
+
649
+ CREATE TRIGGER _900_notify_worker
650
+ AFTER INSERT
651
+ ON app_jobs.jobs
652
+ FOR EACH ROW
653
+ EXECUTE PROCEDURE app_jobs.do_notify('jobs:insert');
654
+
655
+ CREATE FUNCTION app_jobs.tg_update_timestamps() RETURNS trigger AS $EOFCODE$
656
+ BEGIN
657
+ IF TG_OP = 'INSERT' THEN
658
+ NEW.created_at = NOW();
659
+ NEW.updated_at = NOW();
660
+ ELSIF TG_OP = 'UPDATE' THEN
661
+ NEW.created_at = OLD.created_at;
662
+ NEW.updated_at = greatest (now(), OLD.updated_at + interval '1 millisecond');
663
+ END IF;
664
+ RETURN NEW;
665
+ END;
666
+ $EOFCODE$ LANGUAGE plpgsql;
667
+
668
+ ALTER TABLE app_jobs.jobs
669
+ ADD COLUMN created_at timestamptz;
670
+
671
+ ALTER TABLE app_jobs.jobs
672
+ ALTER COLUMN created_at SET DEFAULT now();
673
+
674
+ ALTER TABLE app_jobs.jobs
675
+ ADD COLUMN updated_at timestamptz;
676
+
677
+ ALTER TABLE app_jobs.jobs
678
+ ALTER COLUMN updated_at SET DEFAULT now();
679
+
680
+ CREATE TRIGGER _100_update_jobs_modtime_tg
681
+ BEFORE INSERT OR UPDATE
682
+ ON app_jobs.jobs
683
+ FOR EACH ROW
684
+ EXECUTE PROCEDURE app_jobs.tg_update_timestamps();
685
+
686
+ GRANT SELECT, INSERT, UPDATE, DELETE ON app_jobs.scheduled_jobs TO administrator;
687
+
688
+ CREATE INDEX scheduled_jobs_locked_by_idx ON app_jobs.scheduled_jobs (locked_by);
689
+
690
+ CREATE INDEX scheduled_jobs_priority_id_idx ON app_jobs.scheduled_jobs (priority, id);
691
+
692
+ CREATE TRIGGER _900_notify_scheduled_job
693
+ AFTER INSERT
694
+ ON app_jobs.scheduled_jobs
695
+ FOR EACH ROW
696
+ EXECUTE PROCEDURE app_jobs.do_notify('scheduled_jobs:insert');
697
+
698
+ CREATE FUNCTION app_jobs.trigger_job_with_fields() RETURNS trigger AS $EOFCODE$
699
+ DECLARE
700
+ arg text;
701
+ fn text;
702
+ i int;
703
+ args text[];
704
+ BEGIN
705
+ FOR i IN
706
+ SELECT
707
+ *
708
+ FROM
709
+ generate_series(1, TG_NARGS) g (i)
710
+ LOOP
711
+ IF (i = 1) THEN
712
+ fn = TG_ARGV[i - 1];
713
+ ELSE
714
+ args = array_append(args, TG_ARGV[i - 1]);
715
+ IF (TG_OP = 'INSERT' OR TG_OP = 'UPDATE') THEN
716
+ EXECUTE format('SELECT ($1).%s::text', TG_ARGV[i - 1])
717
+ USING NEW INTO arg;
718
+ END IF;
719
+ IF (TG_OP = 'DELETE') THEN
720
+ EXECUTE format('SELECT ($1).%s::text', TG_ARGV[i - 1])
721
+ USING OLD INTO arg;
722
+ END IF;
723
+ args = array_append(args, arg);
724
+ END IF;
725
+ END LOOP;
726
+ PERFORM
727
+ app_jobs.add_job (jwt_private.current_database_id(), fn, app_jobs.json_build_object_apply (args));
728
+ IF (TG_OP = 'INSERT' OR TG_OP = 'UPDATE') THEN
729
+ RETURN NEW;
730
+ END IF;
731
+ IF (TG_OP = 'DELETE') THEN
732
+ RETURN OLD;
733
+ END IF;
734
+ END;
735
+ $EOFCODE$ LANGUAGE plpgsql VOLATILE SECURITY DEFINER;
736
+
737
+ CREATE FUNCTION app_jobs.tg_add_job_with_row_id() RETURNS trigger AS $EOFCODE$
738
+ BEGIN
739
+ IF (TG_OP = 'INSERT' OR TG_OP = 'UPDATE') THEN
740
+ PERFORM
741
+ app_jobs.add_job (jwt_private.current_database_id(), tg_argv[0], json_build_object('id', NEW.id));
742
+ RETURN NEW;
743
+ END IF;
744
+ IF (TG_OP = 'DELETE') THEN
745
+ PERFORM
746
+ app_jobs.add_job (jwt_private.current_database_id(), tg_argv[0], json_build_object('id', OLD.id));
747
+ RETURN OLD;
748
+ END IF;
749
+ END;
750
+ $EOFCODE$ LANGUAGE plpgsql VOLATILE SECURITY DEFINER;
751
+
752
+ COMMENT ON FUNCTION app_jobs.tg_add_job_with_row_id IS 'Useful shortcut to create a job on insert or update. Pass the task name as the trigger argument, and the record id will automatically be available on the JSON payload.';
753
+
754
+ CREATE FUNCTION app_jobs.tg_add_job_with_row() RETURNS trigger AS $EOFCODE$
755
+ BEGIN
756
+ IF (TG_OP = 'INSERT' OR TG_OP = 'UPDATE') THEN
757
+ PERFORM
758
+ app_jobs.add_job (jwt_private.current_database_id(), TG_ARGV[0], to_json(NEW));
759
+ RETURN NEW;
760
+ END IF;
761
+ IF (TG_OP = 'DELETE') THEN
762
+ PERFORM
763
+ app_jobs.add_job (jwt_private.current_database_id(), TG_ARGV[0], to_json(OLD));
764
+ RETURN OLD;
765
+ END IF;
766
+ END;
767
+ $EOFCODE$ LANGUAGE plpgsql VOLATILE SECURITY DEFINER;
768
+
769
+ COMMENT ON FUNCTION app_jobs.tg_add_job_with_row IS 'Useful shortcut to create a job on insert or update. Pass the task name as the trigger argument, and the record data will automatically be available on the JSON payload.';