@pgpm/jobs 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/LICENSE +22 -0
  2. package/Makefile +6 -0
  3. package/README.md +5 -0
  4. package/__tests__/__snapshots__/jobs.test.ts.snap +16 -0
  5. package/__tests__/jobs.test.ts +139 -0
  6. package/deploy/schemas/app_jobs/helpers/json_build_object_apply.sql +28 -0
  7. package/deploy/schemas/app_jobs/procedures/add_job.sql +65 -0
  8. package/deploy/schemas/app_jobs/procedures/add_scheduled_job.sql +66 -0
  9. package/deploy/schemas/app_jobs/procedures/complete_job.sql +32 -0
  10. package/deploy/schemas/app_jobs/procedures/complete_jobs.sql +19 -0
  11. package/deploy/schemas/app_jobs/procedures/do_notify.sql +16 -0
  12. package/deploy/schemas/app_jobs/procedures/fail_job.sql +41 -0
  13. package/deploy/schemas/app_jobs/procedures/get_job.sql +77 -0
  14. package/deploy/schemas/app_jobs/procedures/get_scheduled_job.sql +46 -0
  15. package/deploy/schemas/app_jobs/procedures/permanently_fail_jobs.sql +24 -0
  16. package/deploy/schemas/app_jobs/procedures/release_jobs.sql +34 -0
  17. package/deploy/schemas/app_jobs/procedures/release_scheduled_jobs.sql +26 -0
  18. package/deploy/schemas/app_jobs/procedures/reschedule_jobs.sql +26 -0
  19. package/deploy/schemas/app_jobs/procedures/run_scheduled_job.sql +67 -0
  20. package/deploy/schemas/app_jobs/schema.sql +7 -0
  21. package/deploy/schemas/app_jobs/tables/job_queues/grants/grant_select_insert_update_delete_to_administrator.sql +12 -0
  22. package/deploy/schemas/app_jobs/tables/job_queues/indexes/job_queues_locked_by_idx.sql +8 -0
  23. package/deploy/schemas/app_jobs/tables/job_queues/table.sql +12 -0
  24. package/deploy/schemas/app_jobs/tables/jobs/grants/grant_select_insert_update_delete_to_administrator.sql +12 -0
  25. package/deploy/schemas/app_jobs/tables/jobs/indexes/jobs_locked_by_idx.sql +8 -0
  26. package/deploy/schemas/app_jobs/tables/jobs/indexes/priority_run_at_id_idx.sql +8 -0
  27. package/deploy/schemas/app_jobs/tables/jobs/table.sql +26 -0
  28. package/deploy/schemas/app_jobs/tables/jobs/triggers/decrease_job_queue_count.sql +42 -0
  29. package/deploy/schemas/app_jobs/tables/jobs/triggers/increase_job_queue_count.sql +32 -0
  30. package/deploy/schemas/app_jobs/tables/jobs/triggers/notify_worker.sql +13 -0
  31. package/deploy/schemas/app_jobs/tables/jobs/triggers/timestamps.sql +20 -0
  32. package/deploy/schemas/app_jobs/tables/scheduled_jobs/grants/grant_select_insert_update_delete_to_administrator.sql +12 -0
  33. package/deploy/schemas/app_jobs/tables/scheduled_jobs/indexes/scheduled_jobs_locked_by_idx.sql +8 -0
  34. package/deploy/schemas/app_jobs/tables/scheduled_jobs/indexes/scheduled_jobs_priority_id_idx.sql +8 -0
  35. package/deploy/schemas/app_jobs/tables/scheduled_jobs/table.sql +26 -0
  36. package/deploy/schemas/app_jobs/tables/scheduled_jobs/triggers/notify_scheduled_job.sql +12 -0
  37. package/deploy/schemas/app_jobs/triggers/tg_add_job_with_fields.sql +50 -0
  38. package/deploy/schemas/app_jobs/triggers/tg_add_job_with_row.sql +26 -0
  39. package/deploy/schemas/app_jobs/triggers/tg_add_job_with_row_id.sql +26 -0
  40. package/deploy/schemas/app_jobs/triggers/tg_update_timestamps.sql +21 -0
  41. package/jest.config.js +15 -0
  42. package/launchql-jobs.control +8 -0
  43. package/launchql.plan +38 -0
  44. package/package.json +29 -0
  45. package/revert/schemas/app_jobs/helpers/json_build_object_apply.sql +7 -0
  46. package/revert/schemas/app_jobs/procedures/add_job.sql +7 -0
  47. package/revert/schemas/app_jobs/procedures/add_scheduled_job.sql +7 -0
  48. package/revert/schemas/app_jobs/procedures/complete_job.sql +7 -0
  49. package/revert/schemas/app_jobs/procedures/complete_jobs.sql +7 -0
  50. package/revert/schemas/app_jobs/procedures/do_notify.sql +7 -0
  51. package/revert/schemas/app_jobs/procedures/fail_job.sql +7 -0
  52. package/revert/schemas/app_jobs/procedures/get_job.sql +7 -0
  53. package/revert/schemas/app_jobs/procedures/get_scheduled_job.sql +7 -0
  54. package/revert/schemas/app_jobs/procedures/permanently_fail_jobs.sql +7 -0
  55. package/revert/schemas/app_jobs/procedures/release_jobs.sql +7 -0
  56. package/revert/schemas/app_jobs/procedures/release_scheduled_jobs.sql +7 -0
  57. package/revert/schemas/app_jobs/procedures/reschedule_jobs.sql +7 -0
  58. package/revert/schemas/app_jobs/procedures/run_scheduled_job.sql +7 -0
  59. package/revert/schemas/app_jobs/schema.sql +7 -0
  60. package/revert/schemas/app_jobs/tables/job_queues/grants/grant_select_insert_update_delete_to_administrator.sql +7 -0
  61. package/revert/schemas/app_jobs/tables/job_queues/indexes/job_queues_locked_by_idx.sql +7 -0
  62. package/revert/schemas/app_jobs/tables/job_queues/table.sql +7 -0
  63. package/revert/schemas/app_jobs/tables/jobs/grants/grant_select_insert_update_delete_to_administrator.sql +7 -0
  64. package/revert/schemas/app_jobs/tables/jobs/indexes/jobs_locked_by_idx.sql +7 -0
  65. package/revert/schemas/app_jobs/tables/jobs/indexes/priority_run_at_id_idx.sql +7 -0
  66. package/revert/schemas/app_jobs/tables/jobs/table.sql +7 -0
  67. package/revert/schemas/app_jobs/tables/jobs/triggers/decrease_job_queue_count.sql +7 -0
  68. package/revert/schemas/app_jobs/tables/jobs/triggers/increase_job_queue_count.sql +7 -0
  69. package/revert/schemas/app_jobs/tables/jobs/triggers/notify_worker.sql +5 -0
  70. package/revert/schemas/app_jobs/tables/jobs/triggers/timestamps.sql +9 -0
  71. package/revert/schemas/app_jobs/tables/scheduled_jobs/grants/grant_select_insert_update_delete_to_administrator.sql +7 -0
  72. package/revert/schemas/app_jobs/tables/scheduled_jobs/indexes/scheduled_jobs_locked_by_idx.sql +7 -0
  73. package/revert/schemas/app_jobs/tables/scheduled_jobs/indexes/scheduled_jobs_priority_id_idx.sql +7 -0
  74. package/revert/schemas/app_jobs/tables/scheduled_jobs/table.sql +7 -0
  75. package/revert/schemas/app_jobs/tables/scheduled_jobs/triggers/notify_scheduled_job.sql +8 -0
  76. package/revert/schemas/app_jobs/triggers/tg_add_job_with_fields.sql +7 -0
  77. package/revert/schemas/app_jobs/triggers/tg_add_job_with_row.sql +7 -0
  78. package/revert/schemas/app_jobs/triggers/tg_add_job_with_row_id.sql +5 -0
  79. package/revert/schemas/app_jobs/triggers/tg_update_timestamps.sql +7 -0
  80. package/sql/launchql-jobs--0.4.6.sql +658 -0
  81. package/verify/schemas/app_jobs/helpers/json_build_object_apply.sql +7 -0
  82. package/verify/schemas/app_jobs/procedures/add_job.sql +7 -0
  83. package/verify/schemas/app_jobs/procedures/add_scheduled_job.sql +7 -0
  84. package/verify/schemas/app_jobs/procedures/complete_job.sql +7 -0
  85. package/verify/schemas/app_jobs/procedures/complete_jobs.sql +7 -0
  86. package/verify/schemas/app_jobs/procedures/do_notify.sql +7 -0
  87. package/verify/schemas/app_jobs/procedures/fail_job.sql +7 -0
  88. package/verify/schemas/app_jobs/procedures/get_job.sql +7 -0
  89. package/verify/schemas/app_jobs/procedures/get_scheduled_job.sql +7 -0
  90. package/verify/schemas/app_jobs/procedures/permanently_fail_jobs.sql +7 -0
  91. package/verify/schemas/app_jobs/procedures/release_jobs.sql +7 -0
  92. package/verify/schemas/app_jobs/procedures/release_scheduled_jobs.sql +7 -0
  93. package/verify/schemas/app_jobs/procedures/reschedule_jobs.sql +7 -0
  94. package/verify/schemas/app_jobs/procedures/run_scheduled_job.sql +7 -0
  95. package/verify/schemas/app_jobs/schema.sql +7 -0
  96. package/verify/schemas/app_jobs/tables/job_queues/grants/grant_select_insert_update_delete_to_administrator.sql +10 -0
  97. package/verify/schemas/app_jobs/tables/job_queues/indexes/job_queues_locked_by_idx.sql +7 -0
  98. package/verify/schemas/app_jobs/tables/job_queues/table.sql +7 -0
  99. package/verify/schemas/app_jobs/tables/jobs/grants/grant_select_insert_update_delete_to_administrator.sql +10 -0
  100. package/verify/schemas/app_jobs/tables/jobs/indexes/jobs_locked_by_idx.sql +7 -0
  101. package/verify/schemas/app_jobs/tables/jobs/indexes/priority_run_at_id_idx.sql +7 -0
  102. package/verify/schemas/app_jobs/tables/jobs/table.sql +7 -0
  103. package/verify/schemas/app_jobs/tables/jobs/triggers/decrease_job_queue_count.sql +10 -0
  104. package/verify/schemas/app_jobs/tables/jobs/triggers/increase_job_queue_count.sql +10 -0
  105. package/verify/schemas/app_jobs/tables/jobs/triggers/notify_worker.sql +6 -0
  106. package/verify/schemas/app_jobs/tables/jobs/triggers/timestamps.sql +16 -0
  107. package/verify/schemas/app_jobs/tables/scheduled_jobs/grants/grant_select_insert_update_delete_to_administrator.sql +10 -0
  108. package/verify/schemas/app_jobs/tables/scheduled_jobs/indexes/scheduled_jobs_locked_by_idx.sql +7 -0
  109. package/verify/schemas/app_jobs/tables/scheduled_jobs/indexes/scheduled_jobs_priority_id_idx.sql +7 -0
  110. package/verify/schemas/app_jobs/tables/scheduled_jobs/table.sql +7 -0
  111. package/verify/schemas/app_jobs/tables/scheduled_jobs/triggers/notify_scheduled_job.sql +8 -0
  112. package/verify/schemas/app_jobs/triggers/tg_add_job_with_fields.sql +7 -0
  113. package/verify/schemas/app_jobs/triggers/tg_add_job_with_row.sql +7 -0
  114. package/verify/schemas/app_jobs/triggers/tg_add_job_with_row_id.sql +6 -0
  115. package/verify/schemas/app_jobs/triggers/tg_update_timestamps.sql +7 -0
@@ -0,0 +1,658 @@
1
+ \echo Use "CREATE EXTENSION launchql-jobs" to load this file. \quit
2
+ CREATE SCHEMA IF NOT EXISTS app_jobs;
3
+
4
+ GRANT USAGE ON SCHEMA app_jobs TO administrator;
5
+
6
+ ALTER DEFAULT PRIVILEGES IN SCHEMA app_jobs
7
+ GRANT EXECUTE ON FUNCTIONS TO administrator;
8
+
9
+ CREATE FUNCTION app_jobs.json_build_object_apply(arguments text[]) RETURNS pg_catalog.json AS $EOFCODE$
10
+ DECLARE
11
+ arg text;
12
+ _sql text;
13
+ _res json;
14
+ args text[];
15
+ BEGIN
16
+ _sql = 'SELECT json_build_object(';
17
+ FOR arg IN
18
+ SELECT
19
+ unnest(arguments)
20
+ LOOP
21
+ args = array_append(args, format('''%s''', arg));
22
+ END LOOP;
23
+ _sql = _sql || format('%s);', array_to_string(args, ','));
24
+ EXECUTE _sql INTO _res;
25
+ RETURN _res;
26
+ END;
27
+ $EOFCODE$ LANGUAGE plpgsql;
28
+
29
+ CREATE TABLE app_jobs.jobs (
30
+ id bigserial PRIMARY KEY,
31
+ queue_name text DEFAULT public.gen_random_uuid()::text,
32
+ task_identifier text NOT NULL,
33
+ payload pg_catalog.json DEFAULT '{}'::json NOT NULL,
34
+ priority int DEFAULT 0 NOT NULL,
35
+ run_at timestamptz DEFAULT now() NOT NULL,
36
+ attempts int DEFAULT 0 NOT NULL,
37
+ max_attempts int DEFAULT 25 NOT NULL,
38
+ key text,
39
+ last_error text,
40
+ locked_at timestamptz,
41
+ locked_by text,
42
+ CHECK (length(key) < 513),
43
+ CHECK (length(task_identifier) < 127),
44
+ CHECK (max_attempts > 0),
45
+ CHECK (length(queue_name) < 127),
46
+ CHECK (length(locked_by) > 3),
47
+ UNIQUE (key)
48
+ );
49
+
50
+ CREATE TABLE app_jobs.job_queues (
51
+ queue_name text NOT NULL PRIMARY KEY,
52
+ job_count int DEFAULT 0 NOT NULL,
53
+ locked_at timestamptz,
54
+ locked_by text
55
+ );
56
+
57
+ CREATE FUNCTION app_jobs.add_job(identifier text, payload pg_catalog.json DEFAULT '{}'::json, job_key text DEFAULT NULL, queue_name text DEFAULT NULL, run_at timestamptz DEFAULT now(), max_attempts int DEFAULT 25, priority int DEFAULT 0) RETURNS app_jobs.jobs AS $EOFCODE$
58
+ DECLARE
59
+ v_job app_jobs.jobs;
60
+ BEGIN
61
+ IF job_key IS NOT NULL THEN
62
+ -- Upsert job
63
+ INSERT INTO app_jobs.jobs (task_identifier, payload, queue_name, run_at, max_attempts, KEY, priority)
64
+ VALUES (identifier, coalesce(payload, '{}'::json), queue_name, coalesce(run_at, now()), coalesce(max_attempts, 25), job_key, coalesce(priority, 0))
65
+ ON CONFLICT (KEY)
66
+ DO UPDATE SET
67
+ task_identifier = excluded.task_identifier, payload = excluded.payload, queue_name = excluded.queue_name, max_attempts = excluded.max_attempts, run_at = excluded.run_at, priority = excluded.priority,
68
+ -- always reset error/retry state
69
+ attempts = 0, last_error = NULL
70
+ WHERE
71
+ jobs.locked_at IS NULL
72
+ RETURNING
73
+ * INTO v_job;
74
+
75
+ -- If upsert succeeded (insert or update), return early
76
+
77
+ IF NOT (v_job IS NULL) THEN
78
+ RETURN v_job;
79
+ END IF;
80
+
81
+ -- Upsert failed -> there must be an existing job that is locked. Remove
82
+ -- existing key to allow a new one to be inserted, and prevent any
83
+ -- subsequent retries by bumping attempts to the max allowed.
84
+
85
+ UPDATE
86
+ app_jobs.jobs
87
+ SET
88
+ KEY = NULL,
89
+ attempts = jobs.max_attempts
90
+ WHERE
91
+ KEY = job_key;
92
+ END IF;
93
+
94
+ INSERT INTO app_jobs.jobs (task_identifier, payload, queue_name, run_at, max_attempts, priority)
95
+ VALUES (identifier, payload, queue_name, run_at, max_attempts, priority)
96
+ RETURNING
97
+ * INTO v_job;
98
+ RETURN v_job;
99
+ END;
100
+ $EOFCODE$ LANGUAGE plpgsql VOLATILE SECURITY DEFINER;
101
+
102
+ CREATE TABLE app_jobs.scheduled_jobs (
103
+ id bigserial PRIMARY KEY,
104
+ queue_name text DEFAULT public.gen_random_uuid()::text,
105
+ task_identifier text NOT NULL,
106
+ payload pg_catalog.json DEFAULT '{}'::json NOT NULL,
107
+ priority int DEFAULT 0 NOT NULL,
108
+ max_attempts int DEFAULT 25 NOT NULL,
109
+ key text,
110
+ locked_at timestamptz,
111
+ locked_by text,
112
+ schedule_info pg_catalog.json NOT NULL,
113
+ last_scheduled timestamptz,
114
+ last_scheduled_id bigint,
115
+ CHECK (length(key) < 513),
116
+ CHECK (length(task_identifier) < 127),
117
+ CHECK (max_attempts > 0),
118
+ CHECK (length(queue_name) < 127),
119
+ CHECK (length(locked_by) > 3),
120
+ UNIQUE (key)
121
+ );
122
+
123
+ CREATE FUNCTION app_jobs.add_scheduled_job(identifier text, payload pg_catalog.json DEFAULT '{}'::json, schedule_info pg_catalog.json DEFAULT '{}'::json, job_key text DEFAULT NULL, queue_name text DEFAULT NULL, max_attempts int DEFAULT 25, priority int DEFAULT 0) RETURNS app_jobs.scheduled_jobs AS $EOFCODE$
124
+ DECLARE
125
+ v_job app_jobs.scheduled_jobs;
126
+ BEGIN
127
+ IF job_key IS NOT NULL THEN
128
+
129
+ -- Upsert job
130
+ INSERT INTO app_jobs.scheduled_jobs (task_identifier, payload, queue_name, schedule_info, max_attempts, KEY, priority)
131
+ VALUES (identifier, coalesce(payload, '{}'::json), queue_name, schedule_info, coalesce(max_attempts, 25), job_key, coalesce(priority, 0))
132
+ ON CONFLICT (KEY)
133
+ DO UPDATE SET
134
+ task_identifier = excluded.task_identifier,
135
+ payload = excluded.payload,
136
+ queue_name = excluded.queue_name,
137
+ max_attempts = excluded.max_attempts,
138
+ schedule_info = excluded.schedule_info,
139
+ priority = excluded.priority
140
+ WHERE
141
+ scheduled_jobs.locked_at IS NULL
142
+ RETURNING
143
+ * INTO v_job;
144
+
145
+ -- If upsert succeeded (insert or update), return early
146
+
147
+ IF NOT (v_job IS NULL) THEN
148
+ RETURN v_job;
149
+ END IF;
150
+
151
+ -- Upsert failed -> there must be an existing scheduled job that is locked. Remove
152
+ -- and allow a new one to be inserted
153
+
154
+ DELETE FROM
155
+ app_jobs.scheduled_jobs
156
+ WHERE
157
+ KEY = job_key;
158
+ END IF;
159
+
160
+ INSERT INTO app_jobs.scheduled_jobs (task_identifier, payload, queue_name, schedule_info, max_attempts, priority)
161
+ VALUES (identifier, payload, queue_name, schedule_info, max_attempts, priority)
162
+ RETURNING
163
+ * INTO v_job;
164
+ RETURN v_job;
165
+ END;
166
+ $EOFCODE$ LANGUAGE plpgsql VOLATILE SECURITY DEFINER;
167
+
168
+ CREATE FUNCTION app_jobs.complete_job(worker_id text, job_id bigint) RETURNS app_jobs.jobs LANGUAGE plpgsql AS $EOFCODE$
169
+ DECLARE
170
+ v_row app_jobs.jobs;
171
+ BEGIN
172
+ DELETE FROM app_jobs.jobs
173
+ WHERE id = job_id
174
+ RETURNING
175
+ * INTO v_row;
176
+ IF v_row.queue_name IS NOT NULL THEN
177
+ UPDATE
178
+ app_jobs.job_queues
179
+ SET
180
+ locked_by = NULL,
181
+ locked_at = NULL
182
+ WHERE
183
+ queue_name = v_row.queue_name
184
+ AND locked_by = worker_id;
185
+ END IF;
186
+ RETURN v_row;
187
+ END;
188
+ $EOFCODE$;
189
+
190
+ CREATE FUNCTION app_jobs.complete_jobs(job_ids bigint[]) RETURNS SETOF app_jobs.jobs LANGUAGE sql AS $EOFCODE$
191
+ DELETE FROM app_jobs.jobs
192
+ WHERE id = ANY (job_ids)
193
+ AND (locked_by IS NULL
194
+ OR locked_at < NOW() - interval '4 hours')
195
+ RETURNING
196
+ *;
197
+ $EOFCODE$;
198
+
199
+ CREATE FUNCTION app_jobs.do_notify() RETURNS trigger AS $EOFCODE$
200
+ BEGIN
201
+ PERFORM
202
+ pg_notify(TG_ARGV[0], '');
203
+ RETURN NEW;
204
+ END;
205
+ $EOFCODE$ LANGUAGE plpgsql;
206
+
207
+ CREATE FUNCTION app_jobs.fail_job(worker_id text, job_id bigint, error_message text) RETURNS app_jobs.jobs LANGUAGE plpgsql STRICT AS $EOFCODE$
208
+ DECLARE
209
+ v_row app_jobs.jobs;
210
+ BEGIN
211
+ UPDATE
212
+ app_jobs.jobs
213
+ SET
214
+ last_error = error_message,
215
+ run_at = greatest (now(), run_at) + (exp(least (attempts, 10))::text || ' seconds')::interval,
216
+ locked_by = NULL,
217
+ locked_at = NULL
218
+ WHERE
219
+ id = job_id
220
+ AND locked_by = worker_id
221
+ RETURNING
222
+ * INTO v_row;
223
+ IF v_row.queue_name IS NOT NULL THEN
224
+ UPDATE
225
+ app_jobs.job_queues
226
+ SET
227
+ locked_by = NULL,
228
+ locked_at = NULL
229
+ WHERE
230
+ queue_name = v_row.queue_name
231
+ AND locked_by = worker_id;
232
+ END IF;
233
+ RETURN v_row;
234
+ END;
235
+ $EOFCODE$;
236
+
237
+ CREATE FUNCTION app_jobs.get_job(worker_id text, task_identifiers text[] DEFAULT NULL, job_expiry interval DEFAULT '4 hours') RETURNS app_jobs.jobs LANGUAGE plpgsql AS $EOFCODE$
238
+ DECLARE
239
+ v_job_id bigint;
240
+ v_queue_name text;
241
+ v_row app_jobs.jobs;
242
+ v_now timestamptz = now();
243
+ BEGIN
244
+ IF worker_id IS NULL THEN
245
+ RAISE exception 'INVALID_WORKER_ID';
246
+ END IF;
247
+ SELECT
248
+ jobs.queue_name,
249
+ jobs.id INTO v_queue_name,
250
+ v_job_id
251
+ FROM
252
+ app_jobs.jobs
253
+ WHERE (jobs.locked_at IS NULL
254
+ OR jobs.locked_at < (v_now - job_expiry))
255
+ AND (jobs.queue_name IS NULL
256
+ OR EXISTS (
257
+ SELECT
258
+ 1
259
+ FROM
260
+ app_jobs.job_queues
261
+ WHERE
262
+ job_queues.queue_name = jobs.queue_name
263
+ AND (job_queues.locked_at IS NULL
264
+ OR job_queues.locked_at < (v_now - job_expiry))
265
+ FOR UPDATE
266
+ SKIP LOCKED))
267
+ AND run_at <= v_now
268
+ AND attempts < max_attempts
269
+ AND (task_identifiers IS NULL
270
+ OR task_identifier = ANY (task_identifiers))
271
+ ORDER BY
272
+ priority ASC,
273
+ run_at ASC,
274
+ id ASC
275
+ LIMIT 1
276
+ FOR UPDATE
277
+ SKIP LOCKED;
278
+ IF v_job_id IS NULL THEN
279
+ RETURN NULL;
280
+ END IF;
281
+ IF v_queue_name IS NOT NULL THEN
282
+ UPDATE
283
+ app_jobs.job_queues
284
+ SET
285
+ locked_by = worker_id,
286
+ locked_at = v_now
287
+ WHERE
288
+ job_queues.queue_name = v_queue_name;
289
+ END IF;
290
+ UPDATE
291
+ app_jobs.jobs
292
+ SET
293
+ attempts = attempts + 1,
294
+ locked_by = worker_id,
295
+ locked_at = v_now
296
+ WHERE
297
+ id = v_job_id
298
+ RETURNING
299
+ * INTO v_row;
300
+ RETURN v_row;
301
+ END;
302
+ $EOFCODE$;
303
+
304
+ CREATE FUNCTION app_jobs.get_scheduled_job(worker_id text, task_identifiers text[] DEFAULT NULL) RETURNS app_jobs.scheduled_jobs LANGUAGE plpgsql AS $EOFCODE$
305
+ DECLARE
306
+ v_job_id bigint;
307
+ v_row app_jobs.scheduled_jobs;
308
+ BEGIN
309
+ IF worker_id IS NULL THEN
310
+ RAISE exception 'INVALID_WORKER_ID';
311
+ END IF;
312
+ SELECT
313
+ scheduled_jobs.id INTO v_job_id
314
+ FROM
315
+ app_jobs.scheduled_jobs
316
+ WHERE (scheduled_jobs.locked_at IS NULL)
317
+ AND (task_identifiers IS NULL
318
+ OR task_identifier = ANY (task_identifiers))
319
+ ORDER BY
320
+ priority ASC,
321
+ id ASC
322
+ LIMIT 1
323
+ FOR UPDATE
324
+ SKIP LOCKED;
325
+ IF v_job_id IS NULL THEN
326
+ RETURN NULL;
327
+ END IF;
328
+ UPDATE
329
+ app_jobs.scheduled_jobs
330
+ SET
331
+ locked_by = worker_id,
332
+ locked_at = NOW()
333
+ WHERE
334
+ id = v_job_id
335
+ RETURNING
336
+ * INTO v_row;
337
+ RETURN v_row;
338
+ END;
339
+ $EOFCODE$;
340
+
341
+ CREATE FUNCTION app_jobs.permanently_fail_jobs(job_ids bigint[], error_message text DEFAULT NULL) RETURNS SETOF app_jobs.jobs LANGUAGE sql AS $EOFCODE$
342
+ UPDATE
343
+ app_jobs.jobs
344
+ SET
345
+ last_error = coalesce(error_message, 'Manually marked as failed'),
346
+ attempts = max_attempts
347
+ WHERE
348
+ id = ANY (job_ids)
349
+ AND (locked_by IS NULL
350
+ OR locked_at < NOW() - interval '4 hours')
351
+ RETURNING
352
+ *;
353
+ $EOFCODE$;
354
+
355
+ CREATE FUNCTION app_jobs.release_jobs(worker_id text) RETURNS void AS $EOFCODE$
356
+ DECLARE
357
+ BEGIN
358
+ -- clear the job
359
+ UPDATE
360
+ app_jobs.jobs
361
+ SET
362
+ locked_at = NULL,
363
+ locked_by = NULL,
364
+ attempts = GREATEST (attempts - 1, 0)
365
+ WHERE
366
+ locked_by = worker_id;
367
+ -- clear the queue
368
+ UPDATE
369
+ app_jobs.job_queues
370
+ SET
371
+ locked_at = NULL,
372
+ locked_by = NULL
373
+ WHERE
374
+ locked_by = worker_id;
375
+ END;
376
+ $EOFCODE$ LANGUAGE plpgsql VOLATILE;
377
+
378
+ CREATE FUNCTION app_jobs.release_scheduled_jobs(worker_id text, ids bigint[] DEFAULT NULL) RETURNS void AS $EOFCODE$
379
+ DECLARE
380
+ BEGIN
381
+ -- clear the scheduled job
382
+ UPDATE
383
+ app_jobs.scheduled_jobs s
384
+ SET
385
+ locked_at = NULL,
386
+ locked_by = NULL
387
+ WHERE
388
+ locked_by = worker_id
389
+ AND (ids IS NULL
390
+ OR s.id = ANY (ids));
391
+ END;
392
+ $EOFCODE$ LANGUAGE plpgsql VOLATILE;
393
+
394
+ CREATE FUNCTION app_jobs.reschedule_jobs(job_ids bigint[], run_at timestamptz DEFAULT NULL, priority int DEFAULT NULL, attempts int DEFAULT NULL, max_attempts int DEFAULT NULL) RETURNS SETOF app_jobs.jobs LANGUAGE sql AS $EOFCODE$
395
+ UPDATE
396
+ app_jobs.jobs
397
+ SET
398
+ run_at = coalesce(reschedule_jobs.run_at, jobs.run_at),
399
+ priority = coalesce(reschedule_jobs.priority, jobs.priority),
400
+ attempts = coalesce(reschedule_jobs.attempts, jobs.attempts),
401
+ max_attempts = coalesce(reschedule_jobs.max_attempts, jobs.max_attempts)
402
+ WHERE
403
+ id = ANY (job_ids)
404
+ AND (locked_by IS NULL
405
+ OR locked_at < NOW() - interval '4 hours')
406
+ RETURNING
407
+ *;
408
+ $EOFCODE$;
409
+
410
+ CREATE FUNCTION app_jobs.run_scheduled_job(id bigint, job_expiry interval DEFAULT '1 hours') RETURNS app_jobs.jobs AS $EOFCODE$
411
+ DECLARE
412
+ j app_jobs.jobs;
413
+ last_id bigint;
414
+ lkd_by text;
415
+ BEGIN
416
+ -- check last scheduled
417
+ SELECT
418
+ last_scheduled_id
419
+ FROM
420
+ app_jobs.scheduled_jobs s
421
+ WHERE
422
+ s.id = run_scheduled_job.id INTO last_id;
423
+ -- if it's been scheduled check if it's been run
424
+ IF (last_id IS NOT NULL) THEN
425
+ SELECT
426
+ locked_by
427
+ FROM
428
+ app_jobs.jobs js
429
+ WHERE
430
+ js.id = last_id
431
+ AND (js.locked_at IS NULL -- never been run
432
+ OR js.locked_at >= (NOW() - job_expiry)
433
+ -- still running within a safe interval
434
+ ) INTO lkd_by;
435
+ IF (FOUND) THEN
436
+ RAISE EXCEPTION 'ALREADY_SCHEDULED';
437
+ END IF;
438
+ END IF;
439
+ -- insert new job
440
+ INSERT INTO app_jobs.jobs (queue_name, task_identifier, payload, priority, max_attempts, key)
441
+ SELECT
442
+ queue_name,
443
+ task_identifier,
444
+ payload,
445
+ priority,
446
+ max_attempts,
447
+ key
448
+ FROM
449
+ app_jobs.scheduled_jobs s
450
+ WHERE
451
+ s.id = run_scheduled_job.id
452
+ RETURNING
453
+ * INTO j;
454
+ -- update the scheduled job
455
+ UPDATE
456
+ app_jobs.scheduled_jobs s
457
+ SET
458
+ last_scheduled = NOW(),
459
+ last_scheduled_id = j.id
460
+ WHERE
461
+ s.id = run_scheduled_job.id;
462
+ RETURN j;
463
+ END;
464
+ $EOFCODE$ LANGUAGE plpgsql VOLATILE;
465
+
466
+ GRANT SELECT, INSERT, UPDATE, DELETE ON app_jobs.job_queues TO administrator;
467
+
468
+ CREATE INDEX job_queues_locked_by_idx ON app_jobs.job_queues (locked_by);
469
+
470
+ GRANT SELECT, INSERT, UPDATE, DELETE ON app_jobs.jobs TO administrator;
471
+
472
+ CREATE INDEX jobs_locked_by_idx ON app_jobs.jobs (locked_by);
473
+
474
+ CREATE INDEX priority_run_at_id_idx ON app_jobs.jobs (priority, run_at, id);
475
+
476
+ CREATE FUNCTION app_jobs.tg_decrease_job_queue_count() RETURNS trigger AS $EOFCODE$
477
+ DECLARE
478
+ v_new_job_count int;
479
+ BEGIN
480
+ UPDATE
481
+ app_jobs.job_queues
482
+ SET
483
+ job_count = job_queues.job_count - 1
484
+ WHERE
485
+ queue_name = OLD.queue_name
486
+ RETURNING
487
+ job_count INTO v_new_job_count;
488
+ IF v_new_job_count <= 0 THEN
489
+ DELETE FROM app_jobs.job_queues
490
+ WHERE queue_name = OLD.queue_name
491
+ AND job_count <= 0;
492
+ END IF;
493
+ RETURN OLD;
494
+ END;
495
+ $EOFCODE$ LANGUAGE plpgsql VOLATILE;
496
+
497
+ CREATE TRIGGER decrease_job_queue_count_on_delete
498
+ AFTER DELETE
499
+ ON app_jobs.jobs
500
+ FOR EACH ROW
501
+ WHEN (old.queue_name IS NOT NULL)
502
+ EXECUTE PROCEDURE app_jobs.tg_decrease_job_queue_count();
503
+
504
+ CREATE TRIGGER decrease_job_queue_count_on_update
505
+ AFTER UPDATE OF queue_name
506
+ ON app_jobs.jobs
507
+ FOR EACH ROW
508
+ WHEN (new.queue_name IS DISTINCT FROM old.queue_name
509
+ AND old.queue_name IS NOT NULL)
510
+ EXECUTE PROCEDURE app_jobs.tg_decrease_job_queue_count();
511
+
512
+ CREATE FUNCTION app_jobs.tg_increase_job_queue_count() RETURNS trigger AS $EOFCODE$
513
+ BEGIN
514
+ INSERT INTO app_jobs.job_queues (queue_name, job_count)
515
+ VALUES (NEW.queue_name, 1)
516
+ ON CONFLICT (queue_name)
517
+ DO UPDATE SET
518
+ job_count = job_queues.job_count + 1;
519
+ RETURN NEW;
520
+ END;
521
+ $EOFCODE$ LANGUAGE plpgsql VOLATILE;
522
+
523
+ CREATE TRIGGER _500_increase_job_queue_count_on_insert
524
+ AFTER INSERT
525
+ ON app_jobs.jobs
526
+ FOR EACH ROW
527
+ WHEN (new.queue_name IS NOT NULL)
528
+ EXECUTE PROCEDURE app_jobs.tg_increase_job_queue_count();
529
+
530
+ CREATE TRIGGER _500_increase_job_queue_count_on_update
531
+ AFTER UPDATE OF queue_name
532
+ ON app_jobs.jobs
533
+ FOR EACH ROW
534
+ WHEN (new.queue_name IS DISTINCT FROM old.queue_name
535
+ AND new.queue_name IS NOT NULL)
536
+ EXECUTE PROCEDURE app_jobs.tg_increase_job_queue_count();
537
+
538
+ CREATE TRIGGER _900_notify_worker
539
+ AFTER INSERT
540
+ ON app_jobs.jobs
541
+ FOR EACH ROW
542
+ EXECUTE PROCEDURE app_jobs.do_notify('jobs:insert');
543
+
544
+ CREATE FUNCTION app_jobs.tg_update_timestamps() RETURNS trigger AS $EOFCODE$
545
+ BEGIN
546
+ IF TG_OP = 'INSERT' THEN
547
+ NEW.created_at = NOW();
548
+ NEW.updated_at = NOW();
549
+ ELSIF TG_OP = 'UPDATE' THEN
550
+ NEW.created_at = OLD.created_at;
551
+ NEW.updated_at = greatest (now(), OLD.updated_at + interval '1 millisecond');
552
+ END IF;
553
+ RETURN NEW;
554
+ END;
555
+ $EOFCODE$ LANGUAGE plpgsql;
556
+
557
+ ALTER TABLE app_jobs.jobs
558
+ ADD COLUMN created_at timestamptz;
559
+
560
+ ALTER TABLE app_jobs.jobs
561
+ ALTER COLUMN created_at SET DEFAULT now();
562
+
563
+ ALTER TABLE app_jobs.jobs
564
+ ADD COLUMN updated_at timestamptz;
565
+
566
+ ALTER TABLE app_jobs.jobs
567
+ ALTER COLUMN updated_at SET DEFAULT now();
568
+
569
+ CREATE TRIGGER _100_update_jobs_modtime_tg
570
+ BEFORE INSERT OR UPDATE
571
+ ON app_jobs.jobs
572
+ FOR EACH ROW
573
+ EXECUTE PROCEDURE app_jobs.tg_update_timestamps();
574
+
575
+ GRANT SELECT, INSERT, UPDATE, DELETE ON app_jobs.scheduled_jobs TO administrator;
576
+
577
+ CREATE INDEX scheduled_jobs_locked_by_idx ON app_jobs.scheduled_jobs (locked_by);
578
+
579
+ CREATE INDEX scheduled_jobs_priority_id_idx ON app_jobs.scheduled_jobs (priority, id);
580
+
581
+ CREATE TRIGGER _900_notify_scheduled_job
582
+ AFTER INSERT
583
+ ON app_jobs.scheduled_jobs
584
+ FOR EACH ROW
585
+ EXECUTE PROCEDURE app_jobs.do_notify('scheduled_jobs:insert');
586
+
587
+ CREATE FUNCTION app_jobs.trigger_job_with_fields() RETURNS trigger AS $EOFCODE$
588
+ DECLARE
589
+ arg text;
590
+ fn text;
591
+ i int;
592
+ args text[];
593
+ BEGIN
594
+ FOR i IN
595
+ SELECT
596
+ *
597
+ FROM
598
+ generate_series(1, TG_NARGS) g (i)
599
+ LOOP
600
+ IF (i = 1) THEN
601
+ fn = TG_ARGV[i - 1];
602
+ ELSE
603
+ args = array_append(args, TG_ARGV[i - 1]);
604
+ IF (TG_OP = 'INSERT' OR TG_OP = 'UPDATE') THEN
605
+ EXECUTE format('SELECT ($1).%s::text', TG_ARGV[i - 1])
606
+ USING NEW INTO arg;
607
+ END IF;
608
+ IF (TG_OP = 'DELETE') THEN
609
+ EXECUTE format('SELECT ($1).%s::text', TG_ARGV[i - 1])
610
+ USING OLD INTO arg;
611
+ END IF;
612
+ args = array_append(args, arg);
613
+ END IF;
614
+ END LOOP;
615
+ PERFORM
616
+ app_jobs.add_job (fn, app_jobs.json_build_object_apply (args));
617
+ IF (TG_OP = 'INSERT' OR TG_OP = 'UPDATE') THEN
618
+ RETURN NEW;
619
+ END IF;
620
+ IF (TG_OP = 'DELETE') THEN
621
+ RETURN OLD;
622
+ END IF;
623
+ END;
624
+ $EOFCODE$ LANGUAGE plpgsql VOLATILE SECURITY DEFINER;
625
+
626
+ CREATE FUNCTION app_jobs.tg_add_job_with_row_id() RETURNS trigger AS $EOFCODE$
627
+ BEGIN
628
+ IF (TG_OP = 'INSERT' OR TG_OP = 'UPDATE') THEN
629
+ PERFORM
630
+ app_jobs.add_job (tg_argv[0], json_build_object('id', NEW.id));
631
+ RETURN NEW;
632
+ END IF;
633
+ IF (TG_OP = 'DELETE') THEN
634
+ PERFORM
635
+ app_jobs.add_job (tg_argv[0], json_build_object('id', OLD.id));
636
+ RETURN OLD;
637
+ END IF;
638
+ END;
639
+ $EOFCODE$ LANGUAGE plpgsql VOLATILE SECURITY DEFINER;
640
+
641
+ COMMENT ON FUNCTION app_jobs.tg_add_job_with_row_id IS 'Useful shortcut to create a job on insert or update. Pass the task name as the trigger argument, and the record id will automatically be available on the JSON payload.';
642
+
643
+ CREATE FUNCTION app_jobs.tg_add_job_with_row() RETURNS trigger AS $EOFCODE$
644
+ BEGIN
645
+ IF (TG_OP = 'INSERT' OR TG_OP = 'UPDATE') THEN
646
+ PERFORM
647
+ app_jobs.add_job (TG_ARGV[0], to_json(NEW));
648
+ RETURN NEW;
649
+ END IF;
650
+ IF (TG_OP = 'DELETE') THEN
651
+ PERFORM
652
+ app_jobs.add_job (TG_ARGV[0], to_json(OLD));
653
+ RETURN OLD;
654
+ END IF;
655
+ END;
656
+ $EOFCODE$ LANGUAGE plpgsql VOLATILE SECURITY DEFINER;
657
+
658
+ COMMENT ON FUNCTION app_jobs.tg_add_job_with_row IS 'Useful shortcut to create a job on insert or update. Pass the task name as the trigger argument, and the record data will automatically be available on the JSON payload.';
@@ -0,0 +1,7 @@
1
+ -- Verify schemas/app_jobs/helpers/json_build_object_apply on pg
2
+
3
+ BEGIN;
4
+
5
+ SELECT verify_function ('app_jobs.json_build_object_apply');
6
+
7
+ ROLLBACK;
@@ -0,0 +1,7 @@
1
+ -- Verify schemas/app_jobs/procedures/add_job on pg
2
+
3
+ BEGIN;
4
+
5
+ SELECT verify_function ('app_jobs.add_job');
6
+
7
+ ROLLBACK;
@@ -0,0 +1,7 @@
1
+ -- Verify schemas/app_jobs/procedures/add_scheduled_job on pg
2
+
3
+ BEGIN;
4
+
5
+ SELECT verify_function ('app_jobs.add_scheduled_job');
6
+
7
+ ROLLBACK;
@@ -0,0 +1,7 @@
1
+ -- Verify schemas/app_jobs/procedures/complete_job on pg
2
+
3
+ BEGIN;
4
+
5
+ SELECT verify_function ('app_jobs.complete_job');
6
+
7
+ ROLLBACK;
@@ -0,0 +1,7 @@
1
+ -- Verify schemas/app_jobs/procedures/complete_jobs on pg
2
+
3
+ BEGIN;
4
+
5
+ SELECT verify_function ('app_jobs.complete_jobs');
6
+
7
+ ROLLBACK;
@@ -0,0 +1,7 @@
1
+ -- Verify schemas/app_jobs/procedures/do_notify on pg
2
+
3
+ BEGIN;
4
+
5
+ SELECT verify_function ('app_jobs.do_notify');
6
+
7
+ ROLLBACK;
@@ -0,0 +1,7 @@
1
+ -- Verify schemas/app_jobs/procedures/fail_job on pg
2
+
3
+ BEGIN;
4
+
5
+ SELECT verify_function ('app_jobs.fail_job');
6
+
7
+ ROLLBACK;