@pgpm/database-jobs 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +22 -0
- package/Makefile +6 -0
- package/README.md +5 -0
- package/__tests__/__snapshots__/jobs.test.ts.snap +19 -0
- package/__tests__/jobs.test.ts +138 -0
- package/deploy/schemas/app_jobs/helpers/json_build_object_apply.sql +28 -0
- package/deploy/schemas/app_jobs/procedures/add_job.sql +103 -0
- package/deploy/schemas/app_jobs/procedures/add_scheduled_job.sql +97 -0
- package/deploy/schemas/app_jobs/procedures/complete_job.sql +32 -0
- package/deploy/schemas/app_jobs/procedures/complete_jobs.sql +19 -0
- package/deploy/schemas/app_jobs/procedures/do_notify.sql +16 -0
- package/deploy/schemas/app_jobs/procedures/fail_job.sql +41 -0
- package/deploy/schemas/app_jobs/procedures/get_job.sql +92 -0
- package/deploy/schemas/app_jobs/procedures/get_scheduled_job.sql +61 -0
- package/deploy/schemas/app_jobs/procedures/permanently_fail_jobs.sql +24 -0
- package/deploy/schemas/app_jobs/procedures/release_jobs.sql +34 -0
- package/deploy/schemas/app_jobs/procedures/release_scheduled_jobs.sql +26 -0
- package/deploy/schemas/app_jobs/procedures/reschedule_jobs.sql +26 -0
- package/deploy/schemas/app_jobs/procedures/run_scheduled_job.sql +78 -0
- package/deploy/schemas/app_jobs/schema.sql +7 -0
- package/deploy/schemas/app_jobs/tables/job_queues/grants/grant_select_insert_update_delete_to_administrator.sql +12 -0
- package/deploy/schemas/app_jobs/tables/job_queues/indexes/job_queues_locked_by_idx.sql +8 -0
- package/deploy/schemas/app_jobs/tables/job_queues/table.sql +12 -0
- package/deploy/schemas/app_jobs/tables/jobs/grants/grant_select_insert_update_delete_to_administrator.sql +12 -0
- package/deploy/schemas/app_jobs/tables/jobs/indexes/jobs_locked_by_idx.sql +8 -0
- package/deploy/schemas/app_jobs/tables/jobs/indexes/priority_run_at_id_idx.sql +8 -0
- package/deploy/schemas/app_jobs/tables/jobs/table.sql +27 -0
- package/deploy/schemas/app_jobs/tables/jobs/triggers/decrease_job_queue_count.sql +45 -0
- package/deploy/schemas/app_jobs/tables/jobs/triggers/increase_job_queue_count.sql +32 -0
- package/deploy/schemas/app_jobs/tables/jobs/triggers/notify_worker.sql +13 -0
- package/deploy/schemas/app_jobs/tables/jobs/triggers/timestamps.sql +20 -0
- package/deploy/schemas/app_jobs/tables/scheduled_jobs/grants/grant_select_insert_update_delete_to_administrator.sql +12 -0
- package/deploy/schemas/app_jobs/tables/scheduled_jobs/indexes/scheduled_jobs_locked_by_idx.sql +8 -0
- package/deploy/schemas/app_jobs/tables/scheduled_jobs/indexes/scheduled_jobs_priority_id_idx.sql +8 -0
- package/deploy/schemas/app_jobs/tables/scheduled_jobs/table.sql +27 -0
- package/deploy/schemas/app_jobs/tables/scheduled_jobs/triggers/notify_scheduled_job.sql +12 -0
- package/deploy/schemas/app_jobs/triggers/tg_add_job_with_fields.sql +50 -0
- package/deploy/schemas/app_jobs/triggers/tg_add_job_with_row.sql +26 -0
- package/deploy/schemas/app_jobs/triggers/tg_add_job_with_row_id.sql +27 -0
- package/deploy/schemas/app_jobs/triggers/tg_update_timestamps.sql +21 -0
- package/jest.config.js +15 -0
- package/launchql-database-jobs.control +8 -0
- package/launchql.plan +38 -0
- package/package.json +29 -0
- package/revert/schemas/app_jobs/helpers/json_build_object_apply.sql +7 -0
- package/revert/schemas/app_jobs/procedures/add_job.sql +7 -0
- package/revert/schemas/app_jobs/procedures/add_scheduled_job.sql +7 -0
- package/revert/schemas/app_jobs/procedures/complete_job.sql +7 -0
- package/revert/schemas/app_jobs/procedures/complete_jobs.sql +7 -0
- package/revert/schemas/app_jobs/procedures/do_notify.sql +7 -0
- package/revert/schemas/app_jobs/procedures/fail_job.sql +7 -0
- package/revert/schemas/app_jobs/procedures/get_job.sql +7 -0
- package/revert/schemas/app_jobs/procedures/get_scheduled_job.sql +7 -0
- package/revert/schemas/app_jobs/procedures/permanently_fail_jobs.sql +7 -0
- package/revert/schemas/app_jobs/procedures/release_jobs.sql +7 -0
- package/revert/schemas/app_jobs/procedures/release_scheduled_jobs.sql +7 -0
- package/revert/schemas/app_jobs/procedures/reschedule_jobs.sql +7 -0
- package/revert/schemas/app_jobs/procedures/run_scheduled_job.sql +7 -0
- package/revert/schemas/app_jobs/schema.sql +7 -0
- package/revert/schemas/app_jobs/tables/job_queues/grants/grant_select_insert_update_delete_to_administrator.sql +7 -0
- package/revert/schemas/app_jobs/tables/job_queues/indexes/job_queues_locked_by_idx.sql +7 -0
- package/revert/schemas/app_jobs/tables/job_queues/table.sql +7 -0
- package/revert/schemas/app_jobs/tables/jobs/grants/grant_select_insert_update_delete_to_administrator.sql +7 -0
- package/revert/schemas/app_jobs/tables/jobs/indexes/jobs_locked_by_idx.sql +7 -0
- package/revert/schemas/app_jobs/tables/jobs/indexes/priority_run_at_id_idx.sql +7 -0
- package/revert/schemas/app_jobs/tables/jobs/table.sql +7 -0
- package/revert/schemas/app_jobs/tables/jobs/triggers/decrease_job_queue_count.sql +7 -0
- package/revert/schemas/app_jobs/tables/jobs/triggers/increase_job_queue_count.sql +7 -0
- package/revert/schemas/app_jobs/tables/jobs/triggers/notify_worker.sql +5 -0
- package/revert/schemas/app_jobs/tables/jobs/triggers/timestamps.sql +9 -0
- package/revert/schemas/app_jobs/tables/scheduled_jobs/grants/grant_select_insert_update_delete_to_administrator.sql +7 -0
- package/revert/schemas/app_jobs/tables/scheduled_jobs/indexes/scheduled_jobs_locked_by_idx.sql +7 -0
- package/revert/schemas/app_jobs/tables/scheduled_jobs/indexes/scheduled_jobs_priority_id_idx.sql +7 -0
- package/revert/schemas/app_jobs/tables/scheduled_jobs/table.sql +7 -0
- package/revert/schemas/app_jobs/tables/scheduled_jobs/triggers/notify_scheduled_job.sql +8 -0
- package/revert/schemas/app_jobs/triggers/tg_add_job_with_fields.sql +7 -0
- package/revert/schemas/app_jobs/triggers/tg_add_job_with_row.sql +7 -0
- package/revert/schemas/app_jobs/triggers/tg_add_job_with_row_id.sql +5 -0
- package/revert/schemas/app_jobs/triggers/tg_update_timestamps.sql +7 -0
- package/sql/launchql-database-jobs--0.4.6.sql +769 -0
- package/verify/schemas/app_jobs/helpers/json_build_object_apply.sql +7 -0
- package/verify/schemas/app_jobs/procedures/add_job.sql +7 -0
- package/verify/schemas/app_jobs/procedures/add_scheduled_job.sql +7 -0
- package/verify/schemas/app_jobs/procedures/complete_job.sql +7 -0
- package/verify/schemas/app_jobs/procedures/complete_jobs.sql +7 -0
- package/verify/schemas/app_jobs/procedures/do_notify.sql +7 -0
- package/verify/schemas/app_jobs/procedures/fail_job.sql +7 -0
- package/verify/schemas/app_jobs/procedures/get_job.sql +7 -0
- package/verify/schemas/app_jobs/procedures/get_scheduled_job.sql +7 -0
- package/verify/schemas/app_jobs/procedures/permanently_fail_jobs.sql +7 -0
- package/verify/schemas/app_jobs/procedures/release_jobs.sql +7 -0
- package/verify/schemas/app_jobs/procedures/release_scheduled_jobs.sql +7 -0
- package/verify/schemas/app_jobs/procedures/reschedule_jobs.sql +7 -0
- package/verify/schemas/app_jobs/procedures/run_scheduled_job.sql +7 -0
- package/verify/schemas/app_jobs/schema.sql +7 -0
- package/verify/schemas/app_jobs/tables/job_queues/grants/grant_select_insert_update_delete_to_administrator.sql +10 -0
- package/verify/schemas/app_jobs/tables/job_queues/indexes/job_queues_locked_by_idx.sql +7 -0
- package/verify/schemas/app_jobs/tables/job_queues/table.sql +7 -0
- package/verify/schemas/app_jobs/tables/jobs/grants/grant_select_insert_update_delete_to_administrator.sql +10 -0
- package/verify/schemas/app_jobs/tables/jobs/indexes/jobs_locked_by_idx.sql +7 -0
- package/verify/schemas/app_jobs/tables/jobs/indexes/priority_run_at_id_idx.sql +7 -0
- package/verify/schemas/app_jobs/tables/jobs/table.sql +7 -0
- package/verify/schemas/app_jobs/tables/jobs/triggers/decrease_job_queue_count.sql +10 -0
- package/verify/schemas/app_jobs/tables/jobs/triggers/increase_job_queue_count.sql +10 -0
- package/verify/schemas/app_jobs/tables/jobs/triggers/notify_worker.sql +6 -0
- package/verify/schemas/app_jobs/tables/jobs/triggers/timestamps.sql +16 -0
- package/verify/schemas/app_jobs/tables/scheduled_jobs/grants/grant_select_insert_update_delete_to_administrator.sql +10 -0
- package/verify/schemas/app_jobs/tables/scheduled_jobs/indexes/scheduled_jobs_locked_by_idx.sql +7 -0
- package/verify/schemas/app_jobs/tables/scheduled_jobs/indexes/scheduled_jobs_priority_id_idx.sql +7 -0
- package/verify/schemas/app_jobs/tables/scheduled_jobs/table.sql +7 -0
- package/verify/schemas/app_jobs/tables/scheduled_jobs/triggers/notify_scheduled_job.sql +8 -0
- package/verify/schemas/app_jobs/triggers/tg_add_job_with_fields.sql +7 -0
- package/verify/schemas/app_jobs/triggers/tg_add_job_with_row.sql +7 -0
- package/verify/schemas/app_jobs/triggers/tg_add_job_with_row_id.sql +6 -0
- package/verify/schemas/app_jobs/triggers/tg_update_timestamps.sql +7 -0
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
-- Deploy schemas/app_jobs/procedures/permanently_fail_jobs to pg
|
|
2
|
+
-- requires: schemas/app_jobs/schema
|
|
3
|
+
-- requires: schemas/app_jobs/tables/job_queues/table
|
|
4
|
+
-- requires: schemas/app_jobs/tables/jobs/table
|
|
5
|
+
|
|
6
|
+
BEGIN;
|
|
7
|
+
CREATE FUNCTION app_jobs.permanently_fail_jobs (job_ids bigint[], error_message text DEFAULT NULL)
|
|
8
|
+
RETURNS SETOF app_jobs.jobs
|
|
9
|
+
LANGUAGE sql
|
|
10
|
+
AS $$
|
|
11
|
+
UPDATE
|
|
12
|
+
app_jobs.jobs
|
|
13
|
+
SET
|
|
14
|
+
last_error = coalesce(error_message, 'Manually marked as failed'),
|
|
15
|
+
attempts = max_attempts
|
|
16
|
+
WHERE
|
|
17
|
+
id = ANY (job_ids)
|
|
18
|
+
AND (locked_by IS NULL
|
|
19
|
+
OR locked_at < NOW() - interval '4 hours')
|
|
20
|
+
RETURNING
|
|
21
|
+
*;
|
|
22
|
+
$$;
|
|
23
|
+
COMMIT;
|
|
24
|
+
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
-- Deploy schemas/app_jobs/procedures/release_jobs to pg
|
|
2
|
+
-- requires: schemas/app_jobs/schema
|
|
3
|
+
-- requires: schemas/app_jobs/tables/jobs/table
|
|
4
|
+
-- requires: schemas/app_jobs/tables/job_queues/table
|
|
5
|
+
|
|
6
|
+
BEGIN;
|
|
7
|
+
CREATE FUNCTION app_jobs.release_jobs (worker_id text)
|
|
8
|
+
RETURNS void
|
|
9
|
+
AS $$
|
|
10
|
+
DECLARE
|
|
11
|
+
BEGIN
|
|
12
|
+
-- clear the job
|
|
13
|
+
UPDATE
|
|
14
|
+
app_jobs.jobs
|
|
15
|
+
SET
|
|
16
|
+
locked_at = NULL,
|
|
17
|
+
locked_by = NULL,
|
|
18
|
+
attempts = GREATEST (attempts - 1, 0)
|
|
19
|
+
WHERE
|
|
20
|
+
locked_by = worker_id;
|
|
21
|
+
-- clear the queue
|
|
22
|
+
UPDATE
|
|
23
|
+
app_jobs.job_queues
|
|
24
|
+
SET
|
|
25
|
+
locked_at = NULL,
|
|
26
|
+
locked_by = NULL
|
|
27
|
+
WHERE
|
|
28
|
+
locked_by = worker_id;
|
|
29
|
+
END;
|
|
30
|
+
$$
|
|
31
|
+
LANGUAGE 'plpgsql'
|
|
32
|
+
VOLATILE;
|
|
33
|
+
COMMIT;
|
|
34
|
+
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
-- Deploy schemas/app_jobs/procedures/release_scheduled_jobs to pg
|
|
2
|
+
-- requires: schemas/app_jobs/schema
|
|
3
|
+
-- requires: schemas/app_jobs/tables/scheduled_jobs/table
|
|
4
|
+
|
|
5
|
+
BEGIN;
|
|
6
|
+
CREATE FUNCTION app_jobs.release_scheduled_jobs (worker_id text, ids bigint[] DEFAULT NULL)
|
|
7
|
+
RETURNS void
|
|
8
|
+
AS $$
|
|
9
|
+
DECLARE
|
|
10
|
+
BEGIN
|
|
11
|
+
-- clear the scheduled job
|
|
12
|
+
UPDATE
|
|
13
|
+
app_jobs.scheduled_jobs s
|
|
14
|
+
SET
|
|
15
|
+
locked_at = NULL,
|
|
16
|
+
locked_by = NULL
|
|
17
|
+
WHERE
|
|
18
|
+
locked_by = worker_id
|
|
19
|
+
AND (ids IS NULL
|
|
20
|
+
OR s.id = ANY (ids));
|
|
21
|
+
END;
|
|
22
|
+
$$
|
|
23
|
+
LANGUAGE 'plpgsql'
|
|
24
|
+
VOLATILE;
|
|
25
|
+
COMMIT;
|
|
26
|
+
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
-- Deploy schemas/app_jobs/procedures/reschedule_jobs to pg
|
|
2
|
+
-- requires: schemas/app_jobs/schema
|
|
3
|
+
-- requires: schemas/app_jobs/tables/jobs/table
|
|
4
|
+
|
|
5
|
+
BEGIN;
|
|
6
|
+
-- NOTE this should be renamed to reset_jobs to avoid confusion of scheduled jobs
|
|
7
|
+
CREATE FUNCTION app_jobs.reschedule_jobs (job_ids bigint[], run_at timestamptz DEFAULT NULL, priority integer DEFAULT NULL, attempts integer DEFAULT NULL, max_attempts integer DEFAULT NULL)
|
|
8
|
+
RETURNS SETOF app_jobs.jobs
|
|
9
|
+
LANGUAGE sql
|
|
10
|
+
AS $$
|
|
11
|
+
UPDATE
|
|
12
|
+
app_jobs.jobs
|
|
13
|
+
SET
|
|
14
|
+
run_at = coalesce(reschedule_jobs.run_at, jobs.run_at),
|
|
15
|
+
priority = coalesce(reschedule_jobs.priority, jobs.priority),
|
|
16
|
+
attempts = coalesce(reschedule_jobs.attempts, jobs.attempts),
|
|
17
|
+
max_attempts = coalesce(reschedule_jobs.max_attempts, jobs.max_attempts)
|
|
18
|
+
WHERE
|
|
19
|
+
id = ANY (job_ids)
|
|
20
|
+
AND (locked_by IS NULL
|
|
21
|
+
OR locked_at < NOW() - interval '4 hours')
|
|
22
|
+
RETURNING
|
|
23
|
+
*;
|
|
24
|
+
$$;
|
|
25
|
+
COMMIT;
|
|
26
|
+
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
-- Deploy schemas/app_jobs/procedures/run_scheduled_job to pg
|
|
2
|
+
-- requires: schemas/app_jobs/schema
|
|
3
|
+
-- requires: schemas/app_jobs/tables/jobs/table
|
|
4
|
+
-- requires: schemas/app_jobs/tables/scheduled_jobs/table
|
|
5
|
+
|
|
6
|
+
BEGIN;
|
|
7
|
+
CREATE FUNCTION app_jobs.run_scheduled_job (id bigint, job_expiry interval DEFAULT '1 hours')
|
|
8
|
+
RETURNS app_jobs.jobs
|
|
9
|
+
AS $$
|
|
10
|
+
DECLARE
|
|
11
|
+
j app_jobs.jobs;
|
|
12
|
+
last_id bigint;
|
|
13
|
+
lkd_by text;
|
|
14
|
+
BEGIN
|
|
15
|
+
-- check last scheduled
|
|
16
|
+
SELECT
|
|
17
|
+
last_scheduled_id
|
|
18
|
+
FROM
|
|
19
|
+
app_jobs.scheduled_jobs s
|
|
20
|
+
WHERE
|
|
21
|
+
s.id = run_scheduled_job.id INTO last_id;
|
|
22
|
+
|
|
23
|
+
-- if it's been scheduled check if it's been run
|
|
24
|
+
|
|
25
|
+
IF (last_id IS NOT NULL) THEN
|
|
26
|
+
SELECT
|
|
27
|
+
locked_by
|
|
28
|
+
FROM
|
|
29
|
+
app_jobs.jobs js
|
|
30
|
+
WHERE
|
|
31
|
+
js.id = last_id
|
|
32
|
+
AND (js.locked_at IS NULL -- never been run
|
|
33
|
+
OR js.locked_at >= (NOW() - job_expiry)
|
|
34
|
+
-- still running within a safe interval
|
|
35
|
+
) INTO lkd_by;
|
|
36
|
+
IF (FOUND) THEN
|
|
37
|
+
RAISE EXCEPTION 'ALREADY_SCHEDULED';
|
|
38
|
+
END IF;
|
|
39
|
+
END IF;
|
|
40
|
+
|
|
41
|
+
-- insert new job
|
|
42
|
+
INSERT INTO app_jobs.jobs (
|
|
43
|
+
database_id,
|
|
44
|
+
queue_name,
|
|
45
|
+
task_identifier,
|
|
46
|
+
payload,
|
|
47
|
+
priority,
|
|
48
|
+
max_attempts,
|
|
49
|
+
key
|
|
50
|
+
) SELECT
|
|
51
|
+
database_id,
|
|
52
|
+
queue_name,
|
|
53
|
+
task_identifier,
|
|
54
|
+
payload,
|
|
55
|
+
priority,
|
|
56
|
+
max_attempts,
|
|
57
|
+
key
|
|
58
|
+
FROM
|
|
59
|
+
app_jobs.scheduled_jobs s
|
|
60
|
+
WHERE
|
|
61
|
+
s.id = run_scheduled_job.id
|
|
62
|
+
RETURNING
|
|
63
|
+
* INTO j;
|
|
64
|
+
-- update the scheduled job
|
|
65
|
+
UPDATE
|
|
66
|
+
app_jobs.scheduled_jobs s
|
|
67
|
+
SET
|
|
68
|
+
last_scheduled = NOW(),
|
|
69
|
+
last_scheduled_id = j.id
|
|
70
|
+
WHERE
|
|
71
|
+
s.id = run_scheduled_job.id;
|
|
72
|
+
RETURN j;
|
|
73
|
+
END;
|
|
74
|
+
$$
|
|
75
|
+
LANGUAGE 'plpgsql'
|
|
76
|
+
VOLATILE;
|
|
77
|
+
COMMIT;
|
|
78
|
+
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
-- Deploy schemas/app_jobs/tables/job_queues/grants/grant_select_insert_update_delete_to_administrator to pg
|
|
2
|
+
|
|
3
|
+
-- requires: schemas/app_jobs/schema
|
|
4
|
+
-- requires: schemas/app_jobs/tables/job_queues/table
|
|
5
|
+
|
|
6
|
+
BEGIN;
|
|
7
|
+
|
|
8
|
+
-- TODO make sure to require any policies on this table!
|
|
9
|
+
|
|
10
|
+
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE app_jobs.job_queues TO administrator;
|
|
11
|
+
|
|
12
|
+
COMMIT;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
-- Deploy schemas/app_jobs/tables/job_queues/indexes/job_queues_locked_by_idx to pg
|
|
2
|
+
-- requires: schemas/app_jobs/schema
|
|
3
|
+
-- requires: schemas/app_jobs/tables/job_queues/table
|
|
4
|
+
|
|
5
|
+
BEGIN;
|
|
6
|
+
CREATE INDEX job_queues_locked_by_idx ON app_jobs.job_queues (locked_by);
|
|
7
|
+
COMMIT;
|
|
8
|
+
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
-- Deploy schemas/app_jobs/tables/job_queues/table to pg
|
|
2
|
+
-- requires: schemas/app_jobs/schema
|
|
3
|
+
|
|
4
|
+
BEGIN;
|
|
5
|
+
CREATE TABLE app_jobs.job_queues (
|
|
6
|
+
queue_name text NOT NULL PRIMARY KEY,
|
|
7
|
+
job_count int DEFAULT 0 NOT NULL,
|
|
8
|
+
locked_at timestamptz,
|
|
9
|
+
locked_by text
|
|
10
|
+
);
|
|
11
|
+
COMMIT;
|
|
12
|
+
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
-- Deploy schemas/app_jobs/tables/jobs/grants/grant_select_insert_update_delete_to_administrator to pg
|
|
2
|
+
|
|
3
|
+
-- requires: schemas/app_jobs/schema
|
|
4
|
+
-- requires: schemas/app_jobs/tables/jobs/table
|
|
5
|
+
|
|
6
|
+
BEGIN;
|
|
7
|
+
|
|
8
|
+
-- TODO make sure to require any policies on this table!
|
|
9
|
+
|
|
10
|
+
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE app_jobs.jobs TO administrator;
|
|
11
|
+
|
|
12
|
+
COMMIT;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
-- Deploy schemas/app_jobs/tables/jobs/indexes/priority_run_at_id_idx to pg
|
|
2
|
+
-- requires: schemas/app_jobs/schema
|
|
3
|
+
-- requires: schemas/app_jobs/tables/jobs/table
|
|
4
|
+
|
|
5
|
+
BEGIN;
|
|
6
|
+
CREATE INDEX priority_run_at_id_idx ON app_jobs.jobs (priority, run_at, id);
|
|
7
|
+
COMMIT;
|
|
8
|
+
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
-- Deploy schemas/app_jobs/tables/jobs/table to pg
|
|
2
|
+
-- requires: schemas/app_jobs/schema
|
|
3
|
+
|
|
4
|
+
BEGIN;
|
|
5
|
+
CREATE TABLE app_jobs.jobs (
|
|
6
|
+
id bigserial PRIMARY KEY,
|
|
7
|
+
database_id uuid NOT NULL,
|
|
8
|
+
queue_name text DEFAULT (public.gen_random_uuid ()) ::text,
|
|
9
|
+
task_identifier text NOT NULL,
|
|
10
|
+
payload json DEFAULT '{}' ::json NOT NULL,
|
|
11
|
+
priority integer DEFAULT 0 NOT NULL,
|
|
12
|
+
run_at timestamptz DEFAULT now() NOT NULL,
|
|
13
|
+
attempts integer DEFAULT 0 NOT NULL,
|
|
14
|
+
max_attempts integer DEFAULT 25 NOT NULL,
|
|
15
|
+
key text,
|
|
16
|
+
last_error text,
|
|
17
|
+
locked_at timestamptz,
|
|
18
|
+
locked_by text,
|
|
19
|
+
CHECK (length(key) < 513),
|
|
20
|
+
CHECK (length(task_identifier) < 127),
|
|
21
|
+
CHECK (max_attempts > 0),
|
|
22
|
+
CHECK (length(queue_name) < 127),
|
|
23
|
+
CHECK (length(locked_by) > 3),
|
|
24
|
+
UNIQUE (key)
|
|
25
|
+
);
|
|
26
|
+
COMMIT;
|
|
27
|
+
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
-- Deploy schemas/app_jobs/tables/jobs/triggers/decrease_job_queue_count to pg
|
|
2
|
+
-- requires: schemas/app_jobs/schema
|
|
3
|
+
-- requires: schemas/app_jobs/tables/jobs/table
|
|
4
|
+
|
|
5
|
+
BEGIN;
|
|
6
|
+
CREATE FUNCTION app_jobs.tg_decrease_job_queue_count ()
|
|
7
|
+
RETURNS TRIGGER
|
|
8
|
+
AS $$
|
|
9
|
+
DECLARE
|
|
10
|
+
v_new_job_count int;
|
|
11
|
+
BEGIN
|
|
12
|
+
UPDATE
|
|
13
|
+
app_jobs.job_queues
|
|
14
|
+
SET
|
|
15
|
+
job_count = job_queues.job_count - 1
|
|
16
|
+
WHERE
|
|
17
|
+
queue_name = OLD.queue_name
|
|
18
|
+
RETURNING
|
|
19
|
+
job_count INTO v_new_job_count;
|
|
20
|
+
IF v_new_job_count <= 0 THEN
|
|
21
|
+
DELETE FROM app_jobs.job_queues
|
|
22
|
+
WHERE queue_name = OLD.queue_name
|
|
23
|
+
AND job_count <= 0;
|
|
24
|
+
END IF;
|
|
25
|
+
RETURN OLD;
|
|
26
|
+
END;
|
|
27
|
+
$$
|
|
28
|
+
LANGUAGE 'plpgsql'
|
|
29
|
+
VOLATILE;
|
|
30
|
+
|
|
31
|
+
CREATE TRIGGER decrease_job_queue_count_on_delete
|
|
32
|
+
AFTER DELETE ON app_jobs.jobs
|
|
33
|
+
FOR EACH ROW
|
|
34
|
+
WHEN ((OLD.queue_name IS NOT NULL))
|
|
35
|
+
EXECUTE PROCEDURE app_jobs.tg_decrease_job_queue_count ();
|
|
36
|
+
|
|
37
|
+
-- only a person would do this...
|
|
38
|
+
CREATE TRIGGER decrease_job_queue_count_on_update
|
|
39
|
+
AFTER UPDATE OF queue_name ON app_jobs.jobs
|
|
40
|
+
FOR EACH ROW
|
|
41
|
+
WHEN (((NEW.queue_name IS DISTINCT FROM OLD.queue_name) AND (OLD.queue_name IS NOT NULL)))
|
|
42
|
+
EXECUTE PROCEDURE app_jobs.tg_decrease_job_queue_count ();
|
|
43
|
+
|
|
44
|
+
COMMIT;
|
|
45
|
+
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
-- Deploy schemas/app_jobs/tables/jobs/triggers/increase_job_queue_count to pg
|
|
2
|
+
-- requires: schemas/app_jobs/schema
|
|
3
|
+
-- requires: schemas/app_jobs/tables/jobs/table
|
|
4
|
+
|
|
5
|
+
BEGIN;
|
|
6
|
+
CREATE FUNCTION app_jobs.tg_increase_job_queue_count ()
|
|
7
|
+
RETURNS TRIGGER
|
|
8
|
+
AS $$
|
|
9
|
+
BEGIN
|
|
10
|
+
INSERT INTO app_jobs.job_queues (queue_name, job_count)
|
|
11
|
+
VALUES (NEW.queue_name, 1)
|
|
12
|
+
ON CONFLICT (queue_name)
|
|
13
|
+
DO UPDATE SET
|
|
14
|
+
job_count = job_queues.job_count + 1;
|
|
15
|
+
RETURN NEW;
|
|
16
|
+
END;
|
|
17
|
+
$$
|
|
18
|
+
LANGUAGE 'plpgsql'
|
|
19
|
+
VOLATILE;
|
|
20
|
+
CREATE TRIGGER _500_increase_job_queue_count_on_insert
|
|
21
|
+
AFTER INSERT ON app_jobs.jobs
|
|
22
|
+
FOR EACH ROW
|
|
23
|
+
WHEN ((NEW.queue_name IS NOT NULL))
|
|
24
|
+
EXECUTE PROCEDURE app_jobs.tg_increase_job_queue_count ();
|
|
25
|
+
-- only a person would do this
|
|
26
|
+
CREATE TRIGGER _500_increase_job_queue_count_on_update
|
|
27
|
+
AFTER UPDATE OF queue_name ON app_jobs.jobs
|
|
28
|
+
FOR EACH ROW
|
|
29
|
+
WHEN (((NEW.queue_name IS DISTINCT FROM OLD.queue_name) AND (NEW.queue_name IS NOT NULL)))
|
|
30
|
+
EXECUTE PROCEDURE app_jobs.tg_increase_job_queue_count ();
|
|
31
|
+
COMMIT;
|
|
32
|
+
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
-- Deploy schemas/app_jobs/tables/jobs/triggers/notify_worker to pg
|
|
2
|
+
-- requires: schemas/app_jobs/schema
|
|
3
|
+
-- requires: schemas/app_jobs/tables/jobs/table
|
|
4
|
+
-- requires: schemas/app_jobs/procedures/do_notify
|
|
5
|
+
-- requires: schemas/app_jobs/tables/jobs/triggers/increase_job_queue_count
|
|
6
|
+
|
|
7
|
+
BEGIN;
|
|
8
|
+
CREATE TRIGGER _900_notify_worker
|
|
9
|
+
AFTER INSERT ON app_jobs.jobs
|
|
10
|
+
FOR EACH ROW
|
|
11
|
+
EXECUTE PROCEDURE app_jobs.do_notify ('jobs:insert');
|
|
12
|
+
COMMIT;
|
|
13
|
+
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
-- Deploy schemas/app_jobs/tables/jobs/triggers/timestamps to pg
|
|
2
|
+
-- requires: schemas/app_jobs/schema
|
|
3
|
+
-- requires: schemas/app_jobs/tables/jobs/table
|
|
4
|
+
-- requires: schemas/app_jobs/triggers/tg_update_timestamps
|
|
5
|
+
|
|
6
|
+
BEGIN;
|
|
7
|
+
ALTER TABLE app_jobs.jobs
|
|
8
|
+
ADD COLUMN created_at timestamptz;
|
|
9
|
+
ALTER TABLE app_jobs.jobs
|
|
10
|
+
ALTER COLUMN created_at SET DEFAULT NOW();
|
|
11
|
+
ALTER TABLE app_jobs.jobs
|
|
12
|
+
ADD COLUMN updated_at timestamptz;
|
|
13
|
+
ALTER TABLE app_jobs.jobs
|
|
14
|
+
ALTER COLUMN updated_at SET DEFAULT NOW();
|
|
15
|
+
CREATE TRIGGER _100_update_jobs_modtime_tg
|
|
16
|
+
BEFORE UPDATE OR INSERT ON app_jobs.jobs
|
|
17
|
+
FOR EACH ROW
|
|
18
|
+
EXECUTE PROCEDURE app_jobs.tg_update_timestamps ();
|
|
19
|
+
COMMIT;
|
|
20
|
+
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
-- Deploy schemas/app_jobs/tables/scheduled_jobs/grants/grant_select_insert_update_delete_to_administrator to pg
|
|
2
|
+
|
|
3
|
+
-- requires: schemas/app_jobs/schema
|
|
4
|
+
-- requires: schemas/app_jobs/tables/scheduled_jobs/table
|
|
5
|
+
|
|
6
|
+
BEGIN;
|
|
7
|
+
|
|
8
|
+
-- TODO make sure to require any policies on this table!
|
|
9
|
+
|
|
10
|
+
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE app_jobs.scheduled_jobs TO administrator;
|
|
11
|
+
|
|
12
|
+
COMMIT;
|
package/deploy/schemas/app_jobs/tables/scheduled_jobs/indexes/scheduled_jobs_locked_by_idx.sql
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
-- Deploy schemas/app_jobs/tables/scheduled_jobs/indexes/scheduled_jobs_locked_by_idx to pg
|
|
2
|
+
-- requires: schemas/app_jobs/schema
|
|
3
|
+
-- requires: schemas/app_jobs/tables/scheduled_jobs/table
|
|
4
|
+
|
|
5
|
+
BEGIN;
|
|
6
|
+
CREATE INDEX scheduled_jobs_locked_by_idx ON app_jobs.scheduled_jobs (locked_by);
|
|
7
|
+
COMMIT;
|
|
8
|
+
|
package/deploy/schemas/app_jobs/tables/scheduled_jobs/indexes/scheduled_jobs_priority_id_idx.sql
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
-- Deploy schemas/app_jobs/tables/scheduled_jobs/indexes/scheduled_jobs_priority_id_idx to pg
|
|
2
|
+
-- requires: schemas/app_jobs/schema
|
|
3
|
+
-- requires: schemas/app_jobs/tables/scheduled_jobs/table
|
|
4
|
+
|
|
5
|
+
BEGIN;
|
|
6
|
+
CREATE INDEX scheduled_jobs_priority_id_idx ON app_jobs.scheduled_jobs (priority, id);
|
|
7
|
+
COMMIT;
|
|
8
|
+
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
-- Deploy schemas/app_jobs/tables/scheduled_jobs/table to pg
|
|
2
|
+
-- requires: schemas/app_jobs/schema
|
|
3
|
+
|
|
4
|
+
BEGIN;
|
|
5
|
+
CREATE TABLE app_jobs.scheduled_jobs (
|
|
6
|
+
id bigserial PRIMARY KEY,
|
|
7
|
+
database_id uuid NOT NULL,
|
|
8
|
+
queue_name text DEFAULT (public.gen_random_uuid ()) ::text,
|
|
9
|
+
task_identifier text NOT NULL,
|
|
10
|
+
payload json DEFAULT '{}' ::json NOT NULL,
|
|
11
|
+
priority integer DEFAULT 0 NOT NULL,
|
|
12
|
+
max_attempts integer DEFAULT 25 NOT NULL,
|
|
13
|
+
key text,
|
|
14
|
+
locked_at timestamptz,
|
|
15
|
+
locked_by text,
|
|
16
|
+
schedule_info json NOT NULL,
|
|
17
|
+
last_scheduled timestamptz,
|
|
18
|
+
last_scheduled_id bigint,
|
|
19
|
+
CHECK (length(key) < 513),
|
|
20
|
+
CHECK (length(task_identifier) < 127),
|
|
21
|
+
CHECK (max_attempts > 0),
|
|
22
|
+
CHECK (length(queue_name) < 127),
|
|
23
|
+
CHECK (length(locked_by) > 3),
|
|
24
|
+
UNIQUE (key)
|
|
25
|
+
);
|
|
26
|
+
COMMIT;
|
|
27
|
+
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
-- Deploy schemas/app_jobs/tables/scheduled_jobs/triggers/notify_scheduled_job to pg
|
|
2
|
+
-- requires: schemas/app_jobs/schema
|
|
3
|
+
-- requires: schemas/app_jobs/tables/scheduled_jobs/table
|
|
4
|
+
-- requires: schemas/app_jobs/procedures/do_notify
|
|
5
|
+
|
|
6
|
+
BEGIN;
|
|
7
|
+
CREATE TRIGGER _900_notify_scheduled_job
|
|
8
|
+
AFTER INSERT ON app_jobs.scheduled_jobs
|
|
9
|
+
FOR EACH ROW
|
|
10
|
+
EXECUTE PROCEDURE app_jobs.do_notify ('scheduled_jobs:insert');
|
|
11
|
+
COMMIT;
|
|
12
|
+
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
-- Deploy schemas/app_jobs/triggers/tg_add_job_with_fields to pg
|
|
2
|
+
-- requires: schemas/app_jobs/schema
|
|
3
|
+
-- requires: schemas/app_jobs/helpers/json_build_object_apply
|
|
4
|
+
|
|
5
|
+
BEGIN;
|
|
6
|
+
CREATE FUNCTION app_jobs.trigger_job_with_fields ()
|
|
7
|
+
RETURNS TRIGGER
|
|
8
|
+
AS $$
|
|
9
|
+
DECLARE
|
|
10
|
+
arg text;
|
|
11
|
+
fn text;
|
|
12
|
+
i int;
|
|
13
|
+
args text[];
|
|
14
|
+
BEGIN
|
|
15
|
+
FOR i IN
|
|
16
|
+
SELECT
|
|
17
|
+
*
|
|
18
|
+
FROM
|
|
19
|
+
generate_series(1, TG_NARGS) g (i)
|
|
20
|
+
LOOP
|
|
21
|
+
IF (i = 1) THEN
|
|
22
|
+
fn = TG_ARGV[i - 1];
|
|
23
|
+
ELSE
|
|
24
|
+
args = array_append(args, TG_ARGV[i - 1]);
|
|
25
|
+
IF (TG_OP = 'INSERT' OR TG_OP = 'UPDATE') THEN
|
|
26
|
+
EXECUTE format('SELECT ($1).%s::text', TG_ARGV[i - 1])
|
|
27
|
+
USING NEW INTO arg;
|
|
28
|
+
END IF;
|
|
29
|
+
IF (TG_OP = 'DELETE') THEN
|
|
30
|
+
EXECUTE format('SELECT ($1).%s::text', TG_ARGV[i - 1])
|
|
31
|
+
USING OLD INTO arg;
|
|
32
|
+
END IF;
|
|
33
|
+
args = array_append(args, arg);
|
|
34
|
+
END IF;
|
|
35
|
+
END LOOP;
|
|
36
|
+
PERFORM
|
|
37
|
+
app_jobs.add_job (jwt_private.current_database_id(), fn, app_jobs.json_build_object_apply (args));
|
|
38
|
+
IF (TG_OP = 'INSERT' OR TG_OP = 'UPDATE') THEN
|
|
39
|
+
RETURN NEW;
|
|
40
|
+
END IF;
|
|
41
|
+
IF (TG_OP = 'DELETE') THEN
|
|
42
|
+
RETURN OLD;
|
|
43
|
+
END IF;
|
|
44
|
+
END;
|
|
45
|
+
$$
|
|
46
|
+
LANGUAGE plpgsql
|
|
47
|
+
VOLATILE
|
|
48
|
+
SECURITY DEFINER;
|
|
49
|
+
COMMIT;
|
|
50
|
+
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
-- Deploy schemas/app_jobs/triggers/tg_add_job_with_row to pg
|
|
2
|
+
-- requires: schemas/app_jobs/schema
|
|
3
|
+
|
|
4
|
+
BEGIN;
|
|
5
|
+
CREATE FUNCTION app_jobs.tg_add_job_with_row ()
|
|
6
|
+
RETURNS TRIGGER
|
|
7
|
+
AS $$
|
|
8
|
+
BEGIN
|
|
9
|
+
IF (TG_OP = 'INSERT' OR TG_OP = 'UPDATE') THEN
|
|
10
|
+
PERFORM
|
|
11
|
+
app_jobs.add_job (jwt_private.current_database_id(), TG_ARGV[0], to_json(NEW));
|
|
12
|
+
RETURN NEW;
|
|
13
|
+
END IF;
|
|
14
|
+
IF (TG_OP = 'DELETE') THEN
|
|
15
|
+
PERFORM
|
|
16
|
+
app_jobs.add_job (jwt_private.current_database_id(), TG_ARGV[0], to_json(OLD));
|
|
17
|
+
RETURN OLD;
|
|
18
|
+
END IF;
|
|
19
|
+
END;
|
|
20
|
+
$$
|
|
21
|
+
LANGUAGE plpgsql
|
|
22
|
+
VOLATILE
|
|
23
|
+
SECURITY DEFINER;
|
|
24
|
+
COMMENT ON FUNCTION app_jobs.tg_add_job_with_row IS E'Useful shortcut to create a job on insert or update. Pass the task name as the trigger argument, and the record data will automatically be available on the JSON payload.';
|
|
25
|
+
COMMIT;
|
|
26
|
+
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
-- Deploy schemas/app_jobs/triggers/tg_add_job_with_row_id to pg
|
|
2
|
+
|
|
3
|
+
-- requires: schemas/app_jobs/schema
|
|
4
|
+
|
|
5
|
+
BEGIN;
|
|
6
|
+
CREATE FUNCTION app_jobs.tg_add_job_with_row_id ()
|
|
7
|
+
RETURNS TRIGGER
|
|
8
|
+
AS $$
|
|
9
|
+
BEGIN
|
|
10
|
+
IF (TG_OP = 'INSERT' OR TG_OP = 'UPDATE') THEN
|
|
11
|
+
PERFORM
|
|
12
|
+
app_jobs.add_job (jwt_private.current_database_id(), tg_argv[0], json_build_object('id', NEW.id));
|
|
13
|
+
RETURN NEW;
|
|
14
|
+
END IF;
|
|
15
|
+
IF (TG_OP = 'DELETE') THEN
|
|
16
|
+
PERFORM
|
|
17
|
+
app_jobs.add_job (jwt_private.current_database_id(), tg_argv[0], json_build_object('id', OLD.id));
|
|
18
|
+
RETURN OLD;
|
|
19
|
+
END IF;
|
|
20
|
+
END;
|
|
21
|
+
$$
|
|
22
|
+
LANGUAGE plpgsql
|
|
23
|
+
VOLATILE
|
|
24
|
+
SECURITY DEFINER;
|
|
25
|
+
COMMENT ON FUNCTION app_jobs.tg_add_job_with_row_id IS E'Useful shortcut to create a job on insert or update. Pass the task name as the trigger argument, and the record id will automatically be available on the JSON payload.';
|
|
26
|
+
COMMIT;
|
|
27
|
+
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
-- Deploy schemas/app_jobs/triggers/tg_update_timestamps to pg
|
|
2
|
+
-- requires: schemas/app_jobs/schema
|
|
3
|
+
|
|
4
|
+
BEGIN;
|
|
5
|
+
CREATE FUNCTION app_jobs.tg_update_timestamps ()
|
|
6
|
+
RETURNS TRIGGER
|
|
7
|
+
AS $$
|
|
8
|
+
BEGIN
|
|
9
|
+
IF TG_OP = 'INSERT' THEN
|
|
10
|
+
NEW.created_at = NOW();
|
|
11
|
+
NEW.updated_at = NOW();
|
|
12
|
+
ELSIF TG_OP = 'UPDATE' THEN
|
|
13
|
+
NEW.created_at = OLD.created_at;
|
|
14
|
+
NEW.updated_at = greatest (now(), OLD.updated_at + interval '1 millisecond');
|
|
15
|
+
END IF;
|
|
16
|
+
RETURN NEW;
|
|
17
|
+
END;
|
|
18
|
+
$$
|
|
19
|
+
LANGUAGE 'plpgsql';
|
|
20
|
+
COMMIT;
|
|
21
|
+
|
package/jest.config.js
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/** @type {import('ts-jest').JestConfigWithTsJest} */
|
|
2
|
+
module.exports = {
|
|
3
|
+
preset: 'ts-jest',
|
|
4
|
+
testEnvironment: 'node',
|
|
5
|
+
|
|
6
|
+
// Match both __tests__ and colocated test files
|
|
7
|
+
testMatch: ['**/?(*.)+(test|spec).{ts,tsx,js,jsx}'],
|
|
8
|
+
|
|
9
|
+
// Ignore build artifacts and type declarations
|
|
10
|
+
testPathIgnorePatterns: ['/dist/', '\\.d\\.ts$'],
|
|
11
|
+
modulePathIgnorePatterns: ['<rootDir>/dist/'],
|
|
12
|
+
watchPathIgnorePatterns: ['/dist/'],
|
|
13
|
+
|
|
14
|
+
moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'],
|
|
15
|
+
};
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# launchql-database-jobs extension
|
|
2
|
+
comment = 'launchql-database-jobs extension'
|
|
3
|
+
default_version = '0.4.6'
|
|
4
|
+
module_pathname = '$libdir/launchql-database-jobs'
|
|
5
|
+
requires = 'plpgsql,uuid-ossp,pgcrypto,launchql-default-roles,launchql-verify'
|
|
6
|
+
relocatable = false
|
|
7
|
+
superuser = false
|
|
8
|
+
|