@ultimat3/jobs 22.14.0 → 23.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +129 -87
- package/README.md +329 -19
- package/package.json +6 -6
- package/src/concurrency.ts +144 -0
- package/src/counters.ts +105 -0
- package/src/describe.ts +29 -0
- package/src/driver-memory-operator.ts +297 -0
- package/src/driver-memory.ts +79 -37
- package/src/driver-nats.ts +2 -2
- package/src/driver-pg-ddl.ts +71 -0
- package/src/driver-pg-jobs-sql.ts +71 -8
- package/src/driver-pg-operator-sql.ts +303 -0
- package/src/driver-pg-operator.ts +300 -0
- package/src/driver-pg-outbox-sql.ts +110 -0
- package/src/driver-pg-rows.ts +31 -0
- package/src/driver-pg-settle-sql.ts +71 -0
- package/src/driver-pg-sql.ts +80 -129
- package/src/driver-pg-wake-sql.ts +76 -0
- package/src/driver-pg.ts +74 -34
- package/src/driver-redis.ts +2 -2
- package/src/driver.ts +96 -34
- package/src/enqueue-signal.ts +50 -0
- package/src/errors-concurrency.ts +91 -0
- package/src/errors-operator.ts +94 -0
- package/src/errors.ts +13 -17
- package/src/events-pg.ts +38 -28
- package/src/events.ts +25 -0
- package/src/execute.ts +160 -81
- package/src/export.ts +23 -4
- package/src/heartbeat.ts +9 -3
- package/src/idle-backoff.ts +74 -0
- package/src/index.ts +105 -12
- package/src/inspect-operator.ts +49 -0
- package/src/inspect.ts +42 -2
- package/src/introspection.ts +262 -0
- package/src/job.ts +78 -15
- package/src/leases.ts +49 -9
- package/src/metrics.ts +6 -1
- package/src/outbox-pg.ts +3 -0
- package/src/outbox-relay.ts +114 -39
- package/src/outbox.ts +29 -6
- package/src/progress.ts +78 -0
- package/src/queue-wake.ts +220 -0
- package/src/redact-input.ts +26 -0
- package/src/renewal-timer.ts +28 -10
- package/src/retry-classification.ts +13 -1
- package/src/retry.ts +10 -2
- package/src/run-deadline.ts +57 -0
- package/src/scheduler-leader.ts +26 -0
- package/src/scheduler-occurrences.ts +76 -0
- package/src/scheduler-pg.ts +55 -2
- package/src/scheduler-state.ts +66 -0
- package/src/scheduler.ts +143 -103
- package/src/settled.ts +161 -0
- package/src/worker-admit.ts +163 -0
- package/src/worker-fleet-slots.ts +107 -38
- package/src/worker-hand-back.ts +6 -1
- package/src/worker-key-busy.ts +79 -0
- package/src/worker-loop.ts +145 -0
- package/src/worker-registry.ts +103 -0
- package/src/worker-run.ts +9 -0
- package/src/worker-types.ts +23 -0
- package/src/worker.ts +103 -134
package/src/driver-nats.ts
CHANGED
|
@@ -70,10 +70,10 @@ export function createNatsDriver(_options: NatsDriverOptions = {}): JobDriver {
|
|
|
70
70
|
claim(_options: ClaimOptions): Promise<readonly ClaimedJob[]> {
|
|
71
71
|
return unavailable('claim');
|
|
72
72
|
},
|
|
73
|
-
ack(_jobId: string): Promise<
|
|
73
|
+
ack(_jobId: string): Promise<boolean> {
|
|
74
74
|
return unavailable('ack');
|
|
75
75
|
},
|
|
76
|
-
nack(_jobId: string, _options: NackOptions): Promise<
|
|
76
|
+
nack(_jobId: string, _options: NackOptions): Promise<boolean> {
|
|
77
77
|
return unavailable('nack');
|
|
78
78
|
},
|
|
79
79
|
heartbeat(_jobId: string, _options: HeartbeatOptions): Promise<boolean> {
|
package/src/driver-pg-ddl.ts
CHANGED
|
@@ -132,6 +132,12 @@ alter table x_outbox add column if not exists claimed_at timestamptz;
|
|
|
132
132
|
|
|
133
133
|
alter table x_outbox add column if not exists claimed_by text;
|
|
134
134
|
|
|
135
|
+
-- The run the published job carries, allocated when the row is STAGED so the enqueue that staged
|
|
136
|
+
-- it can answer which run it started. The row id is the published job id for the same reason.
|
|
137
|
+
-- Added by alter because x_outbox shipped without it. A row staged before this column existed is
|
|
138
|
+
-- given a run id by the default, which is what the driver would have minted at its publish.
|
|
139
|
+
alter table x_outbox add column if not exists run_id uuid not null default gen_random_uuid();
|
|
140
|
+
|
|
135
141
|
-- The scheduler watermark. Without a durable one a redeployed scheduler has no idea what the
|
|
136
142
|
-- pod it replaced already fired, so runRound takes the arming branch and every occurrence
|
|
137
143
|
-- between the two processes is dropped with nothing logged.
|
|
@@ -141,6 +147,13 @@ create table if not exists x_scheduler_state (
|
|
|
141
147
|
updated_at timestamptz not null default now()
|
|
142
148
|
);
|
|
143
149
|
|
|
150
|
+
-- The last occurrence that DISPATCHED, and when. Apart from the watermark, which arming a task or
|
|
151
|
+
-- skipping missed occurrences also moves: an operator asking when a task last ran is asking about
|
|
152
|
+
-- jobs that were queued. Added by alter because x_scheduler_state shipped without them.
|
|
153
|
+
alter table x_scheduler_state add column if not exists fired_occurrence_at timestamptz;
|
|
154
|
+
|
|
155
|
+
alter table x_scheduler_state add column if not exists fired_at timestamptz;
|
|
156
|
+
|
|
144
157
|
-- Leader election as an EXPIRING LEASE rather than a session advisory lock: the executor this
|
|
145
158
|
-- package is handed is a pool, and a session-level pg_try_advisory_lock is released the moment
|
|
146
159
|
-- that connection goes back to it. A row with an expiry needs no connection affinity at all.
|
|
@@ -176,4 +189,62 @@ create table if not exists x_job_events (
|
|
|
176
189
|
|
|
177
190
|
create index if not exists x_job_events_lookup_idx
|
|
178
191
|
on x_job_events (name, published_at);
|
|
192
|
+
|
|
193
|
+
-- What the body last reported through progress, as one value: done, total, note and the instant.
|
|
194
|
+
-- A column and not a table, because it is read with the row and dies with it. Added by alter
|
|
195
|
+
-- because x_jobs shipped without it.
|
|
196
|
+
alter table x_jobs add column if not exists progress jsonb;
|
|
197
|
+
|
|
198
|
+
-- The stack of the thrown value last_error describes, bounded by the driver before it is bound.
|
|
199
|
+
alter table x_jobs add column if not exists last_error_stack text;
|
|
200
|
+
|
|
201
|
+
-- How many times the row has been claimed, moved by the claim itself and never reset. A settle is
|
|
202
|
+
-- fenced on it as well as on claimed_by: a worker that takes back its own lapsed job has the same
|
|
203
|
+
-- id as the body still unwinding from the first claim, and only the ordinal tells the two apart.
|
|
204
|
+
alter table x_jobs add column if not exists claims int not null default 0;
|
|
205
|
+
|
|
206
|
+
-- The keyset a paged listing seeks by. Newest first, id as the tiebreak, so a page is an index
|
|
207
|
+
-- range and never a sort of the whole table.
|
|
208
|
+
create index if not exists x_jobs_created_idx on x_jobs (created_at desc, id desc);
|
|
209
|
+
|
|
210
|
+
-- A paused queue or task, one row each. A queue pause is a fact about the QUEUE and not a column
|
|
211
|
+
-- on every one of its rows: the claim reads this table with not exists, and a pause that had to
|
|
212
|
+
-- rewrite a million rows would not take effect within one poll interval.
|
|
213
|
+
create table if not exists x_job_pauses (
|
|
214
|
+
kind text not null,
|
|
215
|
+
name text not null,
|
|
216
|
+
paused_at timestamptz not null default now(),
|
|
217
|
+
primary key (kind, name)
|
|
218
|
+
);
|
|
219
|
+
|
|
220
|
+
-- The worker registry. One row per live worker, rewritten on its heartbeat and expired by time
|
|
221
|
+
-- like a lease: a killed worker announces nothing and its row stops being read. Never cleaned up
|
|
222
|
+
-- by anything that has to be running.
|
|
223
|
+
create table if not exists x_job_workers (
|
|
224
|
+
id text primary key,
|
|
225
|
+
host text not null,
|
|
226
|
+
queues jsonb not null,
|
|
227
|
+
concurrency int not null,
|
|
228
|
+
in_flight jsonb not null,
|
|
229
|
+
started_at timestamptz not null,
|
|
230
|
+
heartbeat_at timestamptz not null default now(),
|
|
231
|
+
expires_at timestamptz not null
|
|
232
|
+
);
|
|
233
|
+
|
|
234
|
+
-- Per-job history, in buckets. A settle adds to the one-minute bucket in the SAME statement that
|
|
235
|
+
-- moves the row, and the scheduler leader folds old buckets into wider ones, so the key count per
|
|
236
|
+
-- job name is fixed whatever the throughput.
|
|
237
|
+
create table if not exists x_job_counters (
|
|
238
|
+
job text not null,
|
|
239
|
+
bucket_ms int not null,
|
|
240
|
+
bucket_start timestamptz not null,
|
|
241
|
+
done bigint not null default 0,
|
|
242
|
+
retried bigint not null default 0,
|
|
243
|
+
failed bigint not null default 0,
|
|
244
|
+
dead bigint not null default 0,
|
|
245
|
+
duration_ms bigint not null default 0,
|
|
246
|
+
primary key (job, bucket_ms, bucket_start)
|
|
247
|
+
);
|
|
248
|
+
|
|
249
|
+
create index if not exists x_job_counters_age_idx on x_job_counters (bucket_ms, bucket_start);
|
|
179
250
|
`.trim();
|
|
@@ -3,34 +3,95 @@
|
|
|
3
3
|
// none decodes `timestamptz` as TEXT, so `toJobRecord` read `Number('2026-01-01 00:00:00+00')`
|
|
4
4
|
// and `x jobs ls` / `show` / `cancel` printed `NaN` for every timestamp.
|
|
5
5
|
|
|
6
|
+
import { JOBS_WAKE_CHANNEL } from './driver-pg-wake-sql';
|
|
7
|
+
|
|
6
8
|
/**
|
|
7
9
|
* The `JobRow` shape as a projection. Asking Postgres for epoch ms is what makes the decoding
|
|
8
10
|
* independent of the client's type map — never `select *`, whose correctness is the driver's
|
|
9
11
|
* opinion about `timestamptz` rather than this statement's.
|
|
10
12
|
*/
|
|
11
13
|
export const JOB_ROW_COLUMNS = `id, name, queue, input, idempotency_key, run_id, attempt,
|
|
12
|
-
max_attempts, state, tenant_id, last_error, claimed_by,
|
|
13
|
-
traceparent, enqueued_by,
|
|
14
|
+
max_attempts, state, tenant_id, last_error, last_error_stack, claimed_by, claims,
|
|
15
|
+
traceparent, enqueued_by, progress,
|
|
14
16
|
(extract(epoch from run_at) * 1000)::bigint as run_at,
|
|
15
17
|
(extract(epoch from visible_at) * 1000)::bigint as visible_at,
|
|
16
18
|
(extract(epoch from created_at) * 1000)::bigint as created_at,
|
|
17
19
|
(extract(epoch from updated_at) * 1000)::bigint as updated_at`;
|
|
18
20
|
|
|
21
|
+
/** `JOB_ROW_COLUMNS` by their output names — what a statement reading a derived table selects. */
|
|
22
|
+
const JOB_ROW_NAMES = `id, name, queue, input, idempotency_key, run_id, attempt, max_attempts,
|
|
23
|
+
state, tenant_id, last_error, last_error_stack, claimed_by, claims, traceparent,
|
|
24
|
+
enqueued_by, progress, run_at, visible_at, created_at, updated_at`;
|
|
25
|
+
|
|
19
26
|
export const SQL_JOB_GET = `
|
|
20
27
|
select ${JOB_ROW_COLUMNS}
|
|
21
28
|
from x_jobs where id = $1
|
|
22
29
|
`.trim();
|
|
23
30
|
|
|
31
|
+
/**
|
|
32
|
+
* The predicates a page shares, whichever way it reads. $1 queue, $2 name, $3 state, $5 id prefix,
|
|
33
|
+
* $6 created from (ms), $7 created to (ms), $10 tenant. The prefix is compared, never `like`d: a
|
|
34
|
+
* `_` or `%` in it is a character, as `startsWith` reads it in the memory driver.
|
|
35
|
+
*/
|
|
36
|
+
const LIST_MATCH = `($1::text is null or queue = $1)
|
|
37
|
+
and ($2::text is null or name = $2)
|
|
38
|
+
and ($3::text is null or state = $3)
|
|
39
|
+
and ($5::text is null or left(id::text, length($5::text)) = $5::text)
|
|
40
|
+
and ($6::bigint is null or created_at >= to_timestamp($6::bigint / 1000.0))
|
|
41
|
+
and ($7::bigint is null or created_at < to_timestamp($7::bigint / 1000.0))
|
|
42
|
+
and ($10::text is null or tenant_id = $10)`;
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* One page, newest first, sought by KEYSET. `(created_at, id) < (cursor)` is a row comparison the
|
|
46
|
+
* `x_jobs_created_idx` range answers directly; an offset would re-read every row before it and
|
|
47
|
+
* shift under an insert.
|
|
48
|
+
*
|
|
49
|
+
* The cursor carries the last row's `createdAt` in epoch MS, and the column holds microseconds —
|
|
50
|
+
* so the seek reads the cursor row's own `created_at` by id rather than trusting a rounded value,
|
|
51
|
+
* which would skip or repeat its millisecond-mates. A cursor row deleted mid-walk falls back to
|
|
52
|
+
* the millisecond after it: rows of that one millisecond may then repeat, and none is skipped.
|
|
53
|
+
*
|
|
54
|
+
* `LIST_MATCH`'s parameters, plus $4 limit, $8 cursor createdAt (ms), $9 cursor id.
|
|
55
|
+
*/
|
|
24
56
|
export const SQL_JOB_LIST = `
|
|
25
57
|
select ${JOB_ROW_COLUMNS}
|
|
26
58
|
from x_jobs
|
|
27
|
-
where
|
|
28
|
-
and ($
|
|
29
|
-
|
|
30
|
-
|
|
59
|
+
where ${LIST_MATCH}
|
|
60
|
+
and ($9::uuid is null or (created_at, id) < (
|
|
61
|
+
coalesce(
|
|
62
|
+
(select c.created_at from x_jobs c where c.id = $9::uuid),
|
|
63
|
+
to_timestamp(($8::bigint + 1) / 1000.0)
|
|
64
|
+
),
|
|
65
|
+
$9::uuid
|
|
66
|
+
))
|
|
67
|
+
order by created_at desc, id desc
|
|
31
68
|
limit $4
|
|
32
69
|
`.trim();
|
|
33
70
|
|
|
71
|
+
/**
|
|
72
|
+
* The page BEFORE a cursor: the `limit` rows nearest it on the newer side — read oldest first so
|
|
73
|
+
* `limit` keeps the nearest, then turned newest first like every page. The same parameters as
|
|
74
|
+
* `SQL_JOB_LIST`, $8/$9 being the first row of the page the reader is leaving. A cursor row
|
|
75
|
+
* deleted meanwhile falls back to its millisecond: a row of it may repeat, none is skipped.
|
|
76
|
+
*/
|
|
77
|
+
export const SQL_JOB_LIST_BEFORE = `
|
|
78
|
+
select ${JOB_ROW_NAMES} from (
|
|
79
|
+
select ${JOB_ROW_COLUMNS}
|
|
80
|
+
from x_jobs
|
|
81
|
+
where ${LIST_MATCH}
|
|
82
|
+
and (created_at, id) > (
|
|
83
|
+
coalesce(
|
|
84
|
+
(select c.created_at from x_jobs c where c.id = $9::uuid),
|
|
85
|
+
to_timestamp($8::bigint / 1000.0)
|
|
86
|
+
),
|
|
87
|
+
$9::uuid
|
|
88
|
+
)
|
|
89
|
+
order by created_at asc, id asc
|
|
90
|
+
limit $4
|
|
91
|
+
) page
|
|
92
|
+
order by created_at desc, id desc
|
|
93
|
+
`.trim();
|
|
94
|
+
|
|
34
95
|
export const SQL_JOB_DEAD_LETTERS = `
|
|
35
96
|
select ${JOB_ROW_COLUMNS}
|
|
36
97
|
from x_jobs where state = 'dead' order by updated_at desc limit $1
|
|
@@ -41,14 +102,16 @@ select ${JOB_ROW_COLUMNS}
|
|
|
41
102
|
* to FINISHED states and releasing the claim: requeueing a running row left `claimed_by` set with
|
|
42
103
|
* state `ready`, and a second worker claimed it — the job ran twice. `REQUEUEABLE_STATES`
|
|
43
104
|
* (`driver.ts`) is the same set; a row this matches nothing for was refused before it was sent,
|
|
44
|
-
* or raced to live in between.
|
|
105
|
+
* or raced to live in between. `woke` announces the row in the statement that made it claimable
|
|
106
|
+
* (`driver-pg-wake-sql.ts`): `x jobs retry` is a job starting now, not at an idle worker's poll.
|
|
45
107
|
*/
|
|
46
108
|
export const SQL_JOB_REQUEUE = `
|
|
47
109
|
update x_jobs
|
|
48
110
|
set state = 'ready', attempt = 0, run_at = now(), updated_at = now(),
|
|
49
111
|
claimed_by = null, visible_at = null
|
|
50
112
|
where id = $1 and state in ('dead', 'cancelled', 'done', 'failed')
|
|
51
|
-
returning ${JOB_ROW_COLUMNS}
|
|
113
|
+
returning ${JOB_ROW_COLUMNS},
|
|
114
|
+
pg_notify('${JOBS_WAKE_CHANNEL}', queue)::text as woke
|
|
52
115
|
`.trim();
|
|
53
116
|
|
|
54
117
|
/**
|
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
// Every statement behind the operator surface (`introspection.ts`) and the scheduler's atomic
|
|
2
|
+
// fire. One decision per statement throughout: a bulk call, a fold and a fire each read and write
|
|
3
|
+
// in ONE round trip, so nothing here counts rows and then acts on the count.
|
|
4
|
+
//
|
|
5
|
+
// Text is ordered `collate "C"` wherever a list is ordered by name: the memory driver compares by
|
|
6
|
+
// code unit, and a database collation would order the same names differently.
|
|
7
|
+
|
|
8
|
+
import { JOB_ROW_COLUMNS } from './driver-pg-jobs-sql';
|
|
9
|
+
import { JOBS_WAKE_CHANNEL } from './driver-pg-wake-sql';
|
|
10
|
+
|
|
11
|
+
const LIVE = `('ready', 'delayed', 'running', 'suspended')`;
|
|
12
|
+
|
|
13
|
+
/** $1 state, $2 queue, $3 name, $5 tenant — the predicates every bulk statement shares. */
|
|
14
|
+
const bulkMatch = (alias = ''): string =>
|
|
15
|
+
`${alias}state = $1 and ($2::text is null or ${alias}queue = $2) and ($3::text is null or ${alias}name = $3) and ($5::text is null or ${alias}tenant_id = $5)`;
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Re-queue up to $4 finished rows. `distinct on` the idempotency namespace, because two dead rows
|
|
19
|
+
* of one key would both go `ready` and the second would break the live index — and `not exists`
|
|
20
|
+
* a live holder, the single requeue's `X_JOB_DUPLICATE`, answered here as "not this row".
|
|
21
|
+
* `matching` is counted in the same statement, before the update, so the caller's `remaining` is
|
|
22
|
+
* `matching - affected` with no second read.
|
|
23
|
+
*/
|
|
24
|
+
export const SQL_JOB_REQUEUE_MANY = `
|
|
25
|
+
with candidates as (
|
|
26
|
+
select distinct on (j.name, coalesce(j.tenant_id, ''), j.idempotency_key) j.id
|
|
27
|
+
from x_jobs j
|
|
28
|
+
where ${bulkMatch('j.')}
|
|
29
|
+
and not exists (
|
|
30
|
+
select 1 from x_jobs l
|
|
31
|
+
where l.name = j.name
|
|
32
|
+
and coalesce(l.tenant_id, '') = coalesce(j.tenant_id, '')
|
|
33
|
+
and l.idempotency_key = j.idempotency_key
|
|
34
|
+
and l.state in ${LIVE}
|
|
35
|
+
)
|
|
36
|
+
order by j.name, coalesce(j.tenant_id, ''), j.idempotency_key, j.created_at
|
|
37
|
+
limit $4
|
|
38
|
+
), moved as (
|
|
39
|
+
update x_jobs
|
|
40
|
+
set state = 'ready', attempt = 0, run_at = now(), updated_at = now(),
|
|
41
|
+
claimed_by = null, visible_at = null
|
|
42
|
+
where id in (select id from candidates) and state = $1
|
|
43
|
+
returning id
|
|
44
|
+
)
|
|
45
|
+
select (select count(*) from moved)::int as affected,
|
|
46
|
+
(select count(*) from x_jobs where ${bulkMatch()})::int as matching
|
|
47
|
+
`.trim();
|
|
48
|
+
|
|
49
|
+
/** Delete up to $4 rows and their step records. `running` is refused before this is sent. */
|
|
50
|
+
export const SQL_JOB_REMOVE_MANY = `
|
|
51
|
+
with doomed as (
|
|
52
|
+
select id, run_id from x_jobs
|
|
53
|
+
where ${bulkMatch()} and state <> 'running'
|
|
54
|
+
order by created_at
|
|
55
|
+
limit $4
|
|
56
|
+
for update skip locked
|
|
57
|
+
), steps as (
|
|
58
|
+
delete from x_job_steps where run_id in (select run_id from doomed)
|
|
59
|
+
), gone as (
|
|
60
|
+
delete from x_jobs where id in (select id from doomed) returning id
|
|
61
|
+
)
|
|
62
|
+
select (select count(*) from gone)::int as affected,
|
|
63
|
+
(select count(*) from x_jobs where ${bulkMatch()})::int as matching
|
|
64
|
+
`.trim();
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Up to $4 rows of the filter still waiting on `run_at`, due now — `SQL_JOB_PROMOTE` over a set.
|
|
68
|
+
* `remaining` counts the rows of the filter STILL waiting afterwards: a `ready` row already due has
|
|
69
|
+
* nothing to promote, so it is neither affected nor remaining. `woke` announces the queues it made
|
|
70
|
+
* claimable, once each (`driver-pg-wake-sql.ts`).
|
|
71
|
+
*/
|
|
72
|
+
export const SQL_JOB_PROMOTE_MANY = `
|
|
73
|
+
with due as (
|
|
74
|
+
select id from x_jobs
|
|
75
|
+
where ${bulkMatch()} and run_at > now()
|
|
76
|
+
order by created_at
|
|
77
|
+
limit $4
|
|
78
|
+
for update skip locked
|
|
79
|
+
), moved as (
|
|
80
|
+
update x_jobs
|
|
81
|
+
set state = 'ready', run_at = now(), updated_at = now()
|
|
82
|
+
where id in (select id from due) and run_at > now()
|
|
83
|
+
returning id, queue
|
|
84
|
+
), woke as (
|
|
85
|
+
select pg_notify('${JOBS_WAKE_CHANNEL}', queue) from (select distinct queue from moved) q
|
|
86
|
+
)
|
|
87
|
+
select (select count(*) from moved)::int as affected,
|
|
88
|
+
(select count(*) from woke)::int as woken,
|
|
89
|
+
(select count(*) from x_jobs where ${bulkMatch()} and run_at > now()
|
|
90
|
+
and id not in (select id from moved))::int as matching
|
|
91
|
+
`.trim();
|
|
92
|
+
|
|
93
|
+
/** One row and its steps. Fenced on not-running, so a claim that raced the delete wins. */
|
|
94
|
+
export const SQL_JOB_REMOVE = `
|
|
95
|
+
with gone as (
|
|
96
|
+
delete from x_jobs where id = $1 and state <> 'running' returning run_id
|
|
97
|
+
), steps as (
|
|
98
|
+
delete from x_job_steps where run_id in (select run_id from gone)
|
|
99
|
+
)
|
|
100
|
+
select run_id from gone
|
|
101
|
+
`.trim();
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Due now. Only a row waiting on its `run_at` — delayed at enqueue, or `ready` and backing off —
|
|
105
|
+
* and never a `suspended` one, whose wake time is its step record and not this column. `woke`
|
|
106
|
+
* announces the row in the statement that made it due (`driver-pg-wake-sql.ts`).
|
|
107
|
+
*/
|
|
108
|
+
export const SQL_JOB_PROMOTE = `
|
|
109
|
+
update x_jobs
|
|
110
|
+
set state = 'ready', run_at = now(), updated_at = now()
|
|
111
|
+
where id = $1 and state in ('ready', 'delayed') and run_at > now()
|
|
112
|
+
returning ${JOB_ROW_COLUMNS},
|
|
113
|
+
pg_notify('${JOBS_WAKE_CHANNEL}', queue)::text as woke
|
|
114
|
+
`.trim();
|
|
115
|
+
|
|
116
|
+
/** Fenced on the claim, as both settles are. $1 id, $2 worker, $3 the progress value, $4 claim. */
|
|
117
|
+
export const SQL_JOB_PROGRESS = `
|
|
118
|
+
update x_jobs set progress = $3::jsonb
|
|
119
|
+
where id = $1 and state = 'running' and claimed_by = $2 and claims = $4::int
|
|
120
|
+
`.trim();
|
|
121
|
+
|
|
122
|
+
/** The FIRST pause is the one recorded: pausing a paused queue does not move `paused_at`. */
|
|
123
|
+
export const SQL_PAUSE = `
|
|
124
|
+
insert into x_job_pauses (kind, name) values ($1, $2)
|
|
125
|
+
on conflict (kind, name) do nothing
|
|
126
|
+
`.trim();
|
|
127
|
+
|
|
128
|
+
export const SQL_RESUME = `
|
|
129
|
+
delete from x_job_pauses where kind = $1 and name = $2
|
|
130
|
+
`.trim();
|
|
131
|
+
|
|
132
|
+
export const SQL_PAUSED = `
|
|
133
|
+
select name, (extract(epoch from paused_at) * 1000)::bigint as paused_at
|
|
134
|
+
from x_job_pauses where kind = $1 order by name collate "C"
|
|
135
|
+
`.trim();
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* A worker's heartbeat: the whole row rewritten, expiry pushed out. $1 id, $2 host, $3 queues,
|
|
139
|
+
* $4 slots, $5 in-flight ids, $6 started (ms), $7 ttl (ms).
|
|
140
|
+
*/
|
|
141
|
+
export const SQL_WORKER_ANNOUNCE = `
|
|
142
|
+
insert into x_job_workers
|
|
143
|
+
(id, host, queues, concurrency, in_flight, started_at, heartbeat_at, expires_at)
|
|
144
|
+
values
|
|
145
|
+
($1, $2, $3::jsonb, $4::int, $5::jsonb, to_timestamp($6::bigint / 1000.0), now(),
|
|
146
|
+
now() + ($7::bigint * interval '1 millisecond'))
|
|
147
|
+
on conflict (id) do update
|
|
148
|
+
set host = excluded.host, queues = excluded.queues, concurrency = excluded.concurrency,
|
|
149
|
+
in_flight = excluded.in_flight, heartbeat_at = excluded.heartbeat_at,
|
|
150
|
+
expires_at = excluded.expires_at
|
|
151
|
+
`.trim();
|
|
152
|
+
|
|
153
|
+
export const SQL_WORKER_FORGET = `
|
|
154
|
+
delete from x_job_workers where id = $1
|
|
155
|
+
`.trim();
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Live rows only, and the lapsed ones go in the same statement: a killed worker is never read
|
|
159
|
+
* again, and nothing else has to be running for its row to leave.
|
|
160
|
+
*/
|
|
161
|
+
export const SQL_WORKERS = `
|
|
162
|
+
with lapsed as (
|
|
163
|
+
delete from x_job_workers where expires_at <= now()
|
|
164
|
+
)
|
|
165
|
+
select id, host, queues, concurrency, in_flight,
|
|
166
|
+
(extract(epoch from started_at) * 1000)::bigint as started_at,
|
|
167
|
+
(extract(epoch from heartbeat_at) * 1000)::bigint as heartbeat_at
|
|
168
|
+
from x_job_workers
|
|
169
|
+
where expires_at > now()
|
|
170
|
+
order by started_at, id
|
|
171
|
+
`.trim();
|
|
172
|
+
|
|
173
|
+
const COUNTER_COLUMNS = `done, retried, failed, dead, duration_ms`;
|
|
174
|
+
|
|
175
|
+
export const SQL_COUNTERS = `
|
|
176
|
+
select job, bucket_ms, (extract(epoch from bucket_start) * 1000)::bigint as bucket_start,
|
|
177
|
+
${COUNTER_COLUMNS}
|
|
178
|
+
from x_job_counters
|
|
179
|
+
where job = $1 and bucket_start >= to_timestamp($2::bigint / 1000.0)
|
|
180
|
+
order by bucket_start, bucket_ms
|
|
181
|
+
`.trim();
|
|
182
|
+
|
|
183
|
+
export const SQL_COUNTER_TOTALS = `
|
|
184
|
+
select job, sum(done)::bigint as done, sum(retried)::bigint as retried,
|
|
185
|
+
sum(failed)::bigint as failed, sum(dead)::bigint as dead,
|
|
186
|
+
sum(duration_ms)::bigint as duration_ms
|
|
187
|
+
from x_job_counters
|
|
188
|
+
where bucket_start >= to_timestamp($1::bigint / 1000.0)
|
|
189
|
+
group by job
|
|
190
|
+
order by job collate "C"
|
|
191
|
+
`.trim();
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* Fold every bucket of width $1 older than $2 ms into the bucket of width $3 that contains it.
|
|
195
|
+
* Delete-and-insert in one statement, so a fold that two nodes run at once moves each bucket
|
|
196
|
+
* once: the second node's `delete` finds the rows gone.
|
|
197
|
+
*/
|
|
198
|
+
export const SQL_COUNTER_FOLD = `
|
|
199
|
+
with old as (
|
|
200
|
+
delete from x_job_counters
|
|
201
|
+
where bucket_ms = $1::int and bucket_start < now() - ($2::bigint * interval '1 millisecond')
|
|
202
|
+
returning job, bucket_start, ${COUNTER_COLUMNS}
|
|
203
|
+
), folded as (
|
|
204
|
+
insert into x_job_counters (job, bucket_ms, bucket_start, ${COUNTER_COLUMNS})
|
|
205
|
+
select job, $3::int,
|
|
206
|
+
to_timestamp(floor(extract(epoch from bucket_start) * 1000 / $3::int) * $3::int / 1000.0),
|
|
207
|
+
sum(done), sum(retried), sum(failed), sum(dead), sum(duration_ms)
|
|
208
|
+
from old
|
|
209
|
+
group by 1, 3
|
|
210
|
+
on conflict (job, bucket_ms, bucket_start) do update
|
|
211
|
+
set done = x_job_counters.done + excluded.done,
|
|
212
|
+
retried = x_job_counters.retried + excluded.retried,
|
|
213
|
+
failed = x_job_counters.failed + excluded.failed,
|
|
214
|
+
dead = x_job_counters.dead + excluded.dead,
|
|
215
|
+
duration_ms = x_job_counters.duration_ms + excluded.duration_ms
|
|
216
|
+
)
|
|
217
|
+
select count(*)::int as moved from old
|
|
218
|
+
`.trim();
|
|
219
|
+
|
|
220
|
+
/** The last tier has nowhere to fold into: past its `keepMs` a bucket is dropped. */
|
|
221
|
+
export const SQL_COUNTER_DROP = `
|
|
222
|
+
with old as (
|
|
223
|
+
delete from x_job_counters
|
|
224
|
+
where bucket_ms = $1::int and bucket_start < now() - ($2::bigint * interval '1 millisecond')
|
|
225
|
+
returning job
|
|
226
|
+
)
|
|
227
|
+
select count(*)::int as moved from old
|
|
228
|
+
`.trim();
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* The record of a fire that went through `driver.enqueue` — a store that could not write it in
|
|
232
|
+
* the firing statement. The watermark only ever moves forward here too. $1 task, $2 occurrence.
|
|
233
|
+
*/
|
|
234
|
+
export const SQL_TASK_FIRE_RECORD = `
|
|
235
|
+
insert into x_scheduler_state
|
|
236
|
+
(task_name, last_fired_at, updated_at, fired_occurrence_at, fired_at)
|
|
237
|
+
values ($1, to_timestamp($2::bigint / 1000.0), now(), to_timestamp($2::bigint / 1000.0), now())
|
|
238
|
+
on conflict (task_name) do update
|
|
239
|
+
set last_fired_at = greatest(x_scheduler_state.last_fired_at, excluded.last_fired_at),
|
|
240
|
+
updated_at = now(),
|
|
241
|
+
fired_occurrence_at = excluded.fired_occurrence_at, fired_at = excluded.fired_at
|
|
242
|
+
`.trim();
|
|
243
|
+
|
|
244
|
+
/** Every task that has dispatched. A row that was only ever armed has no `fired_at`. $1 limit. */
|
|
245
|
+
export const SQL_TASK_FIRES = `
|
|
246
|
+
select task_name,
|
|
247
|
+
(extract(epoch from fired_occurrence_at) * 1000)::bigint as occurrence,
|
|
248
|
+
(extract(epoch from fired_at) * 1000)::bigint as fired_at
|
|
249
|
+
from x_scheduler_state
|
|
250
|
+
where fired_at is not null
|
|
251
|
+
order by task_name collate "C"
|
|
252
|
+
limit $1
|
|
253
|
+
`.trim();
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* One occurrence, fired ATOMICALLY: the watermark moves onto it and its jobs are queued in one
|
|
257
|
+
* statement, or neither happens.
|
|
258
|
+
*
|
|
259
|
+
* It was two round trips — enqueue each job, then mark — and anything between them (a rejected
|
|
260
|
+
* mark, a dead process, job 2 of 3 failing) left the watermark behind the jobs already queued. The
|
|
261
|
+
* next round fired the occurrence again, and the occurrence-scoped idempotency key only absorbs
|
|
262
|
+
* that while the first job is still LIVE: the unique index is partial, so once job 1 is `done` the
|
|
263
|
+
* repeat inserts a second row and the work runs twice.
|
|
264
|
+
*
|
|
265
|
+
* The watermark is the fence. `moved` is empty when it is already at or past this occurrence —
|
|
266
|
+
* another dispatcher fired it — and `queued` inserts only `where exists (select 1 from moved)`.
|
|
267
|
+
* `woken` announces each queue that received a job, once, when the statement commits — read in
|
|
268
|
+
* the final select because a plain-select CTE nothing reads is never run.
|
|
269
|
+
*
|
|
270
|
+
* $1 task, $2 occurrence (ms), $3 the jobs as a JSON array.
|
|
271
|
+
*/
|
|
272
|
+
export const SQL_SCHEDULER_FIRE = `
|
|
273
|
+
with moved as (
|
|
274
|
+
insert into x_scheduler_state
|
|
275
|
+
(task_name, last_fired_at, updated_at, fired_occurrence_at, fired_at)
|
|
276
|
+
values ($1, to_timestamp($2::bigint / 1000.0), now(), to_timestamp($2::bigint / 1000.0), now())
|
|
277
|
+
on conflict (task_name) do update
|
|
278
|
+
set last_fired_at = excluded.last_fired_at, updated_at = now(),
|
|
279
|
+
fired_occurrence_at = excluded.fired_occurrence_at, fired_at = excluded.fired_at
|
|
280
|
+
where x_scheduler_state.last_fired_at < excluded.last_fired_at
|
|
281
|
+
returning task_name
|
|
282
|
+
), queued as (
|
|
283
|
+
insert into x_jobs
|
|
284
|
+
(id, name, queue, input, idempotency_key, run_id, max_attempts, state, run_at)
|
|
285
|
+
select j.id, j.name, j.queue, coalesce(j.input, 'null'::jsonb), j.idempotency_key, j.run_id, j.max_attempts,
|
|
286
|
+
case when to_timestamp($2::bigint / 1000.0) > now() then 'delayed' else 'ready' end,
|
|
287
|
+
to_timestamp($2::bigint / 1000.0)
|
|
288
|
+
from jsonb_to_recordset($3::jsonb) as j(
|
|
289
|
+
id uuid, name text, queue text, input jsonb, idempotency_key text,
|
|
290
|
+
run_id uuid, max_attempts int)
|
|
291
|
+
where exists (select 1 from moved)
|
|
292
|
+
on conflict (name, (coalesce(tenant_id, '')), idempotency_key)
|
|
293
|
+
where state in ${LIVE}
|
|
294
|
+
do nothing
|
|
295
|
+
returning id, run_id, idempotency_key, queue
|
|
296
|
+
), woken as (
|
|
297
|
+
select pg_notify('${JOBS_WAKE_CHANNEL}', w.queue) from (select distinct queue from queued) w
|
|
298
|
+
)
|
|
299
|
+
select (select count(*) from moved)::int as fired, q.id, q.run_id, q.idempotency_key,
|
|
300
|
+
(select count(*) from woken)::int as woken
|
|
301
|
+
from (select 1) as one
|
|
302
|
+
left join queued q on true
|
|
303
|
+
`.trim();
|