@ultimat3/jobs 22.14.0 → 23.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/CLAUDE.md +129 -87
  2. package/README.md +329 -19
  3. package/package.json +6 -6
  4. package/src/concurrency.ts +144 -0
  5. package/src/counters.ts +105 -0
  6. package/src/describe.ts +29 -0
  7. package/src/driver-memory-operator.ts +297 -0
  8. package/src/driver-memory.ts +79 -37
  9. package/src/driver-nats.ts +2 -2
  10. package/src/driver-pg-ddl.ts +71 -0
  11. package/src/driver-pg-jobs-sql.ts +71 -8
  12. package/src/driver-pg-operator-sql.ts +303 -0
  13. package/src/driver-pg-operator.ts +300 -0
  14. package/src/driver-pg-outbox-sql.ts +110 -0
  15. package/src/driver-pg-rows.ts +31 -0
  16. package/src/driver-pg-settle-sql.ts +71 -0
  17. package/src/driver-pg-sql.ts +80 -129
  18. package/src/driver-pg-wake-sql.ts +76 -0
  19. package/src/driver-pg.ts +74 -34
  20. package/src/driver-redis.ts +2 -2
  21. package/src/driver.ts +96 -34
  22. package/src/enqueue-signal.ts +50 -0
  23. package/src/errors-concurrency.ts +91 -0
  24. package/src/errors-operator.ts +94 -0
  25. package/src/errors.ts +13 -17
  26. package/src/events-pg.ts +38 -28
  27. package/src/events.ts +25 -0
  28. package/src/execute.ts +160 -81
  29. package/src/export.ts +23 -4
  30. package/src/heartbeat.ts +9 -3
  31. package/src/idle-backoff.ts +74 -0
  32. package/src/index.ts +105 -12
  33. package/src/inspect-operator.ts +49 -0
  34. package/src/inspect.ts +42 -2
  35. package/src/introspection.ts +262 -0
  36. package/src/job.ts +78 -15
  37. package/src/leases.ts +49 -9
  38. package/src/metrics.ts +6 -1
  39. package/src/outbox-pg.ts +3 -0
  40. package/src/outbox-relay.ts +114 -39
  41. package/src/outbox.ts +29 -6
  42. package/src/progress.ts +78 -0
  43. package/src/queue-wake.ts +220 -0
  44. package/src/redact-input.ts +26 -0
  45. package/src/renewal-timer.ts +28 -10
  46. package/src/retry-classification.ts +13 -1
  47. package/src/retry.ts +10 -2
  48. package/src/run-deadline.ts +57 -0
  49. package/src/scheduler-leader.ts +26 -0
  50. package/src/scheduler-occurrences.ts +76 -0
  51. package/src/scheduler-pg.ts +55 -2
  52. package/src/scheduler-state.ts +66 -0
  53. package/src/scheduler.ts +143 -103
  54. package/src/settled.ts +161 -0
  55. package/src/worker-admit.ts +163 -0
  56. package/src/worker-fleet-slots.ts +107 -38
  57. package/src/worker-hand-back.ts +6 -1
  58. package/src/worker-key-busy.ts +79 -0
  59. package/src/worker-loop.ts +145 -0
  60. package/src/worker-registry.ts +103 -0
  61. package/src/worker-run.ts +9 -0
  62. package/src/worker-types.ts +23 -0
  63. package/src/worker.ts +103 -134
@@ -70,10 +70,10 @@ export function createNatsDriver(_options: NatsDriverOptions = {}): JobDriver {
70
70
  claim(_options: ClaimOptions): Promise<readonly ClaimedJob[]> {
71
71
  return unavailable('claim');
72
72
  },
73
- ack(_jobId: string): Promise<void> {
73
+ ack(_jobId: string): Promise<boolean> {
74
74
  return unavailable('ack');
75
75
  },
76
- nack(_jobId: string, _options: NackOptions): Promise<void> {
76
+ nack(_jobId: string, _options: NackOptions): Promise<boolean> {
77
77
  return unavailable('nack');
78
78
  },
79
79
  heartbeat(_jobId: string, _options: HeartbeatOptions): Promise<boolean> {
@@ -132,6 +132,12 @@ alter table x_outbox add column if not exists claimed_at timestamptz;
132
132
 
133
133
  alter table x_outbox add column if not exists claimed_by text;
134
134
 
135
+ -- The run the published job carries, allocated when the row is STAGED so the enqueue that staged
136
+ -- it can answer which run it started. The row id is the published job id for the same reason.
137
+ -- Added by alter because x_outbox shipped without it. A row staged before this column existed is
138
+ -- given a run id by the default, which is what the driver would have minted at its publish.
139
+ alter table x_outbox add column if not exists run_id uuid not null default gen_random_uuid();
140
+
135
141
  -- The scheduler watermark. Without a durable one a redeployed scheduler has no idea what the
136
142
  -- pod it replaced already fired, so runRound takes the arming branch and every occurrence
137
143
  -- between the two processes is dropped with nothing logged.
@@ -141,6 +147,13 @@ create table if not exists x_scheduler_state (
141
147
  updated_at timestamptz not null default now()
142
148
  );
143
149
 
150
+ -- The last occurrence that DISPATCHED, and when. Apart from the watermark, which arming a task or
151
+ -- skipping missed occurrences also moves: an operator asking when a task last ran is asking about
152
+ -- jobs that were queued. Added by alter because x_scheduler_state shipped without them.
153
+ alter table x_scheduler_state add column if not exists fired_occurrence_at timestamptz;
154
+
155
+ alter table x_scheduler_state add column if not exists fired_at timestamptz;
156
+
144
157
  -- Leader election as an EXPIRING LEASE rather than a session advisory lock: the executor this
145
158
  -- package is handed is a pool, and a session-level pg_try_advisory_lock is released the moment
146
159
  -- that connection goes back to it. A row with an expiry needs no connection affinity at all.
@@ -176,4 +189,62 @@ create table if not exists x_job_events (
176
189
 
177
190
  create index if not exists x_job_events_lookup_idx
178
191
  on x_job_events (name, published_at);
192
+
193
+ -- What the body last reported through progress, as one value: done, total, note and the instant.
194
+ -- A column and not a table, because it is read with the row and dies with it. Added by alter
195
+ -- because x_jobs shipped without it.
196
+ alter table x_jobs add column if not exists progress jsonb;
197
+
198
+ -- The stack of the thrown value last_error describes, bounded by the driver before it is bound.
199
+ alter table x_jobs add column if not exists last_error_stack text;
200
+
201
+ -- How many times the row has been claimed, moved by the claim itself and never reset. A settle is
202
+ -- fenced on it as well as on claimed_by: a worker that takes back its own lapsed job has the same
203
+ -- id as the body still unwinding from the first claim, and only the ordinal tells the two apart.
204
+ alter table x_jobs add column if not exists claims int not null default 0;
205
+
206
+ -- The keyset a paged listing seeks by. Newest first, id as the tiebreak, so a page is an index
207
+ -- range and never a sort of the whole table.
208
+ create index if not exists x_jobs_created_idx on x_jobs (created_at desc, id desc);
209
+
210
+ -- A paused queue or task, one row each. A queue pause is a fact about the QUEUE and not a column
211
+ -- on every one of its rows: the claim reads this table with not exists, and a pause that had to
212
+ -- rewrite a million rows would not take effect within one poll interval.
213
+ create table if not exists x_job_pauses (
214
+ kind text not null,
215
+ name text not null,
216
+ paused_at timestamptz not null default now(),
217
+ primary key (kind, name)
218
+ );
219
+
220
+ -- The worker registry. One row per live worker, rewritten on its heartbeat and expired by time
221
+ -- like a lease: a killed worker announces nothing and its row stops being read. Never cleaned up
222
+ -- by anything that has to be running.
223
+ create table if not exists x_job_workers (
224
+ id text primary key,
225
+ host text not null,
226
+ queues jsonb not null,
227
+ concurrency int not null,
228
+ in_flight jsonb not null,
229
+ started_at timestamptz not null,
230
+ heartbeat_at timestamptz not null default now(),
231
+ expires_at timestamptz not null
232
+ );
233
+
234
+ -- Per-job history, in buckets. A settle adds to the one-minute bucket in the SAME statement that
235
+ -- moves the row, and the scheduler leader folds old buckets into wider ones, so the key count per
236
+ -- job name is fixed whatever the throughput.
237
+ create table if not exists x_job_counters (
238
+ job text not null,
239
+ bucket_ms int not null,
240
+ bucket_start timestamptz not null,
241
+ done bigint not null default 0,
242
+ retried bigint not null default 0,
243
+ failed bigint not null default 0,
244
+ dead bigint not null default 0,
245
+ duration_ms bigint not null default 0,
246
+ primary key (job, bucket_ms, bucket_start)
247
+ );
248
+
249
+ create index if not exists x_job_counters_age_idx on x_job_counters (bucket_ms, bucket_start);
179
250
  `.trim();
@@ -3,34 +3,95 @@
3
3
  // none decodes `timestamptz` as TEXT, so `toJobRecord` read `Number('2026-01-01 00:00:00+00')`
4
4
  // and `x jobs ls` / `show` / `cancel` printed `NaN` for every timestamp.
5
5
 
6
+ import { JOBS_WAKE_CHANNEL } from './driver-pg-wake-sql';
7
+
6
8
  /**
7
9
  * The `JobRow` shape as a projection. Asking Postgres for epoch ms is what makes the decoding
8
10
  * independent of the client's type map — never `select *`, whose correctness is the driver's
9
11
  * opinion about `timestamptz` rather than this statement's.
10
12
  */
11
13
  export const JOB_ROW_COLUMNS = `id, name, queue, input, idempotency_key, run_id, attempt,
12
- max_attempts, state, tenant_id, last_error, claimed_by,
13
- traceparent, enqueued_by,
14
+ max_attempts, state, tenant_id, last_error, last_error_stack, claimed_by, claims,
15
+ traceparent, enqueued_by, progress,
14
16
  (extract(epoch from run_at) * 1000)::bigint as run_at,
15
17
  (extract(epoch from visible_at) * 1000)::bigint as visible_at,
16
18
  (extract(epoch from created_at) * 1000)::bigint as created_at,
17
19
  (extract(epoch from updated_at) * 1000)::bigint as updated_at`;
18
20
 
21
+ /** `JOB_ROW_COLUMNS` by their output names — what a statement reading a derived table selects. */
22
+ const JOB_ROW_NAMES = `id, name, queue, input, idempotency_key, run_id, attempt, max_attempts,
23
+ state, tenant_id, last_error, last_error_stack, claimed_by, claims, traceparent,
24
+ enqueued_by, progress, run_at, visible_at, created_at, updated_at`;
25
+
19
26
  export const SQL_JOB_GET = `
20
27
  select ${JOB_ROW_COLUMNS}
21
28
  from x_jobs where id = $1
22
29
  `.trim();
23
30
 
31
+ /**
32
+ * The predicates a page shares, whichever way it reads. $1 queue, $2 name, $3 state, $5 id prefix,
33
+ * $6 created from (ms), $7 created to (ms), $10 tenant. The prefix is compared, never `like`d: a
34
+ * `_` or `%` in it is a character, as `startsWith` reads it in the memory driver.
35
+ */
36
+ const LIST_MATCH = `($1::text is null or queue = $1)
37
+ and ($2::text is null or name = $2)
38
+ and ($3::text is null or state = $3)
39
+ and ($5::text is null or left(id::text, length($5::text)) = $5::text)
40
+ and ($6::bigint is null or created_at >= to_timestamp($6::bigint / 1000.0))
41
+ and ($7::bigint is null or created_at < to_timestamp($7::bigint / 1000.0))
42
+ and ($10::text is null or tenant_id = $10)`;
43
+
44
+ /**
45
+ * One page, newest first, sought by KEYSET. `(created_at, id) < (cursor)` is a row comparison the
46
+ * `x_jobs_created_idx` range answers directly; an offset would re-read every row before it and
47
+ * shift under an insert.
48
+ *
49
+ * The cursor carries the last row's `createdAt` in epoch MS, and the column holds microseconds —
50
+ * so the seek reads the cursor row's own `created_at` by id rather than trusting a rounded value,
51
+ * which would skip or repeat its millisecond-mates. A cursor row deleted mid-walk falls back to
52
+ * the millisecond after it: rows of that one millisecond may then repeat, and none is skipped.
53
+ *
54
+ * `LIST_MATCH`'s parameters, plus $4 limit, $8 cursor createdAt (ms), $9 cursor id.
55
+ */
24
56
  export const SQL_JOB_LIST = `
25
57
  select ${JOB_ROW_COLUMNS}
26
58
  from x_jobs
27
- where ($1::text is null or queue = $1)
28
- and ($2::text is null or name = $2)
29
- and ($3::text is null or state = $3)
30
- order by created_at desc
59
+ where ${LIST_MATCH}
60
+ and ($9::uuid is null or (created_at, id) < (
61
+ coalesce(
62
+ (select c.created_at from x_jobs c where c.id = $9::uuid),
63
+ to_timestamp(($8::bigint + 1) / 1000.0)
64
+ ),
65
+ $9::uuid
66
+ ))
67
+ order by created_at desc, id desc
31
68
  limit $4
32
69
  `.trim();
33
70
 
71
+ /**
72
+ * The page BEFORE a cursor: the `limit` rows nearest it on the newer side — read oldest first so
73
+ * `limit` keeps the nearest, then turned newest first like every page. The same parameters as
74
+ * `SQL_JOB_LIST`, $8/$9 being the first row of the page the reader is leaving. A cursor row
75
+ * deleted meanwhile falls back to its millisecond: a row of it may repeat, none is skipped.
76
+ */
77
+ export const SQL_JOB_LIST_BEFORE = `
78
+ select ${JOB_ROW_NAMES} from (
79
+ select ${JOB_ROW_COLUMNS}
80
+ from x_jobs
81
+ where ${LIST_MATCH}
82
+ and (created_at, id) > (
83
+ coalesce(
84
+ (select c.created_at from x_jobs c where c.id = $9::uuid),
85
+ to_timestamp($8::bigint / 1000.0)
86
+ ),
87
+ $9::uuid
88
+ )
89
+ order by created_at asc, id asc
90
+ limit $4
91
+ ) page
92
+ order by created_at desc, id desc
93
+ `.trim();
94
+
34
95
  export const SQL_JOB_DEAD_LETTERS = `
35
96
  select ${JOB_ROW_COLUMNS}
36
97
  from x_jobs where state = 'dead' order by updated_at desc limit $1
@@ -41,14 +102,16 @@ select ${JOB_ROW_COLUMNS}
41
102
  * to FINISHED states and releasing the claim: requeueing a running row left `claimed_by` set with
42
103
  * state `ready`, and a second worker claimed it — the job ran twice. `REQUEUEABLE_STATES`
43
104
  * (`driver.ts`) is the same set; a row this matches nothing for was refused before it was sent,
44
- * or raced to live in between.
105
+ * or raced to live in between. `woke` announces the row in the statement that made it claimable
106
+ * (`driver-pg-wake-sql.ts`): `x jobs retry` is a job starting now, not at an idle worker's poll.
45
107
  */
46
108
  export const SQL_JOB_REQUEUE = `
47
109
  update x_jobs
48
110
  set state = 'ready', attempt = 0, run_at = now(), updated_at = now(),
49
111
  claimed_by = null, visible_at = null
50
112
  where id = $1 and state in ('dead', 'cancelled', 'done', 'failed')
51
- returning ${JOB_ROW_COLUMNS}
113
+ returning ${JOB_ROW_COLUMNS},
114
+ pg_notify('${JOBS_WAKE_CHANNEL}', queue)::text as woke
52
115
  `.trim();
53
116
 
54
117
  /**
@@ -0,0 +1,303 @@
1
+ // Every statement behind the operator surface (`introspection.ts`) and the scheduler's atomic
2
+ // fire. One decision per statement throughout: a bulk call, a fold and a fire each read and write
3
+ // in ONE round trip, so nothing here counts rows and then acts on the count.
4
+ //
5
+ // Text is ordered `collate "C"` wherever a list is ordered by name: the memory driver compares by
6
+ // code unit, and a database collation would order the same names differently.
7
+
8
+ import { JOB_ROW_COLUMNS } from './driver-pg-jobs-sql';
9
+ import { JOBS_WAKE_CHANNEL } from './driver-pg-wake-sql';
10
+
11
+ const LIVE = `('ready', 'delayed', 'running', 'suspended')`;
12
+
13
+ /** $1 state, $2 queue, $3 name, $5 tenant — the predicates every bulk statement shares. */
14
+ const bulkMatch = (alias = ''): string =>
15
+ `${alias}state = $1 and ($2::text is null or ${alias}queue = $2) and ($3::text is null or ${alias}name = $3) and ($5::text is null or ${alias}tenant_id = $5)`;
16
+
17
+ /**
18
+ * Re-queue up to $4 finished rows. `distinct on` the idempotency namespace, because two dead rows
19
+ * of one key would both go `ready` and the second would break the live index — and `not exists`
20
+ * a live holder, the single requeue's `X_JOB_DUPLICATE`, answered here as "not this row".
21
+ * `matching` is counted in the same statement, before the update, so the caller's `remaining` is
22
+ * `matching - affected` with no second read.
23
+ */
24
+ export const SQL_JOB_REQUEUE_MANY = `
25
+ with candidates as (
26
+ select distinct on (j.name, coalesce(j.tenant_id, ''), j.idempotency_key) j.id
27
+ from x_jobs j
28
+ where ${bulkMatch('j.')}
29
+ and not exists (
30
+ select 1 from x_jobs l
31
+ where l.name = j.name
32
+ and coalesce(l.tenant_id, '') = coalesce(j.tenant_id, '')
33
+ and l.idempotency_key = j.idempotency_key
34
+ and l.state in ${LIVE}
35
+ )
36
+ order by j.name, coalesce(j.tenant_id, ''), j.idempotency_key, j.created_at
37
+ limit $4
38
+ ), moved as (
39
+ update x_jobs
40
+ set state = 'ready', attempt = 0, run_at = now(), updated_at = now(),
41
+ claimed_by = null, visible_at = null
42
+ where id in (select id from candidates) and state = $1
43
+ returning id
44
+ )
45
+ select (select count(*) from moved)::int as affected,
46
+ (select count(*) from x_jobs where ${bulkMatch()})::int as matching
47
+ `.trim();
48
+
49
+ /** Delete up to $4 rows and their step records. `running` is refused before this is sent. */
50
+ export const SQL_JOB_REMOVE_MANY = `
51
+ with doomed as (
52
+ select id, run_id from x_jobs
53
+ where ${bulkMatch()} and state <> 'running'
54
+ order by created_at
55
+ limit $4
56
+ for update skip locked
57
+ ), steps as (
58
+ delete from x_job_steps where run_id in (select run_id from doomed)
59
+ ), gone as (
60
+ delete from x_jobs where id in (select id from doomed) returning id
61
+ )
62
+ select (select count(*) from gone)::int as affected,
63
+ (select count(*) from x_jobs where ${bulkMatch()})::int as matching
64
+ `.trim();
65
+
66
+ /**
67
+ * Up to $4 rows of the filter still waiting on `run_at`, due now — `SQL_JOB_PROMOTE` over a set.
68
+ * `remaining` counts the rows of the filter STILL waiting afterwards: a `ready` row already due has
69
+ * nothing to promote, so it is neither affected nor remaining. `woke` announces the queues it made
70
+ * claimable, once each (`driver-pg-wake-sql.ts`).
71
+ */
72
+ export const SQL_JOB_PROMOTE_MANY = `
73
+ with due as (
74
+ select id from x_jobs
75
+ where ${bulkMatch()} and run_at > now()
76
+ order by created_at
77
+ limit $4
78
+ for update skip locked
79
+ ), moved as (
80
+ update x_jobs
81
+ set state = 'ready', run_at = now(), updated_at = now()
82
+ where id in (select id from due) and run_at > now()
83
+ returning id, queue
84
+ ), woke as (
85
+ select pg_notify('${JOBS_WAKE_CHANNEL}', queue) from (select distinct queue from moved) q
86
+ )
87
+ select (select count(*) from moved)::int as affected,
88
+ (select count(*) from woke)::int as woken,
89
+ (select count(*) from x_jobs where ${bulkMatch()} and run_at > now()
90
+ and id not in (select id from moved))::int as matching
91
+ `.trim();
92
+
93
+ /** One row and its steps. Fenced on not-running, so a claim that raced the delete wins. */
94
+ export const SQL_JOB_REMOVE = `
95
+ with gone as (
96
+ delete from x_jobs where id = $1 and state <> 'running' returning run_id
97
+ ), steps as (
98
+ delete from x_job_steps where run_id in (select run_id from gone)
99
+ )
100
+ select run_id from gone
101
+ `.trim();
102
+
103
+ /**
104
+ * Due now. Only a row waiting on its `run_at` — delayed at enqueue, or `ready` and backing off —
105
+ * and never a `suspended` one, whose wake time is its step record and not this column. `woke`
106
+ * announces the row in the statement that made it due (`driver-pg-wake-sql.ts`).
107
+ */
108
+ export const SQL_JOB_PROMOTE = `
109
+ update x_jobs
110
+ set state = 'ready', run_at = now(), updated_at = now()
111
+ where id = $1 and state in ('ready', 'delayed') and run_at > now()
112
+ returning ${JOB_ROW_COLUMNS},
113
+ pg_notify('${JOBS_WAKE_CHANNEL}', queue)::text as woke
114
+ `.trim();
115
+
116
+ /** Fenced on the claim, as both settles are. $1 id, $2 worker, $3 the progress value, $4 claim. */
117
+ export const SQL_JOB_PROGRESS = `
118
+ update x_jobs set progress = $3::jsonb
119
+ where id = $1 and state = 'running' and claimed_by = $2 and claims = $4::int
120
+ `.trim();
121
+
122
+ /** The FIRST pause is the one recorded: pausing a paused queue does not move `paused_at`. */
123
+ export const SQL_PAUSE = `
124
+ insert into x_job_pauses (kind, name) values ($1, $2)
125
+ on conflict (kind, name) do nothing
126
+ `.trim();
127
+
128
+ export const SQL_RESUME = `
129
+ delete from x_job_pauses where kind = $1 and name = $2
130
+ `.trim();
131
+
132
+ export const SQL_PAUSED = `
133
+ select name, (extract(epoch from paused_at) * 1000)::bigint as paused_at
134
+ from x_job_pauses where kind = $1 order by name collate "C"
135
+ `.trim();
136
+
137
+ /**
138
+ * A worker's heartbeat: the whole row rewritten, expiry pushed out. $1 id, $2 host, $3 queues,
139
+ * $4 slots, $5 in-flight ids, $6 started (ms), $7 ttl (ms).
140
+ */
141
+ export const SQL_WORKER_ANNOUNCE = `
142
+ insert into x_job_workers
143
+ (id, host, queues, concurrency, in_flight, started_at, heartbeat_at, expires_at)
144
+ values
145
+ ($1, $2, $3::jsonb, $4::int, $5::jsonb, to_timestamp($6::bigint / 1000.0), now(),
146
+ now() + ($7::bigint * interval '1 millisecond'))
147
+ on conflict (id) do update
148
+ set host = excluded.host, queues = excluded.queues, concurrency = excluded.concurrency,
149
+ in_flight = excluded.in_flight, heartbeat_at = excluded.heartbeat_at,
150
+ expires_at = excluded.expires_at
151
+ `.trim();
152
+
153
+ export const SQL_WORKER_FORGET = `
154
+ delete from x_job_workers where id = $1
155
+ `.trim();
156
+
157
+ /**
158
+ * Live rows only, and the lapsed ones go in the same statement: a killed worker is never read
159
+ * again, and nothing else has to be running for its row to leave.
160
+ */
161
+ export const SQL_WORKERS = `
162
+ with lapsed as (
163
+ delete from x_job_workers where expires_at <= now()
164
+ )
165
+ select id, host, queues, concurrency, in_flight,
166
+ (extract(epoch from started_at) * 1000)::bigint as started_at,
167
+ (extract(epoch from heartbeat_at) * 1000)::bigint as heartbeat_at
168
+ from x_job_workers
169
+ where expires_at > now()
170
+ order by started_at, id
171
+ `.trim();
172
+
173
+ const COUNTER_COLUMNS = `done, retried, failed, dead, duration_ms`;
174
+
175
+ export const SQL_COUNTERS = `
176
+ select job, bucket_ms, (extract(epoch from bucket_start) * 1000)::bigint as bucket_start,
177
+ ${COUNTER_COLUMNS}
178
+ from x_job_counters
179
+ where job = $1 and bucket_start >= to_timestamp($2::bigint / 1000.0)
180
+ order by bucket_start, bucket_ms
181
+ `.trim();
182
+
183
+ export const SQL_COUNTER_TOTALS = `
184
+ select job, sum(done)::bigint as done, sum(retried)::bigint as retried,
185
+ sum(failed)::bigint as failed, sum(dead)::bigint as dead,
186
+ sum(duration_ms)::bigint as duration_ms
187
+ from x_job_counters
188
+ where bucket_start >= to_timestamp($1::bigint / 1000.0)
189
+ group by job
190
+ order by job collate "C"
191
+ `.trim();
192
+
193
+ /**
194
+ * Fold every bucket of width $1 older than $2 ms into the bucket of width $3 that contains it.
195
+ * Delete-and-insert in one statement, so a fold that two nodes run at once moves each bucket
196
+ * once: the second node's `delete` finds the rows gone.
197
+ */
198
+ export const SQL_COUNTER_FOLD = `
199
+ with old as (
200
+ delete from x_job_counters
201
+ where bucket_ms = $1::int and bucket_start < now() - ($2::bigint * interval '1 millisecond')
202
+ returning job, bucket_start, ${COUNTER_COLUMNS}
203
+ ), folded as (
204
+ insert into x_job_counters (job, bucket_ms, bucket_start, ${COUNTER_COLUMNS})
205
+ select job, $3::int,
206
+ to_timestamp(floor(extract(epoch from bucket_start) * 1000 / $3::int) * $3::int / 1000.0),
207
+ sum(done), sum(retried), sum(failed), sum(dead), sum(duration_ms)
208
+ from old
209
+ group by 1, 3
210
+ on conflict (job, bucket_ms, bucket_start) do update
211
+ set done = x_job_counters.done + excluded.done,
212
+ retried = x_job_counters.retried + excluded.retried,
213
+ failed = x_job_counters.failed + excluded.failed,
214
+ dead = x_job_counters.dead + excluded.dead,
215
+ duration_ms = x_job_counters.duration_ms + excluded.duration_ms
216
+ )
217
+ select count(*)::int as moved from old
218
+ `.trim();
219
+
220
+ /** The last tier has nowhere to fold into: past its `keepMs` a bucket is dropped. */
221
+ export const SQL_COUNTER_DROP = `
222
+ with old as (
223
+ delete from x_job_counters
224
+ where bucket_ms = $1::int and bucket_start < now() - ($2::bigint * interval '1 millisecond')
225
+ returning job
226
+ )
227
+ select count(*)::int as moved from old
228
+ `.trim();
229
+
230
+ /**
231
+ * The record of a fire that went through `driver.enqueue` — a store that could not write it in
232
+ * the firing statement. The watermark only ever moves forward here too. $1 task, $2 occurrence.
233
+ */
234
+ export const SQL_TASK_FIRE_RECORD = `
235
+ insert into x_scheduler_state
236
+ (task_name, last_fired_at, updated_at, fired_occurrence_at, fired_at)
237
+ values ($1, to_timestamp($2::bigint / 1000.0), now(), to_timestamp($2::bigint / 1000.0), now())
238
+ on conflict (task_name) do update
239
+ set last_fired_at = greatest(x_scheduler_state.last_fired_at, excluded.last_fired_at),
240
+ updated_at = now(),
241
+ fired_occurrence_at = excluded.fired_occurrence_at, fired_at = excluded.fired_at
242
+ `.trim();
243
+
244
+ /** Every task that has dispatched. A row that was only ever armed has no `fired_at`. $1 limit. */
245
+ export const SQL_TASK_FIRES = `
246
+ select task_name,
247
+ (extract(epoch from fired_occurrence_at) * 1000)::bigint as occurrence,
248
+ (extract(epoch from fired_at) * 1000)::bigint as fired_at
249
+ from x_scheduler_state
250
+ where fired_at is not null
251
+ order by task_name collate "C"
252
+ limit $1
253
+ `.trim();
254
+
255
+ /**
256
+ * One occurrence, fired ATOMICALLY: the watermark moves onto it and its jobs are queued in one
257
+ * statement, or neither happens.
258
+ *
259
+ * It was two round trips — enqueue each job, then mark — and anything between them (a rejected
260
+ * mark, a dead process, job 2 of 3 failing) left the watermark behind the jobs already queued. The
261
+ * next round fired the occurrence again, and the occurrence-scoped idempotency key only absorbs
262
+ * that while the first job is still LIVE: the unique index is partial, so once job 1 is `done` the
263
+ * repeat inserts a second row and the work runs twice.
264
+ *
265
+ * The watermark is the fence. `moved` is empty when it is already at or past this occurrence —
266
+ * another dispatcher fired it — and `queued` inserts only `where exists (select 1 from moved)`.
267
+ * `woken` announces each queue that received a job, once, when the statement commits — read in
268
+ * the final select because a plain-select CTE nothing reads is never run.
269
+ *
270
+ * $1 task, $2 occurrence (ms), $3 the jobs as a JSON array.
271
+ */
272
+ export const SQL_SCHEDULER_FIRE = `
273
+ with moved as (
274
+ insert into x_scheduler_state
275
+ (task_name, last_fired_at, updated_at, fired_occurrence_at, fired_at)
276
+ values ($1, to_timestamp($2::bigint / 1000.0), now(), to_timestamp($2::bigint / 1000.0), now())
277
+ on conflict (task_name) do update
278
+ set last_fired_at = excluded.last_fired_at, updated_at = now(),
279
+ fired_occurrence_at = excluded.fired_occurrence_at, fired_at = excluded.fired_at
280
+ where x_scheduler_state.last_fired_at < excluded.last_fired_at
281
+ returning task_name
282
+ ), queued as (
283
+ insert into x_jobs
284
+ (id, name, queue, input, idempotency_key, run_id, max_attempts, state, run_at)
285
+ select j.id, j.name, j.queue, coalesce(j.input, 'null'::jsonb), j.idempotency_key, j.run_id, j.max_attempts,
286
+ case when to_timestamp($2::bigint / 1000.0) > now() then 'delayed' else 'ready' end,
287
+ to_timestamp($2::bigint / 1000.0)
288
+ from jsonb_to_recordset($3::jsonb) as j(
289
+ id uuid, name text, queue text, input jsonb, idempotency_key text,
290
+ run_id uuid, max_attempts int)
291
+ where exists (select 1 from moved)
292
+ on conflict (name, (coalesce(tenant_id, '')), idempotency_key)
293
+ where state in ${LIVE}
294
+ do nothing
295
+ returning id, run_id, idempotency_key, queue
296
+ ), woken as (
297
+ select pg_notify('${JOBS_WAKE_CHANNEL}', w.queue) from (select distinct queue from queued) w
298
+ )
299
+ select (select count(*) from moved)::int as fired, q.id, q.run_id, q.idempotency_key,
300
+ (select count(*) from woken)::int as woken
301
+ from (select 1) as one
302
+ left join queued q on true
303
+ `.trim();