@ultimat3/jobs 22.15.0 → 24.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +165 -124
- package/README.md +360 -21
- package/package.json +6 -6
- package/src/backfill-ledger.ts +24 -2
- package/src/backfill-pass.ts +22 -5
- package/src/claim-exhausted.ts +76 -0
- package/src/concurrency.ts +144 -0
- package/src/counters.ts +105 -0
- package/src/describe.ts +29 -0
- package/src/driver-memory-operator.ts +319 -0
- package/src/driver-memory.ts +146 -59
- package/src/driver-nats.ts +7 -4
- package/src/driver-pg-ddl.ts +71 -0
- package/src/driver-pg-jobs-sql.ts +96 -25
- package/src/driver-pg-operator-sql.ts +323 -0
- package/src/driver-pg-operator.ts +316 -0
- package/src/driver-pg-outbox-sql.ts +110 -0
- package/src/driver-pg-rows.ts +34 -3
- package/src/driver-pg-settle-sql.ts +71 -0
- package/src/driver-pg-sql.ts +171 -151
- package/src/driver-pg-wake-sql.ts +76 -0
- package/src/driver-pg.ts +150 -105
- package/src/driver-redis.ts +7 -4
- package/src/driver.ts +133 -35
- package/src/enqueue-signal.ts +50 -0
- package/src/errors-concurrency.ts +91 -0
- package/src/errors-operator.ts +102 -0
- package/src/errors-requeue.ts +21 -3
- package/src/errors.ts +18 -30
- package/src/events-pg.ts +45 -45
- package/src/events.ts +38 -2
- package/src/execute.ts +161 -81
- package/src/export.ts +23 -4
- package/src/heartbeat.ts +9 -3
- package/src/idle-backoff.ts +74 -0
- package/src/index.ts +115 -16
- package/src/inspect-operator.ts +49 -0
- package/src/inspect.ts +46 -5
- package/src/introspection.ts +274 -0
- package/src/job.ts +83 -21
- package/src/leases.ts +49 -9
- package/src/limits.ts +33 -7
- package/src/metrics.ts +6 -1
- package/src/outbox-pg.ts +4 -2
- package/src/outbox-relay.ts +114 -39
- package/src/outbox.ts +59 -8
- package/src/progress.ts +78 -0
- package/src/purge.ts +1 -1
- package/src/queue-wake.ts +219 -0
- package/src/redact-input.ts +26 -0
- package/src/renewal-timer.ts +28 -10
- package/src/retry-classification.ts +13 -1
- package/src/retry.ts +10 -2
- package/src/run-deadline.ts +57 -0
- package/src/scheduler-leader.ts +26 -0
- package/src/scheduler-occurrences.ts +76 -0
- package/src/scheduler-pg.ts +58 -4
- package/src/scheduler-state.ts +73 -0
- package/src/scheduler.ts +150 -111
- package/src/settled.ts +161 -0
- package/src/steps-suspension.ts +46 -0
- package/src/steps-timeout.ts +33 -0
- package/src/steps.ts +55 -81
- package/src/worker-admit.ts +167 -0
- package/src/worker-fleet-slots.ts +157 -54
- package/src/worker-hand-back.ts +6 -1
- package/src/worker-key-busy.ts +107 -0
- package/src/worker-loop.ts +169 -0
- package/src/worker-registry.ts +118 -0
- package/src/worker-run.ts +9 -0
- package/src/worker-types.ts +23 -0
- package/src/worker.ts +142 -130
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
// Every statement the `x_outbox` store runs. Split off `driver-pg-sql.ts` at its size ceiling and
|
|
2
|
+
// re-exported from it, the way the settle and whole-row statements are.
|
|
3
|
+
|
|
4
|
+
import { NOTIFY_STAGED } from './driver-pg-wake-sql';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* The stage, and its wake. One statement on the CALLER's connection, so the notification is in
|
|
8
|
+
* the business transaction with the row: delivered when it commits, never when it rolls back.
|
|
9
|
+
*/
|
|
10
|
+
export const SQL_OUTBOX_STAGE = `
|
|
11
|
+
with staged as (
|
|
12
|
+
insert into x_outbox
|
|
13
|
+
(id, job, queue, input, idempotency_key, max_attempts, run_at, staged_at, tenant_id,
|
|
14
|
+
traceparent, enqueued_by, run_id)
|
|
15
|
+
values ($1, $2, $3, $4::jsonb, $5, $6, to_timestamp($7 / 1000.0), to_timestamp($8 / 1000.0),
|
|
16
|
+
$9, $10, $11, $12)
|
|
17
|
+
returning id
|
|
18
|
+
)
|
|
19
|
+
select id,${NOTIFY_STAGED}::text as woke
|
|
20
|
+
from staged
|
|
21
|
+
`.trim();
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* The claim, and it has to be ONE statement. `for update skip locked` in a bare select holds its
|
|
25
|
+
* row locks only until that statement ends — under autocommit, before `claim()` even resolves — so
|
|
26
|
+
* two relays polling 200ms apart read the same unpublished rows and both hand them to `enqueue`.
|
|
27
|
+
* `SQL_ENQUEUE` collapses the repeat only while the first job is still LIVE: its conflict target
|
|
28
|
+
* is a partial index over the live states, so a second publish landing after that job reached a
|
|
29
|
+
* terminal state inserts a second row and the handler runs again. (The mechanism is Postgres
|
|
30
|
+
* semantics; how often the two orderings line up in a deployment was never measured.)
|
|
31
|
+
*
|
|
32
|
+
* So the lock and the claim commit together, the CTE shape `SQL_CLAIM` already uses, and
|
|
33
|
+
* `claimed_at` is a LEASE: `$2` is the window after which a row a dead relay was holding is
|
|
34
|
+
* claimable again, because a claim nothing can expire strands its rows forever.
|
|
35
|
+
*
|
|
36
|
+
* The outer `select ... order by staged_at, id` is not cosmetic. `update ... returning` has no
|
|
37
|
+
* defined row order and the relay publishes in the order it is handed rows, so an app staging
|
|
38
|
+
* `createInvoice` then `chargeCard` in one transaction depends on this line.
|
|
39
|
+
*
|
|
40
|
+
* `, id` is what makes that key TOTAL, and the CTE needs it as much as the projection does: every
|
|
41
|
+
* row staged in one transaction shares a `staged_at`, so `staged_at` alone leaves the tie to the
|
|
42
|
+
* planner — which rows a `limit` takes, and in which order they publish, then differ between two
|
|
43
|
+
* relays and between two runs of one relay. No column was added for it: `id` is a UUIDv7 minted by
|
|
44
|
+
* `uuid()`, monotonic and already the primary key, so the tiebreak IS stage order.
|
|
45
|
+
*/
|
|
46
|
+
export const SQL_OUTBOX_CLAIM = `
|
|
47
|
+
with claimable as (
|
|
48
|
+
select id
|
|
49
|
+
from x_outbox
|
|
50
|
+
where published_at is null
|
|
51
|
+
and (claimed_at is null
|
|
52
|
+
or claimed_at <= now() - ($2::bigint * interval '1 millisecond'))
|
|
53
|
+
order by staged_at, id
|
|
54
|
+
limit $1
|
|
55
|
+
for update skip locked
|
|
56
|
+
), claimed as (
|
|
57
|
+
update x_outbox o
|
|
58
|
+
set claimed_at = now(), claimed_by = $3
|
|
59
|
+
from claimable c
|
|
60
|
+
where o.id = c.id
|
|
61
|
+
returning o.id, o.run_id, o.job, o.queue, o.input, o.idempotency_key, o.max_attempts,
|
|
62
|
+
o.tenant_id, o.traceparent, o.enqueued_by, o.claimed_by, o.run_at, o.staged_at
|
|
63
|
+
)
|
|
64
|
+
select id, run_id, job, queue, input, idempotency_key, max_attempts, tenant_id,
|
|
65
|
+
traceparent, enqueued_by, claimed_by,
|
|
66
|
+
(extract(epoch from run_at) * 1000)::bigint as run_at,
|
|
67
|
+
(extract(epoch from staged_at) * 1000)::bigint as staged_at
|
|
68
|
+
from claimed
|
|
69
|
+
order by staged_at, id
|
|
70
|
+
`.trim();
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Hand a claim back early. The relay stops its batch on the first publish that fails, and without
|
|
74
|
+
* this the rows behind it would wait out the whole lease before any relay could retry them — a
|
|
75
|
+
* pool blip during a failover becoming tens of seconds of unpublished, committed work.
|
|
76
|
+
*
|
|
77
|
+
* Fenced on `published_at is null` so it can never unclaim a row some other pass already
|
|
78
|
+
* published, AND on `claimed_by` so it can never unclaim one a NEWER claimant now holds: a relay
|
|
79
|
+
* whose lease lapsed while it stalled wakes into a world where its batch is another relay's, and
|
|
80
|
+
* an unfenced release frees rows that relay is mid-publish on — a third relay claims them and
|
|
81
|
+
* publishes them again, which is the duplicate the lease exists to prevent.
|
|
82
|
+
*/
|
|
83
|
+
export const SQL_OUTBOX_RELEASE = `
|
|
84
|
+
update x_outbox
|
|
85
|
+
set claimed_at = null, claimed_by = null
|
|
86
|
+
where id = any($1::uuid[]) and published_at is null and claimed_by = $2
|
|
87
|
+
`.trim();
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Same fence, and here it is the more expensive one to miss: marking a row published is LOSING it,
|
|
91
|
+
* so a lapsed claimant stamping a row the current one has not published yet drops that job with
|
|
92
|
+
* nothing to notice. `published_at is null` makes the stamp first-writer-wins rather than a
|
|
93
|
+
* rewrite of an audit timestamp.
|
|
94
|
+
*/
|
|
95
|
+
export const SQL_OUTBOX_MARK_PUBLISHED = `
|
|
96
|
+
update x_outbox set published_at = to_timestamp($2 / 1000.0)
|
|
97
|
+
where id = $1 and published_at is null and claimed_by = $3
|
|
98
|
+
`.trim();
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* The job a staged row's publish made, by the id the two share. Read when `SQL_ENQUEUE` inserted
|
|
102
|
+
* nothing for a request that named its id: the answer is that publish, repeated.
|
|
103
|
+
*/
|
|
104
|
+
export const SQL_OUTBOX_PUBLISHED_JOB = `
|
|
105
|
+
select id, run_id from x_jobs where id = $1
|
|
106
|
+
`.trim();
|
|
107
|
+
|
|
108
|
+
export const SQL_OUTBOX_PENDING_COUNT = `
|
|
109
|
+
select count(*)::bigint as pending from x_outbox where published_at is null
|
|
110
|
+
`.trim();
|
package/src/driver-pg-rows.ts
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
import { BACKFILL_STATUSES, type BackfillRun, isBackfillStatus } from './backfill-ledger';
|
|
7
7
|
import { isJobState, JOB_STATES, type JobRecord, type JobState } from './driver';
|
|
8
8
|
import { JobRowStatusUnknownError } from './errors';
|
|
9
|
+
import type { JobProgress } from './introspection';
|
|
9
10
|
import { isStepStatus, STEP_STATUSES, type StepRecord, type StepStatus } from './steps';
|
|
10
11
|
|
|
11
12
|
export interface StepRow {
|
|
@@ -47,12 +48,17 @@ export interface JobRow {
|
|
|
47
48
|
readonly tenant_id: string | null;
|
|
48
49
|
readonly last_error: string | null;
|
|
49
50
|
readonly claimed_by: string | null;
|
|
51
|
+
/** Absent from a row an older statement projected. */
|
|
52
|
+
readonly claims?: number | string | null;
|
|
50
53
|
readonly run_at: number | string;
|
|
51
54
|
readonly visible_at: number | string | null;
|
|
52
55
|
readonly created_at: number | string;
|
|
53
56
|
readonly updated_at: number | string;
|
|
54
57
|
readonly traceparent?: string | null;
|
|
55
58
|
readonly enqueued_by?: string | null;
|
|
59
|
+
/** `jsonb`: an object from a client with a type map, its text from one without. */
|
|
60
|
+
readonly progress?: unknown;
|
|
61
|
+
readonly last_error_stack?: string | null;
|
|
56
62
|
}
|
|
57
63
|
|
|
58
64
|
/**
|
|
@@ -88,7 +94,7 @@ export function toJobRecord(row: JobRow): JobRecord {
|
|
|
88
94
|
attempt: row.attempt,
|
|
89
95
|
maxAttempts: row.max_attempts,
|
|
90
96
|
state: statusIn<JobState>(JOB_STATES, isJobState, {
|
|
91
|
-
table: '
|
|
97
|
+
table: 'x_jobs',
|
|
92
98
|
column: 'state',
|
|
93
99
|
value: row.state,
|
|
94
100
|
}),
|
|
@@ -98,6 +104,7 @@ export function toJobRecord(row: JobRow): JobRecord {
|
|
|
98
104
|
...(row.tenant_id === null ? {} : { tenantId: row.tenant_id }),
|
|
99
105
|
...(row.last_error === null ? {} : { lastError: row.last_error }),
|
|
100
106
|
...(row.claimed_by === null ? {} : { claimedBy: row.claimed_by }),
|
|
107
|
+
...(num(row.claims ?? 0) > 0 ? { claim: num(row.claims ?? 0) } : {}),
|
|
101
108
|
...(visibleAt === undefined ? {} : { visibleAt }),
|
|
102
109
|
...(row.traceparent === null || row.traceparent === undefined
|
|
103
110
|
? {}
|
|
@@ -105,9 +112,33 @@ export function toJobRecord(row: JobRow): JobRecord {
|
|
|
105
112
|
...(row.enqueued_by === null || row.enqueued_by === undefined
|
|
106
113
|
? {}
|
|
107
114
|
: { enqueuedBy: row.enqueued_by }),
|
|
115
|
+
...(row.last_error_stack === null || row.last_error_stack === undefined
|
|
116
|
+
? {}
|
|
117
|
+
: { lastErrorStack: row.last_error_stack }),
|
|
118
|
+
...progressOf(row.progress),
|
|
108
119
|
};
|
|
109
120
|
}
|
|
110
121
|
|
|
122
|
+
/**
|
|
123
|
+
* The `progress` column, or nothing. Total: the value was written by `recordProgress` in whatever
|
|
124
|
+
* build was deployed then, and a row that carries something else has no progress, not a throw.
|
|
125
|
+
*/
|
|
126
|
+
function progressOf(raw: unknown): { readonly progress?: JobProgress } {
|
|
127
|
+
const value: unknown = typeof raw === 'string' ? parseJson(raw) : raw;
|
|
128
|
+
if (typeof value !== 'object' || value === null) return {};
|
|
129
|
+
const { done, total, note, at } = value as Readonly<Record<string, unknown>>;
|
|
130
|
+
if (typeof done !== 'number' || typeof total !== 'number' || typeof at !== 'number') return {};
|
|
131
|
+
return { progress: { done, total, at, ...(typeof note === 'string' ? { note } : {}) } };
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function parseJson(text: string): unknown {
|
|
135
|
+
try {
|
|
136
|
+
return JSON.parse(text) as unknown;
|
|
137
|
+
} catch {
|
|
138
|
+
return undefined;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
111
142
|
export function toStepRecord(row: StepRow): StepRecord {
|
|
112
143
|
const completedAt = optionalNum(row.completed_at);
|
|
113
144
|
const wakeAt = optionalNum(row.wake_at);
|
|
@@ -115,7 +146,7 @@ export function toStepRecord(row: StepRow): StepRecord {
|
|
|
115
146
|
runId: row.run_id,
|
|
116
147
|
name: row.name,
|
|
117
148
|
status: statusIn<StepStatus>(STEP_STATUSES, isStepStatus, {
|
|
118
|
-
table: '
|
|
149
|
+
table: 'x_job_steps',
|
|
119
150
|
column: 'status',
|
|
120
151
|
value: row.status,
|
|
121
152
|
}),
|
|
@@ -137,7 +168,7 @@ export function toBackfillRun(row: BackfillRow): BackfillRun {
|
|
|
137
168
|
name: row.name,
|
|
138
169
|
checksum: row.checksum,
|
|
139
170
|
status: statusIn(BACKFILL_STATUSES, isBackfillStatus, {
|
|
140
|
-
table: '
|
|
171
|
+
table: 'x_backfills',
|
|
141
172
|
column: 'status',
|
|
142
173
|
value: row.status,
|
|
143
174
|
}),
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
// The two statements that SETTLE a claimed row, each also adding the settle to its job's
|
|
2
|
+
// one-minute counter bucket. One statement, not two: a count that can disagree with the rows is
|
|
3
|
+
// worse than no count, and a second round trip per job is a cost every job would pay. Split off
|
|
4
|
+
// `driver-pg-sql.ts` at its size ceiling and re-exported from it.
|
|
5
|
+
|
|
6
|
+
import { COUNTER_BUCKET_MS } from './introspection';
|
|
7
|
+
|
|
8
|
+
/** `x_job_counters`'s conflict target and the bucket every settle writes — spelled once. */
|
|
9
|
+
const BUCKET = `${COUNTER_BUCKET_MS}, date_trunc('minute', now())`;
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* `and state = 'running'`, `and claimed_by = $2` and `and claims = $4` are all FENCES, not
|
|
13
|
+
* filters. Without the first, an ack from a worker that was cancelled overwrites the
|
|
14
|
+
* cancellation. Without the second, an ack from a worker whose lease lapsed marks `done` a run
|
|
15
|
+
* ANOTHER worker has since claimed and is still executing — the row is `running` again, so the
|
|
16
|
+
* state fence alone lets it through. Without the third, the same thing happens when the worker
|
|
17
|
+
* that claimed it again is the SAME one: its id did not change, only the claim's ordinal did.
|
|
18
|
+
* `returning` is how the caller learns whether its settle landed.
|
|
19
|
+
*
|
|
20
|
+
* $1 id, $2 worker, $3 the attempt's duration in ms, $4 the claim, $5 whether it is counted — a
|
|
21
|
+
* row `x jobs drain` moved is `done` here and is not a completed run.
|
|
22
|
+
*/
|
|
23
|
+
export const SQL_ACK = `
|
|
24
|
+
with settled as (
|
|
25
|
+
update x_jobs
|
|
26
|
+
set state = 'done', visible_at = null, claimed_by = null, updated_at = now()
|
|
27
|
+
where id = $1 and state = 'running' and claimed_by = $2 and claims = $4::int
|
|
28
|
+
returning name
|
|
29
|
+
), counted as (
|
|
30
|
+
insert into x_job_counters (job, bucket_ms, bucket_start, done, duration_ms)
|
|
31
|
+
select name, ${BUCKET}, 1, $3::bigint from settled where $5::boolean
|
|
32
|
+
on conflict (job, bucket_ms, bucket_start) do update
|
|
33
|
+
set done = x_job_counters.done + 1,
|
|
34
|
+
duration_ms = x_job_counters.duration_ms + excluded.duration_ms
|
|
35
|
+
)
|
|
36
|
+
select name from settled
|
|
37
|
+
`.trim();
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* The same three fences. The counter row is written only when one of the three counts is set: a
|
|
41
|
+
* shed, a suspension and a drained attempt are handed back UNCOUNTED and are not history.
|
|
42
|
+
*
|
|
43
|
+
* $1 id, $2 state, $3 counts-as-attempt, $4 delay ms, $5 error, $6 worker, $7 stack,
|
|
44
|
+
* $8 retried, $9 failed, $10 dead (each 0 or 1), $11 the attempt's duration in ms, $12 the claim.
|
|
45
|
+
*/
|
|
46
|
+
export const SQL_NACK = `
|
|
47
|
+
with settled as (
|
|
48
|
+
update x_jobs
|
|
49
|
+
set state = $2,
|
|
50
|
+
attempt = case when $3::boolean then attempt else greatest(attempt - 1, 0) end,
|
|
51
|
+
run_at = now() + ($4::bigint * interval '1 millisecond'),
|
|
52
|
+
visible_at = null,
|
|
53
|
+
claimed_by = null,
|
|
54
|
+
last_error = coalesce($5::text, last_error),
|
|
55
|
+
last_error_stack = case when $5::text is null then last_error_stack else $7::text end,
|
|
56
|
+
updated_at = now()
|
|
57
|
+
where id = $1 and state = 'running' and claimed_by = $6 and claims = $12::int
|
|
58
|
+
returning name
|
|
59
|
+
), counted as (
|
|
60
|
+
insert into x_job_counters (job, bucket_ms, bucket_start, retried, failed, dead, duration_ms)
|
|
61
|
+
select name, ${BUCKET}, $8::int, $9::int, $10::int, $11::bigint
|
|
62
|
+
from settled
|
|
63
|
+
where $8::int + $9::int + $10::int > 0
|
|
64
|
+
on conflict (job, bucket_ms, bucket_start) do update
|
|
65
|
+
set retried = x_job_counters.retried + excluded.retried,
|
|
66
|
+
failed = x_job_counters.failed + excluded.failed,
|
|
67
|
+
dead = x_job_counters.dead + excluded.dead,
|
|
68
|
+
duration_ms = x_job_counters.duration_ms + excluded.duration_ms
|
|
69
|
+
)
|
|
70
|
+
select name from settled
|
|
71
|
+
`.trim();
|