@ultimat3/jobs 30.3.0 → 30.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +5 -6
- package/package.json +7 -7
- package/src/errors.ts +9 -3
- package/src/execute-result.ts +43 -0
- package/src/execute-unwind.ts +10 -3
- package/src/execute.ts +43 -48
- package/src/index.ts +12 -2
- package/src/job.ts +10 -0
- package/src/metrics.ts +1 -1
- package/src/step-scope.ts +13 -0
- package/src/steps.ts +10 -3
- package/src/worker-admit.ts +1 -1
- package/src/worker-key-busy.ts +1 -1
- package/src/worker-run.ts +1 -1
- package/src/worker-tally.ts +1 -1
- package/src/worker-types.ts +1 -1
- package/src/worker.ts +1 -1
package/CLAUDE.md
CHANGED
|
@@ -122,7 +122,7 @@ Tier 3. The `job` + `task` primitives, durable steps, transactional outbox, queu
|
|
|
122
122
|
`backoffDelayMs`): `docs/history/jobs.md`, "Moved 2026-10-01 — retry and bounds".
|
|
123
123
|
- **The claim loop re-arms on the PASS, never the jobs**; `tick()` resolves with that pass's executions.
|
|
124
124
|
- **A deadline CANCELS, then fails the attempt**, never the reverse (`raceTimeout`, `withStepTimeout`);
|
|
125
|
-
the
|
|
125
|
+
`ctx.signal` aborts unless the body completed AND its ack landed; steps fence on `ended`.
|
|
126
126
|
- **A cancelled runner writes NOTHING** — `put()` in `steps.ts` (`X_ABORTED`) — save a fenced
|
|
127
127
|
`X_DRAINING` on a live attempt: a step done in the drain is recorded once. The STORE fences
|
|
128
128
|
the claim (`StepFence`, `SQL_STEP_PUT` as `SQL_ACK`: `X_JOB_LEASE_LOST`). `executeJob` passes it.
|
|
@@ -143,13 +143,12 @@ Tier 3. The `job` + `task` primitives, durable steps, transactional outbox, queu
|
|
|
143
143
|
- **`step.run` hydrates the run's steps from ONE `store.list(runId)`**; `sleep` and `waitForEvent`
|
|
144
144
|
record replays through `trace()`. **`claimName` reads a Set**; `usedNames()` keeps
|
|
145
145
|
`MAX_TRACE_NAMES` (200).
|
|
146
|
-
- **`x jobs cancel` binds to `cancelJob(driver, id, reason?)`, which refuses** a finished job or a
|
|
147
|
-
driver with no `cancel` (`X_JOB_NOT_CANCELLABLE`).
|
|
148
146
|
|
|
149
147
|
## Rules — long form in `docs/history/jobs.md`
|
|
150
148
|
|
|
151
149
|
- "Moved out 2026-11-03": operator verbs on BOTH drivers; an idle round touches no store.
|
|
152
150
|
- "Moved out 2026-11-06": `demand` with no worker registry; how the claim seeks.
|
|
151
|
+
- "Moved out 2026-11-07": `cancelJob`'s refusals; why a completed attempt's signal stays live.
|
|
153
152
|
- "Plan 101 wave 6c": compensate ONLY failed for good; `once` writes `started` BEFORE `fn`; a
|
|
154
153
|
waking publish is ONE statement; payload org = enqueue org (`X_JOB_TENANT_MISMATCH`).
|
|
155
154
|
|
|
@@ -213,8 +212,8 @@ Long form: [`docs/history/jobs.md`](../../docs/history/jobs.md), "Moved 2026-10-
|
|
|
213
212
|
| `consumer.ts` / `consumer-loop.ts` / `consumer-types.ts` | `consumer()` — a job + `.supervisor` task; one run's turns; the shapes |
|
|
214
213
|
| `consumer-registry.ts` / `consumer-guard.ts` / `consumer-errors.ts` | the `origin` stamp + `registeredConsumers`; the self-enqueue refusal; `X_CONSUMER_*` |
|
|
215
214
|
| `register.ts` | `registerJobs`/`registerTasks` over a module namespace + the registrar announcements; an `action-job` projection is `X_ACTION_JOB_UNBRIDGED` |
|
|
216
|
-
| `describe.ts` |
|
|
217
|
-
| `steps.ts` / `-core.ts` / `-wait.ts` / `-unwind.ts` | replay runner;
|
|
215
|
+
| `describe.ts` | one handle's JSON projection; `describeJobs()` maps it |
|
|
216
|
+
| `steps.ts` / `-core.ts` / `-wait.ts` / `-unwind.ts` / `step-scope.ts` | replay runner; core; `waitForEvent`; compensation; `currentStep()` |
|
|
218
217
|
| `steps-timeout.ts` / `steps-suspension.ts` | a step's ceiling; `StepSuspension`, `UnwindReached`, timeout marker |
|
|
219
218
|
| `steps-api.ts` / `steps-store.ts` / `steps-output.ts` | `StepApi`, `EventLookup`; `StepRecord`, `StepFence`, `StepStore`; what a step answers, `StepOutput` and `jsonForm` |
|
|
220
219
|
| `type-pins.ts` | compile-time pins for `StepOutput` and `waitForEvent` |
|
|
@@ -238,7 +237,7 @@ Long form: [`docs/history/jobs.md`](../../docs/history/jobs.md), "Moved 2026-10-
|
|
|
238
237
|
| `retry.ts` | the dead-letter decision, and this package's option names over core's `backoffDelay` — no curve of its own |
|
|
239
238
|
| `retry-classification.ts` | the other half: what the thrown error says, and the stop reason |
|
|
240
239
|
| `execute-unwind.ts` / `execute-ctx.ts` | a saga failed for good; caller ctx |
|
|
241
|
-
| `execute.ts` / `run-deadline.ts` | `executeJob
|
|
240
|
+
| `execute.ts` (+`-result.ts`) / `run-deadline.ts` | `executeJob`: one claimed job run and settled; its deadline: cancel, then fail |
|
|
242
241
|
| `heartbeat.ts` | one claimed job's lease: the renewal interval and the loss it reports |
|
|
243
242
|
| `renewal-timer.ts` | the renewal interval and its `stopped()` latch |
|
|
244
243
|
| `worker.ts` | `worker` role, claim round, drain |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ultimat3/jobs",
|
|
3
|
-
"version": "30.
|
|
3
|
+
"version": "30.5.0",
|
|
4
4
|
"description": "Durable background work: steps, transactional outbox, cron tasks, one driver interface",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"ultimate",
|
|
@@ -44,11 +44,11 @@
|
|
|
44
44
|
"test": "bun test"
|
|
45
45
|
},
|
|
46
46
|
"dependencies": {
|
|
47
|
-
"@ultimat3/core": "30.
|
|
48
|
-
"@ultimat3/db": "30.
|
|
49
|
-
"@ultimat3/entity": "30.
|
|
50
|
-
"@ultimat3/http": "30.
|
|
51
|
-
"@ultimat3/schema": "30.
|
|
52
|
-
"@ultimat3/time": "30.
|
|
47
|
+
"@ultimat3/core": "30.5.0",
|
|
48
|
+
"@ultimat3/db": "30.5.0",
|
|
49
|
+
"@ultimat3/entity": "30.5.0",
|
|
50
|
+
"@ultimat3/http": "30.5.0",
|
|
51
|
+
"@ultimat3/schema": "30.5.0",
|
|
52
|
+
"@ultimat3/time": "30.5.0"
|
|
53
53
|
}
|
|
54
54
|
}
|
package/src/errors.ts
CHANGED
|
@@ -129,14 +129,20 @@ export class JobTimeoutError extends UltimateError {
|
|
|
129
129
|
* the work inside it stops with.
|
|
130
130
|
*/
|
|
131
131
|
export class JobAbortedError extends UltimateError {
|
|
132
|
-
|
|
132
|
+
/** `ended`: the attempt FINISHED — it was not cancelled — and a step it never awaited came late. */
|
|
133
|
+
constructor(input: { job: string; step?: string; ended?: boolean }) {
|
|
133
134
|
super({
|
|
134
135
|
code: 'X_ABORTED',
|
|
135
136
|
cause:
|
|
136
137
|
input.step === undefined
|
|
137
138
|
? `job "${input.job}" was cancelled — this attempt no longer owns the run`
|
|
138
|
-
:
|
|
139
|
-
|
|
139
|
+
: input.ended === true
|
|
140
|
+
? `job "${input.job}"'s attempt had already ended when step "${input.step}" finished — run() returned without awaiting it`
|
|
141
|
+
: `job "${input.job}" was cancelled before step "${input.step}" could be recorded`,
|
|
142
|
+
fix:
|
|
143
|
+
input.ended === true
|
|
144
|
+
? `await step.run('${input.step ?? '<name>'}', fn) inside run(); work meant to outlive the attempt (a flush, a deferred enqueue): runToCompletion(() => work) from @ultimat3/core`
|
|
145
|
+
: 'add throwIfAborted(ctx) before expensive work, or pass fetch(url, { signal: ctx.signal }) — the queue re-runs the job, so stop at the deadline instead of running past it; work meant to outlive the attempt: runToCompletion(() => work) from @ultimat3/core',
|
|
140
146
|
});
|
|
141
147
|
}
|
|
142
148
|
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
// What one attempt of `executeJob` answers: how it ended and what it left behind. Its own file so
|
|
2
|
+
// the worker, the metrics and the tally read the result's shape without the execution path.
|
|
3
|
+
|
|
4
|
+
import type { JobStopReason } from './retry-classification';
|
|
5
|
+
import type { StepRecord } from './steps';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* How one attempt ended. `interrupted` is the worker's drain cutting the attempt short: the job is
|
|
9
|
+
* back in the ready bucket with the attempt UNCOUNTED, because the process ended it and not the
|
|
10
|
+
* job — filed as `retried`, a deploy would burn an attempt per job it held, and with
|
|
11
|
+
* `attempts: 1` dead-letter it. `dropped` is a run that failed for good on a job declaring
|
|
12
|
+
* `retry.deadLetter: false`: the row is `failed`, where it used to go back `ready` and run
|
|
13
|
+
* forever. `refused` is `whenBusy: 'fail'` over a busy concurrency key: the
|
|
14
|
+
* row is `failed`, the body never ran, and it is neither a retry nor a dead letter
|
|
15
|
+
* (`worker-key-busy.ts`).
|
|
16
|
+
*/
|
|
17
|
+
export type JobOutcome =
|
|
18
|
+
| 'completed'
|
|
19
|
+
| 'suspended'
|
|
20
|
+
| 'retried'
|
|
21
|
+
| 'dead-lettered'
|
|
22
|
+
| 'dropped'
|
|
23
|
+
| 'interrupted'
|
|
24
|
+
| 'refused';
|
|
25
|
+
|
|
26
|
+
export interface JobExecution {
|
|
27
|
+
readonly outcome: JobOutcome;
|
|
28
|
+
readonly jobId: string;
|
|
29
|
+
readonly job: string;
|
|
30
|
+
readonly attempt: number;
|
|
31
|
+
readonly durationMs: number;
|
|
32
|
+
/** Epoch ms the row is claimable again: a suspension's wake, or a retry's backoff ending. */
|
|
33
|
+
readonly resumeAt?: number;
|
|
34
|
+
readonly error?: string;
|
|
35
|
+
/** The stable `X_*` code the attempt's error carried — absent for an uncoded throw. */
|
|
36
|
+
readonly code?: string;
|
|
37
|
+
/** Why this attempt was the last. Absent while the job is still being retried. */
|
|
38
|
+
readonly stopReason?: JobStopReason;
|
|
39
|
+
/** What the body returned — present exactly on `completed`, for a caller driving a run by hand. */
|
|
40
|
+
readonly result?: unknown;
|
|
41
|
+
readonly steps: readonly StepRecord[];
|
|
42
|
+
readonly replayed: readonly string[];
|
|
43
|
+
}
|
package/src/execute-unwind.ts
CHANGED
|
@@ -14,7 +14,7 @@ import {
|
|
|
14
14
|
import { nowMs } from './clock';
|
|
15
15
|
import type { ClaimedJob, JobDriver, SettleBy } from './driver';
|
|
16
16
|
import { LEASE_LAPSED_FINAL_ATTEMPT } from './driver';
|
|
17
|
-
import type { JobExecution, JobOutcome } from './execute';
|
|
17
|
+
import type { JobExecution, JobOutcome } from './execute-result';
|
|
18
18
|
import type { AnyJobHandle } from './job';
|
|
19
19
|
import { backoffDelayMs, isFinalUnwindAttempt } from './retry';
|
|
20
20
|
import type { JobStopReason } from './retry-classification';
|
|
@@ -50,7 +50,13 @@ export interface UnwindSettleOptions {
|
|
|
50
50
|
readonly drained: () => boolean;
|
|
51
51
|
}
|
|
52
52
|
|
|
53
|
-
type Ending = Pick<JobExecution, 'error' | 'resumeAt' | 'stopReason'>;
|
|
53
|
+
type Ending = Pick<JobExecution, 'error' | 'code' | 'resumeAt' | 'stopReason'>;
|
|
54
|
+
|
|
55
|
+
/** The execution's `code`, when the thrown value carried one. */
|
|
56
|
+
const codeOf = (thrown: unknown): Pick<JobExecution, 'code'> => {
|
|
57
|
+
const code = settledCode(thrown);
|
|
58
|
+
return code === undefined ? {} : { code };
|
|
59
|
+
};
|
|
54
60
|
|
|
55
61
|
/** The execution this attempt answers, and `onSettled` after a settle that landed. */
|
|
56
62
|
function endings(options: UnwindSettleOptions) {
|
|
@@ -154,7 +160,7 @@ export async function unwindAndSettle(
|
|
|
154
160
|
scope: { operation: handle.name, extra: { jobId: claimed.id, runId: claimed.runId } },
|
|
155
161
|
});
|
|
156
162
|
if (settled) await announce(settledAs, recorded, settledCode(thrown));
|
|
157
|
-
return finish(settledAs, { error: message, stopReason: 'terminal' });
|
|
163
|
+
return finish(settledAs, { error: message, ...codeOf(thrown), stopReason: 'terminal' });
|
|
158
164
|
}
|
|
159
165
|
|
|
160
166
|
const stack = failure === undefined ? undefined : failureStack(failure.thrown);
|
|
@@ -186,6 +192,7 @@ export async function unwindAndSettle(
|
|
|
186
192
|
if (settled) await announce(settledAs, original, settledCode(failure?.thrown));
|
|
187
193
|
return finish(settledAs, {
|
|
188
194
|
error: original,
|
|
195
|
+
...codeOf(failure?.thrown),
|
|
189
196
|
stopReason: failure?.stop ?? 'attempts-exhausted',
|
|
190
197
|
});
|
|
191
198
|
}
|
package/src/execute.ts
CHANGED
|
@@ -22,12 +22,12 @@ import { claimOf } from './driver';
|
|
|
22
22
|
import { JobAbortedError } from './errors';
|
|
23
23
|
import { eventBus } from './events';
|
|
24
24
|
import { callerActor, callerServices, callerSignal, drainedBy } from './execute-ctx';
|
|
25
|
+
import type { JobExecution } from './execute-result';
|
|
25
26
|
import type { UnwindSettleOptions } from './execute-unwind';
|
|
26
27
|
import { unwindAndSettle } from './execute-unwind';
|
|
27
28
|
import type { AnyJobHandle } from './job';
|
|
28
29
|
import { createProgressReporter } from './progress';
|
|
29
30
|
import { isFinalAttempt, isUnwindAttempt } from './retry';
|
|
30
|
-
import type { JobStopReason } from './retry-classification';
|
|
31
31
|
import {
|
|
32
32
|
failureForRow,
|
|
33
33
|
nextRetryForError,
|
|
@@ -37,47 +37,11 @@ import {
|
|
|
37
37
|
import { raceTimeout } from './run-deadline';
|
|
38
38
|
import { createRunSignal } from './run-signal';
|
|
39
39
|
import { announceSettled, settledCode } from './settled';
|
|
40
|
-
import type { EventLookup
|
|
40
|
+
import type { EventLookup } from './steps';
|
|
41
41
|
import { isStepSuspension, stepRunner } from './steps';
|
|
42
42
|
import { isUnwindReached } from './steps-suspension';
|
|
43
43
|
import { jobRunActor, refuseRunTenant } from './tenant';
|
|
44
44
|
|
|
45
|
-
/**
|
|
46
|
-
* How one attempt ended. `interrupted` is the worker's drain cutting the attempt short: the job is
|
|
47
|
-
* back in the ready bucket with the attempt UNCOUNTED, because the process ended it and not the
|
|
48
|
-
* job — filed as `retried`, a deploy would burn an attempt per job it held, and with
|
|
49
|
-
* `attempts: 1` dead-letter it. `dropped` is a run that failed for good on a job declaring
|
|
50
|
-
* `retry.deadLetter: false`: the row is `failed`, where it used to go back `ready` and run
|
|
51
|
-
* forever. `refused` is `whenBusy: 'fail'` over a busy concurrency key: the
|
|
52
|
-
* row is `failed`, the body never ran, and it is neither a retry nor a dead letter
|
|
53
|
-
* (`worker-key-busy.ts`).
|
|
54
|
-
*/
|
|
55
|
-
export type JobOutcome =
|
|
56
|
-
| 'completed'
|
|
57
|
-
| 'suspended'
|
|
58
|
-
| 'retried'
|
|
59
|
-
| 'dead-lettered'
|
|
60
|
-
| 'dropped'
|
|
61
|
-
| 'interrupted'
|
|
62
|
-
| 'refused';
|
|
63
|
-
|
|
64
|
-
export interface JobExecution {
|
|
65
|
-
readonly outcome: JobOutcome;
|
|
66
|
-
readonly jobId: string;
|
|
67
|
-
readonly job: string;
|
|
68
|
-
readonly attempt: number;
|
|
69
|
-
readonly durationMs: number;
|
|
70
|
-
/** Epoch ms the row is claimable again: a suspension's wake, or a retry's backoff ending. */
|
|
71
|
-
readonly resumeAt?: number;
|
|
72
|
-
readonly error?: string;
|
|
73
|
-
/** Why this attempt was the last. Absent while the job is still being retried. */
|
|
74
|
-
readonly stopReason?: JobStopReason;
|
|
75
|
-
/** What the body returned — present exactly on `completed`, for a caller driving a run by hand. */
|
|
76
|
-
readonly result?: unknown;
|
|
77
|
-
readonly steps: readonly StepRecord[];
|
|
78
|
-
readonly replayed: readonly string[];
|
|
79
|
-
}
|
|
80
|
-
|
|
81
45
|
export interface ExecuteJobOptions {
|
|
82
46
|
readonly driver: JobDriver;
|
|
83
47
|
readonly claimed: ClaimedJob;
|
|
@@ -106,6 +70,13 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
|
|
|
106
70
|
// where every other body does, with no jobs-only parameter to know about. Composed with the
|
|
107
71
|
// caller's signal rather than replacing it: a ctx that was already going away still is.
|
|
108
72
|
const cancel = new AbortController();
|
|
73
|
+
// The attempt's END, apart from its cancellation: aborted in the `finally` however the attempt
|
|
74
|
+
// ended, so the step runner records nothing after it. `cancel` — what `ctx.signal` follows — is
|
|
75
|
+
// aborted only when another attempt may own the run: a failure, a suspension, an ack that did
|
|
76
|
+
// not land. A body that COMPLETED and was acked owns a finished run, and a timer it armed (a log
|
|
77
|
+
// flush, a deferred enqueue) still writes; reading as "cancelled" lost every such write.
|
|
78
|
+
const ended = new AbortController();
|
|
79
|
+
let completed = false;
|
|
109
80
|
// `createRunSignal` and never `AbortSignal.any` — the second of the two sites this package's
|
|
110
81
|
// `CLAUDE.md` states the rule as absolute for. In the worker path `callerSignal` is per-run and
|
|
111
82
|
// nothing leaks; on `@ultimat3/testing`'s job-fixture path, which calls `executeJob` directly,
|
|
@@ -114,6 +85,15 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
|
|
|
114
85
|
// `finally` at the bottom of the try, beside `cancel.abort`.
|
|
115
86
|
const runSignal = createRunSignal([callerSignal(options.ctx), cancel.signal]);
|
|
116
87
|
const signal = runSignal.signal;
|
|
88
|
+
/**
|
|
89
|
+
* This attempt no longer owns the run: `ctx.signal` aborts, so a write still under it is refused
|
|
90
|
+
* (`abandonment()`). A second `abort()` keeps the first reason — a timed-out run still reports
|
|
91
|
+
* the timeout. `dispose` AFTER it: it stops following the sources and never aborts.
|
|
92
|
+
*/
|
|
93
|
+
const release = (): void => {
|
|
94
|
+
cancel.abort(new JobAbortedError({ job: handle.name }));
|
|
95
|
+
runSignal.dispose();
|
|
96
|
+
};
|
|
117
97
|
const ctx: Ctx = Object.freeze({
|
|
118
98
|
...options.ctx,
|
|
119
99
|
signal,
|
|
@@ -127,7 +107,7 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
|
|
|
127
107
|
signal,
|
|
128
108
|
// This attempt's OWN end — the deadline, the `finally` below — apart from the run's signal,
|
|
129
109
|
// whose first reason may be the drain's: a drained step still records, an ended attempt never.
|
|
130
|
-
ended:
|
|
110
|
+
ended: ended.signal,
|
|
131
111
|
// The DECLARED per-step ceiling and event poll. Passed here or nowhere: this is the only
|
|
132
112
|
// production construction of a runner, so a `StepRunnerOptions` field it omits is a feature
|
|
133
113
|
// no `job()` can reach — which both of these were until 2026-08.
|
|
@@ -261,6 +241,7 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
|
|
|
261
241
|
if (handle.compensates && (await runner.unwinding())) {
|
|
262
242
|
return await unwindAndSettle(unwinder(), undefined);
|
|
263
243
|
}
|
|
244
|
+
completed = true;
|
|
264
245
|
} catch (error) {
|
|
265
246
|
// Before ANY settle: the last value a body reported is written while the row is still this
|
|
266
247
|
// worker's, which is the only time `recordProgress`'s fence lets it land.
|
|
@@ -287,6 +268,10 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
|
|
|
287
268
|
}
|
|
288
269
|
|
|
289
270
|
const message = renderThrowable(error);
|
|
271
|
+
// The code beside the message: a caller judging the failure (`x exercise`) reads a coded
|
|
272
|
+
// refusal by its code, as it reads an action's, and the message alone had lost it.
|
|
273
|
+
const thrownCode = settledCode(error);
|
|
274
|
+
const coded = thrownCode === undefined ? {} : { code: thrownCode };
|
|
290
275
|
// Unwinding already: the replay stopped at its first incomplete step (`UnwindReached`) or
|
|
291
276
|
// ended any other way. Never retried forward — a drain still hands the attempt back.
|
|
292
277
|
if (
|
|
@@ -327,6 +312,7 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
|
|
|
327
312
|
attempt: claimed.attempt,
|
|
328
313
|
durationMs: nowMs(options.clock) - startedAt,
|
|
329
314
|
error: message,
|
|
315
|
+
...coded,
|
|
330
316
|
steps: [],
|
|
331
317
|
replayed: [],
|
|
332
318
|
});
|
|
@@ -359,6 +345,7 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
|
|
|
359
345
|
attempt: claimed.attempt,
|
|
360
346
|
durationMs: nowMs(options.clock) - startedAt,
|
|
361
347
|
error: message,
|
|
348
|
+
...coded,
|
|
362
349
|
resumeAt: nowMs(options.clock) + deferMs,
|
|
363
350
|
steps: [],
|
|
364
351
|
replayed: [],
|
|
@@ -445,6 +432,7 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
|
|
|
445
432
|
attempt: claimed.attempt,
|
|
446
433
|
durationMs: nowMs(options.clock) - startedAt,
|
|
447
434
|
error: message,
|
|
435
|
+
...coded,
|
|
448
436
|
// When the retry falls due, on this process's clock — what lets the worker that failed it
|
|
449
437
|
// pass again at that moment instead of finding out on a backed-off poll.
|
|
450
438
|
...(decision.retry ? { resumeAt: nowMs(options.clock) + decision.delayMs } : {}),
|
|
@@ -453,14 +441,11 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
|
|
|
453
441
|
replayed: [],
|
|
454
442
|
});
|
|
455
443
|
} finally {
|
|
456
|
-
// The attempt is over however it ended, so
|
|
457
|
-
//
|
|
458
|
-
// attempt
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
// AFTER the abort, so the runner's fence still sees it: `dispose` stops following the sources,
|
|
462
|
-
// it never aborts, and the signal keeps whatever state the abort above left it in.
|
|
463
|
-
runSignal.dispose();
|
|
444
|
+
// The attempt is over however it ended, so no step from it may still record: a step left in
|
|
445
|
+
// flight by a handler that returned without awaiting it would settle into a run the next
|
|
446
|
+
// attempt may own. The runner fences its writes on `ended`.
|
|
447
|
+
ended.abort(new JobAbortedError({ job: handle.name }));
|
|
448
|
+
if (!completed) release();
|
|
464
449
|
}
|
|
465
450
|
|
|
466
451
|
// Only reachable when the BODY succeeded, and settlement is deliberately outside the catch
|
|
@@ -470,9 +455,19 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
|
|
|
470
455
|
// Let it reach the worker instead, which logs `jobs.worker.settle-failed`; the lease then
|
|
471
456
|
// lapses and the queue re-delivers, which is the honest outcome for "we could not say it ended".
|
|
472
457
|
await progress.flush();
|
|
458
|
+
let acked = false;
|
|
459
|
+
try {
|
|
460
|
+
acked = landed(await driver.ack(claimed.id, by()), 'completed');
|
|
461
|
+
} finally {
|
|
462
|
+
// Unacked, the row is another worker's (or re-delivered once the lease lapses): cancelled.
|
|
463
|
+
if (!acked) release();
|
|
464
|
+
// Acked, the run is finished and owned by nobody: the signal stops following its sources and
|
|
465
|
+
// stays un-aborted, so work the body deliberately left running keeps landing.
|
|
466
|
+
else runSignal.dispose();
|
|
467
|
+
}
|
|
473
468
|
// Announced only when the ack LANDED: a row cancelled or re-delivered under this body is
|
|
474
469
|
// somebody else's to settle, and `completed` would be a claim about a row that is not `done`.
|
|
475
|
-
if (
|
|
470
|
+
if (acked) {
|
|
476
471
|
await announceSettled({
|
|
477
472
|
handle,
|
|
478
473
|
claimed,
|
package/src/index.ts
CHANGED
|
@@ -205,8 +205,9 @@ export {
|
|
|
205
205
|
} from './events';
|
|
206
206
|
export type { PostgresEventBusOptions } from './events-pg';
|
|
207
207
|
export { postgresEventBus } from './events-pg';
|
|
208
|
-
export type { ExecuteJobOptions
|
|
208
|
+
export type { ExecuteJobOptions } from './execute';
|
|
209
209
|
export { executeJob } from './execute';
|
|
210
|
+
export type { JobExecution, JobOutcome } from './execute-result';
|
|
210
211
|
export type { ExportDefinition, ExportReport } from './export';
|
|
211
212
|
export { DEFAULT_EXPORT_BATCH, DEFAULT_EXPORT_MAX_PART_BYTES, exportRows } from './export';
|
|
212
213
|
export { ExportPartTooLargeError, ExportRowInvalidError } from './export-errors';
|
|
@@ -301,7 +302,15 @@ export type {
|
|
|
301
302
|
JobHandle,
|
|
302
303
|
JobRunArgs,
|
|
303
304
|
} from './job';
|
|
304
|
-
export {
|
|
305
|
+
export {
|
|
306
|
+
describeJobs,
|
|
307
|
+
getJob,
|
|
308
|
+
isJobHandle,
|
|
309
|
+
job,
|
|
310
|
+
registeredJobs,
|
|
311
|
+
resetJobs,
|
|
312
|
+
restoreJobs,
|
|
313
|
+
} from './job';
|
|
305
314
|
export type { EnqueueManyOptions } from './job-enqueue-many';
|
|
306
315
|
export { MAX_ENQUEUE_BATCH } from './job-enqueue-many';
|
|
307
316
|
export type { JobsRetentionOptions } from './jobs-retention';
|
|
@@ -421,6 +430,7 @@ export type { ScheduledFire, SchedulerState } from './scheduler-state';
|
|
|
421
430
|
export { fireThroughDriver, memorySchedulerState } from './scheduler-state';
|
|
422
431
|
export type { JobCompleted, JobFailed, JobSettled } from './settled';
|
|
423
432
|
export { ON_SETTLED_ATTEMPTS, ON_SETTLED_TIMEOUT_MS } from './settled';
|
|
433
|
+
export { currentStep } from './step-scope';
|
|
424
434
|
export type {
|
|
425
435
|
CompensateContext,
|
|
426
436
|
EventLookup,
|
package/src/job.ts
CHANGED
|
@@ -278,6 +278,16 @@ export function resetJobs(): void {
|
|
|
278
278
|
registry.clear();
|
|
279
279
|
}
|
|
280
280
|
|
|
281
|
+
/**
|
|
282
|
+
* Put the registry back to exactly `handles` — `restoreTasks`' twin. `job()` registers at module
|
|
283
|
+
* scope, once per `bun test` process, so a test whose premise is "only these jobs" (`x exercise`
|
|
284
|
+
* over a fixture) empties the registry and hands every inherited handle back after it.
|
|
285
|
+
*/
|
|
286
|
+
export function restoreJobs(handles: readonly AnyJobHandle[]): void {
|
|
287
|
+
registry.clear();
|
|
288
|
+
for (const handle of handles) registry.set(handle.name, handle);
|
|
289
|
+
}
|
|
290
|
+
|
|
281
291
|
/**
|
|
282
292
|
* Registered jobs as the manifest, the `/_x` jobs panel and the MCP dev server need them.
|
|
283
293
|
* Name-sorted because `x.manifest.json` is committed and diffed — an iteration-order-dependent
|
package/src/metrics.ts
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
|
|
15
15
|
import type { Gauge } from '@ultimat3/core';
|
|
16
16
|
import { gauge } from '@ultimat3/core';
|
|
17
|
-
import type { JobOutcome } from './execute';
|
|
17
|
+
import type { JobOutcome } from './execute-result';
|
|
18
18
|
|
|
19
19
|
/**
|
|
20
20
|
* `JobOutcome` -> the `jobs_total` label, and `null` for the outcomes that are not one. `suspended`
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
// Which durable step's body is running, ambient across its awaits. Read by an observer that must
|
|
2
|
+
// tell an effect made INSIDE a step (rerun when the step did not complete) from one made between
|
|
3
|
+
// steps (rerun on every replay): `@ultimat3/testing`'s crash proof names the step it ran in.
|
|
4
|
+
|
|
5
|
+
import { asyncContext } from '@ultimat3/core';
|
|
6
|
+
|
|
7
|
+
const scope = asyncContext<string>('a durable step body');
|
|
8
|
+
|
|
9
|
+
/** Run one step body with its name in flight. `steps.ts` is the one caller. */
|
|
10
|
+
export const inStep = <T>(name: string, body: () => T): T => scope.run(name, body);
|
|
11
|
+
|
|
12
|
+
/** The name of the step whose body is running here, or `undefined` outside every step body. */
|
|
13
|
+
export const currentStep = (): string | undefined => scope.get();
|
package/src/steps.ts
CHANGED
|
@@ -21,6 +21,7 @@ import {
|
|
|
21
21
|
UNWINDING_STEP,
|
|
22
22
|
} from './errors-saga';
|
|
23
23
|
import { createRunSignal } from './run-signal';
|
|
24
|
+
import { inStep } from './step-scope';
|
|
24
25
|
import type { EventLookup, StepApi, StepRunOptions } from './steps-api';
|
|
25
26
|
import { DEFAULT_EVENT_POLL_MS, WAKING_EVENT_POLL_MS } from './steps-api';
|
|
26
27
|
import type { RunnerCore } from './steps-core';
|
|
@@ -209,7 +210,9 @@ export function stepRunner(options: StepRunnerOptions): StepRunner {
|
|
|
209
210
|
return occurrence === 1 ? base : `${base}#${occurrence}`;
|
|
210
211
|
};
|
|
211
212
|
|
|
212
|
-
|
|
213
|
+
// The run cancelled, or this attempt over — `ended` also covers one that COMPLETED, whose
|
|
214
|
+
// `ctx.signal` stays un-aborted on purpose (`execute.ts`).
|
|
215
|
+
const cancelled = (): boolean => runSignal.aborted || options.ended?.aborted === true;
|
|
213
216
|
|
|
214
217
|
/**
|
|
215
218
|
* A cancelled run whose step result is still this attempt's to record: the worker's drain cut
|
|
@@ -247,7 +250,11 @@ export function stepRunner(options: StepRunnerOptions): StepRunner {
|
|
|
247
250
|
*/
|
|
248
251
|
const put = async (record: StepRecord): Promise<void> => {
|
|
249
252
|
if (cancelled() && !writesThrough()) {
|
|
250
|
-
throw new JobAbortedError({
|
|
253
|
+
throw new JobAbortedError({
|
|
254
|
+
job: jobName,
|
|
255
|
+
step: record.name,
|
|
256
|
+
...(runSignal.aborted ? {} : { ended: true }),
|
|
257
|
+
});
|
|
251
258
|
}
|
|
252
259
|
await persist(record);
|
|
253
260
|
remember(record);
|
|
@@ -291,7 +298,7 @@ export function stepRunner(options: StepRunnerOptions): StepRunner {
|
|
|
291
298
|
const signal = composed?.signal ?? runSignal;
|
|
292
299
|
try {
|
|
293
300
|
const output = await withStepTimeout(
|
|
294
|
-
fn(signal),
|
|
301
|
+
inStep(name, () => fn(signal)),
|
|
295
302
|
stepTimeoutMs,
|
|
296
303
|
deadline,
|
|
297
304
|
() =>
|
package/src/worker-admit.ts
CHANGED
|
@@ -7,7 +7,7 @@ import type { Clock, Ctx } from '@ultimat3/core';
|
|
|
7
7
|
import { logger } from '@ultimat3/core';
|
|
8
8
|
import type { ClaimedJob, JobDriver } from './driver';
|
|
9
9
|
import { claimOf } from './driver';
|
|
10
|
-
import type { JobExecution } from './execute';
|
|
10
|
+
import type { JobExecution } from './execute-result';
|
|
11
11
|
import type { Lease, Limiter } from './limits';
|
|
12
12
|
import type { FleetSlots, SlotGrant } from './worker-fleet-slots';
|
|
13
13
|
import { handBack } from './worker-hand-back';
|
package/src/worker-key-busy.ts
CHANGED
|
@@ -9,7 +9,7 @@ import { nowMs } from './clock';
|
|
|
9
9
|
import type { ClaimedJob, JobDriver } from './driver';
|
|
10
10
|
import { claimOf } from './driver';
|
|
11
11
|
import { JobKeyBusyError } from './errors-concurrency';
|
|
12
|
-
import type { JobExecution } from './execute';
|
|
12
|
+
import type { JobExecution } from './execute-result';
|
|
13
13
|
import { getJob } from './job';
|
|
14
14
|
import { failureForRow } from './retry-classification';
|
|
15
15
|
import { announceSettled } from './settled';
|
package/src/worker-run.ts
CHANGED
|
@@ -8,8 +8,8 @@ import { parseTraceparent, withSpan } from '@ultimat3/core';
|
|
|
8
8
|
import type { ClaimedJob, JobDriver } from './driver';
|
|
9
9
|
import { claimOf } from './driver';
|
|
10
10
|
import { JobSlotLostError } from './errors';
|
|
11
|
-
import type { JobExecution } from './execute';
|
|
12
11
|
import { executeJob } from './execute';
|
|
12
|
+
import type { JobExecution } from './execute-result';
|
|
13
13
|
import { startLeaseHeartbeat } from './heartbeat';
|
|
14
14
|
import { getJob } from './job';
|
|
15
15
|
import type { IntervalScheduler } from './renewal-timer';
|
package/src/worker-tally.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// What one worker has settled, by outcome — the counters `stats()` reports. Split from `worker.ts`
|
|
2
2
|
// at the file-size ceiling; the claim loop counts, this file says what each count means.
|
|
3
3
|
|
|
4
|
-
import type { JobExecution } from './execute';
|
|
4
|
+
import type { JobExecution } from './execute-result';
|
|
5
5
|
import type { WorkerStats } from './worker-types';
|
|
6
6
|
|
|
7
7
|
export type WorkerTallies = Pick<
|
package/src/worker-types.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
import type { Clock, Ctx } from '@ultimat3/core';
|
|
6
6
|
import type { JobDriver, QueueStats } from './driver';
|
|
7
|
-
import type { JobExecution } from './execute';
|
|
7
|
+
import type { JobExecution } from './execute-result';
|
|
8
8
|
import type { Limiter } from './limits';
|
|
9
9
|
import type { IntervalScheduler } from './renewal-timer';
|
|
10
10
|
import type { EventLookup } from './steps';
|
package/src/worker.ts
CHANGED
|
@@ -18,7 +18,7 @@ import { nowMs } from './clock';
|
|
|
18
18
|
import { createDrainBudget, settleAllBy } from './drain-wait';
|
|
19
19
|
import type { ClaimedJob, JobRecord } from './driver';
|
|
20
20
|
import { DEFAULT_QUEUE } from './driver';
|
|
21
|
-
import type { JobExecution } from './execute';
|
|
21
|
+
import type { JobExecution } from './execute-result';
|
|
22
22
|
import { registeredJobs } from './job';
|
|
23
23
|
import { concurrencyLimiter } from './limits';
|
|
24
24
|
import { JOB_OUTCOME_LABELS } from './metrics';
|