@ultimat3/jobs 9.0.0 → 10.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -5
- package/src/backfill-errors.ts +0 -8
- package/src/backfill-pass.ts +2 -2
- package/src/errors.ts +6 -20
- package/src/events-pg.ts +2 -2
- package/src/execute.ts +15 -5
- package/src/heartbeat.ts +3 -5
- package/src/outbox.ts +3 -3
- package/src/renewal-timer.ts +16 -1
- package/src/scheduler.ts +2 -2
- package/src/steps.ts +17 -6
- package/src/worker-fleet-slots.ts +2 -2
- package/src/worker.ts +11 -4
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ultimat3/jobs",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "10.0.0",
|
|
4
4
|
"description": "Durable background work: steps, transactional outbox, cron tasks, one driver interface",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -32,9 +32,9 @@
|
|
|
32
32
|
"test": "bun test"
|
|
33
33
|
},
|
|
34
34
|
"dependencies": {
|
|
35
|
-
"@ultimat3/core": "
|
|
36
|
-
"@ultimat3/entity": "
|
|
37
|
-
"@ultimat3/schema": "
|
|
38
|
-
"@ultimat3/time": "
|
|
35
|
+
"@ultimat3/core": "10.0.0",
|
|
36
|
+
"@ultimat3/entity": "10.0.0",
|
|
37
|
+
"@ultimat3/schema": "10.0.0",
|
|
38
|
+
"@ultimat3/time": "10.0.0"
|
|
39
39
|
}
|
|
40
40
|
}
|
package/src/backfill-errors.ts
CHANGED
|
@@ -6,7 +6,6 @@
|
|
|
6
6
|
// `backfill-pending.ts` and `backfill-registry.ts`.
|
|
7
7
|
|
|
8
8
|
import { UltimateError } from '@ultimat3/core';
|
|
9
|
-
import { docsFor } from './errors';
|
|
10
9
|
|
|
11
10
|
/**
|
|
12
11
|
* The seven backfill codes below all answer one question — "why is this sweep not running?" — and
|
|
@@ -31,7 +30,6 @@ export class BackfillPendingError extends UltimateError {
|
|
|
31
30
|
code: 'X_BACKFILL_PENDING',
|
|
32
31
|
cause: `backfill "${input.backfill}" is declared and x_backfills holds no completed pass for it in ${input.environment}`,
|
|
33
32
|
fix: `x db backfill ${input.backfill} --write --json`,
|
|
34
|
-
docs: docsFor('X_BACKFILL_PENDING'),
|
|
35
33
|
});
|
|
36
34
|
}
|
|
37
35
|
}
|
|
@@ -43,7 +41,6 @@ export class BackfillAppliedError extends UltimateError {
|
|
|
43
41
|
code: 'X_BACKFILL_APPLIED',
|
|
44
42
|
cause: `backfill "${input.backfill}" completed as run ${input.runId} at ${input.completedAt}; a forced rerun writes a NEW ledger row and never edits that one`,
|
|
45
43
|
fix: `x db backfill ${input.backfill} --write --force --json`,
|
|
46
|
-
docs: docsFor('X_BACKFILL_APPLIED'),
|
|
47
44
|
});
|
|
48
45
|
}
|
|
49
46
|
}
|
|
@@ -68,7 +65,6 @@ export class BackfillEnvironmentError extends UltimateError {
|
|
|
68
65
|
target === undefined
|
|
69
66
|
? 'x db backfill --pending --json'
|
|
70
67
|
: `ULTIMATE_ENV=${target} x db backfill ${input.backfill} --write --json`,
|
|
71
|
-
docs: docsFor('X_BACKFILL_ENVIRONMENT'),
|
|
72
68
|
});
|
|
73
69
|
}
|
|
74
70
|
}
|
|
@@ -84,7 +80,6 @@ export class BackfillMigrationPendingError extends UltimateError {
|
|
|
84
80
|
code: 'X_BACKFILL_MIGRATION_PENDING',
|
|
85
81
|
cause: `backfill "${input.backfill}" requires migration ${input.migration}, which x_migrations does not record as applied`,
|
|
86
82
|
fix: 'x db migrate --json',
|
|
87
|
-
docs: docsFor('X_BACKFILL_MIGRATION_PENDING'),
|
|
88
83
|
});
|
|
89
84
|
}
|
|
90
85
|
}
|
|
@@ -100,7 +95,6 @@ export class BackfillRunningError extends UltimateError {
|
|
|
100
95
|
code: 'X_BACKFILL_RUNNING',
|
|
101
96
|
cause: `backfill "${input.backfill}" already has a live pass queued as ${input.jobId}, and one name holds one live pass; its step trace names the batch it is on, and a pass that is not advancing is a worker that lost its lease`,
|
|
102
97
|
fix: `x jobs show ${input.jobId} --json`,
|
|
103
|
-
docs: docsFor('X_BACKFILL_RUNNING'),
|
|
104
98
|
});
|
|
105
99
|
}
|
|
106
100
|
}
|
|
@@ -116,7 +110,6 @@ export class BackfillStalledError extends UltimateError {
|
|
|
116
110
|
code: 'X_BACKFILL_STALLED',
|
|
117
111
|
cause: `backfill "${input.backfill}" swept ${input.swept} rows, exhausted its source, and count() still matches ${input.remaining} — a WHERE the sweep narrows and the count does not is what leaves rows behind`,
|
|
118
112
|
fix: `make count() select on exactly what source() selects on in backfill("${input.backfill}")`,
|
|
119
|
-
docs: docsFor('X_BACKFILL_STALLED'),
|
|
120
113
|
});
|
|
121
114
|
}
|
|
122
115
|
}
|
|
@@ -131,7 +124,6 @@ export class BackfillUnknownError extends UltimateError {
|
|
|
131
124
|
? `no backfill named "${input.backfill}" is declared, and this app declares none at all`
|
|
132
125
|
: `no backfill named "${input.backfill}" is declared (declared: ${input.known.join(', ')})`,
|
|
133
126
|
fix: 'x db backfill --pending --json',
|
|
134
|
-
docs: docsFor('X_BACKFILL_UNKNOWN'),
|
|
135
127
|
});
|
|
136
128
|
}
|
|
137
129
|
}
|
package/src/backfill-pass.ts
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
// in step with the work and decides where a resumed pass restarts, while the `x_backfills` row is
|
|
14
14
|
// a report an operator reads and the record that a completed name has already been swept.
|
|
15
15
|
|
|
16
|
-
import { appVersion, assert, logger, resolveEnvironment } from '@ultimat3/core';
|
|
16
|
+
import { appVersion, assert, logger, renderThrowable, resolveEnvironment } from '@ultimat3/core';
|
|
17
17
|
import type { BatchIterator } from '@ultimat3/entity';
|
|
18
18
|
import type { BackfillDefinition, BackfillInput, BackfillReport } from './backfill';
|
|
19
19
|
import { BackfillStalledError } from './backfill-errors';
|
|
@@ -101,7 +101,7 @@ async function markFailed(
|
|
|
101
101
|
} catch (error) {
|
|
102
102
|
logger.warn('jobs.backfill.ledger-failed', {
|
|
103
103
|
runId,
|
|
104
|
-
error:
|
|
104
|
+
error: renderThrowable(error),
|
|
105
105
|
});
|
|
106
106
|
}
|
|
107
107
|
}
|
package/src/errors.ts
CHANGED
|
@@ -93,8 +93,12 @@ registerErrorRetry({
|
|
|
93
93
|
X_BACKFILL_APPLIED: 'terminal',
|
|
94
94
|
});
|
|
95
95
|
|
|
96
|
-
|
|
97
|
-
|
|
96
|
+
// No `docs:` on any class below, here or in `backfill-errors.ts`. `UltimateError` fills it from
|
|
97
|
+
// `describeErrorCode(code).docs`, which is `@ultimat3/core`'s `ERROR_DOCS_URL` — one page for every
|
|
98
|
+
// code, never one per code, because `wiki/` is the framework's only public documentation surface
|
|
99
|
+
// and a code lives there in a TABLE ROW, which has no anchor. The `docsFor` that stood here built
|
|
100
|
+
// `https://ultimate.dev/errors/<code>`, which answered 404, host included, on every job failure
|
|
101
|
+
// this package has ever put in a dead-letter row.
|
|
98
102
|
|
|
99
103
|
/** An enqueue collided with a live job holding the same idempotency key under `onConflict: 'error'`. */
|
|
100
104
|
export class JobDuplicateError extends UltimateError {
|
|
@@ -103,7 +107,6 @@ export class JobDuplicateError extends UltimateError {
|
|
|
103
107
|
code: 'X_JOB_DUPLICATE',
|
|
104
108
|
cause: `job "${input.job}" already queued as ${input.existingId} with idempotencyKey "${input.idempotencyKey}"`,
|
|
105
109
|
fix: 'pass onConflict: "dedupe" to enqueue, or make idempotencyKey narrower',
|
|
106
|
-
docs: docsFor('X_JOB_DUPLICATE'),
|
|
107
110
|
});
|
|
108
111
|
}
|
|
109
112
|
}
|
|
@@ -124,7 +127,6 @@ export class JobNameTakenError extends UltimateError {
|
|
|
124
127
|
code: 'X_JOB_DUPLICATE',
|
|
125
128
|
cause: `two ${input.kind}s claim the name "${input.name}"`,
|
|
126
129
|
fix: `x jobs ls --json names the one already seated; rename the other's export, or its "name:" if it declares one — a ${input.kind} name is a durable queue key and is globally unique`,
|
|
127
|
-
docs: docsFor('X_JOB_DUPLICATE'),
|
|
128
130
|
});
|
|
129
131
|
}
|
|
130
132
|
}
|
|
@@ -151,7 +153,6 @@ export class JobRowStatusUnknownError extends UltimateError {
|
|
|
151
153
|
`${input.table}.${input.column} holds "${input.value}", which this build does not know — ` +
|
|
152
154
|
`it reads ${input.known.join(', ')}`,
|
|
153
155
|
fix: `x jobs show --json # then drain the older workers: a status this build cannot read was almost certainly written by a newer deploy`,
|
|
154
|
-
docs: docsFor('X_JOB_ROW_STATUS_UNKNOWN'),
|
|
155
156
|
});
|
|
156
157
|
}
|
|
157
158
|
}
|
|
@@ -175,7 +176,6 @@ export class ActionJobUnbridgedError extends UltimateError {
|
|
|
175
176
|
code: 'X_ACTION_JOB_UNBRIDGED',
|
|
176
177
|
cause: `export "${input.export}" is the action projection "${input.job}", which is not a job handle and cannot be registered as one`,
|
|
177
178
|
fix: `wrap it: agentJob(${input.export}, { name: '${input.export}', tenant, retry }) from @ultimat3/ai — that composes job() and returns a handle the queue accepts`,
|
|
178
|
-
docs: docsFor('X_ACTION_JOB_UNBRIDGED'),
|
|
179
179
|
});
|
|
180
180
|
}
|
|
181
181
|
}
|
|
@@ -187,7 +187,6 @@ export class StepDuplicateError extends UltimateError {
|
|
|
187
187
|
code: 'X_STEP_DUPLICATE',
|
|
188
188
|
cause: `job "${input.job}" used step name "${input.step}" twice in one run`,
|
|
189
189
|
fix: `rename one of them, e.g. step.run('${input.step}-2', ...) — step names are the replay key`,
|
|
190
|
-
docs: docsFor('X_STEP_DUPLICATE'),
|
|
191
190
|
});
|
|
192
191
|
}
|
|
193
192
|
}
|
|
@@ -201,7 +200,6 @@ export class JobTimeoutError extends UltimateError {
|
|
|
201
200
|
? `job "${input.job}" exceeded its ${input.timeoutMs}ms timeout`
|
|
202
201
|
: `job "${input.job}" step "${input.step}" exceeded its ${input.timeoutMs}ms timeout`,
|
|
203
202
|
fix: `raise timeout on the job definition, or split the work into step.run() calls`,
|
|
204
|
-
docs: docsFor('X_JOB_TIMEOUT'),
|
|
205
203
|
});
|
|
206
204
|
}
|
|
207
205
|
}
|
|
@@ -225,7 +223,6 @@ export class JobAbortedError extends UltimateError {
|
|
|
225
223
|
? `job "${input.job}" was cancelled — this attempt no longer owns the run`
|
|
226
224
|
: `job "${input.job}" was cancelled before step "${input.step}" could be recorded`,
|
|
227
225
|
fix: 'add throwIfAborted(ctx) before expensive work, or pass fetch(url, { signal: ctx.signal }) — the queue re-runs the job, so stop at the deadline instead of running past it',
|
|
228
|
-
docs: docsFor('X_ABORTED'),
|
|
229
226
|
});
|
|
230
227
|
}
|
|
231
228
|
}
|
|
@@ -237,7 +234,6 @@ export class JobMaxAttemptsError extends UltimateError {
|
|
|
237
234
|
code: 'X_JOB_MAX_ATTEMPTS',
|
|
238
235
|
cause: `job "${input.job}" failed ${input.attempts} times, last error: ${input.lastError}`,
|
|
239
236
|
fix: `x jobs retry ${input.jobId}`,
|
|
240
|
-
docs: docsFor('X_JOB_MAX_ATTEMPTS'),
|
|
241
237
|
});
|
|
242
238
|
}
|
|
243
239
|
}
|
|
@@ -248,7 +244,6 @@ export class DriverUnavailableError extends UltimateError {
|
|
|
248
244
|
code: 'X_DRIVER_UNAVAILABLE',
|
|
249
245
|
cause: `jobs driver "${input.driver}" is unavailable: ${input.cause}`,
|
|
250
246
|
fix: input.fix,
|
|
251
|
-
docs: docsFor('X_DRIVER_UNAVAILABLE'),
|
|
252
247
|
});
|
|
253
248
|
}
|
|
254
249
|
}
|
|
@@ -263,7 +258,6 @@ export class IdempotencyRequiredError extends UltimateError {
|
|
|
263
258
|
code: 'X_IDEMPOTENCY_REQUIRED',
|
|
264
259
|
cause: `job "${input.job}" has no idempotencyKey — at-least-once delivery would run it twice`,
|
|
265
260
|
fix: `add idempotencyKey: (input) => \`${input.job}:\${input.id}\` to the job definition`,
|
|
266
|
-
docs: docsFor('X_IDEMPOTENCY_REQUIRED'),
|
|
267
261
|
});
|
|
268
262
|
}
|
|
269
263
|
}
|
|
@@ -290,7 +284,6 @@ export class JobTenantRequiredError extends UltimateError {
|
|
|
290
284
|
// tenant, and the pass opens the cross-tenant scope for exactly that declaration. Half the
|
|
291
285
|
// callers of this code arrive through `backfill()`, which forwards its `tenant` to `job()`.
|
|
292
286
|
fix: `add tenant: (input) => input.orgId to job("${input.job}") — or tenant: 'none', which declares NO org: right for a job that touches no tenant-scoped table, and the spelling a backfill() uses to sweep every tenant`,
|
|
293
|
-
docs: docsFor('X_JOB_TENANT_REQUIRED'),
|
|
294
287
|
});
|
|
295
288
|
}
|
|
296
289
|
}
|
|
@@ -307,7 +300,6 @@ export class LeaseLostError extends UltimateError {
|
|
|
307
300
|
code: 'X_JOB_LEASE_LOST',
|
|
308
301
|
cause: `job "${input.job}" (${input.jobId}) is no longer claimed by this worker — it was cancelled, or its visibility lease lapsed and the queue re-delivered it`,
|
|
309
302
|
fix: `x jobs show ${input.jobId} --json`,
|
|
310
|
-
docs: docsFor('X_JOB_LEASE_LOST'),
|
|
311
303
|
});
|
|
312
304
|
}
|
|
313
305
|
}
|
|
@@ -325,7 +317,6 @@ export class JobSlotLostError extends UltimateError {
|
|
|
325
317
|
code: 'X_JOB_SLOT_LOST',
|
|
326
318
|
cause: `job "${input.job}" (${input.jobId}) no longer holds fleet concurrency slot ${input.slot} — its lease expired and another worker took it`,
|
|
327
319
|
fix: `x jobs show ${input.jobId} --json`,
|
|
328
|
-
docs: docsFor('X_JOB_SLOT_LOST'),
|
|
329
320
|
});
|
|
330
321
|
}
|
|
331
322
|
}
|
|
@@ -344,7 +335,6 @@ export class JobNotCancellableError extends UltimateError {
|
|
|
344
335
|
? `no job ${input.jobId} exists in this queue`
|
|
345
336
|
: `job ${input.jobId} is "${input.state}" and only a job that has not finished can be cancelled`,
|
|
346
337
|
fix: `x jobs ls --state running --json`,
|
|
347
|
-
docs: docsFor('X_JOB_NOT_CANCELLABLE'),
|
|
348
338
|
});
|
|
349
339
|
}
|
|
350
340
|
}
|
|
@@ -356,7 +346,6 @@ export class CancelUnsupportedError extends UltimateError {
|
|
|
356
346
|
code: 'X_JOB_NOT_CANCELLABLE',
|
|
357
347
|
cause: `the "${input.driver}" jobs driver cannot cancel a single job`,
|
|
358
348
|
fix: 'call setJobDriver(createPgDriver()) at boot — only the pg driver implements introspect.cancel — then: x jobs cancel <id> --json',
|
|
359
|
-
docs: docsFor('X_JOB_NOT_CANCELLABLE'),
|
|
360
349
|
});
|
|
361
350
|
}
|
|
362
351
|
}
|
|
@@ -373,7 +362,6 @@ export class ConcurrencyUnenforceableError extends UltimateError {
|
|
|
373
362
|
code: 'X_JOB_CONCURRENCY_UNENFORCEABLE',
|
|
374
363
|
cause: `${input.jobs.join(', ')} declare concurrency and the "${input.driver}" jobs driver has no lease store, so the cap would hold per process and the fleet would run concurrency x replicas`,
|
|
375
364
|
fix: `remove concurrency from job("${input.jobs[0] ?? 'the job'}"), or call setJobDriver(createPgDriver()) at boot — the pg driver is the one with a lease store`,
|
|
376
|
-
docs: docsFor('X_JOB_CONCURRENCY_UNENFORCEABLE'),
|
|
377
365
|
});
|
|
378
366
|
}
|
|
379
367
|
}
|
|
@@ -385,7 +373,6 @@ export class OutboxNoTxError extends UltimateError {
|
|
|
385
373
|
code: 'X_OUTBOX_NO_TX',
|
|
386
374
|
cause: `ctx.jobs.enqueue(${input.job}) ran outside a transaction with outbox: 'required'`,
|
|
387
375
|
fix: 'wrap the call in ctx.tx(async (tx) => ...), or enqueue with { outbox: false }',
|
|
388
|
-
docs: docsFor('X_OUTBOX_NO_TX'),
|
|
389
376
|
});
|
|
390
377
|
}
|
|
391
378
|
}
|
|
@@ -396,7 +383,6 @@ export class JobsNotImplementedError extends UltimateError {
|
|
|
396
383
|
code: 'X_NOT_IMPLEMENTED',
|
|
397
384
|
cause: `${input.feature} is declared but not implemented in @ultimat3/jobs`,
|
|
398
385
|
fix: input.fix,
|
|
399
|
-
docs: docsFor('X_NOT_IMPLEMENTED'),
|
|
400
386
|
});
|
|
401
387
|
}
|
|
402
388
|
}
|
package/src/events-pg.ts
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
// resumes at 12:00:30 must still see an event published at 12:00:10.
|
|
8
8
|
|
|
9
9
|
import type { Clock } from '@ultimat3/core';
|
|
10
|
-
import { logger, systemClock, uuid } from '@ultimat3/core';
|
|
10
|
+
import { logger, renderThrowable, systemClock, uuid } from '@ultimat3/core';
|
|
11
11
|
import type { DurationInput } from './clock';
|
|
12
12
|
import { nowMs, toMs } from './clock';
|
|
13
13
|
import type { PgExecutor } from './driver-pg';
|
|
@@ -54,7 +54,7 @@ export function createPgEventBus(options: PgEventBusOptions): EventBus {
|
|
|
54
54
|
void exec.query(SQL_EVENT_PURGE, []).catch((error: unknown) => {
|
|
55
55
|
// Housekeeping never costs a publish: an unpurged row is filtered out of every read.
|
|
56
56
|
logger.warn('jobs.event.purge-failed', {
|
|
57
|
-
error:
|
|
57
|
+
error: renderThrowable(error),
|
|
58
58
|
});
|
|
59
59
|
});
|
|
60
60
|
return 0;
|
package/src/execute.ts
CHANGED
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
anonymousActor,
|
|
11
11
|
isUltimateError,
|
|
12
12
|
logger,
|
|
13
|
+
renderThrowable,
|
|
13
14
|
reportError,
|
|
14
15
|
runWithContext,
|
|
15
16
|
useContext,
|
|
@@ -22,6 +23,7 @@ import { eventBus } from './events';
|
|
|
22
23
|
import type { AnyJobHandle } from './job';
|
|
23
24
|
import type { JobStopReason } from './retry-classification';
|
|
24
25
|
import { nextRetryForError, recordedFailure } from './retry-classification';
|
|
26
|
+
import { createRunSignal } from './run-signal';
|
|
25
27
|
import type { EventLookup, StepRecord } from './steps';
|
|
26
28
|
import { createStepRunner, isStepSuspension } from './steps';
|
|
27
29
|
import { jobRunActor } from './tenant';
|
|
@@ -100,7 +102,14 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
|
|
|
100
102
|
// where every other body does, with no jobs-only parameter to know about. Composed with the
|
|
101
103
|
// caller's signal rather than replacing it: a ctx that was already going away still is.
|
|
102
104
|
const cancel = new AbortController();
|
|
103
|
-
|
|
105
|
+
// `createRunSignal` and never `AbortSignal.any` — the second of the two sites this package's
|
|
106
|
+
// `CLAUDE.md` states the rule as absolute for. In the worker path `callerSignal` is per-run and
|
|
107
|
+
// nothing leaks; on `@ultimat3/testing`'s job-fixture path, which calls `executeJob` directly,
|
|
108
|
+
// the caller's `ctx.signal` may be process-lifetime, and a composite cannot be undone — so every
|
|
109
|
+
// job a fixture ran left a dependent signal on it for the life of the process. Disposed in the
|
|
110
|
+
// `finally` at the bottom of the try, beside `cancel.abort`.
|
|
111
|
+
const runSignal = createRunSignal([callerSignal(options.ctx), cancel.signal]);
|
|
112
|
+
const signal = runSignal.signal;
|
|
104
113
|
const ctx: Ctx = Object.freeze({
|
|
105
114
|
...options.ctx,
|
|
106
115
|
signal,
|
|
@@ -178,7 +187,7 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
|
|
|
178
187
|
});
|
|
179
188
|
}
|
|
180
189
|
|
|
181
|
-
const message =
|
|
190
|
+
const message = renderThrowable(error);
|
|
182
191
|
// The ERROR decides too, not only the attempt count. A `terminal` code — a rotated password,
|
|
183
192
|
// a schema mismatch, a permission denial — fails identically on every remaining attempt, so
|
|
184
193
|
// spending them is a queue slot, a provider bill and, at a site that locks an account after
|
|
@@ -238,6 +247,9 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
|
|
|
238
247
|
// attempt already owns. The runner fences its writes on this signal. A second `abort()` keeps
|
|
239
248
|
// the first reason, so a timed-out run still reports the timeout, not this.
|
|
240
249
|
cancel.abort(new JobAbortedError({ job: handle.name }));
|
|
250
|
+
// AFTER the abort, so the runner's fence still sees it: `dispose` stops following the sources,
|
|
251
|
+
// it never aborts, and the signal keeps whatever state the abort above left it in.
|
|
252
|
+
runSignal.dispose();
|
|
241
253
|
}
|
|
242
254
|
|
|
243
255
|
// Only reachable when the BODY succeeded, and settlement is deliberately outside the catch
|
|
@@ -287,9 +299,7 @@ function raceTimeout(
|
|
|
287
299
|
job,
|
|
288
300
|
timeoutMs,
|
|
289
301
|
ended,
|
|
290
|
-
...(error === undefined
|
|
291
|
-
? {}
|
|
292
|
-
: { error: error instanceof Error ? error.message : String(error) }),
|
|
302
|
+
...(error === undefined ? {} : { error: renderThrowable(error) }),
|
|
293
303
|
});
|
|
294
304
|
};
|
|
295
305
|
work.then(
|
package/src/heartbeat.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// hardest bug in a queue to see from the outside and the easiest to name from in here.
|
|
5
5
|
|
|
6
6
|
import type { Clock } from '@ultimat3/core';
|
|
7
|
-
import { logger, recordLeaseLost } from '@ultimat3/core';
|
|
7
|
+
import { logger, recordLeaseLost, renderThrowable } from '@ultimat3/core';
|
|
8
8
|
import { nowMs } from './clock';
|
|
9
9
|
import type { ClaimedJob, JobDriver } from './driver';
|
|
10
10
|
import { LeaseLostError } from './errors';
|
|
@@ -76,9 +76,7 @@ export function startLeaseHeartbeat(options: LeaseHeartbeatOptions): LeaseHeartb
|
|
|
76
76
|
attempt: claimed.attempt,
|
|
77
77
|
visibilityTimeoutMs,
|
|
78
78
|
reason: reason ?? 'expired',
|
|
79
|
-
...(error === undefined
|
|
80
|
-
? {}
|
|
81
|
-
: { error: error instanceof Error ? error.message : String(error) }),
|
|
79
|
+
...(error === undefined ? {} : { error: renderThrowable(error) }),
|
|
82
80
|
});
|
|
83
81
|
recordLeaseLost(claimed.queue);
|
|
84
82
|
// Cancel LAST, so the loss is logged and counted before the body it unwinds starts throwing.
|
|
@@ -134,7 +132,7 @@ export function startLeaseHeartbeat(options: LeaseHeartbeatOptions): LeaseHeartb
|
|
|
134
132
|
job: claimed.name,
|
|
135
133
|
jobId: claimed.id,
|
|
136
134
|
attempt: claimed.attempt,
|
|
137
|
-
error:
|
|
135
|
+
error: renderThrowable(error),
|
|
138
136
|
});
|
|
139
137
|
if (lapsed()) reportLost(error);
|
|
140
138
|
} finally {
|
package/src/outbox.ts
CHANGED
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
// test and `x dev` must enqueue with nothing wired — but it is a fallback, not the guarantee.
|
|
25
25
|
|
|
26
26
|
import type { Clock } from '@ultimat3/core';
|
|
27
|
-
import { currentSpanContext, logger, traceparent, uuid } from '@ultimat3/core';
|
|
27
|
+
import { currentSpanContext, logger, renderThrowable, traceparent, uuid } from '@ultimat3/core';
|
|
28
28
|
import type { Tx } from '@ultimat3/entity';
|
|
29
29
|
import { nowMs } from './clock';
|
|
30
30
|
import type { EnqueueResult, JobDriver } from './driver';
|
|
@@ -421,7 +421,7 @@ export function createOutboxRelay(options: RelayOptions): OutboxRelay {
|
|
|
421
421
|
id: record.id,
|
|
422
422
|
published,
|
|
423
423
|
remaining: batch.length - published,
|
|
424
|
-
error:
|
|
424
|
+
error: renderThrowable(error),
|
|
425
425
|
});
|
|
426
426
|
// Hand the rest of the batch back rather than sit on a claim nobody is publishing. The
|
|
427
427
|
// claim is a lease now, so without this a single pool timeout parks every committed row
|
|
@@ -456,7 +456,7 @@ export function createOutboxRelay(options: RelayOptions): OutboxRelay {
|
|
|
456
456
|
.then((): void => undefined)
|
|
457
457
|
.catch((error: unknown) => {
|
|
458
458
|
logger.error('jobs.outbox.tick-failed', {
|
|
459
|
-
error:
|
|
459
|
+
error: renderThrowable(error),
|
|
460
460
|
});
|
|
461
461
|
})
|
|
462
462
|
.finally(() => {
|
package/src/renewal-timer.ts
CHANGED
|
@@ -4,6 +4,8 @@
|
|
|
4
4
|
// the interval, which does nothing to the request already on the wire. So a flag is what every
|
|
5
5
|
// branch after an `await` re-reads — the shape `settleWithin`'s `decided` uses in core.
|
|
6
6
|
|
|
7
|
+
import { logger, renderThrowable } from '@ultimat3/core';
|
|
8
|
+
|
|
7
9
|
export interface RenewalTimer {
|
|
8
10
|
/**
|
|
9
11
|
* True once `stop()` has been called. Read AFTER every await in the renewal body: a clean
|
|
@@ -22,7 +24,20 @@ export function startRenewalTimer(
|
|
|
22
24
|
): RenewalTimer {
|
|
23
25
|
let stopped = false;
|
|
24
26
|
const timer = setInterval(() => {
|
|
25
|
-
void renew()
|
|
27
|
+
// `Promise.resolve().then(renew)` and never `void renew()`: a `renew` that throws SYNCHRONOUSLY
|
|
28
|
+
// escapes before any `.catch` its body chained exists. `worker-fleet-slots.ts` guards the
|
|
29
|
+
// promise chain and cannot guard this — `LeaseStore.renew` is an injected seam, and a store
|
|
30
|
+
// that throws on a closed pool throws on the call, not in the chain. Nothing sits above a
|
|
31
|
+
// `setInterval` callback, so that throw is an uncaught exception in the timer that was going
|
|
32
|
+
// to keep the lease alive. The shape `outbox.ts`'s tick loop already uses.
|
|
33
|
+
void Promise.resolve()
|
|
34
|
+
.then(renew)
|
|
35
|
+
.catch((error: unknown) => {
|
|
36
|
+
// A renewal that FAILS is each caller's own business and both handle it. Reaching here
|
|
37
|
+
// means the seam broke its contract, which is a different fact and is worth its own line —
|
|
38
|
+
// swallowed, it would be a lease that stops renewing with nothing anywhere saying so.
|
|
39
|
+
logger.error('jobs.renewal.raised', { error: renderThrowable(error) });
|
|
40
|
+
});
|
|
26
41
|
}, intervalMs);
|
|
27
42
|
return {
|
|
28
43
|
stopped: () => stopped,
|
package/src/scheduler.ts
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
// round rather than opening a second one over the same `lastFiredAt`.
|
|
14
14
|
|
|
15
15
|
import type { Clock } from '@ultimat3/core';
|
|
16
|
-
import { isUltimateError, logger, onShutdown } from '@ultimat3/core';
|
|
16
|
+
import { isUltimateError, logger, onShutdown, renderThrowable } from '@ultimat3/core';
|
|
17
17
|
import { instant, nextCronOccurrence } from '@ultimat3/time';
|
|
18
18
|
import { nowMs } from './clock';
|
|
19
19
|
import type { JobDriver } from './driver';
|
|
@@ -26,7 +26,7 @@ import { registeredTasks } from './task';
|
|
|
26
26
|
* stable code to search on and the `fix:` to run, not a sentence.
|
|
27
27
|
*/
|
|
28
28
|
function failureFields(error: unknown): Record<string, unknown> {
|
|
29
|
-
const message =
|
|
29
|
+
const message = renderThrowable(error);
|
|
30
30
|
return isUltimateError(error)
|
|
31
31
|
? { error: message, code: error.code, cause: error.cause, fix: error.fix }
|
|
32
32
|
: { error: message };
|
package/src/steps.ts
CHANGED
|
@@ -7,10 +7,11 @@
|
|
|
7
7
|
// catches it and re-queues the job for `resumeAt` instead of holding a process for three days.
|
|
8
8
|
|
|
9
9
|
import type { Clock } from '@ultimat3/core';
|
|
10
|
-
import { logger } from '@ultimat3/core';
|
|
10
|
+
import { logger, renderThrowable } from '@ultimat3/core';
|
|
11
11
|
import type { DurationInput } from './clock';
|
|
12
12
|
import { nowMs, toMs } from './clock';
|
|
13
13
|
import { JobAbortedError, JobTimeoutError, StepDuplicateError } from './errors';
|
|
14
|
+
import { createRunSignal } from './run-signal';
|
|
14
15
|
|
|
15
16
|
/**
|
|
16
17
|
* The runtime list is the declaration and `StepStatus` is derived from it, the shape
|
|
@@ -283,10 +284,16 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
|
|
|
283
284
|
// The step's own ceiling, folded into the run's cancellation so the body reads ONE signal and
|
|
284
285
|
// sees whichever deadline lands first. Composed only when there is a second one to compose.
|
|
285
286
|
const deadline = new AbortController();
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
287
|
+
// `createRunSignal` and never `AbortSignal.any`, which is the rule this package's `CLAUDE.md`
|
|
288
|
+
// states as absolute: a composite cannot be undone, so ONE dependent signal per STEP stayed on
|
|
289
|
+
// the run's signal for the whole attempt. A `backfill()` at `batch: 1000` over 5M rows is
|
|
290
|
+
// 5,000 of them held at once, and an app whose `WorkerOptions.context()` carries a
|
|
291
|
+
// process-lifetime signal keeps them past the run. Composed only when there is a second signal
|
|
292
|
+
// to compose, and DISPOSED in the `finally` below, which is the whole reason `run-signal.ts`
|
|
293
|
+
// exists — `worker-run.ts` disposes the run's own the same way.
|
|
294
|
+
const composed =
|
|
295
|
+
options.stepTimeoutMs === undefined ? null : createRunSignal([runSignal, deadline.signal]);
|
|
296
|
+
const signal = composed?.signal ?? runSignal;
|
|
290
297
|
try {
|
|
291
298
|
const output = await withStepTimeout(
|
|
292
299
|
fn(signal),
|
|
@@ -320,12 +327,16 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
|
|
|
320
327
|
status: 'failed',
|
|
321
328
|
startedAt,
|
|
322
329
|
attempts,
|
|
323
|
-
error:
|
|
330
|
+
error: renderThrowable(error),
|
|
324
331
|
};
|
|
325
332
|
await store.put(failure);
|
|
326
333
|
remember(failure);
|
|
327
334
|
}
|
|
328
335
|
throw error;
|
|
336
|
+
} finally {
|
|
337
|
+
// Nothing of the run's is held past the step. Idempotent, and it never aborts: a step that
|
|
338
|
+
// settled leaves its signal in whatever state it ended in.
|
|
339
|
+
composed?.dispose();
|
|
329
340
|
}
|
|
330
341
|
}
|
|
331
342
|
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// Apart from `worker.ts` because the claim loop's question is "may I start this one?" — which job
|
|
4
4
|
// holds which slot, and who gives it back, is bookkeeping of its own.
|
|
5
5
|
|
|
6
|
-
import { logger } from '@ultimat3/core';
|
|
6
|
+
import { logger, renderThrowable } from '@ultimat3/core';
|
|
7
7
|
import type { ClaimedJob } from './driver';
|
|
8
8
|
import { getJob } from './job';
|
|
9
9
|
import type { HeldLease, LeaseStore } from './leases';
|
|
@@ -121,7 +121,7 @@ export function createFleetSlots(options: FleetSlotOptions): FleetSlots {
|
|
|
121
121
|
logger.warn('jobs.worker.lease-release-failed', {
|
|
122
122
|
workerId: options.workerId,
|
|
123
123
|
jobId,
|
|
124
|
-
error:
|
|
124
|
+
error: renderThrowable(error),
|
|
125
125
|
});
|
|
126
126
|
});
|
|
127
127
|
},
|
package/src/worker.ts
CHANGED
|
@@ -4,7 +4,14 @@
|
|
|
4
4
|
// deploy turns "at least once" into "always twice", so draining is on by default.
|
|
5
5
|
|
|
6
6
|
import type { Clock, Ctx } from '@ultimat3/core';
|
|
7
|
-
import {
|
|
7
|
+
import {
|
|
8
|
+
logger,
|
|
9
|
+
onShutdown,
|
|
10
|
+
recordJob,
|
|
11
|
+
recordQueueDepth,
|
|
12
|
+
renderThrowable,
|
|
13
|
+
uuid,
|
|
14
|
+
} from '@ultimat3/core';
|
|
8
15
|
import { nowMs } from './clock';
|
|
9
16
|
import type { ClaimedJob, JobDriver, QueueStats } from './driver';
|
|
10
17
|
import { DEFAULT_QUEUE, DEFAULT_VISIBILITY_TIMEOUT_MS } from './driver';
|
|
@@ -141,7 +148,7 @@ export function createWorker(options: WorkerOptions): Worker {
|
|
|
141
148
|
// Instrumentation never costs a tick: a queue that cannot be measured must still be worked.
|
|
142
149
|
logger.warn('jobs.worker.depth-failed', {
|
|
143
150
|
workerId,
|
|
144
|
-
error:
|
|
151
|
+
error: renderThrowable(error),
|
|
145
152
|
});
|
|
146
153
|
}
|
|
147
154
|
};
|
|
@@ -290,7 +297,7 @@ export function createWorker(options: WorkerOptions): Worker {
|
|
|
290
297
|
workerId,
|
|
291
298
|
job: job.name,
|
|
292
299
|
jobId: job.id,
|
|
293
|
-
error:
|
|
300
|
+
error: renderThrowable(error),
|
|
294
301
|
});
|
|
295
302
|
},
|
|
296
303
|
);
|
|
@@ -332,7 +339,7 @@ export function createWorker(options: WorkerOptions): Worker {
|
|
|
332
339
|
.catch((error: unknown) => {
|
|
333
340
|
logger.error('jobs.worker.tick-failed', {
|
|
334
341
|
workerId,
|
|
335
|
-
error:
|
|
342
|
+
error: renderThrowable(error),
|
|
336
343
|
});
|
|
337
344
|
})
|
|
338
345
|
.finally(() => {
|