@ultimat3/jobs 3.0.0 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +53 -15
- package/package.json +5 -5
- package/src/backfill-errors.ts +137 -0
- package/src/backfill-gate.ts +5 -5
- package/src/backfill-pass.ts +1 -1
- package/src/describe.ts +17 -1
- package/src/driver-memory.ts +34 -8
- package/src/driver-nats.ts +6 -3
- package/src/driver-pg-ddl.ts +20 -6
- package/src/driver-pg-rows.ts +34 -6
- package/src/driver-pg-sql.ts +11 -2
- package/src/driver-pg.ts +15 -5
- package/src/driver-redis.ts +6 -3
- package/src/driver.ts +36 -12
- package/src/errors.ts +59 -131
- package/src/execute.ts +11 -6
- package/src/index.ts +16 -8
- package/src/inspect.ts +1 -1
- package/src/metrics.ts +1 -1
- package/src/outbox.ts +1 -1
- package/src/register.ts +25 -1
- package/src/retry.ts +4 -3
- package/src/steps.ts +14 -1
- package/src/task.ts +18 -1
- package/src/worker-run.ts +3 -0
- package/src/worker.ts +34 -9
package/src/driver.ts
CHANGED
|
@@ -17,15 +17,22 @@ import type { StepStore } from './steps';
|
|
|
17
17
|
* a cancellation is work an operator stopped on purpose and `x jobs retry` must not resurrect by
|
|
18
18
|
* accident. It appears in no claim predicate, so the queue never hands a cancelled row out again.
|
|
19
19
|
*/
|
|
20
|
-
export
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
20
|
+
export const JOB_STATES = [
|
|
21
|
+
'ready',
|
|
22
|
+
'delayed',
|
|
23
|
+
'running',
|
|
24
|
+
'suspended',
|
|
25
|
+
'done',
|
|
26
|
+
'failed',
|
|
27
|
+
'dead',
|
|
28
|
+
'cancelled',
|
|
29
|
+
] as const;
|
|
30
|
+
|
|
31
|
+
export type JobState = (typeof JOB_STATES)[number];
|
|
32
|
+
|
|
33
|
+
/** Narrows a state read back off a queue row. Never a cast — the list decides. */
|
|
34
|
+
export const isJobState = (value: string): value is JobState =>
|
|
35
|
+
(JOB_STATES as readonly string[]).includes(value);
|
|
29
36
|
|
|
30
37
|
export interface JobRecord {
|
|
31
38
|
readonly id: string;
|
|
@@ -114,10 +121,23 @@ export interface NackOptions {
|
|
|
114
121
|
readonly delayMs: number;
|
|
115
122
|
readonly error?: string;
|
|
116
123
|
/**
|
|
117
|
-
*
|
|
118
|
-
* a
|
|
124
|
+
* The ATTEMPT COUNTER, and nothing else. False for a suspension and for a shed alike: neither is
|
|
125
|
+
* a failure, and a 3-day sleep that burned an attempt would dead-letter the job.
|
|
119
126
|
*/
|
|
120
127
|
readonly countsAsAttempt?: boolean;
|
|
128
|
+
/**
|
|
129
|
+
* True for a SUSPENSION — `step.sleep`, or a name this deploy does not know — which leaves the
|
|
130
|
+
* ready bucket and is counted `suspended`. Absent for a limiter or `job.concurrency` shed, which
|
|
131
|
+
* is a job still WAITING to run.
|
|
132
|
+
*
|
|
133
|
+
* The two were one flag until 2026-08: `countsAsAttempt: false` decided the state as well as the
|
|
134
|
+
* counter, so a shed was filed beside a 3-day sleep and `stats()` excluded it from `ready` and
|
|
135
|
+
* from `oldestReadyMs` — the two numbers the worker publishes as `queue_depth` and
|
|
136
|
+
* `queue_oldest_ready_seconds`. Under sustained overload the shed fraction approaches 100%, so
|
|
137
|
+
* the HPA signal and the "oldest job older than 5 minutes" page both went quiet exactly when the
|
|
138
|
+
* queue was saturated.
|
|
139
|
+
*/
|
|
140
|
+
readonly park?: boolean;
|
|
121
141
|
readonly deadLetter?: boolean;
|
|
122
142
|
}
|
|
123
143
|
|
|
@@ -128,7 +148,11 @@ export interface QueueStats {
|
|
|
128
148
|
readonly running: number;
|
|
129
149
|
readonly suspended: number;
|
|
130
150
|
readonly dead: number;
|
|
131
|
-
/**
|
|
151
|
+
/**
|
|
152
|
+
* Age in ms of the oldest job that is READY and due — the number that decides autoscaling.
|
|
153
|
+
* Not "claimable": a `suspended` row is claimable once its `runAt` passes and is deliberately
|
|
154
|
+
* excluded here, which is exactly why a limiter shed may not be filed as a suspension.
|
|
155
|
+
*/
|
|
132
156
|
readonly oldestReadyMs: number;
|
|
133
157
|
}
|
|
134
158
|
|
package/src/errors.ts
CHANGED
|
@@ -23,6 +23,8 @@ export const JOB_OWNED_ERROR_CODES = [
|
|
|
23
23
|
'X_BACKFILL_RUNNING',
|
|
24
24
|
'X_BACKFILL_STALLED',
|
|
25
25
|
'X_BACKFILL_UNKNOWN',
|
|
26
|
+
'X_JOB_ROW_STATUS_UNKNOWN',
|
|
27
|
+
'X_ACTION_JOB_UNBRIDGED',
|
|
26
28
|
] as const;
|
|
27
29
|
|
|
28
30
|
/**
|
|
@@ -58,6 +60,8 @@ export const JOB_ERROR_TITLES: Readonly<Record<JobOwnedErrorCode, string>> = {
|
|
|
58
60
|
X_BACKFILL_RUNNING: 'a pass under this name is already live',
|
|
59
61
|
X_BACKFILL_STALLED: 'the sweep ended with rows its own count still matches',
|
|
60
62
|
X_BACKFILL_UNKNOWN: 'no declaration carries this backfill name',
|
|
63
|
+
X_JOB_ROW_STATUS_UNKNOWN: 'a queue row carries a status this build does not know',
|
|
64
|
+
X_ACTION_JOB_UNBRIDGED: 'an action projection was registered as a job',
|
|
61
65
|
};
|
|
62
66
|
|
|
63
67
|
// One unconditional call, so a second package claiming one of jobs' codes throws
|
|
@@ -89,7 +93,8 @@ registerErrorRetry({
|
|
|
89
93
|
X_BACKFILL_APPLIED: 'terminal',
|
|
90
94
|
});
|
|
91
95
|
|
|
92
|
-
|
|
96
|
+
/** Shared with `backfill-errors.ts`, which holds the seven `X_BACKFILL_*` classes. */
|
|
97
|
+
export const docsFor = (code: JobErrorCode): string => `https://ultimate.dev/errors/${code}`;
|
|
93
98
|
|
|
94
99
|
/** An enqueue collided with a live job holding the same idempotency key under `onConflict: 'error'`. */
|
|
95
100
|
export class JobDuplicateError extends UltimateError {
|
|
@@ -124,6 +129,57 @@ export class JobNameTakenError extends UltimateError {
|
|
|
124
129
|
}
|
|
125
130
|
}
|
|
126
131
|
|
|
132
|
+
/**
|
|
133
|
+
* A `text` status column holds a value outside the vocabulary this build compiled.
|
|
134
|
+
*
|
|
135
|
+
* Refused rather than passed through, because the alternative is what used to happen: the three
|
|
136
|
+
* decoders in `driver-pg-rows.ts` cast the column, and `stepRun`'s `existing?.status ===
|
|
137
|
+
* 'completed'` then read false for the laundered value and RE-EXECUTED the step. A second charge
|
|
138
|
+
* is a worse answer than a failed attempt, and "an unrecognised fact is never a satisfied one" is
|
|
139
|
+
* the rule the rest of the framework already follows.
|
|
140
|
+
*
|
|
141
|
+
* Almost always a NEWER deploy's row, not corruption: a status string only reaches the table
|
|
142
|
+
* because some version of this framework wrote it. A rolling deploy that only ADDS a status is
|
|
143
|
+
* safe in the normal direction — the new build knows every old value — and it is the old build
|
|
144
|
+
* reading the new build's row that lands here, on that one job, loudly.
|
|
145
|
+
*/
|
|
146
|
+
export class JobRowStatusUnknownError extends UltimateError {
|
|
147
|
+
constructor(input: { table: string; column: string; value: string; known: readonly string[] }) {
|
|
148
|
+
super({
|
|
149
|
+
code: 'X_JOB_ROW_STATUS_UNKNOWN',
|
|
150
|
+
cause:
|
|
151
|
+
`${input.table}.${input.column} holds "${input.value}", which this build does not know — ` +
|
|
152
|
+
`it reads ${input.known.join(', ')}`,
|
|
153
|
+
fix: `x jobs show --json # then drain the older workers: a status this build cannot read was almost certainly written by a newer deploy`,
|
|
154
|
+
docs: docsFor('X_JOB_ROW_STATUS_UNKNOWN'),
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* `registerJobs()` was handed `someAction.job()`.
|
|
161
|
+
*
|
|
162
|
+
* That call answers an `ActionJobHandle` — `kind: 'action-job'`, deliberately a different literal
|
|
163
|
+
* from `'job'` — which is the four fields `job()` takes, not a job. It cannot be one: `action` and
|
|
164
|
+
* `jobs` are both tier 3, so neither may import the other, and only `job()` seats a handle the
|
|
165
|
+
* queue, the worker and the manifest accept.
|
|
166
|
+
*
|
|
167
|
+
* Refused BY NAME rather than skipped, which is what used to happen. `registerJobs(module)` is
|
|
168
|
+
* handed a whole module namespace, so silently ignoring a constant or a helper exported beside a
|
|
169
|
+
* job is right — but ignoring this one meant `registerJobs({ publishPost: publishPost.job() })`
|
|
170
|
+
* registered nothing, returned `[]`, and the job never ran, with nothing failing anywhere.
|
|
171
|
+
*/
|
|
172
|
+
export class ActionJobUnbridgedError extends UltimateError {
|
|
173
|
+
constructor(input: { export: string; job: string }) {
|
|
174
|
+
super({
|
|
175
|
+
code: 'X_ACTION_JOB_UNBRIDGED',
|
|
176
|
+
cause: `export "${input.export}" is the action projection "${input.job}", which is not a job handle and cannot be registered as one`,
|
|
177
|
+
fix: `wrap it: agentJob(${input.export}, { name: '${input.export}', tenant, retry }) from @ultimat3/ai — that composes job() and returns a handle the queue accepts`,
|
|
178
|
+
docs: docsFor('X_ACTION_JOB_UNBRIDGED'),
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
127
183
|
/** Two steps in one run share a name, so replay cannot tell their persisted results apart. */
|
|
128
184
|
export class StepDuplicateError extends UltimateError {
|
|
129
185
|
constructor(input: { job: string; step: string }) {
|
|
@@ -299,7 +355,7 @@ export class CancelUnsupportedError extends UltimateError {
|
|
|
299
355
|
super({
|
|
300
356
|
code: 'X_JOB_NOT_CANCELLABLE',
|
|
301
357
|
cause: `the "${input.driver}" jobs driver cannot cancel a single job`,
|
|
302
|
-
fix:
|
|
358
|
+
fix: 'call setJobDriver(createPgDriver()) at boot — only the pg driver implements introspect.cancel — then: x jobs cancel <id> --json',
|
|
303
359
|
docs: docsFor('X_JOB_NOT_CANCELLABLE'),
|
|
304
360
|
});
|
|
305
361
|
}
|
|
@@ -316,7 +372,7 @@ export class ConcurrencyUnenforceableError extends UltimateError {
|
|
|
316
372
|
super({
|
|
317
373
|
code: 'X_JOB_CONCURRENCY_UNENFORCEABLE',
|
|
318
374
|
cause: `${input.jobs.join(', ')} declare concurrency and the "${input.driver}" jobs driver has no lease store, so the cap would hold per process and the fleet would run concurrency x replicas`,
|
|
319
|
-
fix: `remove concurrency from job("${input.jobs[0] ?? 'the job'}"), or
|
|
375
|
+
fix: `remove concurrency from job("${input.jobs[0] ?? 'the job'}"), or call setJobDriver(createPgDriver()) at boot — the pg driver is the one with a lease store`,
|
|
320
376
|
docs: docsFor('X_JOB_CONCURRENCY_UNENFORCEABLE'),
|
|
321
377
|
});
|
|
322
378
|
}
|
|
@@ -334,134 +390,6 @@ export class OutboxNoTxError extends UltimateError {
|
|
|
334
390
|
}
|
|
335
391
|
}
|
|
336
392
|
|
|
337
|
-
/**
|
|
338
|
-
* The seven backfill codes below all answer one question — "why is this sweep not running?" — and
|
|
339
|
-
* each is here because it sends the reader somewhere different: run it, force it, change
|
|
340
|
-
* environment, migrate first, wait, fix the predicates, or fix the name. A code that shared a fix
|
|
341
|
-
* line with another one would be code inflation, which is why `X_BACKFILL_WRITE_UNCONFIRMED` was
|
|
342
|
-
* considered and rejected: a dry run that wrote nothing did exactly what it was asked to.
|
|
343
|
-
*
|
|
344
|
-
* Every `fix` below is ONE line something can execute — a shell command, or the edit to make.
|
|
345
|
-
* Never a command with prose appended: a `fix:` is copied and run verbatim, so a trailing clause
|
|
346
|
-
* turns a working command into a syntax error at the one moment the reader is following it
|
|
347
|
-
* literally. Explanations belong in `cause`, which is read and never run.
|
|
348
|
-
*/
|
|
349
|
-
|
|
350
|
-
/**
|
|
351
|
-
* Declared and never completed. The alarm the framework did not have: an author could
|
|
352
|
-
* `x g backfill`, merge and deploy, and nothing anywhere said the pass had not run.
|
|
353
|
-
*/
|
|
354
|
-
export class BackfillPendingError extends UltimateError {
|
|
355
|
-
constructor(input: { backfill: string; environment: string }) {
|
|
356
|
-
super({
|
|
357
|
-
code: 'X_BACKFILL_PENDING',
|
|
358
|
-
cause: `backfill "${input.backfill}" is declared and x_backfills holds no completed pass for it in ${input.environment}`,
|
|
359
|
-
fix: `x db backfill ${input.backfill} --write --json`,
|
|
360
|
-
docs: docsFor('X_BACKFILL_PENDING'),
|
|
361
|
-
});
|
|
362
|
-
}
|
|
363
|
-
}
|
|
364
|
-
|
|
365
|
-
/** Already swept. A rerun is legitimate, so this names the flag rather than refusing outright. */
|
|
366
|
-
export class BackfillAppliedError extends UltimateError {
|
|
367
|
-
constructor(input: { backfill: string; runId: string; completedAt: string }) {
|
|
368
|
-
super({
|
|
369
|
-
code: 'X_BACKFILL_APPLIED',
|
|
370
|
-
cause: `backfill "${input.backfill}" completed as run ${input.runId} at ${input.completedAt}; a forced rerun writes a NEW ledger row and never edits that one`,
|
|
371
|
-
fix: `x db backfill ${input.backfill} --write --force --json`,
|
|
372
|
-
docs: docsFor('X_BACKFILL_APPLIED'),
|
|
373
|
-
});
|
|
374
|
-
}
|
|
375
|
-
}
|
|
376
|
-
|
|
377
|
-
/**
|
|
378
|
-
* The declaration names the environments it belongs to and this is not one. Declared DATA, never a
|
|
379
|
-
* hardcoded "cleanups are production": a staging rehearsal is correct practice, so which
|
|
380
|
-
* environments a sweep belongs to is the app's convention and this is only the mechanism carrying
|
|
381
|
-
* it (axiom 8).
|
|
382
|
-
*/
|
|
383
|
-
export class BackfillEnvironmentError extends UltimateError {
|
|
384
|
-
constructor(input: { backfill: string; environment: string; declared: readonly string[] }) {
|
|
385
|
-
// The first declared environment, because the fix has to be ONE runnable line and the list is
|
|
386
|
-
// ordered by the author. The empty case cannot arise from `checkBackfillEnvironment`, which
|
|
387
|
-
// treats an empty list as "every environment" — but this constructor is public, so it answers
|
|
388
|
-
// with the command that lists what IS declared rather than an `ULTIMATE_ENV=undefined`.
|
|
389
|
-
const target = input.declared[0];
|
|
390
|
-
super({
|
|
391
|
-
code: 'X_BACKFILL_ENVIRONMENT',
|
|
392
|
-
cause: `backfill "${input.backfill}" declares environments: ${input.declared.join(', ')} and this process resolved ${input.environment} — add "${input.environment}" to that list if this deploy should sweep too`,
|
|
393
|
-
fix:
|
|
394
|
-
target === undefined
|
|
395
|
-
? 'x db backfill --pending --json'
|
|
396
|
-
: `ULTIMATE_ENV=${target} x db backfill ${input.backfill} --write --json`,
|
|
397
|
-
docs: docsFor('X_BACKFILL_ENVIRONMENT'),
|
|
398
|
-
});
|
|
399
|
-
}
|
|
400
|
-
}
|
|
401
|
-
|
|
402
|
-
/**
|
|
403
|
-
* `requires` names a migration the ledger has not applied. Checked where `x_migrations` is
|
|
404
|
-
* readable — this package holds no `@ultimat3/db` dependency and growing one to read a ledger
|
|
405
|
-
* would put the migration engine on the tier-3 queue's import graph.
|
|
406
|
-
*/
|
|
407
|
-
export class BackfillMigrationPendingError extends UltimateError {
|
|
408
|
-
constructor(input: { backfill: string; migration: string }) {
|
|
409
|
-
super({
|
|
410
|
-
code: 'X_BACKFILL_MIGRATION_PENDING',
|
|
411
|
-
cause: `backfill "${input.backfill}" requires migration ${input.migration}, which x_migrations does not record as applied`,
|
|
412
|
-
fix: 'x db migrate --json',
|
|
413
|
-
docs: docsFor('X_BACKFILL_MIGRATION_PENDING'),
|
|
414
|
-
});
|
|
415
|
-
}
|
|
416
|
-
}
|
|
417
|
-
|
|
418
|
-
/**
|
|
419
|
-
* The enqueue deduped: one live pass per name, so this run is the one already going. Distinct from
|
|
420
|
-
* `X_BACKFILL_APPLIED`, which is a pass that finished — the response there is `--force`, and the
|
|
421
|
-
* response here is to look at the run that is holding the key.
|
|
422
|
-
*/
|
|
423
|
-
export class BackfillRunningError extends UltimateError {
|
|
424
|
-
constructor(input: { backfill: string; jobId: string }) {
|
|
425
|
-
super({
|
|
426
|
-
code: 'X_BACKFILL_RUNNING',
|
|
427
|
-
cause: `backfill "${input.backfill}" already has a live pass queued as ${input.jobId}, and one name holds one live pass; its step trace names the batch it is on, and a pass that is not advancing is a worker that lost its lease`,
|
|
428
|
-
fix: `x jobs show ${input.jobId} --json`,
|
|
429
|
-
docs: docsFor('X_BACKFILL_RUNNING'),
|
|
430
|
-
});
|
|
431
|
-
}
|
|
432
|
-
}
|
|
433
|
-
|
|
434
|
-
/**
|
|
435
|
-
* The source ran out of rows and the declaration's own `count()` still matches some. Two
|
|
436
|
-
* predicates that disagree is an authoring bug in any business — the sweep reported success over
|
|
437
|
-
* rows it never visited — so the pass fails rather than writing a completed row nobody can trust.
|
|
438
|
-
*/
|
|
439
|
-
export class BackfillStalledError extends UltimateError {
|
|
440
|
-
constructor(input: { backfill: string; remaining: number; swept: number }) {
|
|
441
|
-
super({
|
|
442
|
-
code: 'X_BACKFILL_STALLED',
|
|
443
|
-
cause: `backfill "${input.backfill}" swept ${input.swept} rows, exhausted its source, and count() still matches ${input.remaining} — a WHERE the sweep narrows and the count does not is what leaves rows behind`,
|
|
444
|
-
fix: `make count() select on exactly what source() selects on in backfill("${input.backfill}")`,
|
|
445
|
-
docs: docsFor('X_BACKFILL_STALLED'),
|
|
446
|
-
});
|
|
447
|
-
}
|
|
448
|
-
}
|
|
449
|
-
|
|
450
|
-
/** A name no declaration in this app carries — a typo, or a backfill whose module was deleted. */
|
|
451
|
-
export class BackfillUnknownError extends UltimateError {
|
|
452
|
-
constructor(input: { backfill: string; known: readonly string[] }) {
|
|
453
|
-
super({
|
|
454
|
-
code: 'X_BACKFILL_UNKNOWN',
|
|
455
|
-
cause:
|
|
456
|
-
input.known.length === 0
|
|
457
|
-
? `no backfill named "${input.backfill}" is declared, and this app declares none at all`
|
|
458
|
-
: `no backfill named "${input.backfill}" is declared (declared: ${input.known.join(', ')})`,
|
|
459
|
-
fix: 'x db backfill --pending --json',
|
|
460
|
-
docs: docsFor('X_BACKFILL_UNKNOWN'),
|
|
461
|
-
});
|
|
462
|
-
}
|
|
463
|
-
}
|
|
464
|
-
|
|
465
393
|
export class JobsNotImplementedError extends UltimateError {
|
|
466
394
|
constructor(input: { feature: string; fix: string }) {
|
|
467
395
|
super({
|
package/src/execute.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
// One claimed job run to completion, suspension or failure and settled with the driver — the
|
|
2
|
-
// single execution path the worker loop
|
|
2
|
+
// single execution path, shared by the worker loop (`worker-run.ts`) and by the one caller outside
|
|
3
|
+
// this package, `@ultimat3/testing`'s job fixture. There is no `x jobs run` to share it with: the
|
|
4
|
+
// subcommands are `ls`, `show`, `retry`, `cancel`, `drain`. It owns the run's deadline, and
|
|
3
5
|
// a deadline here means CANCEL: the nack that follows makes the job claimable again, so a body
|
|
4
6
|
// still running past it would be a second copy of one job, racing the attempt that replaced it.
|
|
5
7
|
|
|
@@ -87,7 +89,8 @@ export interface ExecuteJobOptions {
|
|
|
87
89
|
|
|
88
90
|
/**
|
|
89
91
|
* Run one claimed job to completion, suspension or failure, and settle it with the driver.
|
|
90
|
-
* Shared by the worker loop and `
|
|
92
|
+
* Shared by the worker loop and by `@ultimat3/testing`'s job fixture, so a job under test takes
|
|
93
|
+
* exactly the code path the worker takes.
|
|
91
94
|
*/
|
|
92
95
|
export async function executeJob(options: ExecuteJobOptions): Promise<JobExecution> {
|
|
93
96
|
const { driver, claimed, handle } = options;
|
|
@@ -159,8 +162,10 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
|
|
|
159
162
|
} catch (error) {
|
|
160
163
|
if (isStepSuspension(error)) {
|
|
161
164
|
const delayMs = Math.max(0, error.resumeAt - nowMs(options.clock));
|
|
162
|
-
//
|
|
163
|
-
|
|
165
|
+
// `park: true` is the suspension itself — the row leaves the ready bucket — and
|
|
166
|
+
// `countsAsAttempt: false` only says not to burn an attempt on it. A limiter shed passes the
|
|
167
|
+
// second and not the first: it is a job still waiting, and it belongs in `queue_depth`.
|
|
168
|
+
await driver.nack(claimed.id, { delayMs, countsAsAttempt: false, park: true });
|
|
164
169
|
return settle({
|
|
165
170
|
outcome: 'suspended',
|
|
166
171
|
jobId: claimed.id,
|
|
@@ -200,8 +205,8 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
|
|
|
200
205
|
// This package's ONE error-reporting call site, and it is here rather than in the loop because
|
|
201
206
|
// this is the only frame that still holds the thrown value — the loop sees a message string.
|
|
202
207
|
// A retry is a failure the framework recovered from, so it is a `warning`; a dead letter is
|
|
203
|
-
// one nobody recovered from.
|
|
204
|
-
// execution path means one place a failed job
|
|
208
|
+
// one nobody recovered from. A job driven by `@ultimat3/testing`'s fixture takes this path
|
|
209
|
+
// too, which is the point: one execution path means one place a failed job becomes visible.
|
|
205
210
|
reportError(error, {
|
|
206
211
|
source: 'job',
|
|
207
212
|
severity: decision.retry ? 'warning' : 'error',
|
package/src/index.ts
CHANGED
|
@@ -17,6 +17,15 @@ export type {
|
|
|
17
17
|
BackfillReport,
|
|
18
18
|
} from './backfill';
|
|
19
19
|
export { backfill, DEFAULT_BACKFILL_BATCH } from './backfill';
|
|
20
|
+
export {
|
|
21
|
+
BackfillAppliedError,
|
|
22
|
+
BackfillEnvironmentError,
|
|
23
|
+
BackfillMigrationPendingError,
|
|
24
|
+
BackfillPendingError,
|
|
25
|
+
BackfillRunningError,
|
|
26
|
+
BackfillStalledError,
|
|
27
|
+
BackfillUnknownError,
|
|
28
|
+
} from './backfill-errors';
|
|
20
29
|
export type { BackfillGate, BackfillGateInput } from './backfill-gate';
|
|
21
30
|
export { checkBackfillEnvironment, gateBackfill } from './backfill-gate';
|
|
22
31
|
export type { BackfillProgress } from './backfill-inspect';
|
|
@@ -77,11 +86,13 @@ export type {
|
|
|
77
86
|
export {
|
|
78
87
|
DEFAULT_QUEUE,
|
|
79
88
|
DEFAULT_VISIBILITY_TIMEOUT_MS,
|
|
89
|
+
isJobState,
|
|
90
|
+
JOB_STATES,
|
|
80
91
|
jobDriver,
|
|
81
92
|
resetJobDriver,
|
|
82
93
|
setJobDriver,
|
|
83
94
|
} from './driver';
|
|
84
|
-
export type { MemoryDriverOptions } from './driver-memory';
|
|
95
|
+
export type { MemoryDriverOptions, MemoryJobDriver } from './driver-memory';
|
|
85
96
|
export { createMemoryDriver } from './driver-memory';
|
|
86
97
|
export type { NatsDriverOptions } from './driver-nats';
|
|
87
98
|
export { createNatsDriver } from './driver-nats';
|
|
@@ -121,13 +132,7 @@ export type { RedisDriverOptions } from './driver-redis';
|
|
|
121
132
|
export { createRedisDriver } from './driver-redis';
|
|
122
133
|
export type { JobErrorCode } from './errors';
|
|
123
134
|
export {
|
|
124
|
-
|
|
125
|
-
BackfillEnvironmentError,
|
|
126
|
-
BackfillMigrationPendingError,
|
|
127
|
-
BackfillPendingError,
|
|
128
|
-
BackfillRunningError,
|
|
129
|
-
BackfillStalledError,
|
|
130
|
-
BackfillUnknownError,
|
|
135
|
+
ActionJobUnbridgedError,
|
|
131
136
|
CancelUnsupportedError,
|
|
132
137
|
ConcurrencyUnenforceableError,
|
|
133
138
|
DriverUnavailableError,
|
|
@@ -139,6 +144,7 @@ export {
|
|
|
139
144
|
JobMaxAttemptsError,
|
|
140
145
|
JobNameTakenError,
|
|
141
146
|
JobNotCancellableError,
|
|
147
|
+
JobRowStatusUnknownError,
|
|
142
148
|
JobSlotLostError,
|
|
143
149
|
JobsNotImplementedError,
|
|
144
150
|
JobTenantRequiredError,
|
|
@@ -248,8 +254,10 @@ export type {
|
|
|
248
254
|
export {
|
|
249
255
|
createMemoryStepStore,
|
|
250
256
|
createStepRunner,
|
|
257
|
+
isStepStatus,
|
|
251
258
|
isStepSuspension,
|
|
252
259
|
MAX_TRACE_NAMES,
|
|
260
|
+
STEP_STATUSES,
|
|
253
261
|
StepSuspension,
|
|
254
262
|
} from './steps';
|
|
255
263
|
export type {
|
package/src/inspect.ts
CHANGED
|
@@ -91,7 +91,7 @@ function requireIntrospection(driver: JobDriver): NonNullable<JobDriver['introsp
|
|
|
91
91
|
if (driver.introspect === undefined) {
|
|
92
92
|
throw new JobsNotImplementedError({
|
|
93
93
|
feature: `introspection for the "${driver.name}" jobs driver`,
|
|
94
|
-
fix:
|
|
94
|
+
fix: 'call setJobDriver(createPgDriver()) at boot — only the pg driver implements introspect — then: x jobs ls --json',
|
|
95
95
|
});
|
|
96
96
|
}
|
|
97
97
|
return driver.introspect;
|
package/src/metrics.ts
CHANGED
|
@@ -18,7 +18,7 @@ import { gauge } from '@ultimat3/core';
|
|
|
18
18
|
/** Seconds and not milliseconds: every Prometheus duration is seconds, and the alert is `> 300`. */
|
|
19
19
|
export const queueOldestReady: Gauge = gauge('queue_oldest_ready_seconds', {
|
|
20
20
|
unit: 's',
|
|
21
|
-
description: 'Age of the oldest
|
|
21
|
+
description: 'Age of the oldest job that is ready and due, by queue — 0 when none is',
|
|
22
22
|
});
|
|
23
23
|
|
|
24
24
|
export const queueDeadJobs: Gauge = gauge('queue_dead_jobs', {
|
package/src/outbox.ts
CHANGED
|
@@ -338,7 +338,7 @@ export function jobsFacade(): JobsFacade {
|
|
|
338
338
|
throw new DriverUnavailableError({
|
|
339
339
|
driver: 'none',
|
|
340
340
|
cause: 'no queue driver is installed in this process',
|
|
341
|
-
fix: 'call setJobDriver(createMemoryDriver()) before enqueuing
|
|
341
|
+
fix: 'call setJobDriver(createMemoryDriver()) before enqueuing, or setJobDriver(createPgDriver()) for a real queue',
|
|
342
342
|
});
|
|
343
343
|
}
|
|
344
344
|
return installed;
|
package/src/register.ts
CHANGED
|
@@ -5,9 +5,24 @@
|
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
7
|
import { type RegisteredPrimitive, registerPrimitiveRegistrar } from '@ultimat3/core';
|
|
8
|
+
import { ActionJobUnbridgedError } from './errors';
|
|
8
9
|
import { isJobHandle, registerJob } from './job';
|
|
9
10
|
import { isTaskHandle, registerTask } from './task';
|
|
10
11
|
|
|
12
|
+
/**
|
|
13
|
+
* `@ultimat3/action`'s job PROJECTION, recognised structurally because that package is this tier
|
|
14
|
+
* and may never be imported here. `kind: 'action-job'` is a literal chosen to be distinguishable
|
|
15
|
+
* from `'job'` rather than a near-miss (`packages/action/src/job-handle.ts` says so), which is
|
|
16
|
+
* precisely what makes this check possible without an import.
|
|
17
|
+
*/
|
|
18
|
+
const isActionProjection = (
|
|
19
|
+
value: unknown,
|
|
20
|
+
): value is { readonly kind: 'action-job'; readonly name: string } =>
|
|
21
|
+
typeof value === 'object' &&
|
|
22
|
+
value !== null &&
|
|
23
|
+
'kind' in value &&
|
|
24
|
+
(value as { readonly kind: unknown }).kind === 'action-job';
|
|
25
|
+
|
|
11
26
|
/** `registerJobs(await import('./jobs'))` — export names become job names. */
|
|
12
27
|
export function registerJobs(
|
|
13
28
|
module: Readonly<Record<string, unknown>>,
|
|
@@ -15,7 +30,16 @@ export function registerJobs(
|
|
|
15
30
|
const registered: RegisteredPrimitive[] = [];
|
|
16
31
|
for (const name of Object.keys(module).sort()) {
|
|
17
32
|
const value = module[name];
|
|
18
|
-
if (isJobHandle(value))
|
|
33
|
+
if (isJobHandle(value)) {
|
|
34
|
+
registered.push(registerJob(name, value));
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
// Everything else is skipped in silence — a module namespace is full of constants, types and
|
|
38
|
+
// helpers exported beside the jobs. Everything else EXCEPT this one, which is unambiguously
|
|
39
|
+
// someone trying to queue an action and getting nothing at all.
|
|
40
|
+
if (isActionProjection(value)) {
|
|
41
|
+
throw new ActionJobUnbridgedError({ export: name, job: value.name });
|
|
42
|
+
}
|
|
19
43
|
}
|
|
20
44
|
return registered;
|
|
21
45
|
}
|
package/src/retry.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
// Retry arithmetic, kept pure so the schedule is testable and printable. `x jobs
|
|
2
|
-
// renders `retrySchedule()` verbatim — an agent should be able to see
|
|
3
|
-
// without running the queue.
|
|
1
|
+
// Retry arithmetic, kept pure so the schedule is testable and printable. `x jobs show <id>`
|
|
2
|
+
// renders `retrySchedule()` verbatim as `JobTrace.retryDelaysMs` — an agent should be able to see
|
|
3
|
+
// when attempt 5 lands without running the queue. There is no `x jobs schedule`: the subcommands
|
|
4
|
+
// are `ls`, `show`, `retry`, `cancel`, `drain`.
|
|
4
5
|
|
|
5
6
|
import type { DurationInput } from './clock';
|
|
6
7
|
import { toMs } from './clock';
|
package/src/steps.ts
CHANGED
|
@@ -12,7 +12,20 @@ import type { DurationInput } from './clock';
|
|
|
12
12
|
import { nowMs, toMs } from './clock';
|
|
13
13
|
import { JobAbortedError, JobTimeoutError, StepDuplicateError } from './errors';
|
|
14
14
|
|
|
15
|
-
|
|
15
|
+
/**
|
|
16
|
+
* The runtime list is the declaration and `StepStatus` is derived from it, the shape
|
|
17
|
+
* `BACKFILL_STATUSES` and `PRIMITIVE_KINDS` already have. A bare union cannot narrow a `text`
|
|
18
|
+
* column, so `driver-pg-rows.ts` cast one instead — and a cast that lands an unknown status on a
|
|
19
|
+
* record makes `stepRun`'s `existing?.status === 'completed'` false, which RE-EXECUTES a step
|
|
20
|
+
* this file promises runs once.
|
|
21
|
+
*/
|
|
22
|
+
export const STEP_STATUSES = ['completed', 'sleeping', 'waiting', 'failed'] as const;
|
|
23
|
+
|
|
24
|
+
export type StepStatus = (typeof STEP_STATUSES)[number];
|
|
25
|
+
|
|
26
|
+
/** Narrows a status read back out of a store. Never a cast — the list decides. */
|
|
27
|
+
export const isStepStatus = (value: string): value is StepStatus =>
|
|
28
|
+
(STEP_STATUSES as readonly string[]).includes(value);
|
|
16
29
|
|
|
17
30
|
export interface StepRecord {
|
|
18
31
|
readonly runId: string;
|
package/src/task.ts
CHANGED
|
@@ -54,6 +54,7 @@ export interface TaskDefinition {
|
|
|
54
54
|
*/
|
|
55
55
|
enqueue: (occurrenceMs: number) => readonly TaskEnqueueEntry[];
|
|
56
56
|
readonly catchUp?: CatchUpPolicy;
|
|
57
|
+
/** Whole occurrences per round, one or more. Zero fires nothing at all, and `task()` refuses it. */
|
|
57
58
|
readonly maxCatchUp?: number;
|
|
58
59
|
}
|
|
59
60
|
|
|
@@ -98,6 +99,9 @@ export interface TaskHandle {
|
|
|
98
99
|
const registry = new Map<string, TaskHandle>();
|
|
99
100
|
let anonymous = 0;
|
|
100
101
|
|
|
102
|
+
/** Occurrences one round may fire when neither the declaration nor a catch-up says otherwise. */
|
|
103
|
+
const DEFAULT_MAX_CATCH_UP = 10;
|
|
104
|
+
|
|
101
105
|
/** Job's store, for tasks: proof `task()` built the handle, plus whether it named itself. */
|
|
102
106
|
interface TaskOrigin {
|
|
103
107
|
readonly declaredName: boolean;
|
|
@@ -127,13 +131,26 @@ export function task(definition: TaskDefinition): TaskHandle {
|
|
|
127
131
|
`use the full zone id on task("${name}"), e.g. tz: 'America/Bogota' — list the valid ones with: bun -e "console.log(Intl.supportedValuesOf('timeZone').join('\\n'))"`,
|
|
128
132
|
);
|
|
129
133
|
|
|
134
|
+
// `maxCatchUp: 0` is not "no ceiling" — `occurrencesSince` walks
|
|
135
|
+
// `for (let i = 0; i < handle.maxCatchUp; i += 1)`, so zero (and any negative, and any fraction
|
|
136
|
+
// below one) returns an empty list on every round and the task NEVER fires: no error, no log
|
|
137
|
+
// line, no queue row, forever. Refused where it is written, exactly as `job()` refuses
|
|
138
|
+
// `concurrency: 0` and `createPacer` refuses `rate: 0`. `Number.isInteger` covers `NaN` and
|
|
139
|
+
// `Infinity` in the same predicate — an unbounded catch-up is a burst nobody declared.
|
|
140
|
+
assert(
|
|
141
|
+
definition.maxCatchUp === undefined ||
|
|
142
|
+
(Number.isInteger(definition.maxCatchUp) && definition.maxCatchUp >= 1),
|
|
143
|
+
`task "${name}" declares maxCatchUp ${String(definition.maxCatchUp)}, so no occurrence can ever fire`,
|
|
144
|
+
`set a whole maxCatchUp of 1 or more on task("${name}"), or omit the field for the default of ${DEFAULT_MAX_CATCH_UP}`,
|
|
145
|
+
);
|
|
146
|
+
|
|
130
147
|
const handle: TaskHandle = {
|
|
131
148
|
kind: 'task',
|
|
132
149
|
name,
|
|
133
150
|
cron: definition.cron,
|
|
134
151
|
tz: definition.tz,
|
|
135
152
|
catchUp: definition.catchUp ?? 'skip',
|
|
136
|
-
maxCatchUp: definition.maxCatchUp ??
|
|
153
|
+
maxCatchUp: definition.maxCatchUp ?? DEFAULT_MAX_CATCH_UP,
|
|
137
154
|
// `nowMs()` and not `Date.now()`: every reading of time in this package goes through a
|
|
138
155
|
// Clock so a frozen one cannot be bypassed.
|
|
139
156
|
entries: (occurrenceMs: number = nowMs()) => definition.enqueue(occurrenceMs),
|
package/src/worker-run.ts
CHANGED
|
@@ -48,10 +48,13 @@ export async function runClaimedJob(options: RunClaimedOptions): Promise<JobExec
|
|
|
48
48
|
const handle = getJob(claimed.name);
|
|
49
49
|
if (handle === undefined) {
|
|
50
50
|
// Park it, do not burn attempts: the job may well be registered by the pod next to this one.
|
|
51
|
+
// A genuine park — no worker in this deploy can run it — so it leaves the ready bucket, unlike
|
|
52
|
+
// a limiter shed, which is a job this fleet will pick up on its next pass.
|
|
51
53
|
await driver.nack(claimed.id, {
|
|
52
54
|
delayMs: 30_000,
|
|
53
55
|
error: `no job registered as "${claimed.name}"`,
|
|
54
56
|
countsAsAttempt: false,
|
|
57
|
+
park: true,
|
|
55
58
|
});
|
|
56
59
|
return unknownJob(claimed);
|
|
57
60
|
}
|
package/src/worker.ts
CHANGED
|
@@ -161,6 +161,26 @@ export function createWorker(options: WorkerOptions): Worker {
|
|
|
161
161
|
...(options.events === undefined ? {} : { events: options.events }),
|
|
162
162
|
});
|
|
163
163
|
|
|
164
|
+
/**
|
|
165
|
+
* A claimed job handed straight back over a cap. It is NOT a suspension and NOT a failure: no
|
|
166
|
+
* `park`, so the row stays where `queue_depth` and `queue_oldest_ready_seconds` can see it, and
|
|
167
|
+
* no `error`, so `x jobs show` does not report a `lastError` for a job that never ran. It was
|
|
168
|
+
* both of those until 2026-08 — parked beside a 3-day `step.sleep`, and stamped with a failure
|
|
169
|
+
* it never had — which is why the two sheds go through one function now.
|
|
170
|
+
*/
|
|
171
|
+
const shed = async (
|
|
172
|
+
claimed: ClaimedJob,
|
|
173
|
+
detail: { readonly queue: string; readonly reason: string },
|
|
174
|
+
): Promise<void> => {
|
|
175
|
+
logger.debug('jobs.worker.shed', {
|
|
176
|
+
workerId,
|
|
177
|
+
job: claimed.name,
|
|
178
|
+
jobId: claimed.id,
|
|
179
|
+
...detail,
|
|
180
|
+
});
|
|
181
|
+
await options.driver.nack(claimed.id, { delayMs: pollIntervalMs, countsAsAttempt: false });
|
|
182
|
+
};
|
|
183
|
+
|
|
164
184
|
/** The drain's one question: may this worker still take work off the queue? */
|
|
165
185
|
const claiming = (): boolean => state !== 'draining' && state !== 'stopped';
|
|
166
186
|
|
|
@@ -196,11 +216,17 @@ export function createWorker(options: WorkerOptions): Worker {
|
|
|
196
216
|
...(job.tenantId === undefined ? {} : { tenantId: job.tenantId }),
|
|
197
217
|
});
|
|
198
218
|
if (lease === undefined) {
|
|
199
|
-
// Over a tenant/queue/global cap: hand it straight back for another worker.
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
219
|
+
// Over a tenant/queue/global cap: hand it straight back for another worker. No `park`
|
|
220
|
+
// and no `error` — the row stays in the ready bucket the depth gauge reads, and nothing
|
|
221
|
+
// about this job failed, so `x jobs show` must not report a `lastError` for it. The
|
|
222
|
+
// reason is a log FIELD instead, where it costs nothing when nobody is asking.
|
|
223
|
+
await shed(job, {
|
|
224
|
+
queue,
|
|
225
|
+
reason:
|
|
226
|
+
limiter.blockedBy({
|
|
227
|
+
queue,
|
|
228
|
+
...(job.tenantId === undefined ? {} : { tenantId: job.tenantId }),
|
|
229
|
+
}) ?? 'unknown',
|
|
204
230
|
});
|
|
205
231
|
continue;
|
|
206
232
|
}
|
|
@@ -223,10 +249,9 @@ export function createWorker(options: WorkerOptions): Worker {
|
|
|
223
249
|
}
|
|
224
250
|
if (!granted) {
|
|
225
251
|
lease.release();
|
|
226
|
-
await
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
error: `limited: job concurrency (${getJob(job.name)?.concurrency ?? 0})`,
|
|
252
|
+
await shed(job, {
|
|
253
|
+
queue,
|
|
254
|
+
reason: `job concurrency (${getJob(job.name)?.concurrency ?? 0})`,
|
|
230
255
|
});
|
|
231
256
|
continue;
|
|
232
257
|
}
|