@ultimat3/jobs 23.0.0 → 25.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +72 -75
- package/README.md +83 -38
- package/package.json +7 -7
- package/src/backfill-errors.ts +3 -2
- package/src/backfill-ledger.ts +25 -3
- package/src/backfill-pass.ts +22 -5
- package/src/backfill-rate.ts +2 -2
- package/src/backfill.ts +2 -2
- package/src/claim-exhausted.ts +76 -0
- package/src/counters.ts +1 -1
- package/src/driver-memory-operator.ts +34 -12
- package/src/driver-memory.ts +80 -38
- package/src/driver-pg-jobs-sql.ts +27 -19
- package/src/driver-pg-operator-sql.ts +32 -12
- package/src/driver-pg-operator.ts +19 -3
- package/src/driver-pg-rows.ts +3 -3
- package/src/driver-pg-settle-sql.ts +2 -3
- package/src/driver-pg-sql.ts +99 -30
- package/src/driver-pg.ts +103 -98
- package/src/driver.ts +46 -16
- package/src/errors-concurrency.ts +1 -1
- package/src/errors-operator.ts +12 -4
- package/src/errors-requeue.ts +21 -3
- package/src/errors.ts +9 -17
- package/src/events-pg.ts +11 -21
- package/src/events.ts +26 -15
- package/src/execute.ts +47 -5
- package/src/export.ts +2 -2
- package/src/heartbeat.ts +7 -2
- package/src/index.ts +45 -99
- package/src/inspect-operator.ts +4 -4
- package/src/inspect.ts +5 -4
- package/src/introspection.ts +23 -11
- package/src/job.ts +6 -7
- package/src/leases.ts +1 -1
- package/src/limits.ts +46 -15
- package/src/outbox-lease.ts +1 -1
- package/src/outbox-pg.ts +6 -7
- package/src/outbox-relay.ts +6 -6
- package/src/outbox.ts +47 -19
- package/src/purge.ts +1 -1
- package/src/queue-wake.ts +1 -2
- package/src/renewal-timer.ts +1 -1
- package/src/retry-classification.ts +52 -15
- package/src/scheduler-leader.ts +2 -2
- package/src/scheduler-pg.ts +19 -16
- package/src/scheduler-state.ts +10 -3
- package/src/scheduler.ts +14 -15
- package/src/steps-memory.ts +2 -2
- package/src/steps-suspension.ts +46 -0
- package/src/steps-timeout.ts +33 -0
- package/src/steps.ts +90 -88
- package/src/task.ts +1 -1
- package/src/webhook-attempt.ts +206 -0
- package/src/webhook-ledger.ts +15 -4
- package/src/webhook.ts +64 -118
- package/src/worker-admit.ts +14 -6
- package/src/worker-drain-cutoff.ts +110 -0
- package/src/worker-fleet-slots.ts +82 -45
- package/src/worker-held.ts +100 -0
- package/src/worker-key-busy.ts +38 -10
- package/src/worker-loop.ts +35 -11
- package/src/worker-options.ts +19 -15
- package/src/worker-registry.ts +16 -1
- package/src/worker-run.ts +38 -8
- package/src/worker-tally.ts +53 -0
- package/src/worker-types.ts +7 -4
- package/src/worker.ts +124 -73
- package/src/driver-nats.ts +0 -86
- package/src/driver-redis.ts +0 -89
package/src/events-pg.ts
CHANGED
|
@@ -6,16 +6,16 @@
|
|
|
6
6
|
// Stored and not broadcast, exactly as the memory bus is: a step that suspends at 12:00 and
|
|
7
7
|
// resumes at 12:00:30 must still see an event published at 12:00:10.
|
|
8
8
|
|
|
9
|
-
import {
|
|
9
|
+
import type { PgExecutor } from '@ultimat3/core';
|
|
10
|
+
import { finiteOption, logger, uuidV7 } from '@ultimat3/core';
|
|
10
11
|
import type { DurationInput } from './clock';
|
|
11
12
|
import { finiteDurationMs } from './clock';
|
|
12
|
-
import type { PgExecutor } from './driver-pg';
|
|
13
13
|
import {
|
|
14
14
|
SQL_EVENT_FIND,
|
|
15
15
|
SQL_EVENT_LIST,
|
|
16
16
|
SQL_EVENT_NOW,
|
|
17
17
|
SQL_EVENT_PUBLISH,
|
|
18
|
-
|
|
18
|
+
SQL_EVENT_PURGE_COUNTED,
|
|
19
19
|
} from './driver-pg-sql';
|
|
20
20
|
import type { EventBus } from './events';
|
|
21
21
|
|
|
@@ -28,7 +28,7 @@ interface EventRow {
|
|
|
28
28
|
readonly expires_at: number | string;
|
|
29
29
|
}
|
|
30
30
|
|
|
31
|
-
export interface
|
|
31
|
+
export interface PostgresEventBusOptions {
|
|
32
32
|
/**
|
|
33
33
|
* The only thing this bus is built from. There is no `clock`: every instant it writes or
|
|
34
34
|
* compares is the database's (`SQL_EVENT_PUBLISH`), and a process clock here was the defect.
|
|
@@ -40,14 +40,7 @@ export interface PgEventBusOptions {
|
|
|
40
40
|
readonly listLimit?: number;
|
|
41
41
|
}
|
|
42
42
|
|
|
43
|
-
|
|
44
|
-
* `purgeExpired()` is SYNCHRONOUS in `EventBus` because the memory bus can be — it walks a Map.
|
|
45
|
-
* Over SQL the delete is a round trip, so this fires it and answers 0: the count is a diagnostic
|
|
46
|
-
* the memory bus offers and this one cannot, and blocking a step's resume on a housekeeping
|
|
47
|
-
* DELETE would be a far worse trade than an unanswered number. The index on `(name, published_at)`
|
|
48
|
-
* plus `expires_at > now()` in `find` means an unpurged row costs a filter, never a wrong answer.
|
|
49
|
-
*/
|
|
50
|
-
export function createPgEventBus(options: PgEventBusOptions): EventBus {
|
|
43
|
+
export function postgresEventBus(options: PostgresEventBusOptions): EventBus {
|
|
51
44
|
// TWO screens, for the reason `events.ts` states: `defaultTtl` is the constructor's knob and
|
|
52
45
|
// `ttl` is the publish call's, so one screen over `ttl ?? defaultTtl` names the wrong one for
|
|
53
46
|
// whichever value actually arrived.
|
|
@@ -59,14 +52,11 @@ export function createPgEventBus(options: PgEventBusOptions): EventBus {
|
|
|
59
52
|
const listLimit = finiteOption('the pg event bus', 'listLimit', options.listLimit ?? 1_000);
|
|
60
53
|
const exec = options.executor;
|
|
61
54
|
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
});
|
|
68
|
-
});
|
|
69
|
-
return 0;
|
|
55
|
+
// Awaited and never fired-and-forgotten: it was `void`ed with nothing calling it, so the table
|
|
56
|
+
// only grew. A failure REJECTS — the caller is the retention sweep's step, which retries.
|
|
57
|
+
const purgeExpired = async (): Promise<number> => {
|
|
58
|
+
const rows = await exec.query<{ removed: number | string }>(SQL_EVENT_PURGE_COUNTED, []);
|
|
59
|
+
return Number(rows[0]?.removed ?? 0);
|
|
70
60
|
};
|
|
71
61
|
|
|
72
62
|
return {
|
|
@@ -77,7 +67,7 @@ export function createPgEventBus(options: PgEventBusOptions): EventBus {
|
|
|
77
67
|
},
|
|
78
68
|
|
|
79
69
|
async publish(name, payload, publishOptions = {}) {
|
|
80
|
-
const id =
|
|
70
|
+
const id = uuidV7();
|
|
81
71
|
const ttlMs =
|
|
82
72
|
publishOptions.ttl === undefined
|
|
83
73
|
? defaultTtlMs
|
package/src/events.ts
CHANGED
|
@@ -3,9 +3,10 @@
|
|
|
3
3
|
// 12:00:10, so a fire-and-forget emitter would silently strand every waiting run.
|
|
4
4
|
|
|
5
5
|
import type { Clock } from '@ultimat3/core';
|
|
6
|
-
import { finiteOption, logger, systemClock,
|
|
6
|
+
import { finiteOption, logger, systemClock, uuidV7 } from '@ultimat3/core';
|
|
7
7
|
import type { DurationInput } from './clock';
|
|
8
8
|
import { finiteDurationMs, nowMs } from './clock';
|
|
9
|
+
import type { PurgeTarget } from './purge';
|
|
9
10
|
import type { EventLookup } from './steps';
|
|
10
11
|
|
|
11
12
|
export interface JobEvent {
|
|
@@ -31,16 +32,14 @@ export interface EventBus extends EventLookup {
|
|
|
31
32
|
* split asks, and the reason `eventPrompt()` refuses the in-memory bus outside development.
|
|
32
33
|
*/
|
|
33
34
|
readonly stored: boolean;
|
|
34
|
-
/**
|
|
35
|
-
* The bus's OWN clock, in epoch ms: what a publish issued now would be stamped with. A consumer
|
|
36
|
-
* that wants "published after I asked" takes its "asked at" from here and never from its own
|
|
37
|
-
* process — `publishedAt` is this clock's, and two clocks compared is a skew between pods
|
|
38
|
-
* deciding which answers count.
|
|
39
|
-
*/
|
|
40
|
-
now(): Promise<number>;
|
|
41
35
|
publish(name: string, payload: unknown, options?: PublishOptions): Promise<JobEvent>;
|
|
42
36
|
list(name?: string): Promise<readonly JobEvent[]>;
|
|
43
|
-
|
|
37
|
+
/**
|
|
38
|
+
* Delete every event past its expiry and answer how many went. Measured on the bus's own clock,
|
|
39
|
+
* like every other instant it holds. The memory bus also sweeps on each publish; the stored one
|
|
40
|
+
* has no caller but the retention sweep (`eventsPurgeTarget`).
|
|
41
|
+
*/
|
|
42
|
+
purgeExpired(): Promise<number>;
|
|
44
43
|
size(): number;
|
|
45
44
|
}
|
|
46
45
|
|
|
@@ -50,7 +49,7 @@ export interface MemoryEventBusOptions {
|
|
|
50
49
|
readonly maxEvents?: number;
|
|
51
50
|
}
|
|
52
51
|
|
|
53
|
-
export function
|
|
52
|
+
export function memoryEventBus(options: MemoryEventBusOptions = {}): EventBus {
|
|
54
53
|
const clock = options.clock ?? systemClock;
|
|
55
54
|
// TWO screens, because these are two knobs: the default is declared at construction and belongs
|
|
56
55
|
// to whoever built the bus, `ttl` rides the publish CALL. One screen over `ttl ?? defaultTtl`
|
|
@@ -84,7 +83,7 @@ export function createMemoryEventBus(options: MemoryEventBusOptions = {}): Event
|
|
|
84
83
|
purgeExpired();
|
|
85
84
|
const at = nowMs(clock);
|
|
86
85
|
const event: JobEvent = {
|
|
87
|
-
id:
|
|
86
|
+
id: uuidV7(),
|
|
88
87
|
name,
|
|
89
88
|
payload,
|
|
90
89
|
publishedAt: at,
|
|
@@ -136,15 +135,27 @@ export function createMemoryEventBus(options: MemoryEventBusOptions = {}): Event
|
|
|
136
135
|
return Promise.resolve(all);
|
|
137
136
|
},
|
|
138
137
|
|
|
139
|
-
purgeExpired,
|
|
138
|
+
purgeExpired: () => Promise.resolve(purgeExpired()),
|
|
140
139
|
size: () => events.size,
|
|
141
140
|
};
|
|
142
141
|
}
|
|
143
142
|
|
|
144
|
-
|
|
143
|
+
/** The table the stored bus keeps — the sweep's durable step key, log field and report name. */
|
|
144
|
+
export const EVENTS_PURGE_TARGET = 'x_job_events';
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* The bus as one table of the retention sweep (`purge()`). `x_job_events` takes a row per publish
|
|
148
|
+
* and an expired one is only FILTERED by `find`, so with no sweep the table grows for the life of
|
|
149
|
+
* the deployment. The sweep's `nowMs` is not passed on: a bus holds its own clock.
|
|
150
|
+
*/
|
|
151
|
+
export function eventsPurgeTarget(bus: EventBus): PurgeTarget {
|
|
152
|
+
return { name: EVENTS_PURGE_TARGET, purgeExpired: () => bus.purgeExpired() };
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
let ambientBus: EventBus = memoryEventBus();
|
|
145
156
|
|
|
146
157
|
/**
|
|
147
|
-
* **Install `
|
|
158
|
+
* **Install `postgresEventBus({ executor })` here in any deployment with more than one process.**
|
|
148
159
|
* The default above is one process's heap: the pod that publishes and the pod that resumes are
|
|
149
160
|
* never the same one, so a webhook landing on web-3 strands a run on worker-7 until its 24h
|
|
150
161
|
* timeout dead-letters it, with nothing logged before then. This line used to promise a
|
|
@@ -165,7 +176,7 @@ export function eventBus(): EventBus {
|
|
|
165
176
|
* wait before that test has asked anything — call this between tests.
|
|
166
177
|
*/
|
|
167
178
|
export function resetEventBus(): void {
|
|
168
|
-
ambientBus =
|
|
179
|
+
ambientBus = memoryEventBus();
|
|
169
180
|
}
|
|
170
181
|
|
|
171
182
|
/** The one function app code calls to unblock a waiting step. */
|
package/src/execute.ts
CHANGED
|
@@ -26,12 +26,17 @@ import type { AnyJobHandle } from './job';
|
|
|
26
26
|
import { createProgressReporter } from './progress';
|
|
27
27
|
import { isFinalAttempt } from './retry';
|
|
28
28
|
import type { JobStopReason } from './retry-classification';
|
|
29
|
-
import {
|
|
29
|
+
import {
|
|
30
|
+
failureForRow,
|
|
31
|
+
nextRetryForError,
|
|
32
|
+
rateLimitDeferralMs,
|
|
33
|
+
recordedFailure,
|
|
34
|
+
} from './retry-classification';
|
|
30
35
|
import { raceTimeout } from './run-deadline';
|
|
31
36
|
import { createRunSignal } from './run-signal';
|
|
32
37
|
import { announceSettled, settledCode } from './settled';
|
|
33
38
|
import type { EventLookup, StepRecord } from './steps';
|
|
34
|
-
import {
|
|
39
|
+
import { isStepSuspension, stepRunner } from './steps';
|
|
35
40
|
import { jobRunActor } from './tenant';
|
|
36
41
|
|
|
37
42
|
/**
|
|
@@ -57,7 +62,7 @@ export type JobOutcome =
|
|
|
57
62
|
const NEVER_ABORTED = new AbortController().signal;
|
|
58
63
|
|
|
59
64
|
/**
|
|
60
|
-
* `Ctx.signal` is non-optional in the type and `
|
|
65
|
+
* `Ctx.signal` is non-optional in the type and `ctxOf` always sets it — but a context can
|
|
61
66
|
* still arrive across a cast (`@ultimat3/http`'s `asCtx`, a test's `{} as Ctx`) without one, and
|
|
62
67
|
* a job that crashed on a missing field would be a far worse answer than a job with no caller to
|
|
63
68
|
* follow. Read it, do not assume it.
|
|
@@ -68,7 +73,7 @@ function callerSignal(ctx: Ctx): AbortSignal {
|
|
|
68
73
|
|
|
69
74
|
/**
|
|
70
75
|
* The same defensive read, for the same reason: `Ctx.actor` is non-optional in the type and
|
|
71
|
-
* `
|
|
76
|
+
* `ctxOf` always sets it, but `WorkerOptions.context()` is the app's own function and a
|
|
72
77
|
* cast context (`{} as Ctx`) reaches here without one. Anonymous is the honest stand-in — it
|
|
73
78
|
* carries no org, so the job's declared tenant is the only thing that can put one on the run.
|
|
74
79
|
*/
|
|
@@ -147,11 +152,15 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
|
|
|
147
152
|
signal,
|
|
148
153
|
services: callerServices(options.ctx),
|
|
149
154
|
});
|
|
150
|
-
const runner =
|
|
155
|
+
const runner = stepRunner({
|
|
151
156
|
runId: claimed.runId,
|
|
152
157
|
jobName: handle.name,
|
|
153
158
|
store: driver.steps,
|
|
159
|
+
fence: { job: handle.name, jobId: claimed.id, ...claimOf(claimed) },
|
|
154
160
|
signal,
|
|
161
|
+
// This attempt's OWN end — the deadline, the `finally` below — apart from the run's signal,
|
|
162
|
+
// whose first reason may be the drain's: a drained step still records, an ended attempt never.
|
|
163
|
+
ended: cancel.signal,
|
|
155
164
|
// The DECLARED per-step ceiling and event poll. Passed here or nowhere: this is the only
|
|
156
165
|
// production construction of a runner, so a `StepRunnerOptions` field it omits is a feature
|
|
157
166
|
// no `job()` can reach — which both of these were until 2026-08.
|
|
@@ -309,6 +318,39 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
|
|
|
309
318
|
replayed: [],
|
|
310
319
|
});
|
|
311
320
|
}
|
|
321
|
+
// A rate-limit refusal is "not yet": rescheduled for its Retry-After with the attempt
|
|
322
|
+
// UNCOUNTED, like a suspension — but left in the ready bucket (no park): it is a job still
|
|
323
|
+
// waiting, and it belongs in `queue_depth`. Counted, a backlog of rate-limited `llm()` jobs was
|
|
324
|
+
// dead-lettered for having waited on a bucket that was only ever going to refill.
|
|
325
|
+
const deferMs = rateLimitDeferralMs(handle.retry, claimed.attempt, error);
|
|
326
|
+
if (deferMs !== undefined) {
|
|
327
|
+
landed(
|
|
328
|
+
await driver.nack(claimed.id, {
|
|
329
|
+
...by(),
|
|
330
|
+
delayMs: deferMs,
|
|
331
|
+
error: failureForRow(error),
|
|
332
|
+
countsAsAttempt: false,
|
|
333
|
+
}),
|
|
334
|
+
'retried',
|
|
335
|
+
);
|
|
336
|
+
logger.info('jobs.attempt.rate_limited', {
|
|
337
|
+
job: handle.name,
|
|
338
|
+
jobId: claimed.id,
|
|
339
|
+
attempt: claimed.attempt,
|
|
340
|
+
delayMs: deferMs,
|
|
341
|
+
});
|
|
342
|
+
return settle({
|
|
343
|
+
outcome: 'retried',
|
|
344
|
+
jobId: claimed.id,
|
|
345
|
+
job: handle.name,
|
|
346
|
+
attempt: claimed.attempt,
|
|
347
|
+
durationMs: nowMs(options.clock) - startedAt,
|
|
348
|
+
error: message,
|
|
349
|
+
resumeAt: nowMs(options.clock) + deferMs,
|
|
350
|
+
steps: [],
|
|
351
|
+
replayed: [],
|
|
352
|
+
});
|
|
353
|
+
}
|
|
312
354
|
// The ERROR decides too, not only the attempt count. A `terminal` code — a rotated password,
|
|
313
355
|
// a schema mismatch, a permission denial — fails identically on every remaining attempt, so
|
|
314
356
|
// spending them is a queue slot, a provider bill and, at a site that locks an account after
|
package/src/export.ts
CHANGED
|
@@ -20,7 +20,7 @@ import { assert, systemClock } from '@ultimat3/core';
|
|
|
20
20
|
import type { ReadBuilder } from '@ultimat3/entity';
|
|
21
21
|
import type { StandardSchemaV1 } from '@ultimat3/schema';
|
|
22
22
|
import type { Pacer } from './backfill-rate';
|
|
23
|
-
import {
|
|
23
|
+
import { backfillPacer } from './backfill-rate';
|
|
24
24
|
import type { DurationInput } from './clock';
|
|
25
25
|
import { nowMs } from './clock';
|
|
26
26
|
import type { ExportFormat, ExportRecord } from './export-format';
|
|
@@ -177,7 +177,7 @@ export function exportRows<Row, I>(definition: ExportDefinition<Row, I>): JobHan
|
|
|
177
177
|
const pace =
|
|
178
178
|
definition.rate === undefined
|
|
179
179
|
? undefined
|
|
180
|
-
:
|
|
180
|
+
: backfillPacer({ rate: definition.rate, job: definition.name, clock });
|
|
181
181
|
|
|
182
182
|
// Resolved per write, never here: this line runs at the app's import, before its storage exists.
|
|
183
183
|
const planned: PlannedExport<Row, I> = {
|
package/src/heartbeat.ts
CHANGED
|
@@ -98,6 +98,11 @@ export function startLeaseHeartbeat(options: LeaseHeartbeatOptions): LeaseHeartb
|
|
|
98
98
|
// request per tick onto the connection that is already the thing failing.
|
|
99
99
|
if (renewing) return;
|
|
100
100
|
renewing = true;
|
|
101
|
+
// Stamped BEFORE the call: the driver writes the new lease at the statement, so the window it
|
|
102
|
+
// bought starts no later than this. Stamped at the reply instead, a slow answer would stretch
|
|
103
|
+
// this process's view of the lease past the server's by the reply's latency — and a second
|
|
104
|
+
// worker re-claims the row while this one still believes it holds it.
|
|
105
|
+
const sentAt = now();
|
|
101
106
|
try {
|
|
102
107
|
const held = await options.driver.heartbeat(claimed.id, {
|
|
103
108
|
visibilityTimeoutMs,
|
|
@@ -121,14 +126,14 @@ export function startLeaseHeartbeat(options: LeaseHeartbeatOptions): LeaseHeartb
|
|
|
121
126
|
}
|
|
122
127
|
// Asked again AFTER the call, because a renewal that SUCCEEDS late is still late: an
|
|
123
128
|
// event-loop stall or a driver that answered at the end of a connect timeout can land this
|
|
124
|
-
// past the window it was renewing, and
|
|
129
|
+
// past the window it was renewing, and renewing there would restart the clock on
|
|
125
130
|
// a lease the queue has already re-delivered — the loss hidden by the very call meant to
|
|
126
131
|
// prevent it.
|
|
127
132
|
if (lapsed()) {
|
|
128
133
|
reportLost();
|
|
129
134
|
return;
|
|
130
135
|
}
|
|
131
|
-
renewedAt =
|
|
136
|
+
renewedAt = sentAt;
|
|
132
137
|
} catch (error) {
|
|
133
138
|
// One failed renewal is not a lost lease: there is a whole visibility window left to land
|
|
134
139
|
// the next one, and the default interval gives three tries inside it. Say it at warn and
|
package/src/index.ts
CHANGED
|
@@ -7,23 +7,10 @@
|
|
|
7
7
|
// bypasses `defineApi`'s own result — the ambiguity axiom 1 exists to refuse.
|
|
8
8
|
import './register';
|
|
9
9
|
|
|
10
|
-
// The wire format
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
|
|
14
|
-
export type { WebhookMacInput, WebhookSigningInput } from '@ultimat3/core';
|
|
15
|
-
export {
|
|
16
|
-
isCanonicalWebhookField,
|
|
17
|
-
WEBHOOK_FIELD_MAX,
|
|
18
|
-
WEBHOOK_ID_HEADER,
|
|
19
|
-
WEBHOOK_SIGNATURE_HEADER,
|
|
20
|
-
WEBHOOK_SIGNATURE_VERSION,
|
|
21
|
-
WEBHOOK_TOPIC_HEADER,
|
|
22
|
-
webhookHeaders,
|
|
23
|
-
webhookMac,
|
|
24
|
-
webhookSignature,
|
|
25
|
-
webhookSigningString,
|
|
26
|
-
} from '@ultimat3/core';
|
|
10
|
+
// The webhook wire format (`WEBHOOK_*`, `webhookSignature`, `webhookMac`, …) is `@ultimat3/core`'s
|
|
11
|
+
// and imported from there: one module at the tier this package (the signer) and `@ultimat3/http`
|
|
12
|
+
// (the verifier) both reach, with one import path — never re-exported here (`X_HELPER_COPY`).
|
|
13
|
+
export type { PgExecutor } from '@ultimat3/core';
|
|
27
14
|
/** Re-exported so a `job`/`task` file needs one import, not two. Same object as schema's. */
|
|
28
15
|
export type { Infer } from '@ultimat3/schema';
|
|
29
16
|
export { t } from '@ultimat3/schema';
|
|
@@ -57,9 +44,9 @@ export type {
|
|
|
57
44
|
export {
|
|
58
45
|
BACKFILL_STATUSES,
|
|
59
46
|
backfillChecksum,
|
|
60
|
-
createMemoryBackfillLedger,
|
|
61
47
|
decideBackfill,
|
|
62
48
|
isBackfillStatus,
|
|
49
|
+
memoryBackfillLedger,
|
|
63
50
|
} from './backfill-ledger';
|
|
64
51
|
export type {
|
|
65
52
|
BackfillPendingReport,
|
|
@@ -73,7 +60,7 @@ export {
|
|
|
73
60
|
pendingBackfills,
|
|
74
61
|
} from './backfill-pending';
|
|
75
62
|
export type { Pacer, PacerOptions } from './backfill-rate';
|
|
76
|
-
export {
|
|
63
|
+
export { backfillPacer, DEFAULT_BACKFILL_RATE } from './backfill-rate';
|
|
77
64
|
export type { BackfillCount, BackfillDeclaration, BackfillOrigin } from './backfill-registry';
|
|
78
65
|
// `stampBackfill` is deliberately absent, for the reason `registerJob` is: a second way to make a
|
|
79
66
|
// handle claim it is a backfill would let a plain `job()` inherit the pending diff and the gate.
|
|
@@ -84,6 +71,8 @@ export {
|
|
|
84
71
|
isBackfill,
|
|
85
72
|
registeredBackfills,
|
|
86
73
|
} from './backfill-registry';
|
|
74
|
+
export type { AnnounceExhaustedOptions, ExhaustedCounts } from './claim-exhausted';
|
|
75
|
+
export { announceExhausted } from './claim-exhausted';
|
|
87
76
|
export type { JobConcurrency, KeyedConcurrency, WhenBusy } from './concurrency';
|
|
88
77
|
export { MAX_CONCURRENCY_KEY_LENGTH, WHEN_BUSY } from './concurrency';
|
|
89
78
|
export type { JobConcurrencyDescriptor, JobDescriptor } from './describe';
|
|
@@ -112,66 +101,20 @@ export {
|
|
|
112
101
|
isJobState,
|
|
113
102
|
JOB_STATES,
|
|
114
103
|
jobDriver,
|
|
104
|
+
LEASE_LAPSED_FINAL_ATTEMPT,
|
|
105
|
+
LIVE_STATES,
|
|
115
106
|
nackState,
|
|
116
107
|
resetJobDriver,
|
|
117
108
|
setJobDriver,
|
|
118
109
|
} from './driver';
|
|
119
|
-
export type {
|
|
120
|
-
export {
|
|
121
|
-
export type {
|
|
122
|
-
export {
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
SQL_COUNTER_FOLD,
|
|
128
|
-
SQL_COUNTER_TOTALS,
|
|
129
|
-
SQL_COUNTERS,
|
|
130
|
-
SQL_JOB_PROGRESS,
|
|
131
|
-
SQL_JOB_PROMOTE,
|
|
132
|
-
SQL_JOB_REMOVE,
|
|
133
|
-
SQL_JOB_REMOVE_MANY,
|
|
134
|
-
SQL_JOB_REQUEUE_MANY,
|
|
135
|
-
SQL_PAUSE,
|
|
136
|
-
SQL_PAUSED,
|
|
137
|
-
SQL_RESUME,
|
|
138
|
-
SQL_SCHEDULER_FIRE,
|
|
139
|
-
SQL_WORKER_ANNOUNCE,
|
|
140
|
-
SQL_WORKER_FORGET,
|
|
141
|
-
SQL_WORKERS,
|
|
142
|
-
} from './driver-pg-operator-sql';
|
|
143
|
-
export {
|
|
144
|
-
SQL_ACK,
|
|
145
|
-
SQL_ADVISORY_UNLOCK,
|
|
146
|
-
SQL_BACKFILL_FINISH,
|
|
147
|
-
SQL_BACKFILL_LIST,
|
|
148
|
-
SQL_BACKFILL_PROGRESS,
|
|
149
|
-
SQL_BACKFILL_START,
|
|
150
|
-
SQL_CANCEL,
|
|
151
|
-
SQL_CLAIM,
|
|
152
|
-
SQL_ENQUEUE,
|
|
153
|
-
SQL_HEARTBEAT,
|
|
154
|
-
SQL_JOBS_TABLE,
|
|
155
|
-
SQL_LEADER_ACQUIRE,
|
|
156
|
-
SQL_LEADER_RELEASE,
|
|
157
|
-
SQL_LEASE_ACQUIRE,
|
|
158
|
-
SQL_LEASE_HOLDERS,
|
|
159
|
-
SQL_LEASE_RELEASE,
|
|
160
|
-
SQL_LEASE_RENEW,
|
|
161
|
-
SQL_NACK,
|
|
162
|
-
SQL_OUTBOX_CLAIM,
|
|
163
|
-
SQL_OUTBOX_MARK_PUBLISHED,
|
|
164
|
-
SQL_OUTBOX_RELEASE,
|
|
165
|
-
SQL_OUTBOX_STAGE,
|
|
166
|
-
SQL_SCHEDULER_STATE_GET,
|
|
167
|
-
SQL_SCHEDULER_STATE_MARK,
|
|
168
|
-
SQL_STATS,
|
|
169
|
-
SQL_STEP_GET,
|
|
170
|
-
SQL_STEP_PUT,
|
|
171
|
-
SQL_TRY_ADVISORY_LOCK,
|
|
172
|
-
} from './driver-pg-sql';
|
|
173
|
-
export type { RedisDriverOptions } from './driver-redis';
|
|
174
|
-
export { createRedisDriver } from './driver-redis';
|
|
110
|
+
export type { MemoryJobDriver, MemoryJobDriverOptions } from './driver-memory';
|
|
111
|
+
export { memoryJobDriver } from './driver-memory';
|
|
112
|
+
export type { PostgresJobDriverOptions } from './driver-pg';
|
|
113
|
+
export { postgresJobDriver, postgresLeader } from './driver-pg';
|
|
114
|
+
// Only the statement another workspace runs: the install (`cli/src/framework-schema.ts`, action,
|
|
115
|
+
// notify). The rest are this package's internals, read where they are defined, and tested there
|
|
116
|
+
// too — the array-parameter proofs are `driver-pg-array.live.test.ts` (25.0.0).
|
|
117
|
+
export { SQL_JOBS_TABLE } from './driver-pg-sql';
|
|
175
118
|
export { signalEnqueued, signalStaged } from './enqueue-signal';
|
|
176
119
|
export type { JobErrorCode } from './errors';
|
|
177
120
|
export {
|
|
@@ -190,7 +133,6 @@ export {
|
|
|
190
133
|
JobNotCancellableError,
|
|
191
134
|
JobRowStatusUnknownError,
|
|
192
135
|
JobSlotLostError,
|
|
193
|
-
JobsNotImplementedError,
|
|
194
136
|
JobTenantRequiredError,
|
|
195
137
|
JobTimeoutError,
|
|
196
138
|
LeaseLostError,
|
|
@@ -210,17 +152,19 @@ export {
|
|
|
210
152
|
JobOnSettledFailedError,
|
|
211
153
|
JobPageInvalidError,
|
|
212
154
|
} from './errors-operator';
|
|
213
|
-
export { JobNotRequeueableError } from './errors-requeue';
|
|
155
|
+
export { JobNotFoundError, JobNotRequeueableError } from './errors-requeue';
|
|
214
156
|
export type { EventBus, JobEvent, MemoryEventBusOptions, PublishOptions } from './events';
|
|
215
157
|
export {
|
|
216
|
-
|
|
158
|
+
EVENTS_PURGE_TARGET,
|
|
217
159
|
eventBus,
|
|
160
|
+
eventsPurgeTarget,
|
|
161
|
+
memoryEventBus,
|
|
218
162
|
publishEvent,
|
|
219
163
|
resetEventBus,
|
|
220
164
|
setEventBus,
|
|
221
165
|
} from './events';
|
|
222
|
-
export type {
|
|
223
|
-
export {
|
|
166
|
+
export type { PostgresEventBusOptions } from './events-pg';
|
|
167
|
+
export { postgresEventBus } from './events-pg';
|
|
224
168
|
export type { ExecuteJobOptions, JobExecution, JobOutcome } from './execute';
|
|
225
169
|
export { executeJob } from './execute';
|
|
226
170
|
export type { ExportDefinition, ExportReport } from './export';
|
|
@@ -291,7 +235,7 @@ export type {
|
|
|
291
235
|
} from './job';
|
|
292
236
|
export { describeJobs, getJob, isJobHandle, job, registeredJobs, resetJobs } from './job';
|
|
293
237
|
export type { HeldLease, LeaseStore, MemoryLeaseStore, MemoryLeaseStoreOptions } from './leases';
|
|
294
|
-
export {
|
|
238
|
+
export { jobLeaseKey, memoryLeaseStore } from './leases';
|
|
295
239
|
export type {
|
|
296
240
|
Lease,
|
|
297
241
|
LimitConfig,
|
|
@@ -301,7 +245,7 @@ export type {
|
|
|
301
245
|
LimitSnapshot,
|
|
302
246
|
RateLimit,
|
|
303
247
|
} from './limits';
|
|
304
|
-
export {
|
|
248
|
+
export { concurrencyLimiter, NO_TENANT, tenantKeyFrom } from './limits';
|
|
305
249
|
export {
|
|
306
250
|
queueDeadJobs,
|
|
307
251
|
queueOldestReady,
|
|
@@ -311,27 +255,27 @@ export {
|
|
|
311
255
|
export type {
|
|
312
256
|
EnqueueOptions,
|
|
313
257
|
JobsFacade,
|
|
314
|
-
MemoryOutboxOptions,
|
|
315
258
|
MemoryOutboxStore,
|
|
259
|
+
MemoryOutboxStoreOptions,
|
|
316
260
|
OutboxDeps,
|
|
317
261
|
OutboxRecord,
|
|
318
262
|
OutboxStore,
|
|
319
263
|
} from './outbox';
|
|
320
264
|
export {
|
|
321
|
-
createJobsFacade,
|
|
322
|
-
createMemoryOutboxStore,
|
|
323
265
|
enqueueInTx,
|
|
324
266
|
jobsFacade,
|
|
267
|
+
memoryOutboxStore,
|
|
268
|
+
outboxJobsFacade,
|
|
325
269
|
resetJobsFacade,
|
|
326
270
|
setJobsFacade,
|
|
327
271
|
} from './outbox';
|
|
328
272
|
// One definition of the lease, consumed by both stores — a memory default and a pg default that
|
|
329
273
|
// could drift are two answers to "how long is a claim mine for", and the shorter one duplicates.
|
|
330
274
|
export { DEFAULT_OUTBOX_CLAIM_LEASE_MS } from './outbox-lease';
|
|
331
|
-
export type {
|
|
332
|
-
export {
|
|
275
|
+
export type { PostgresOutboxStoreOptions } from './outbox-pg';
|
|
276
|
+
export { postgresOutboxStore } from './outbox-pg';
|
|
333
277
|
export type { OutboxRelay, RelayOptions } from './outbox-relay';
|
|
334
|
-
export {
|
|
278
|
+
export { outboxRelay } from './outbox-relay';
|
|
335
279
|
export type { ProgressFn } from './progress';
|
|
336
280
|
export type {
|
|
337
281
|
PurgeDefinition,
|
|
@@ -347,7 +291,7 @@ export type { IntervalScheduler } from './renewal-timer';
|
|
|
347
291
|
export type { BackoffStrategy, Random, RetryDecision, RetryPolicy } from './retry';
|
|
348
292
|
export { backoffDelayMs, DEFAULT_RETRY, isFinalAttempt, nextRetry, retrySchedule } from './retry';
|
|
349
293
|
export type { JobRetryDecision, JobStopReason } from './retry-classification';
|
|
350
|
-
export {
|
|
294
|
+
export { failureForRow, nextRetryForError } from './retry-classification';
|
|
351
295
|
export type {
|
|
352
296
|
CronResolver,
|
|
353
297
|
DispatchedOccurrence,
|
|
@@ -356,27 +300,28 @@ export type {
|
|
|
356
300
|
} from './scheduler';
|
|
357
301
|
export {
|
|
358
302
|
COUNTER_ROLLUP_INTERVAL_MS,
|
|
359
|
-
|
|
303
|
+
jobScheduler,
|
|
360
304
|
PAUSE_RECHECK_MS,
|
|
361
305
|
} from './scheduler';
|
|
362
306
|
export type { LeaderElection } from './scheduler-leader';
|
|
363
307
|
export { soleLeader } from './scheduler-leader';
|
|
364
308
|
export { nextTaskRun } from './scheduler-occurrences';
|
|
365
|
-
export type {
|
|
309
|
+
export type { PostgresLeaseLeaderOptions } from './scheduler-pg';
|
|
366
310
|
export {
|
|
367
|
-
createPgLeaseLeader,
|
|
368
311
|
currentLeader,
|
|
369
312
|
DEFAULT_LEADER_TTL_MS,
|
|
370
313
|
LEASE_RENEWALS_PER_TTL,
|
|
371
|
-
|
|
314
|
+
postgresLeaseLeader,
|
|
315
|
+
postgresSchedulerState,
|
|
372
316
|
} from './scheduler-pg';
|
|
373
317
|
export type { ScheduledFire, SchedulerState } from './scheduler-state';
|
|
374
|
-
export {
|
|
318
|
+
export { fireThroughDriver, memorySchedulerState } from './scheduler-state';
|
|
375
319
|
export type { JobCompleted, JobFailed, JobSettled } from './settled';
|
|
376
320
|
export { ON_SETTLED_ATTEMPTS } from './settled';
|
|
377
321
|
export type {
|
|
378
322
|
EventLookup,
|
|
379
323
|
StepApi,
|
|
324
|
+
StepFence,
|
|
380
325
|
StepRecord,
|
|
381
326
|
StepRunner,
|
|
382
327
|
StepRunnerOptions,
|
|
@@ -385,14 +330,14 @@ export type {
|
|
|
385
330
|
WaitForEventOptions,
|
|
386
331
|
} from './steps';
|
|
387
332
|
export {
|
|
388
|
-
createStepRunner,
|
|
389
333
|
isStepStatus,
|
|
390
334
|
isStepSuspension,
|
|
391
335
|
MAX_TRACE_NAMES,
|
|
392
336
|
STEP_STATUSES,
|
|
393
337
|
StepSuspension,
|
|
338
|
+
stepRunner,
|
|
394
339
|
} from './steps';
|
|
395
|
-
export {
|
|
340
|
+
export { memoryStepStore } from './steps-memory';
|
|
396
341
|
export type {
|
|
397
342
|
CatchUpPolicy,
|
|
398
343
|
TaskDefinition,
|
|
@@ -415,10 +360,11 @@ export type {
|
|
|
415
360
|
WebhookDeliveryInput,
|
|
416
361
|
WebhookEndpoint,
|
|
417
362
|
WebhookEvent,
|
|
418
|
-
WebhookFetch,
|
|
419
363
|
WebhookReport,
|
|
420
364
|
} from './webhook';
|
|
421
|
-
export { DEFAULT_WEBHOOK_DISABLE_AFTER,
|
|
365
|
+
export { DEFAULT_WEBHOOK_DISABLE_AFTER, webhook } from './webhook';
|
|
366
|
+
export type { WebhookFetch } from './webhook-attempt';
|
|
367
|
+
export { WEBHOOK_CONTENT_TYPE } from './webhook-attempt';
|
|
422
368
|
export {
|
|
423
369
|
WebhookDeliveryFailedError,
|
|
424
370
|
WebhookDeliveryRejectedError,
|
|
@@ -432,4 +378,4 @@ export {
|
|
|
432
378
|
export type { MemoryWebhookLedger, WebhookAttempt, WebhookLedger } from './webhook-ledger';
|
|
433
379
|
export { DEFAULT_MAX_WEBHOOK_ATTEMPTS, memoryWebhookLedger } from './webhook-ledger';
|
|
434
380
|
export type { Worker, WorkerOptions, WorkerStats } from './worker';
|
|
435
|
-
export {
|
|
381
|
+
export { jobWorker } from './worker';
|
package/src/inspect-operator.ts
CHANGED
|
@@ -3,16 +3,16 @@
|
|
|
3
3
|
// know whether the job is gone or was never theirs to remove. Apart from `inspect.ts`, which is
|
|
4
4
|
// the read side.
|
|
5
5
|
|
|
6
|
+
import { NotImplementedError } from '@ultimat3/core';
|
|
6
7
|
import type { JobDriver, JobRecord } from './driver';
|
|
7
|
-
import { JobsNotImplementedError } from './errors';
|
|
8
8
|
import { JobNotPromotableError } from './errors-operator';
|
|
9
9
|
import type { JobIntrospection, PausedName } from './introspection';
|
|
10
10
|
|
|
11
11
|
function operatorOf(driver: JobDriver): JobIntrospection {
|
|
12
12
|
if (driver.introspect === undefined) {
|
|
13
|
-
throw new
|
|
14
|
-
|
|
15
|
-
fix: 'call setJobDriver(
|
|
13
|
+
throw new NotImplementedError({
|
|
14
|
+
cause: `the operator surface of the "${driver.name}" jobs driver is not implemented: the driver has no introspect`,
|
|
15
|
+
fix: 'call setJobDriver(postgresJobDriver()) at boot — the pg driver implements introspect — then: x jobs ls --json',
|
|
16
16
|
});
|
|
17
17
|
}
|
|
18
18
|
return driver.introspect;
|
package/src/inspect.ts
CHANGED
|
@@ -2,11 +2,12 @@
|
|
|
2
2
|
// object so `x jobs ... --json` and the MCP tool share one shape — an agent debugging a stuck
|
|
3
3
|
// queue reads exactly what the dashboard renders.
|
|
4
4
|
|
|
5
|
+
import { NotImplementedError } from '@ultimat3/core';
|
|
5
6
|
import type { BackfillProgress } from './backfill-inspect';
|
|
6
7
|
import { backfillForRun } from './backfill-inspect';
|
|
7
8
|
import type { WhenBusy } from './concurrency';
|
|
8
9
|
import type { JobDriver, JobRecord, QueueStats } from './driver';
|
|
9
|
-
import { CancelUnsupportedError, JobNotCancellableError
|
|
10
|
+
import { CancelUnsupportedError, JobNotCancellableError } from './errors';
|
|
10
11
|
import type { JobFilter, JobProgress } from './introspection';
|
|
11
12
|
import type { AnyJobHandle } from './job';
|
|
12
13
|
import { registeredJobs } from './job';
|
|
@@ -111,9 +112,9 @@ const iso = (ms: number | undefined): string | null =>
|
|
|
111
112
|
|
|
112
113
|
function requireIntrospection(driver: JobDriver): NonNullable<JobDriver['introspect']> {
|
|
113
114
|
if (driver.introspect === undefined) {
|
|
114
|
-
throw new
|
|
115
|
-
|
|
116
|
-
fix: 'call setJobDriver(
|
|
115
|
+
throw new NotImplementedError({
|
|
116
|
+
cause: `introspection for the "${driver.name}" jobs driver is not implemented: the driver has no introspect`,
|
|
117
|
+
fix: 'call setJobDriver(postgresJobDriver()) at boot — only the pg driver implements introspect — then: x jobs ls --json',
|
|
117
118
|
});
|
|
118
119
|
}
|
|
119
120
|
return driver.introspect;
|