@ultimat3/jobs 30.9.0 → 31.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +72 -107
- package/README.md +202 -55
- package/package.json +8 -7
- package/src/adapter.ts +37 -0
- package/src/backfill-ledger.ts +2 -2
- package/src/backfill-rate.ts +1 -0
- package/src/clock.ts +2 -2
- package/src/drain-wait.ts +1 -0
- package/src/driver-memory.ts +2 -2
- package/src/driver-pg-ddl.ts +19 -0
- package/src/driver-pg-event-wake-sql.ts +43 -22
- package/src/driver-pg-sql.ts +68 -27
- package/src/driver-pg.ts +11 -10
- package/src/driver-redis.ts +2 -2
- package/src/driver.ts +14 -8
- package/src/error-codes.ts +27 -0
- package/src/errors-concurrency.ts +1 -1
- package/src/errors.ts +1 -1
- package/src/events-cursor.ts +31 -0
- package/src/events-pg.ts +52 -6
- package/src/events.ts +72 -36
- package/src/export.ts +2 -2
- package/src/gdpr-audit.ts +75 -0
- package/src/gdpr-coverage.ts +94 -0
- package/src/gdpr-errors.ts +35 -0
- package/src/import.ts +2 -2
- package/src/index.ts +45 -206
- package/src/inspect-operator.ts +2 -2
- package/src/inspect.ts +1 -1
- package/src/job-instrument.ts +115 -0
- package/src/job.ts +41 -4
- package/src/jobs-retention.ts +23 -6
- package/src/leases.ts +13 -3
- package/src/ledger-book.ts +168 -0
- package/src/ledger-errors.ts +48 -0
- package/src/ledger.ts +145 -0
- package/src/limits.ts +57 -19
- package/src/lock-errors.ts +54 -0
- package/src/lock.ts +176 -0
- package/src/outbox-relay.ts +4 -0
- package/src/outbox.ts +3 -3
- package/src/partitions.ts +119 -0
- package/src/reindex-sweep.ts +94 -0
- package/src/reindex.ts +147 -0
- package/src/retry-classification.ts +3 -2
- package/src/retry-jitter.ts +32 -0
- package/src/retry.ts +13 -16
- package/src/scheduler.ts +10 -6
- package/src/steps-api.ts +44 -5
- package/src/steps-suspension.ts +34 -3
- package/src/steps-wait.ts +52 -4
- package/src/steps.ts +1 -0
- package/src/subject-jobs.ts +12 -4
- package/src/webhook.ts +2 -2
- package/src/worker-admit.ts +8 -5
- package/src/worker-drain-cutoff.ts +1 -0
- package/src/worker-loop.ts +28 -5
- package/src/worker-registry.ts +7 -5
- package/src/worker-run.ts +53 -9
- package/src/worker.ts +8 -8
package/src/retry.ts
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// when attempt 5 lands without running the queue. There is no `x jobs schedule`: the subcommands
|
|
4
4
|
// are `ls`, `show`, `retry`, `cancel`, `drain`.
|
|
5
5
|
|
|
6
|
-
import type { BackoffCurve, Random } from '@ultimat3/core';
|
|
6
|
+
import type { BackoffCurve, JitterMode, Random } from '@ultimat3/core';
|
|
7
7
|
import { backoffDelay } from '@ultimat3/core';
|
|
8
8
|
import type { DurationInput } from './clock';
|
|
9
9
|
import { finiteDurationMs } from './clock';
|
|
@@ -24,10 +24,11 @@ export interface RetryPolicy {
|
|
|
24
24
|
/** Ceiling for any single delay. Default 1h. */
|
|
25
25
|
readonly maxDelay?: DurationInput;
|
|
26
26
|
/**
|
|
27
|
-
*
|
|
28
|
-
* retries in lockstep
|
|
27
|
+
* Core's `JitterMode`. `'equal'` (half fixed, half random) by default: a job that has already
|
|
28
|
+
* failed twice keeps a floor. `'none'` retries a burst in lockstep — the thundering herd that
|
|
29
|
+
* killed the dependency — and is for tests and printable schedules.
|
|
29
30
|
*/
|
|
30
|
-
readonly jitter?:
|
|
31
|
+
readonly jitter?: JitterMode;
|
|
31
32
|
/** Default true: an exhausted job is parked, never dropped. */
|
|
32
33
|
readonly deadLetter?: boolean;
|
|
33
34
|
}
|
|
@@ -37,7 +38,7 @@ export const DEFAULT_RETRY = {
|
|
|
37
38
|
backoff: 'exponential',
|
|
38
39
|
delay: 1_000,
|
|
39
40
|
maxDelay: 3_600_000,
|
|
40
|
-
jitter:
|
|
41
|
+
jitter: 'equal',
|
|
41
42
|
deadLetter: true,
|
|
42
43
|
} satisfies RetryPolicy;
|
|
43
44
|
|
|
@@ -48,14 +49,12 @@ export type { Random };
|
|
|
48
49
|
*
|
|
49
50
|
* The arithmetic is core's — one curve-and-jitter function for the framework, because four
|
|
50
51
|
* packages shipped four of them. What stays here is the part that is this package's and not the
|
|
51
|
-
* framework's: `DurationInput` (`'30s'` as well as a number)
|
|
52
|
-
*
|
|
52
|
+
* framework's: `DurationInput` (`'30s'` as well as a number) and the `DEFAULT_RETRY` fallbacks.
|
|
53
|
+
* `jitter` is core's `JitterMode` itself since 31.0.0, never a flag mapped onto two of its modes.
|
|
53
54
|
*
|
|
54
|
-
* `?? DEFAULT_RETRY.jitter`, like every other option
|
|
55
|
-
* `
|
|
56
|
-
*
|
|
57
|
-
* Masked for jobs declared through `job()` (it merges the defaults) and live for every direct
|
|
58
|
-
* caller of this exported function, `retrySchedule` included.
|
|
55
|
+
* `?? DEFAULT_RETRY.jitter`, like every other option: an omitted `jitter` is the documented
|
|
56
|
+
* `'equal'`, never off — masked for jobs declared through `job()` (it merges the defaults) and live
|
|
57
|
+
* for every direct caller of this exported function, `retrySchedule` included.
|
|
59
58
|
*/
|
|
60
59
|
export function backoffDelayMs(policy: RetryPolicy, attempt: number, random?: Random): number {
|
|
61
60
|
return backoffDelay({
|
|
@@ -63,9 +62,7 @@ export function backoffDelayMs(policy: RetryPolicy, attempt: number, random?: Ra
|
|
|
63
62
|
base: finiteDurationMs(policy.delay ?? DEFAULT_RETRY.delay, 'retry', 'delay'),
|
|
64
63
|
max: finiteDurationMs(policy.maxDelay ?? DEFAULT_RETRY.maxDelay, 'retry', 'maxDelay'),
|
|
65
64
|
curve: policy.backoff ?? DEFAULT_RETRY.backoff,
|
|
66
|
-
|
|
67
|
-
// and this package's `jitter: true` has meant "half fixed, half random" since it shipped.
|
|
68
|
-
jitter: (policy.jitter ?? DEFAULT_RETRY.jitter) === true ? 'equal' : 'none',
|
|
65
|
+
jitter: policy.jitter ?? DEFAULT_RETRY.jitter,
|
|
69
66
|
random,
|
|
70
67
|
});
|
|
71
68
|
}
|
|
@@ -132,7 +129,7 @@ export function nextRetry(policy: RetryPolicy, attempt: number, random?: Random)
|
|
|
132
129
|
|
|
133
130
|
/** Every delay in the policy, jitter off, for docs and `--json` output. */
|
|
134
131
|
export function retrySchedule(policy: RetryPolicy): readonly number[] {
|
|
135
|
-
const deterministic: RetryPolicy = { ...policy, jitter:
|
|
132
|
+
const deterministic: RetryPolicy = { ...policy, jitter: 'none' };
|
|
136
133
|
const out: number[] = [];
|
|
137
134
|
for (let attempt = 1; attempt < policy.attempts; attempt += 1) {
|
|
138
135
|
out.push(backoffDelayMs(deterministic, attempt));
|
package/src/scheduler.ts
CHANGED
|
@@ -27,6 +27,7 @@ import type { DrainBudget } from './drain-wait';
|
|
|
27
27
|
import { createDrainBudget, settleAllBy } from './drain-wait';
|
|
28
28
|
import type { JobDriver } from './driver';
|
|
29
29
|
import { signalEnqueued } from './enqueue-signal';
|
|
30
|
+
import { instrumentedFire } from './job-instrument';
|
|
30
31
|
import { enqueuedAttempts } from './retry';
|
|
31
32
|
import type { LeaderElection } from './scheduler-leader';
|
|
32
33
|
import { soleLeader } from './scheduler-leader';
|
|
@@ -165,12 +166,15 @@ export function jobScheduler(options: SchedulerOptions): Scheduler {
|
|
|
165
166
|
});
|
|
166
167
|
// Jobs and watermark in ONE step (`SchedulerState.fire`). `undefined` is an occurrence some
|
|
167
168
|
// other dispatcher already fired: nothing was queued, and nothing is reported as dispatched.
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
169
|
+
// Under the fire's producer span, which stamps each request's `traceparent` (`job-instrument.ts`).
|
|
170
|
+
const results = await instrumentedFire(handle.name, requests, (jobs) =>
|
|
171
|
+
schedulerState.fire(options.driver, {
|
|
172
|
+
task: handle.name,
|
|
173
|
+
occurrenceMs,
|
|
174
|
+
...(watermarkMs === undefined ? {} : { watermarkMs }),
|
|
175
|
+
jobs,
|
|
176
|
+
}),
|
|
177
|
+
);
|
|
174
178
|
if (results === undefined) {
|
|
175
179
|
logger.info('jobs.scheduler.already-fired', {
|
|
176
180
|
task: handle.name,
|
package/src/steps-api.ts
CHANGED
|
@@ -14,6 +14,18 @@ export const DEFAULT_EVENT_POLL_MS = 30_000;
|
|
|
14
14
|
*/
|
|
15
15
|
export const WAKING_EVENT_POLL_MS = 900_000;
|
|
16
16
|
|
|
17
|
+
/**
|
|
18
|
+
* Where a wait begins — its stamp, and what the bus could see then (`EventLookup.mark`) — and the
|
|
19
|
+
* events it refused BEHIND its cursor: one owed through the horizon sorts before the cursor, so
|
|
20
|
+
* the cursor stays and its id is excluded by name. Moving the cursor back to it re-opened every
|
|
21
|
+
* event between the two, the one refused before it included.
|
|
22
|
+
*/
|
|
23
|
+
export interface EventSince {
|
|
24
|
+
readonly at: number;
|
|
25
|
+
readonly horizon?: string | undefined;
|
|
26
|
+
readonly refused?: readonly string[] | undefined;
|
|
27
|
+
}
|
|
28
|
+
|
|
17
29
|
/** The slice of the event bus a waiting step needs. Implemented by `events.ts`. */
|
|
18
30
|
export interface EventLookup {
|
|
19
31
|
/**
|
|
@@ -31,18 +43,45 @@ export interface EventLookup {
|
|
|
31
43
|
*/
|
|
32
44
|
now(): Promise<number>;
|
|
33
45
|
/**
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
46
|
+
* A NEW wait's starting line, read in ONE statement: the bus's `now()`, and the `horizon` — what
|
|
47
|
+
* the bus could see at that instant. Over a database an event is stamped at INSERT and visible
|
|
48
|
+
* at COMMIT, so a publish whose transaction was still open when the wait began carries a stamp
|
|
49
|
+
* BEFORE it: by `afterMs` alone every later search skips it and the wait runs to its timeout.
|
|
50
|
+
* `find` given the horizon also matches every event the horizon could not see. Absent on a bus
|
|
51
|
+
* with no transactions (the memory one): there every publish is visible as it is stamped.
|
|
52
|
+
*/
|
|
53
|
+
mark?(): Promise<{ readonly at: number; readonly horizon?: string }>;
|
|
54
|
+
/**
|
|
55
|
+
* The earliest match past the cursor in the bus's own `(published, id)` order
|
|
56
|
+
* (`events-cursor.ts`): published at or after `afterMs` — and, given `afterId`, strictly past
|
|
57
|
+
* `(afterMs, afterId)`. The id is the cursor a refused event leaves: a floor of
|
|
58
|
+
* `publishedAt + 1` stepped over every valid event stamped in the same millisecond. Compared,
|
|
59
|
+
* never looked up, so a cursor whose event was purged still excludes what it excluded.
|
|
60
|
+
*
|
|
61
|
+
* Given a `horizon` (from `mark`, or a hit's own), ALSO every event that horizon could not see,
|
|
62
|
+
* whatever its stamp. A hit answers the horizon of the search that found it, which is the one a
|
|
63
|
+
* refusal's cursor must carry: anything that search could not see is still owed to the wait.
|
|
64
|
+
*
|
|
65
|
+
* `since` is the WAIT's own line, which a refusal's cursor never moves: only an event stamped at
|
|
66
|
+
* or after `since.at`, or unseen by `since.horizon` (the mark's), is the wait's at all. Absent,
|
|
67
|
+
* it is `{ at: afterMs, horizon }` — right for a search that never refused anything. A cursor
|
|
68
|
+
* alone let an event committed BEFORE the wait sort past a refused one and be handed out.
|
|
38
69
|
*/
|
|
39
70
|
find(
|
|
40
71
|
event: string,
|
|
41
72
|
correlationKey: string | undefined,
|
|
42
73
|
afterMs: number,
|
|
43
74
|
afterId?: string,
|
|
75
|
+
horizon?: string,
|
|
76
|
+
since?: EventSince,
|
|
44
77
|
): Promise<
|
|
45
|
-
|
|
78
|
+
| {
|
|
79
|
+
readonly id: string;
|
|
80
|
+
readonly payload: unknown;
|
|
81
|
+
readonly publishedAt: number;
|
|
82
|
+
readonly horizon?: string;
|
|
83
|
+
}
|
|
84
|
+
| undefined
|
|
46
85
|
>;
|
|
47
86
|
}
|
|
48
87
|
|
package/src/steps-suspension.ts
CHANGED
|
@@ -43,11 +43,22 @@ export function isWaitTimedOut(output: unknown): boolean {
|
|
|
43
43
|
|
|
44
44
|
const WAIT_FLOOR_KEY = '~ultimate.waitFloor';
|
|
45
45
|
const WAIT_FLOOR_ID_KEY = '~ultimate.waitFloorId';
|
|
46
|
+
const WAIT_HORIZON_KEY = '~ultimate.waitHorizon';
|
|
47
|
+
const WAIT_SINCE_HORIZON_KEY = '~ultimate.waitSinceHorizon';
|
|
48
|
+
const WAIT_REFUSED_KEY = '~ultimate.waitRefused';
|
|
46
49
|
|
|
47
|
-
/**
|
|
50
|
+
/**
|
|
51
|
+
* Where a wait's next search starts: at `afterMs`, past the event `afterId` names — and, given a
|
|
52
|
+
* `horizon` (`EventLookup.mark`), at every event that horizon could not see, whatever its stamp.
|
|
53
|
+
*/
|
|
48
54
|
export interface WaitFloor {
|
|
49
55
|
readonly afterMs: number;
|
|
50
56
|
readonly afterId?: string;
|
|
57
|
+
readonly horizon?: string;
|
|
58
|
+
/** The MARK's horizon: the wait's own line, kept by every refusal (`EventSince`). */
|
|
59
|
+
readonly sinceHorizon?: string;
|
|
60
|
+
/** Events refused BEHIND the cursor, excluded by id (`EventSince.refused`). */
|
|
61
|
+
readonly refused?: readonly string[];
|
|
51
62
|
}
|
|
52
63
|
|
|
53
64
|
/**
|
|
@@ -56,10 +67,19 @@ export interface WaitFloor {
|
|
|
56
67
|
* event, so without this a repaired publisher's event sat behind the refused one until its TTL.
|
|
57
68
|
* The refused event's id rides with its instant: a floor of `publishedAt + 1` skipped every valid
|
|
58
69
|
* event stamped in the same millisecond. A record written before the id has `afterMs` alone.
|
|
70
|
+
* Over a bus that marks (`EventLookup.mark`), every parked wait holds one from its first park: the
|
|
71
|
+
* horizon rides here, since a re-poll must search under the line its wait began at.
|
|
59
72
|
*/
|
|
60
|
-
export const waitFloor = (
|
|
73
|
+
export const waitFloor = (
|
|
74
|
+
floor: WaitFloor,
|
|
75
|
+
): Record<string, number | string | readonly string[]> => ({
|
|
61
76
|
[WAIT_FLOOR_KEY]: floor.afterMs,
|
|
62
77
|
...(floor.afterId === undefined ? {} : { [WAIT_FLOOR_ID_KEY]: floor.afterId }),
|
|
78
|
+
...(floor.horizon === undefined ? {} : { [WAIT_HORIZON_KEY]: floor.horizon }),
|
|
79
|
+
...(floor.sinceHorizon === undefined ? {} : { [WAIT_SINCE_HORIZON_KEY]: floor.sinceHorizon }),
|
|
80
|
+
...(floor.refused === undefined || floor.refused.length === 0
|
|
81
|
+
? {}
|
|
82
|
+
: { [WAIT_REFUSED_KEY]: floor.refused }),
|
|
63
83
|
});
|
|
64
84
|
|
|
65
85
|
export function waitFloorOf(output: unknown): WaitFloor | undefined {
|
|
@@ -68,7 +88,18 @@ export function waitFloorOf(output: unknown): WaitFloor | undefined {
|
|
|
68
88
|
const afterMs = record[WAIT_FLOOR_KEY];
|
|
69
89
|
if (typeof afterMs !== 'number') return undefined;
|
|
70
90
|
const afterId = record[WAIT_FLOOR_ID_KEY];
|
|
71
|
-
|
|
91
|
+
const horizon = record[WAIT_HORIZON_KEY];
|
|
92
|
+
const sinceHorizon = record[WAIT_SINCE_HORIZON_KEY];
|
|
93
|
+
const refused = record[WAIT_REFUSED_KEY];
|
|
94
|
+
return {
|
|
95
|
+
afterMs,
|
|
96
|
+
...(typeof afterId === 'string' ? { afterId } : {}),
|
|
97
|
+
...(typeof horizon === 'string' ? { horizon } : {}),
|
|
98
|
+
...(typeof sinceHorizon === 'string' ? { sinceHorizon } : {}),
|
|
99
|
+
...(Array.isArray(refused)
|
|
100
|
+
? { refused: refused.filter((id): id is string => typeof id === 'string') }
|
|
101
|
+
: {}),
|
|
102
|
+
};
|
|
72
103
|
}
|
|
73
104
|
|
|
74
105
|
/**
|
package/src/steps-wait.ts
CHANGED
|
@@ -8,10 +8,12 @@ import { formatIssues, validateAsync } from '@ultimat3/schema';
|
|
|
8
8
|
import { finiteDurationMs } from './clock';
|
|
9
9
|
import { JobTimeoutError } from './errors';
|
|
10
10
|
import { JobEventInvalidError } from './errors-steps';
|
|
11
|
+
import { eventBefore } from './events-cursor';
|
|
11
12
|
import type { WaitForEventOptions } from './steps-api';
|
|
12
13
|
import type { RunnerCore } from './steps-core';
|
|
13
14
|
import { jsonForm } from './steps-output';
|
|
14
15
|
import type { StepRecord } from './steps-store';
|
|
16
|
+
import type { WaitFloor } from './steps-suspension';
|
|
15
17
|
import {
|
|
16
18
|
isWaitTimedOut,
|
|
17
19
|
StepSuspension,
|
|
@@ -22,7 +24,12 @@ import {
|
|
|
22
24
|
|
|
23
25
|
const DEFAULT_WAIT_MS = 86_400_000;
|
|
24
26
|
|
|
25
|
-
type Hit = {
|
|
27
|
+
type Hit = {
|
|
28
|
+
readonly id: string;
|
|
29
|
+
readonly payload: unknown;
|
|
30
|
+
readonly publishedAt: number;
|
|
31
|
+
readonly horizon?: string;
|
|
32
|
+
};
|
|
26
33
|
|
|
27
34
|
export function waitForEventStep(core: RunnerCore) {
|
|
28
35
|
const { runId, jobName } = core;
|
|
@@ -50,7 +57,10 @@ export function waitForEventStep(core: RunnerCore) {
|
|
|
50
57
|
const at = core.now();
|
|
51
58
|
// A NEW wait is stamped by the bus, whose clock stamps every `publishedAt` it is compared
|
|
52
59
|
// with; the runner's own only when there is no bus to ask. A re-poll keeps what it persisted.
|
|
53
|
-
|
|
60
|
+
// Its horizon is read in the same statement and persisted as the floor: a publish whose
|
|
61
|
+
// transaction was open at this instant is stamped before it, and is owed to this wait.
|
|
62
|
+
const mark = existing === undefined ? await core.events?.mark?.() : undefined;
|
|
63
|
+
const startedAt = existing?.startedAt ?? mark?.at ?? (await core.events?.now()) ?? at;
|
|
54
64
|
const timeoutMs = finiteDurationMs(
|
|
55
65
|
waitOptions.timeout ?? DEFAULT_WAIT_MS,
|
|
56
66
|
'step.waitForEvent',
|
|
@@ -61,7 +71,41 @@ export function waitForEventStep(core: RunnerCore) {
|
|
|
61
71
|
const attempts = (existing?.attempts ?? 0) + 1;
|
|
62
72
|
const keyed = correlationKey === undefined ? {} : { correlationKey };
|
|
63
73
|
|
|
64
|
-
const floor =
|
|
74
|
+
const floor =
|
|
75
|
+
waitFloorOf(existing?.output) ??
|
|
76
|
+
(mark?.horizon === undefined
|
|
77
|
+
? undefined
|
|
78
|
+
: { afterMs: startedAt, horizon: mark.horizon, sinceHorizon: mark.horizon });
|
|
79
|
+
// The wait's own line, which a refusal's cursor never moves: `startedAt` and the mark's horizon.
|
|
80
|
+
const since = { at: startedAt, horizon: floor?.sinceHorizon, refused: floor?.refused };
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* The floor after `hit` is refused — monotone, so no refused event can match again. A hit past
|
|
84
|
+
* the cursor moves it there, under the horizon of the search that found it: what that search
|
|
85
|
+
* could not see is still owed, and everything it saw up to the hit is spent. A hit BEHIND the
|
|
86
|
+
* cursor (owed through a horizon) leaves the cursor where it is and is excluded by id: moving
|
|
87
|
+
* the cursor back re-opened every event between the two, the one refused before it included.
|
|
88
|
+
*/
|
|
89
|
+
const refusedFloor = (hit: Hit): WaitFloor => {
|
|
90
|
+
const cursor = { publishedAt: floor?.afterMs ?? startedAt, id: floor?.afterId ?? '' };
|
|
91
|
+
const sinceHorizon =
|
|
92
|
+
floor?.sinceHorizon === undefined ? {} : { sinceHorizon: floor.sinceHorizon };
|
|
93
|
+
if (eventBefore(cursor, hit)) {
|
|
94
|
+
return {
|
|
95
|
+
afterMs: hit.publishedAt,
|
|
96
|
+
afterId: hit.id,
|
|
97
|
+
...(hit.horizon === undefined ? {} : { horizon: hit.horizon }),
|
|
98
|
+
...sinceHorizon,
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
return {
|
|
102
|
+
afterMs: cursor.publishedAt,
|
|
103
|
+
...(floor?.afterId === undefined ? {} : { afterId: floor.afterId }),
|
|
104
|
+
...(floor?.horizon === undefined ? {} : { horizon: floor.horizon }),
|
|
105
|
+
...sinceHorizon,
|
|
106
|
+
refused: [...(floor?.refused ?? []), hit.id],
|
|
107
|
+
};
|
|
108
|
+
};
|
|
65
109
|
const parked = {
|
|
66
110
|
runId,
|
|
67
111
|
name,
|
|
@@ -81,7 +125,7 @@ export function waitForEventStep(core: RunnerCore) {
|
|
|
81
125
|
await core
|
|
82
126
|
.put({
|
|
83
127
|
...parked,
|
|
84
|
-
output: waitFloor(
|
|
128
|
+
output: waitFloor(refusedFloor(hit)),
|
|
85
129
|
wakeAt: at,
|
|
86
130
|
})
|
|
87
131
|
.catch((unwritten: unknown) =>
|
|
@@ -108,6 +152,8 @@ export function waitForEventStep(core: RunnerCore) {
|
|
|
108
152
|
correlationKey,
|
|
109
153
|
floor?.afterMs ?? startedAt,
|
|
110
154
|
floor?.afterId,
|
|
155
|
+
floor?.horizon,
|
|
156
|
+
since,
|
|
111
157
|
);
|
|
112
158
|
if (hit !== undefined) return consume(hit);
|
|
113
159
|
|
|
@@ -155,6 +201,8 @@ export function waitForEventStep(core: RunnerCore) {
|
|
|
155
201
|
correlationKey,
|
|
156
202
|
floor?.afterMs ?? startedAt,
|
|
157
203
|
floor?.afterId,
|
|
204
|
+
floor?.horizon,
|
|
205
|
+
since,
|
|
158
206
|
);
|
|
159
207
|
if (late !== undefined) return consume(late);
|
|
160
208
|
}
|
package/src/steps.ts
CHANGED
package/src/subject-jobs.ts
CHANGED
|
@@ -3,13 +3,15 @@
|
|
|
3
3
|
// table before a row is touched. Who may call them is the app's action: no route, no MCP tool.
|
|
4
4
|
|
|
5
5
|
import type { Clock } from '@ultimat3/core';
|
|
6
|
-
import {
|
|
6
|
+
import { liveClock } from '@ultimat3/core';
|
|
7
7
|
import { registeredEntities } from '@ultimat3/entity';
|
|
8
8
|
import type { StandardSchemaV1 } from '@ultimat3/schema';
|
|
9
9
|
import type { DurationInput } from './clock';
|
|
10
10
|
import { nowMs } from './clock';
|
|
11
11
|
import type { ExportFormat } from './export-format';
|
|
12
12
|
import type { ExportSink } from './export-sink';
|
|
13
|
+
import { auditedErase } from './gdpr-audit';
|
|
14
|
+
import { unerasedPersonalData } from './gdpr-coverage';
|
|
13
15
|
import type { JobHandle } from './job';
|
|
14
16
|
import { job } from './job';
|
|
15
17
|
import type { RetryPolicy } from './retry';
|
|
@@ -57,6 +59,10 @@ export function plannedSubject<I>(subject: Subject<I>): SubjectPlan {
|
|
|
57
59
|
});
|
|
58
60
|
const [first] = problems;
|
|
59
61
|
if (first !== undefined) throw first;
|
|
62
|
+
// A reached table whose treatment would leave a `.personalData()` column standing: refused before
|
|
63
|
+
// a row is touched, as an untreated table is (`gdpr-coverage.ts`).
|
|
64
|
+
const [leak] = unerasedPersonalData(plan);
|
|
65
|
+
if (leak !== undefined) throw leak;
|
|
60
66
|
return plan;
|
|
61
67
|
}
|
|
62
68
|
|
|
@@ -78,14 +84,16 @@ export function subjectErase<I>(definition: SubjectJobDefinition<I>): JobHandle<
|
|
|
78
84
|
...common(definition),
|
|
79
85
|
// One live erase per person: a second request while one runs is the same run.
|
|
80
86
|
idempotencyKey: (input) => `${definition.name}:${subject.init.key(input)}`,
|
|
81
|
-
run: ({ step, input }): Promise<SubjectEraseReport> =>
|
|
82
|
-
|
|
87
|
+
run: ({ step, input, ctx, runId }): Promise<SubjectEraseReport> =>
|
|
88
|
+
auditedErase({ job: definition.name, subject: subject.entity, runId, ctx }, step, () =>
|
|
89
|
+
subjectErasePass(step, readerFor(subject, input), subject.init.targets ?? []),
|
|
90
|
+
),
|
|
83
91
|
});
|
|
84
92
|
}
|
|
85
93
|
|
|
86
94
|
export function subjectExport<I>(definition: SubjectExportDefinition<I>): JobHandle<I> {
|
|
87
95
|
const { subject } = definition;
|
|
88
|
-
const clock = definition.clock ??
|
|
96
|
+
const clock = definition.clock ?? liveClock;
|
|
89
97
|
return job<I>({
|
|
90
98
|
...common(definition),
|
|
91
99
|
// One live pass per artifact, derived as `exportRows()` derives it.
|
package/src/webhook.ts
CHANGED
|
@@ -20,7 +20,7 @@ import {
|
|
|
20
20
|
finiteOption,
|
|
21
21
|
isCanonicalWebhookField,
|
|
22
22
|
isUltimateError,
|
|
23
|
-
|
|
23
|
+
liveClock,
|
|
24
24
|
throwIfAborted,
|
|
25
25
|
WEBHOOK_FIELD_MAX,
|
|
26
26
|
} from '@ultimat3/core';
|
|
@@ -189,7 +189,7 @@ export interface WebhookDefinition {
|
|
|
189
189
|
}
|
|
190
190
|
|
|
191
191
|
export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliveryInput> {
|
|
192
|
-
const clock = definition.clock ??
|
|
192
|
+
const clock = definition.clock ?? liveClock;
|
|
193
193
|
const disableAfter = finiteOption(
|
|
194
194
|
'webhook()',
|
|
195
195
|
'disableAfter',
|
package/src/worker-admit.ts
CHANGED
|
@@ -34,11 +34,14 @@ export type Admission =
|
|
|
34
34
|
| { readonly kind: 'waiting' }
|
|
35
35
|
| { readonly kind: 'refused'; readonly execution: JobExecution };
|
|
36
36
|
|
|
37
|
-
/**
|
|
37
|
+
/**
|
|
38
|
+
* `behind` is the rest of the claimed batch: handed back before a store failure is rethrown. A
|
|
39
|
+
* thunk, read only on that failure: a slice per admitted job made a batch O(batch²) to admit.
|
|
40
|
+
*/
|
|
38
41
|
export type Admit = (
|
|
39
42
|
job: ClaimedJob,
|
|
40
43
|
queue: string,
|
|
41
|
-
behind: readonly ClaimedJob[],
|
|
44
|
+
behind: () => readonly ClaimedJob[],
|
|
42
45
|
) => Promise<Admission>;
|
|
43
46
|
|
|
44
47
|
export function createAdmission(options: AdmissionOptions): Admit {
|
|
@@ -79,7 +82,7 @@ export function createAdmission(options: AdmissionOptions): Admit {
|
|
|
79
82
|
// The nack itself failed, so THIS job went nowhere either: it goes back with the jobs
|
|
80
83
|
// behind it, one more best-effort try, before the round reports. Left out, it sat
|
|
81
84
|
// `running` until its lease lapsed — an attempt spent on a body that never started.
|
|
82
|
-
await handBack(driver, [job, ...behind], back);
|
|
85
|
+
await handBack(driver, [job, ...behind()], back);
|
|
83
86
|
throw error;
|
|
84
87
|
}
|
|
85
88
|
return { kind: 'waiting' };
|
|
@@ -101,7 +104,7 @@ export function createAdmission(options: AdmissionOptions): Admit {
|
|
|
101
104
|
lease.abandon();
|
|
102
105
|
// This job and every one behind it go BACK, unburned — rethrowing alone stranded them in
|
|
103
106
|
// `running` with an attempt spent on work that never started.
|
|
104
|
-
await handBack(driver, [job, ...behind], back);
|
|
107
|
+
await handBack(driver, [job, ...behind()], back);
|
|
105
108
|
throw error;
|
|
106
109
|
}
|
|
107
110
|
if (grant.outcome === 'granted') return { kind: 'run', lease };
|
|
@@ -132,7 +135,7 @@ export function createAdmission(options: AdmissionOptions): Admit {
|
|
|
132
135
|
} catch (error) {
|
|
133
136
|
// This job too, for the reason above. A refusal that DID settle before throwing is safe to
|
|
134
137
|
// repeat: every nack is fenced on `running` and the claim, so a settled row ignores it.
|
|
135
|
-
await handBack(driver, [job, ...behind], back);
|
|
138
|
+
await handBack(driver, [job, ...behind()], back);
|
|
136
139
|
throw error;
|
|
137
140
|
}
|
|
138
141
|
return { kind: 'waiting' };
|
|
@@ -51,6 +51,7 @@ export function armDrainCutoff(
|
|
|
51
51
|
for (const timer of timers.splice(0)) clearTimeout(timer);
|
|
52
52
|
};
|
|
53
53
|
const at = (instant: number, fire: () => void): void => {
|
|
54
|
+
// Real time, not `liveClock`: this races a real `setTimeout`, and a frozen clock under it never fires.
|
|
54
55
|
const timer = setTimeout(fire, Math.max(0, instant - systemClock.monotonic()));
|
|
55
56
|
timer.unref?.();
|
|
56
57
|
timers.push(timer);
|
package/src/worker-loop.ts
CHANGED
|
@@ -42,19 +42,34 @@ export interface ClaimLoop {
|
|
|
42
42
|
stop(): void;
|
|
43
43
|
/** True once a pass has come back empty. */
|
|
44
44
|
idle(): boolean;
|
|
45
|
-
/**
|
|
46
|
-
|
|
45
|
+
/**
|
|
46
|
+
* A pass ended: `found` decides the next wait. `full` — some ask came back with every row it
|
|
47
|
+
* asked for — means a backlog is likely, so a slot freeing before the next pass claims at once.
|
|
48
|
+
*/
|
|
49
|
+
passed(found: boolean, full?: boolean): void;
|
|
47
50
|
/** The wait the loop will arm, or has armed. */
|
|
48
51
|
delayMs(): number;
|
|
49
52
|
/**
|
|
50
53
|
* A slot came free, or a handed-back row is due: one pass NOW if the wait in hand is longer
|
|
51
|
-
* than the floor — and no reset, so a pass that finds nothing
|
|
54
|
+
* than the floor, or the last pass came back full — and no reset, so a pass that finds nothing
|
|
55
|
+
* costs one statement, not a ramp.
|
|
52
56
|
*/
|
|
53
57
|
kick(): void;
|
|
54
58
|
/** A row this worker settled back onto the queue is claimable again in `ms`. */
|
|
55
59
|
dueIn(ms: number): void;
|
|
56
60
|
}
|
|
57
61
|
|
|
62
|
+
/**
|
|
63
|
+
* Whether one ask came back FULL — every row it asked for STARTED. Counted on what a pass admitted,
|
|
64
|
+
* never on what it claimed: rows a concurrency or rate cap sent back are not a backlog this worker
|
|
65
|
+
* can run, and kicking on them is a claim plus N nacks per completion.
|
|
66
|
+
*/
|
|
67
|
+
export function fillsAsk(admitted: ReadonlyMap<string, number>, limit: number): boolean {
|
|
68
|
+
let ran = 0;
|
|
69
|
+
for (const count of admitted.values()) ran += count;
|
|
70
|
+
return ran > 0 && ran >= limit;
|
|
71
|
+
}
|
|
72
|
+
|
|
58
73
|
/** A due time further out than this is the poll's to find: no timer is held for a 3-day sleep. */
|
|
59
74
|
const DUE_HINT_MAX_MS = 60_000;
|
|
60
75
|
/** After the due time, never on it: the row's `run_at` is the database's clock, not this one. */
|
|
@@ -74,6 +89,12 @@ export function createClaimLoop(options: ClaimLoopOptions): ClaimLoop {
|
|
|
74
89
|
/** The timer's own pass is in flight. A wake landing now asks for ONE more, never a second chain. */
|
|
75
90
|
let passing = false;
|
|
76
91
|
let again = false;
|
|
92
|
+
/**
|
|
93
|
+
* The last pass came back full. A pass that found work waits the FLOOR, and a kick shorter than
|
|
94
|
+
* that did nothing — so a backlog drained at one batch per floor: 8 slots / 250 ms = 32 claims/s,
|
|
95
|
+
* which `scripts/bench/worker-throughput.bench.ts` measured exactly. Cleared by the next pass.
|
|
96
|
+
*/
|
|
97
|
+
let backlog = false;
|
|
77
98
|
let unsubscribe: (() => void) | undefined;
|
|
78
99
|
/** One timer per floor-wide bucket of due times, so a burst of retries holds a bounded few. */
|
|
79
100
|
const due = new Map<number, unknown>();
|
|
@@ -126,7 +147,7 @@ export function createClaimLoop(options: ClaimLoopOptions): ClaimLoop {
|
|
|
126
147
|
};
|
|
127
148
|
|
|
128
149
|
const kick = (): void => {
|
|
129
|
-
if (delayMs > floor) now();
|
|
150
|
+
if (delayMs > floor || backlog) now();
|
|
130
151
|
};
|
|
131
152
|
|
|
132
153
|
return {
|
|
@@ -140,6 +161,7 @@ export function createClaimLoop(options: ClaimLoopOptions): ClaimLoop {
|
|
|
140
161
|
stop() {
|
|
141
162
|
active = false;
|
|
142
163
|
again = false;
|
|
164
|
+
backlog = false;
|
|
143
165
|
if (timer !== undefined) timers.clear(timer);
|
|
144
166
|
timer = undefined;
|
|
145
167
|
for (const hint of due.values()) timers.clear(hint);
|
|
@@ -148,7 +170,8 @@ export function createClaimLoop(options: ClaimLoopOptions): ClaimLoop {
|
|
|
148
170
|
unsubscribe = undefined;
|
|
149
171
|
},
|
|
150
172
|
idle: () => backoff.idle(),
|
|
151
|
-
passed(found) {
|
|
173
|
+
passed(found, full = false) {
|
|
174
|
+
backlog = full;
|
|
152
175
|
delayMs = backoff.next(found);
|
|
153
176
|
},
|
|
154
177
|
delayMs: () => delayMs,
|
package/src/worker-registry.ts
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
import type { Clock } from '@ultimat3/core';
|
|
7
7
|
import { logger, renderThrowable } from '@ultimat3/core';
|
|
8
8
|
import { nowMs } from './clock';
|
|
9
|
-
import type
|
|
9
|
+
import { DEFAULT_QUEUE, type JobDriver } from './driver';
|
|
10
10
|
import { MAX_WORKER_IN_FLIGHT } from './introspection';
|
|
11
11
|
import { registeredJobs } from './job';
|
|
12
12
|
import { type IntervalScheduler, startRenewalTimer } from './renewal-timer';
|
|
@@ -122,13 +122,15 @@ export function startWorkerRegistry(options: WorkerRegistryOptions): WorkerRegis
|
|
|
122
122
|
|
|
123
123
|
/**
|
|
124
124
|
* A queue this worker polls that NO registered job names. Not refused: a per-call
|
|
125
|
-
* `enqueue({ queue })` may still land work there
|
|
126
|
-
* queues it covers. Said once, at boot, because it is
|
|
127
|
-
*
|
|
125
|
+
* `enqueue({ queue })` may still land work there — `jobs.queues` exists to name exactly those — and
|
|
126
|
+
* an idle pass is one statement however many queues it covers. Said once, at boot, because it is
|
|
127
|
+
* almost always a config leftover (`x new` configured `['<app>-default']` until 31.0.0).
|
|
128
|
+
* `DEFAULT_QUEUE` is never one: it is where every `job()` naming no queue lands and what a worker
|
|
129
|
+
* with no list serves, so an app with no such job yet logged this on every boot of every pod.
|
|
128
130
|
*/
|
|
129
131
|
export function reportUnregisteredQueues(workerId: string, queues: readonly string[]): void {
|
|
130
132
|
const named = new Set(registeredJobs().map((handle) => handle.queue));
|
|
131
|
-
const unnamed = queues.filter((queue) => !named.has(queue));
|
|
133
|
+
const unnamed = queues.filter((queue) => queue !== DEFAULT_QUEUE && !named.has(queue));
|
|
132
134
|
if (unnamed.length === 0) return;
|
|
133
135
|
logger.info('jobs.worker.queue-unregistered', {
|
|
134
136
|
workerId,
|