@ultimat3/jobs 29.3.2 → 30.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +68 -89
- package/README.md +304 -10
- package/package.json +18 -6
- package/src/backfill-definition.ts +184 -0
- package/src/backfill-errors.ts +18 -2
- package/src/backfill-pass.ts +43 -40
- package/src/backfill-pending.ts +40 -17
- package/src/backfill-reader.ts +94 -0
- package/src/backfill-registry.ts +5 -0
- package/src/backfill-scope.ts +1 -2
- package/src/backfill-source.ts +113 -0
- package/src/backfill.ts +54 -135
- package/src/consumer-errors.ts +42 -0
- package/src/consumer-guard.ts +19 -0
- package/src/consumer-loop.ts +171 -0
- package/src/consumer-registry.ts +103 -0
- package/src/consumer-types.ts +81 -0
- package/src/consumer.ts +147 -0
- package/src/driver-memory-operator.ts +25 -2
- package/src/driver-memory.ts +35 -1
- package/src/driver-pg-claim-sql.ts +128 -0
- package/src/driver-pg-ddl.ts +11 -2
- package/src/driver-pg-demand-sql.ts +27 -0
- package/src/driver-pg-enqueue-many.ts +153 -0
- package/src/driver-pg-event-wake-sql.ts +51 -0
- package/src/driver-pg-operator-sql.ts +34 -6
- package/src/driver-pg-operator.ts +36 -2
- package/src/driver-pg-outbox-sql.ts +34 -0
- package/src/driver-pg-sql.ts +24 -91
- package/src/driver-pg.ts +17 -1
- package/src/driver-redis-refusal.ts +25 -0
- package/src/driver-redis-scripts.ts +79 -30
- package/src/driver-redis.ts +70 -42
- package/src/driver.ts +38 -1
- package/src/enqueue-record.ts +137 -0
- package/src/error-codes.ts +233 -0
- package/src/errors-concurrency.ts +2 -2
- package/src/errors-declaration.ts +15 -2
- package/src/errors-operator.ts +1 -1
- package/src/errors-requeue.ts +1 -1
- package/src/errors-saga.ts +87 -0
- package/src/errors-steps.ts +35 -0
- package/src/errors-tenant.ts +1 -1
- package/src/errors.ts +25 -165
- package/src/events-pg.ts +20 -8
- package/src/events.ts +14 -3
- package/src/execute-ctx.ts +51 -0
- package/src/execute-unwind.ts +191 -0
- package/src/execute.ts +50 -51
- package/src/export-errors.ts +1 -1
- package/src/failure-row.ts +43 -0
- package/src/import-errors.ts +102 -0
- package/src/import-format.ts +232 -0
- package/src/import-pass.ts +255 -0
- package/src/import-source.ts +49 -0
- package/src/import.ts +175 -0
- package/src/index.ts +163 -7
- package/src/inspect-operator.ts +24 -3
- package/src/inspect.ts +17 -4
- package/src/introspection.ts +24 -1
- package/src/job-enqueue-many.ts +108 -0
- package/src/job-types.ts +229 -0
- package/src/job.ts +19 -208
- package/src/metrics.ts +10 -0
- package/src/outbox-pg.ts +35 -0
- package/src/outbox-relay.ts +58 -19
- package/src/outbox.ts +65 -73
- package/src/purge-entities.ts +92 -0
- package/src/purge.ts +10 -2
- package/src/queue-wake.ts +1 -1
- package/src/refresh-view.ts +76 -0
- package/src/retry-classification.ts +27 -8
- package/src/retry.ts +26 -0
- package/src/rollup-errors.ts +27 -0
- package/src/rollup.ts +190 -0
- package/src/run-deadline.ts +11 -7
- package/src/scheduler.ts +2 -1
- package/src/settled.ts +56 -15
- package/src/step-resolve.ts +37 -0
- package/src/steps-api.ts +111 -0
- package/src/steps-core.ts +26 -0
- package/src/steps-output.ts +142 -0
- package/src/steps-store.ts +68 -0
- package/src/steps-suspension.ts +51 -0
- package/src/steps-timeout.ts +10 -5
- package/src/steps-unwind.ts +94 -0
- package/src/steps-wait.ts +163 -0
- package/src/steps.ts +174 -210
- package/src/subject-erase-pass.ts +134 -0
- package/src/subject-errors.ts +48 -0
- package/src/subject-export-pass.ts +228 -0
- package/src/subject-jobs.ts +121 -0
- package/src/subject-plan.ts +287 -0
- package/src/subject-select.ts +106 -0
- package/src/subject.ts +119 -0
- package/src/tenant.ts +69 -1
- package/src/type-pins.ts +90 -0
- package/src/webhook-errors.ts +1 -1
- package/src/webhook-receiver-errors.ts +59 -0
- package/src/webhook-receiver-route.ts +127 -0
- package/src/webhook-receiver-types.ts +97 -0
- package/src/webhook-receiver.ts +132 -0
- package/src/webhook-target.ts +20 -43
- package/src/webhook.ts +7 -4
- package/src/worker-queue-demand.ts +57 -0
- package/src/worker-queue-depth.ts +15 -3
- package/src/worker-registry.ts +4 -1
- package/src/worker-types.ts +2 -0
- package/src/worker.ts +2 -2
package/src/steps.ts
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
//
|
|
6
6
|
// Suspension (sleep / waitForEvent) unwinds the run by throwing a StepSuspension. The worker
|
|
7
7
|
// catches it and re-queues the job for `resumeAt` instead of holding a process for three days.
|
|
8
|
+
// `waitForEvent` is `steps-wait.ts`; compensation, `steps-unwind.ts`; both over `steps-core.ts`.
|
|
8
9
|
|
|
9
10
|
import type { Clock } from '@ultimat3/core';
|
|
10
11
|
import { finiteOption, isUltimateError, logger, renderThrowable } from '@ultimat3/core';
|
|
@@ -12,114 +13,38 @@ import { expectedQueryLoop } from '@ultimat3/db';
|
|
|
12
13
|
import type { DurationInput } from './clock';
|
|
13
14
|
import { finiteDurationMs, nowMs } from './clock';
|
|
14
15
|
import { JobAbortedError, JobTimeoutError, StepDuplicateError } from './errors';
|
|
15
|
-
import { createRunSignal } from './run-signal';
|
|
16
16
|
import {
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
17
|
+
isReservedStepName,
|
|
18
|
+
StepCompensateUndeclaredError,
|
|
19
|
+
StepNameReservedError,
|
|
20
|
+
StepOutcomeUnknownError,
|
|
21
|
+
UNWINDING_STEP,
|
|
22
|
+
} from './errors-saga';
|
|
23
|
+
import { createRunSignal } from './run-signal';
|
|
24
|
+
import type { EventLookup, StepApi, StepRunOptions } from './steps-api';
|
|
25
|
+
import { DEFAULT_EVENT_POLL_MS, WAKING_EVENT_POLL_MS } from './steps-api';
|
|
26
|
+
import type { RunnerCore } from './steps-core';
|
|
27
|
+
import type { StepOutput } from './steps-output';
|
|
28
|
+
import { jsonForm } from './steps-output';
|
|
29
|
+
import type { StepFence, StepRecord, StepStore } from './steps-store';
|
|
30
|
+
import { isStepSuspension, StepSuspension, UnwindReached } from './steps-suspension';
|
|
22
31
|
import { withStepTimeout } from './steps-timeout';
|
|
23
|
-
|
|
32
|
+
import { compensationStack } from './steps-unwind';
|
|
33
|
+
import { waitForEventStep } from './steps-wait';
|
|
34
|
+
|
|
35
|
+
export type {
|
|
36
|
+
CompensateContext,
|
|
37
|
+
EventLookup,
|
|
38
|
+
StepApi,
|
|
39
|
+
StepRunOptions,
|
|
40
|
+
WaitForEventOptions,
|
|
41
|
+
} from './steps-api';
|
|
42
|
+
export { DEFAULT_EVENT_POLL_MS, WAKING_EVENT_POLL_MS } from './steps-api';
|
|
43
|
+
export type { StepOutput } from './steps-output';
|
|
44
|
+
export type { StepFence, StepRecord, StepStatus, StepStore } from './steps-store';
|
|
45
|
+
export { isStepStatus, STEP_STATUSES } from './steps-store';
|
|
24
46
|
export { isStepSuspension, StepSuspension } from './steps-suspension';
|
|
25
47
|
|
|
26
|
-
/**
|
|
27
|
-
* The runtime list is the declaration and `StepStatus` is derived from it, the shape
|
|
28
|
-
* `BACKFILL_STATUSES` and `PRIMITIVE_KINDS` already have. A bare union cannot narrow a `text`
|
|
29
|
-
* column, so `driver-pg-rows.ts` cast one instead — and a cast that lands an unknown status on a
|
|
30
|
-
* record makes `stepRun`'s `existing?.status === 'completed'` false, which RE-EXECUTES a step
|
|
31
|
-
* this file promises runs once.
|
|
32
|
-
*/
|
|
33
|
-
export const STEP_STATUSES = ['completed', 'sleeping', 'waiting', 'failed'] as const;
|
|
34
|
-
|
|
35
|
-
export type StepStatus = (typeof STEP_STATUSES)[number];
|
|
36
|
-
|
|
37
|
-
/** Narrows a status read back out of a store. Never a cast — the list decides. */
|
|
38
|
-
export const isStepStatus = (value: string): value is StepStatus =>
|
|
39
|
-
(STEP_STATUSES as readonly string[]).includes(value);
|
|
40
|
-
|
|
41
|
-
export interface StepRecord {
|
|
42
|
-
readonly runId: string;
|
|
43
|
-
readonly name: string;
|
|
44
|
-
readonly status: StepStatus;
|
|
45
|
-
/** The memoized return value. Present only when `status === 'completed'`. */
|
|
46
|
-
readonly output?: unknown;
|
|
47
|
-
readonly startedAt: number;
|
|
48
|
-
readonly completedAt?: number;
|
|
49
|
-
/** Epoch ms at which a sleeping/waiting step becomes runnable. */
|
|
50
|
-
readonly wakeAt?: number;
|
|
51
|
-
readonly event?: string;
|
|
52
|
-
readonly correlationKey?: string;
|
|
53
|
-
readonly attempts: number;
|
|
54
|
-
readonly error?: string;
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
/** The claim a step write is made under: `claimOf(claimed)`, plus the row it names. */
|
|
58
|
-
export interface StepFence {
|
|
59
|
-
readonly job: string;
|
|
60
|
-
readonly jobId: string;
|
|
61
|
-
readonly workerId: string;
|
|
62
|
-
readonly claim: number;
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
export interface StepStore {
|
|
66
|
-
get(runId: string, name: string): Promise<StepRecord | undefined>;
|
|
67
|
-
/**
|
|
68
|
-
* With `by`, the write lands only while that claim still holds the row (`running`, same worker,
|
|
69
|
-
* same claim ordinal — `SQL_ACK`'s fence) and is `X_JOB_LEASE_LOST` otherwise: the in-process
|
|
70
|
-
* signal cannot stop a body in ANOTHER process whose lease lapsed. Without it the write is
|
|
71
|
-
* unfenced — a transfer (`x jobs drain`) or a runner driven by hand, with no row to hold.
|
|
72
|
-
*/
|
|
73
|
-
put(record: StepRecord, by?: StepFence): Promise<void>;
|
|
74
|
-
list(runId: string): Promise<readonly StepRecord[]>;
|
|
75
|
-
del(runId: string, name: string): Promise<void>;
|
|
76
|
-
clear(runId: string): Promise<void>;
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
/** The slice of the event bus a waiting step needs. Implemented by `events.ts`. */
|
|
80
|
-
export interface EventLookup {
|
|
81
|
-
/**
|
|
82
|
-
* The bus's OWN clock, in epoch ms: what a publish issued now would be stamped with. A consumer
|
|
83
|
-
* that wants "published after I asked" takes its "asked at" from here and never from its own
|
|
84
|
-
* process — `publishedAt` is this clock's, and two clocks compared is a skew between pods
|
|
85
|
-
* deciding which answers count. `waitForEvent` stamps a NEW wait with it.
|
|
86
|
-
*/
|
|
87
|
-
now(): Promise<number>;
|
|
88
|
-
find(
|
|
89
|
-
event: string,
|
|
90
|
-
correlationKey: string | undefined,
|
|
91
|
-
afterMs: number,
|
|
92
|
-
): Promise<{ readonly payload: unknown; readonly publishedAt: number } | undefined>;
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
export interface WaitForEventOptions {
|
|
96
|
-
/** How long to wait before giving up. Default 24h. */
|
|
97
|
-
readonly timeout?: DurationInput;
|
|
98
|
-
/** Correlation key the published event must carry — usually an entity id. */
|
|
99
|
-
readonly match?: string;
|
|
100
|
-
/** Default false: a timeout resolves `undefined`. `true` fails the job with X_JOB_TIMEOUT. */
|
|
101
|
-
readonly required?: boolean;
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
export interface StepApi {
|
|
105
|
-
/**
|
|
106
|
-
* Run once, ever. On replay the persisted output is returned and `fn` is not called.
|
|
107
|
-
*
|
|
108
|
-
* `fn` receives the step's `AbortSignal` — the run's cancellation and this step's own ceiling,
|
|
109
|
-
* whichever fires first. Hand it to `fetch`, or read `.aborted` in a loop: past it, this step
|
|
110
|
-
* may no longer write, because the attempt that replaced this one owns the run.
|
|
111
|
-
*/
|
|
112
|
-
run<T>(name: string, fn: (signal: AbortSignal) => Promise<T> | T): Promise<T>;
|
|
113
|
-
/** Suspend the run. `sleep(duration)` derives the step name from the duration. */
|
|
114
|
-
sleep(name: string, duration: DurationInput): Promise<void>;
|
|
115
|
-
sleep(duration: DurationInput): Promise<void>;
|
|
116
|
-
waitForEvent<T>(
|
|
117
|
-
name: string,
|
|
118
|
-
event: string,
|
|
119
|
-
options?: WaitForEventOptions,
|
|
120
|
-
): Promise<T | undefined>;
|
|
121
|
-
}
|
|
122
|
-
|
|
123
48
|
export interface StepRunnerOptions {
|
|
124
49
|
readonly runId: string;
|
|
125
50
|
readonly jobName: string;
|
|
@@ -128,8 +53,19 @@ export interface StepRunnerOptions {
|
|
|
128
53
|
readonly fence?: StepFence;
|
|
129
54
|
readonly clock?: Clock;
|
|
130
55
|
readonly events?: EventLookup;
|
|
131
|
-
/**
|
|
56
|
+
/**
|
|
57
|
+
* How long a waiting step stays parked between event polls. Default `DEFAULT_EVENT_POLL_MS`
|
|
58
|
+
* (30 s), or `WAKING_EVENT_POLL_MS` (15 min) when `events.wakes`: a publish then brings the run
|
|
59
|
+
* forward, and the poll is only the backstop. A value set here wins either way.
|
|
60
|
+
*/
|
|
132
61
|
readonly eventPollMs?: number;
|
|
62
|
+
/** The job declares `compensates: true`: `step.run(…, { compensate })` is legal. */
|
|
63
|
+
readonly compensates?: boolean;
|
|
64
|
+
/**
|
|
65
|
+
* This attempt is past the policy's last (`isUnwindAttempt`): the run is unwinding whether or
|
|
66
|
+
* not `unwinding` was written — the final attempt crashed before it could be.
|
|
67
|
+
*/
|
|
68
|
+
readonly unwind?: boolean;
|
|
133
69
|
/** Per-step ceiling; the job-level timeout is enforced by the worker. */
|
|
134
70
|
readonly stepTimeoutMs?: number;
|
|
135
71
|
/**
|
|
@@ -157,6 +93,13 @@ export interface StepRunner {
|
|
|
157
93
|
usedNames(): readonly string[];
|
|
158
94
|
/** Names that were served from storage instead of executed. Bounded the same way. */
|
|
159
95
|
replayedNames(): readonly string[];
|
|
96
|
+
/** Whether the run is unwinding: `options.unwind`, or `unwinding` found among its steps. */
|
|
97
|
+
unwinding(): Promise<boolean>;
|
|
98
|
+
/**
|
|
99
|
+
* Run this attempt's registered compensators, last-in first-out (`steps-unwind.ts`). Answers
|
|
100
|
+
* the failure the run unwinds from — the one already recorded, else `failure`.
|
|
101
|
+
*/
|
|
102
|
+
unwind(failure: string): Promise<string>;
|
|
160
103
|
}
|
|
161
104
|
|
|
162
105
|
/** Stands in for an absent run signal, so the fence has one shape and no `undefined` branch. */
|
|
@@ -182,7 +125,12 @@ export function stepRunner(options: StepRunnerOptions): StepRunner {
|
|
|
182
125
|
const used: string[] = [];
|
|
183
126
|
const replayed: string[] = [];
|
|
184
127
|
const clock = options.clock;
|
|
185
|
-
const pollMs = finiteOption(
|
|
128
|
+
const pollMs = finiteOption(
|
|
129
|
+
'step.waitForEvent',
|
|
130
|
+
'eventPollMs',
|
|
131
|
+
options.eventPollMs ??
|
|
132
|
+
(options.events?.wakes === true ? WAKING_EVENT_POLL_MS : DEFAULT_EVENT_POLL_MS),
|
|
133
|
+
);
|
|
186
134
|
// `job()` refuses a non-finite `stepTimeout` at declaration (`job.ts`); screened again here, where
|
|
187
135
|
// it is read, because this runner is also built directly and `withStepTimeout` reads a `NaN`
|
|
188
136
|
// ceiling as none at all — the backstop `pollMs` above already is.
|
|
@@ -203,23 +151,39 @@ export function stepRunner(options: StepRunnerOptions): StepRunner {
|
|
|
203
151
|
* Sound because the fence in `put()` is: only the attempt that owns the run may write to it, so
|
|
204
152
|
* within one attempt an absent name stays absent unless this runner writes it. Writes go into
|
|
205
153
|
* the map as they go into the store, never the other way round — the store is still the record.
|
|
154
|
+
*
|
|
155
|
+
* The PROMISE is the memo, and the map is published only when the list has answered: a map
|
|
156
|
+
* assigned before the await let a second caller read it empty, so `Promise.all` over completed
|
|
157
|
+
* steps re-ran one on replay and answered a value the store did not hold. A rejected list clears
|
|
158
|
+
* the memo, so the next call asks again rather than this attempt failing on one blip for good.
|
|
206
159
|
*/
|
|
207
160
|
let hydrated: Map<string, StepRecord> | undefined;
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
161
|
+
let hydrating: Promise<Map<string, StepRecord>> | undefined;
|
|
162
|
+
|
|
163
|
+
const hydrate = (): Promise<Map<string, StepRecord>> => {
|
|
164
|
+
if (hydrating !== undefined) return hydrating;
|
|
165
|
+
const pending = (async () => {
|
|
166
|
+
const records = await store.list(runId);
|
|
167
|
+
hydrated = new Map(records.map((record) => [record.name, record]));
|
|
168
|
+
return hydrated;
|
|
169
|
+
})();
|
|
170
|
+
hydrating = pending;
|
|
171
|
+
pending.catch(() => {
|
|
172
|
+
if (hydrating === pending) hydrating = undefined;
|
|
173
|
+
});
|
|
174
|
+
return pending;
|
|
215
175
|
};
|
|
216
176
|
|
|
217
|
-
|
|
177
|
+
const load = async (name: string): Promise<StepRecord | undefined> => (await hydrate()).get(name);
|
|
178
|
+
|
|
179
|
+
/** Keep the hydrated view in step. Every write follows a `load`, so the list has answered. */
|
|
218
180
|
const remember = (record: StepRecord): void => {
|
|
219
181
|
hydrated?.set(record.name, record);
|
|
220
182
|
};
|
|
221
183
|
|
|
222
184
|
const claimName = (name: string): void => {
|
|
185
|
+
// Refused for EVERY job, so one that gains `compensates` later cannot meet a step it wrote.
|
|
186
|
+
if (isReservedStepName(name)) throw new StepNameReservedError({ job: jobName, step: name });
|
|
223
187
|
// The Set decides, the array only reports. `Array.includes` made a long run quadratic —
|
|
224
188
|
// `backfill({ batch: 50 })` over a million rows is 20,000 steps and ~200M string compares.
|
|
225
189
|
if (claimed.has(name)) throw new StepDuplicateError({ job: jobName, step: name });
|
|
@@ -291,14 +255,25 @@ export function stepRunner(options: StepRunnerOptions): StepRunner {
|
|
|
291
255
|
|
|
292
256
|
const now = (): number => nowMs(clock);
|
|
293
257
|
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
258
|
+
/**
|
|
259
|
+
* The run is unwinding: this attempt is past the policy's last, or `unwinding` is among the
|
|
260
|
+
* steps the first `load` hydrated. Sticky — once true for an attempt it stays true.
|
|
261
|
+
*/
|
|
262
|
+
let unwindMode = options.unwind === true;
|
|
263
|
+
const unwinding = async (): Promise<boolean> => {
|
|
264
|
+
if (!unwindMode) unwindMode = (await hydrate()).has(UNWINDING_STEP);
|
|
265
|
+
return unwindMode;
|
|
266
|
+
};
|
|
301
267
|
|
|
268
|
+
/**
|
|
269
|
+
* One step body, run under its ceiling and recorded: `completed` before it returns, `failed` on
|
|
270
|
+
* the way out. Shared by `step.run` and by every compensator (`steps-unwind.ts`).
|
|
271
|
+
*/
|
|
272
|
+
const execute = async <T>(
|
|
273
|
+
name: string,
|
|
274
|
+
existing: StepRecord | undefined,
|
|
275
|
+
fn: (signal: AbortSignal) => Promise<T> | T,
|
|
276
|
+
): Promise<StepOutput<T>> => {
|
|
302
277
|
const startedAt = existing?.startedAt ?? now();
|
|
303
278
|
const attempts = (existing?.attempts ?? 0) + 1;
|
|
304
279
|
// The step's own ceiling, folded into the run's cancellation so the body reads ONE signal and
|
|
@@ -319,24 +294,31 @@ export function stepRunner(options: StepRunnerOptions): StepRunner {
|
|
|
319
294
|
fn(signal),
|
|
320
295
|
stepTimeoutMs,
|
|
321
296
|
deadline,
|
|
322
|
-
() =>
|
|
297
|
+
() =>
|
|
298
|
+
new JobTimeoutError({
|
|
299
|
+
job: jobName,
|
|
300
|
+
step: name,
|
|
301
|
+
timeoutMs: stepTimeoutMs ?? 0,
|
|
302
|
+
fix: `job({ name: '${jobName}', stepTimeout: '5m' }) # raises the ceiling; or split step.run('${name}', …) into smaller steps`,
|
|
303
|
+
}),
|
|
323
304
|
);
|
|
305
|
+
// What goes in the store is what comes back, so both attempts see one value.
|
|
306
|
+
const stored = jsonForm(output as T, { job: jobName, step: name });
|
|
324
307
|
// Persist BEFORE returning: a crash one line later must not re-run this step.
|
|
325
308
|
await put({
|
|
326
309
|
runId,
|
|
327
310
|
name,
|
|
328
311
|
status: 'completed',
|
|
329
|
-
output,
|
|
312
|
+
output: stored,
|
|
330
313
|
startedAt,
|
|
331
314
|
completedAt: now(),
|
|
332
315
|
attempts,
|
|
333
316
|
});
|
|
334
|
-
return
|
|
317
|
+
return stored;
|
|
335
318
|
} catch (error) {
|
|
336
319
|
if (isStepSuspension(error)) throw error;
|
|
337
320
|
// The failure is this run's own history, so it is recorded on the way out — unless the
|
|
338
|
-
// attempt was cancelled, in which case the history is no longer ours to write.
|
|
339
|
-
// error is what the caller has to see, never one raised by the bookkeeping.
|
|
321
|
+
// attempt was cancelled, in which case the history is no longer ours to write.
|
|
340
322
|
if (!cancelled()) {
|
|
341
323
|
// Deliberately NOT through `put`: the caller has to see the original error, never one
|
|
342
324
|
// raised by the bookkeeping. The hydrated view is updated by hand for the same reason.
|
|
@@ -365,6 +347,68 @@ export function stepRunner(options: StepRunnerOptions): StepRunner {
|
|
|
365
347
|
// settled leaves its signal in whatever state it ended in.
|
|
366
348
|
composed?.dispose();
|
|
367
349
|
}
|
|
350
|
+
};
|
|
351
|
+
|
|
352
|
+
const core: RunnerCore = {
|
|
353
|
+
runId,
|
|
354
|
+
jobName,
|
|
355
|
+
events: options.events,
|
|
356
|
+
pollMs,
|
|
357
|
+
claim: claimName,
|
|
358
|
+
load,
|
|
359
|
+
put,
|
|
360
|
+
now,
|
|
361
|
+
replayed: (name) => trace(replayed, name),
|
|
362
|
+
forwardOnly: async (name) => {
|
|
363
|
+
if (await unwinding()) throw new UnwindReached(name);
|
|
364
|
+
},
|
|
365
|
+
};
|
|
366
|
+
const compensations = compensationStack(core, execute);
|
|
367
|
+
|
|
368
|
+
async function run<T>(
|
|
369
|
+
name: string,
|
|
370
|
+
fn: (signal: AbortSignal) => Promise<T> | T,
|
|
371
|
+
runOptions: StepRunOptions<T> = {},
|
|
372
|
+
): Promise<StepOutput<T>> {
|
|
373
|
+
const { compensate, once = false } = runOptions;
|
|
374
|
+
if (compensate !== undefined && options.compensates !== true) {
|
|
375
|
+
throw new StepCompensateUndeclaredError({ job: jobName, step: name });
|
|
376
|
+
}
|
|
377
|
+
claimName(name);
|
|
378
|
+
const undo =
|
|
379
|
+
compensate === undefined
|
|
380
|
+
? undefined
|
|
381
|
+
: (output: StepOutput<T>) => async (signal: AbortSignal) => {
|
|
382
|
+
// A compensator's answer is not the run's to keep: `null` is what its step records.
|
|
383
|
+
await compensate(output, { signal });
|
|
384
|
+
return null;
|
|
385
|
+
};
|
|
386
|
+
const existing = await load(name);
|
|
387
|
+
if (existing?.status === 'completed') {
|
|
388
|
+
trace(replayed, name);
|
|
389
|
+
const output = existing.output as StepOutput<T>;
|
|
390
|
+
if (undo !== undefined) compensations.done(name, undo(output));
|
|
391
|
+
return output;
|
|
392
|
+
}
|
|
393
|
+
// `started` or `failed` and never `completed`: a crash or a throw, and nobody knows which side
|
|
394
|
+
// of the side effect it stopped on. `cleared` is an operator's "it did not run".
|
|
395
|
+
const unknown = once && (existing?.status === 'started' || existing?.status === 'failed');
|
|
396
|
+
if (unknown && undo !== undefined) compensations.unknown(name);
|
|
397
|
+
await core.forwardOnly(name);
|
|
398
|
+
if (unknown) throw new StepOutcomeUnknownError({ job: jobName, step: name });
|
|
399
|
+
if (once) {
|
|
400
|
+
// BEFORE `fn`: from here on, a replay cannot tell whether `fn` ran, and so never runs it.
|
|
401
|
+
await put({
|
|
402
|
+
runId,
|
|
403
|
+
name,
|
|
404
|
+
status: 'started',
|
|
405
|
+
startedAt: existing?.startedAt ?? now(),
|
|
406
|
+
attempts: existing?.attempts ?? 0,
|
|
407
|
+
});
|
|
408
|
+
}
|
|
409
|
+
const output = await execute(name, once ? await load(name) : existing, fn);
|
|
410
|
+
if (undo !== undefined) compensations.done(name, undo(output));
|
|
411
|
+
return output;
|
|
368
412
|
}
|
|
369
413
|
|
|
370
414
|
async function sleep(a: string | DurationInput, b?: DurationInput): Promise<void> {
|
|
@@ -378,6 +422,7 @@ export function stepRunner(options: StepRunnerOptions): StepRunner {
|
|
|
378
422
|
trace(replayed, name);
|
|
379
423
|
return;
|
|
380
424
|
}
|
|
425
|
+
await core.forwardOnly(name);
|
|
381
426
|
|
|
382
427
|
const at = now();
|
|
383
428
|
if (existing?.status === 'sleeping' && existing.wakeAt !== undefined) {
|
|
@@ -401,98 +446,17 @@ export function stepRunner(options: StepRunnerOptions): StepRunner {
|
|
|
401
446
|
throw new StepSuspension({ step: name, resumeAt: wakeAt, reason: 'sleep' });
|
|
402
447
|
}
|
|
403
448
|
|
|
404
|
-
async function waitForEvent<T>(
|
|
405
|
-
name: string,
|
|
406
|
-
event: string,
|
|
407
|
-
waitOptions: WaitForEventOptions = {},
|
|
408
|
-
): Promise<T | undefined> {
|
|
409
|
-
claimName(name);
|
|
410
|
-
const existing = await load(name);
|
|
411
|
-
if (existing?.status === 'completed') {
|
|
412
|
-
trace(replayed, name);
|
|
413
|
-
return isWaitTimedOut(existing.output) ? undefined : (existing.output as T | undefined);
|
|
414
|
-
}
|
|
415
|
-
|
|
416
|
-
const at = now();
|
|
417
|
-
// A NEW wait is stamped by the bus, whose clock stamps every `publishedAt` it is compared
|
|
418
|
-
// with; the runner's own only when there is no bus to ask. A re-poll keeps what it persisted.
|
|
419
|
-
const startedAt = existing?.startedAt ?? (await options.events?.now()) ?? at;
|
|
420
|
-
const deadline =
|
|
421
|
-
startedAt +
|
|
422
|
-
finiteDurationMs(waitOptions.timeout ?? 86_400_000, 'step.waitForEvent', 'timeout');
|
|
423
|
-
const correlationKey = waitOptions.match;
|
|
424
|
-
|
|
425
|
-
const hit = await options.events?.find(event, correlationKey, startedAt);
|
|
426
|
-
if (hit !== undefined) {
|
|
427
|
-
await put({
|
|
428
|
-
runId,
|
|
429
|
-
name,
|
|
430
|
-
status: 'completed',
|
|
431
|
-
output: hit.payload,
|
|
432
|
-
startedAt,
|
|
433
|
-
completedAt: at,
|
|
434
|
-
event,
|
|
435
|
-
attempts: (existing?.attempts ?? 0) + 1,
|
|
436
|
-
...(correlationKey === undefined ? {} : { correlationKey }),
|
|
437
|
-
});
|
|
438
|
-
return hit.payload as T;
|
|
439
|
-
}
|
|
440
|
-
|
|
441
|
-
// `deadline` is on the BUS's clock, so the bus calls the timeout: asked only once this runner
|
|
442
|
-
// believes the time is up, which costs a wait one statement at its end. A runner AHEAD of the
|
|
443
|
-
// bus used to give up early by the skew — on an answer still inside its window. (A runner
|
|
444
|
-
// BEHIND waits the skew longer; knowing that sooner would cost a round trip on every poll.)
|
|
445
|
-
let remaining = deadline - at;
|
|
446
|
-
if (remaining <= 0) remaining = deadline - ((await options.events?.now()) ?? at);
|
|
447
|
-
if (remaining <= 0) {
|
|
448
|
-
if (waitOptions.required === true) {
|
|
449
|
-
throw new JobTimeoutError({
|
|
450
|
-
job: jobName,
|
|
451
|
-
step: name,
|
|
452
|
-
timeoutMs: finiteDurationMs(
|
|
453
|
-
waitOptions.timeout ?? 86_400_000,
|
|
454
|
-
'step.waitForEvent',
|
|
455
|
-
'timeout',
|
|
456
|
-
),
|
|
457
|
-
});
|
|
458
|
-
}
|
|
459
|
-
logger.warn('jobs.step.wait-timeout', { job: jobName, step: name, event });
|
|
460
|
-
await put({
|
|
461
|
-
runId,
|
|
462
|
-
name,
|
|
463
|
-
status: 'completed',
|
|
464
|
-
output: WAIT_TIMED_OUT,
|
|
465
|
-
startedAt,
|
|
466
|
-
completedAt: at,
|
|
467
|
-
event,
|
|
468
|
-
attempts: (existing?.attempts ?? 0) + 1,
|
|
469
|
-
});
|
|
470
|
-
return undefined;
|
|
471
|
-
}
|
|
472
|
-
|
|
473
|
-
const resumeAt = at + Math.min(remaining, pollMs);
|
|
474
|
-
await put({
|
|
475
|
-
runId,
|
|
476
|
-
name,
|
|
477
|
-
status: 'waiting',
|
|
478
|
-
startedAt,
|
|
479
|
-
wakeAt: resumeAt,
|
|
480
|
-
event,
|
|
481
|
-
attempts: (existing?.attempts ?? 0) + 1,
|
|
482
|
-
...(correlationKey === undefined ? {} : { correlationKey }),
|
|
483
|
-
});
|
|
484
|
-
throw new StepSuspension({ step: name, resumeAt, reason: 'event' });
|
|
485
|
-
}
|
|
486
|
-
|
|
487
449
|
const step: StepApi = {
|
|
488
450
|
run,
|
|
489
451
|
sleep: sleep as StepApi['sleep'],
|
|
490
|
-
waitForEvent,
|
|
452
|
+
waitForEvent: waitForEventStep(core),
|
|
491
453
|
};
|
|
492
454
|
|
|
493
455
|
return {
|
|
494
456
|
step,
|
|
495
457
|
usedNames: () => [...used],
|
|
496
458
|
replayedNames: () => [...replayed],
|
|
459
|
+
unwinding,
|
|
460
|
+
unwind: (failure) => compensations.unwind(failure),
|
|
497
461
|
};
|
|
498
462
|
}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
// One `subjectErase()` pass: the framework targets first (they read the subject's row), then every
|
|
2
|
+
// loop-closing key cleared, then each table children first, the subject last. Every page of
|
|
3
|
+
// parent keys is its own step, so a killed run resumes at the page it stopped on; each statement
|
|
4
|
+
// is an equality on one key, so a replayed step deletes or overwrites nothing twice.
|
|
5
|
+
|
|
6
|
+
import type { StepApi } from './steps-api';
|
|
7
|
+
import type { SubjectRef, SubjectTarget, SubjectTargetEntry } from './subject';
|
|
8
|
+
import { isRetainedTarget } from './subject';
|
|
9
|
+
import type { SubjectStep } from './subject-plan';
|
|
10
|
+
import type { SubjectReader } from './subject-select';
|
|
11
|
+
import { rowIdentity } from './subject-select';
|
|
12
|
+
|
|
13
|
+
/** What one table or target went through. Bounded and JSON-safe: it is the job's result. */
|
|
14
|
+
export interface SubjectTally {
|
|
15
|
+
readonly deleted: number;
|
|
16
|
+
readonly anonymized: number;
|
|
17
|
+
readonly retained: number;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export interface SubjectEraseReport {
|
|
21
|
+
readonly tables: Readonly<Record<string, SubjectTally>>;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const tally = (deleted: number, anonymized: number, retained: number): SubjectTally => ({
|
|
25
|
+
deleted,
|
|
26
|
+
anonymized,
|
|
27
|
+
retained,
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
/** The subject as a target reads it, fresh: its row is never written into a checkpoint. */
|
|
31
|
+
const refOf = async (reader: SubjectReader): Promise<SubjectRef> => ({
|
|
32
|
+
key: reader.key,
|
|
33
|
+
row: (await reader.rootRow()) ?? {},
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
async function eraseTarget(step: StepApi, reader: SubjectReader, target: SubjectTarget) {
|
|
37
|
+
let total = 0;
|
|
38
|
+
for (let n = 0; ; n += 1) {
|
|
39
|
+
const removed = await step.run(`target:${target.name}:${n}`, async () =>
|
|
40
|
+
target.erase(await refOf(reader), reader.size),
|
|
41
|
+
);
|
|
42
|
+
total += removed;
|
|
43
|
+
if (removed < reader.size) return total;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** One write per parent key in the page: `deleteWhere`/`updateWhere`, the repository's own. */
|
|
48
|
+
async function treatPage(
|
|
49
|
+
reader: SubjectReader,
|
|
50
|
+
step: SubjectStep,
|
|
51
|
+
property: string,
|
|
52
|
+
values: readonly unknown[],
|
|
53
|
+
): Promise<number> {
|
|
54
|
+
const table = reader.table(step.entity);
|
|
55
|
+
let count = 0;
|
|
56
|
+
for (const value of values) {
|
|
57
|
+
const filter = { [property]: value };
|
|
58
|
+
count +=
|
|
59
|
+
step.treatment === 'delete'
|
|
60
|
+
? await table.deleteWhere(filter)
|
|
61
|
+
: typeof step.treatment === 'object' && 'anonymize' in step.treatment
|
|
62
|
+
? await table.updateWhere(filter, step.treatment.anonymize)
|
|
63
|
+
: 0;
|
|
64
|
+
}
|
|
65
|
+
return count;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
async function countRows(reader: SubjectReader, entity: string): Promise<number> {
|
|
69
|
+
const core = reader.core(entity);
|
|
70
|
+
const seen = new Set<string>();
|
|
71
|
+
for await (const page of reader.pages(entity)) {
|
|
72
|
+
for (const row of page) seen.add(rowIdentity(core, row));
|
|
73
|
+
}
|
|
74
|
+
return seen.size;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
async function eraseTable(
|
|
78
|
+
stepApi: StepApi,
|
|
79
|
+
reader: SubjectReader,
|
|
80
|
+
step: SubjectStep,
|
|
81
|
+
): Promise<SubjectTally> {
|
|
82
|
+
const { entity, treatment } = step;
|
|
83
|
+
if (typeof treatment === 'object' && 'retain' in treatment) {
|
|
84
|
+
return tally(0, 0, await stepApi.run(`retain:${entity}`, () => countRows(reader, entity)));
|
|
85
|
+
}
|
|
86
|
+
let count = 0;
|
|
87
|
+
if (entity === reader.plan.subject) {
|
|
88
|
+
count = await stepApi.run(`${entity}:subject`, () =>
|
|
89
|
+
treatPage(reader, step, reader.plan.key, [reader.key]),
|
|
90
|
+
);
|
|
91
|
+
}
|
|
92
|
+
for (const via of step.via) {
|
|
93
|
+
let n = 0;
|
|
94
|
+
for await (const values of reader.parentKeys(via)) {
|
|
95
|
+
count += await stepApi.run(`${entity}.${via.property}:${n}`, () =>
|
|
96
|
+
treatPage(reader, step, via.property, values),
|
|
97
|
+
);
|
|
98
|
+
n += 1;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
return treatment === 'delete' ? tally(count, 0, 0) : tally(0, count, 0);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
export async function subjectErasePass(
|
|
105
|
+
stepApi: StepApi,
|
|
106
|
+
reader: SubjectReader,
|
|
107
|
+
targets: readonly SubjectTargetEntry[],
|
|
108
|
+
): Promise<SubjectEraseReport> {
|
|
109
|
+
const tables = new Map<string, SubjectTally>();
|
|
110
|
+
for (const entry of targets) {
|
|
111
|
+
if (isRetainedTarget(entry)) {
|
|
112
|
+
tables.set(entry.target.name, tally(0, 0, 0));
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
tables.set(entry.name, tally(await eraseTarget(stepApi, reader, entry), 0, 0));
|
|
116
|
+
}
|
|
117
|
+
for (const unlink of reader.plan.unlink) {
|
|
118
|
+
let n = 0;
|
|
119
|
+
for await (const values of reader.parentKeys(unlink)) {
|
|
120
|
+
await stepApi.run(`unlink:${unlink.entity}.${unlink.property}:${n}`, async () => {
|
|
121
|
+
const table = reader.table(unlink.entity);
|
|
122
|
+
for (const value of values) {
|
|
123
|
+
await table.updateWhere({ [unlink.property]: value }, { [unlink.property]: null });
|
|
124
|
+
}
|
|
125
|
+
return null;
|
|
126
|
+
});
|
|
127
|
+
n += 1;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
for (const step of reader.plan.steps) {
|
|
131
|
+
tables.set(step.entity, await eraseTable(stepApi, reader, step));
|
|
132
|
+
}
|
|
133
|
+
return { tables: Object.fromEntries(tables) };
|
|
134
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
// The three `X_SUBJECT_*` refusals a subject's plan can end on — each a refusal of the
|
|
2
|
+
// DECLARATION, so a run raises the first and `x exercise` lists every one. The codes, titles and
|
|
3
|
+
// retry classes are `error-codes.ts`'s; the values printed are entity and column names, never data.
|
|
4
|
+
|
|
5
|
+
import { renderFixShellArg, UltimateError } from '@ultimat3/core';
|
|
6
|
+
|
|
7
|
+
/** A table reaches the subject and `tables:` gives it no treatment. `path` is how it reaches. */
|
|
8
|
+
export class SubjectTableUndeclaredError extends UltimateError {
|
|
9
|
+
constructor(input: { subject: string; table: string; path: readonly string[] }) {
|
|
10
|
+
super({
|
|
11
|
+
code: 'X_SUBJECT_TABLE_UNDECLARED',
|
|
12
|
+
cause: `${input.table} reaches the subject ${input.subject} (${input.path.join(' <- ')}) and defineSubject(${input.subject}) gives it no treatment, so an erase would leave its rows and an export would miss them`,
|
|
13
|
+
fix: `tables: { ${input.table}: 'delete' } # in defineSubject(${input.subject}, …) — or { anonymize: { <column>: <value> } }, or { retain: '<why it is kept>' }`,
|
|
14
|
+
meta: { subject: input.subject, table: input.table, path: [...input.path] },
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** A treatment that cannot be carried out. `field` names what to change. */
|
|
20
|
+
export class SubjectTreatmentInvalidError extends UltimateError {
|
|
21
|
+
constructor(input: {
|
|
22
|
+
subject: string;
|
|
23
|
+
table: string;
|
|
24
|
+
field: string;
|
|
25
|
+
reason: string;
|
|
26
|
+
/** The treatment the repair is: `'delete'` for a kept row a delete would block. */
|
|
27
|
+
instead?: string;
|
|
28
|
+
}) {
|
|
29
|
+
super({
|
|
30
|
+
code: 'X_SUBJECT_TREATMENT_INVALID',
|
|
31
|
+
cause: `defineSubject(${input.subject}) treats ${input.table}: ${input.field} ${input.reason}`,
|
|
32
|
+
fix: `tables: { ${input.table}: ${input.instead ?? "{ retain: '<why it is kept>' }"} } # in defineSubject(${input.subject}, …), or correct ${input.field} as the cause says`,
|
|
33
|
+
meta: { subject: input.subject, table: input.table, field: input.field },
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** Tables the erase deletes reference each other through keys that are all `not null`. */
|
|
39
|
+
export class SubjectCycleError extends UltimateError {
|
|
40
|
+
constructor(input: { subject: string; table: string; column: string; parent: string }) {
|
|
41
|
+
super({
|
|
42
|
+
code: 'X_SUBJECT_CYCLE',
|
|
43
|
+
cause: `${input.table}.${input.column} references ${input.parent}, which the erase of ${input.subject} also deletes, and closes a loop — the key is not nullable, so neither side can go first`,
|
|
44
|
+
fix: `x db gen ${renderFixShellArg(`nullable ${input.table} ${input.column}`, '<description>')} # after ${input.column}: uuid().nullable() in entity('${input.table}'): the erase clears it before deleting either side`,
|
|
45
|
+
meta: { subject: input.subject, table: input.table, column: input.column },
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
}
|