@ultimat3/jobs 29.3.2 → 30.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/CLAUDE.md +68 -89
  2. package/README.md +304 -10
  3. package/package.json +18 -6
  4. package/src/backfill-definition.ts +184 -0
  5. package/src/backfill-errors.ts +18 -2
  6. package/src/backfill-pass.ts +43 -40
  7. package/src/backfill-pending.ts +40 -17
  8. package/src/backfill-reader.ts +94 -0
  9. package/src/backfill-registry.ts +5 -0
  10. package/src/backfill-scope.ts +1 -2
  11. package/src/backfill-source.ts +113 -0
  12. package/src/backfill.ts +54 -135
  13. package/src/consumer-errors.ts +42 -0
  14. package/src/consumer-guard.ts +19 -0
  15. package/src/consumer-loop.ts +171 -0
  16. package/src/consumer-registry.ts +103 -0
  17. package/src/consumer-types.ts +81 -0
  18. package/src/consumer.ts +147 -0
  19. package/src/driver-memory-operator.ts +25 -2
  20. package/src/driver-memory.ts +35 -1
  21. package/src/driver-pg-claim-sql.ts +128 -0
  22. package/src/driver-pg-ddl.ts +11 -2
  23. package/src/driver-pg-demand-sql.ts +27 -0
  24. package/src/driver-pg-enqueue-many.ts +153 -0
  25. package/src/driver-pg-event-wake-sql.ts +51 -0
  26. package/src/driver-pg-operator-sql.ts +34 -6
  27. package/src/driver-pg-operator.ts +36 -2
  28. package/src/driver-pg-outbox-sql.ts +34 -0
  29. package/src/driver-pg-sql.ts +24 -91
  30. package/src/driver-pg.ts +17 -1
  31. package/src/driver-redis-refusal.ts +25 -0
  32. package/src/driver-redis-scripts.ts +79 -30
  33. package/src/driver-redis.ts +70 -42
  34. package/src/driver.ts +38 -1
  35. package/src/enqueue-record.ts +137 -0
  36. package/src/error-codes.ts +233 -0
  37. package/src/errors-concurrency.ts +2 -2
  38. package/src/errors-declaration.ts +15 -2
  39. package/src/errors-operator.ts +1 -1
  40. package/src/errors-requeue.ts +1 -1
  41. package/src/errors-saga.ts +87 -0
  42. package/src/errors-steps.ts +35 -0
  43. package/src/errors-tenant.ts +1 -1
  44. package/src/errors.ts +25 -165
  45. package/src/events-pg.ts +20 -8
  46. package/src/events.ts +14 -3
  47. package/src/execute-ctx.ts +51 -0
  48. package/src/execute-unwind.ts +191 -0
  49. package/src/execute.ts +50 -51
  50. package/src/export-errors.ts +1 -1
  51. package/src/failure-row.ts +43 -0
  52. package/src/import-errors.ts +102 -0
  53. package/src/import-format.ts +232 -0
  54. package/src/import-pass.ts +255 -0
  55. package/src/import-source.ts +49 -0
  56. package/src/import.ts +175 -0
  57. package/src/index.ts +163 -7
  58. package/src/inspect-operator.ts +24 -3
  59. package/src/inspect.ts +17 -4
  60. package/src/introspection.ts +24 -1
  61. package/src/job-enqueue-many.ts +108 -0
  62. package/src/job-types.ts +229 -0
  63. package/src/job.ts +19 -208
  64. package/src/metrics.ts +10 -0
  65. package/src/outbox-pg.ts +35 -0
  66. package/src/outbox-relay.ts +58 -19
  67. package/src/outbox.ts +65 -73
  68. package/src/purge-entities.ts +92 -0
  69. package/src/purge.ts +10 -2
  70. package/src/queue-wake.ts +1 -1
  71. package/src/refresh-view.ts +76 -0
  72. package/src/retry-classification.ts +27 -8
  73. package/src/retry.ts +26 -0
  74. package/src/rollup-errors.ts +27 -0
  75. package/src/rollup.ts +190 -0
  76. package/src/run-deadline.ts +11 -7
  77. package/src/scheduler.ts +2 -1
  78. package/src/settled.ts +56 -15
  79. package/src/step-resolve.ts +37 -0
  80. package/src/steps-api.ts +111 -0
  81. package/src/steps-core.ts +26 -0
  82. package/src/steps-output.ts +142 -0
  83. package/src/steps-store.ts +68 -0
  84. package/src/steps-suspension.ts +51 -0
  85. package/src/steps-timeout.ts +10 -5
  86. package/src/steps-unwind.ts +94 -0
  87. package/src/steps-wait.ts +163 -0
  88. package/src/steps.ts +174 -210
  89. package/src/subject-erase-pass.ts +134 -0
  90. package/src/subject-errors.ts +48 -0
  91. package/src/subject-export-pass.ts +228 -0
  92. package/src/subject-jobs.ts +121 -0
  93. package/src/subject-plan.ts +287 -0
  94. package/src/subject-select.ts +106 -0
  95. package/src/subject.ts +119 -0
  96. package/src/tenant.ts +69 -1
  97. package/src/type-pins.ts +90 -0
  98. package/src/webhook-errors.ts +1 -1
  99. package/src/webhook-receiver-errors.ts +59 -0
  100. package/src/webhook-receiver-route.ts +127 -0
  101. package/src/webhook-receiver-types.ts +97 -0
  102. package/src/webhook-receiver.ts +132 -0
  103. package/src/webhook-target.ts +20 -43
  104. package/src/webhook.ts +7 -4
  105. package/src/worker-queue-demand.ts +57 -0
  106. package/src/worker-queue-depth.ts +15 -3
  107. package/src/worker-registry.ts +4 -1
  108. package/src/worker-types.ts +2 -0
  109. package/src/worker.ts +2 -2
package/src/steps.ts CHANGED
@@ -5,6 +5,7 @@
5
5
  //
6
6
  // Suspension (sleep / waitForEvent) unwinds the run by throwing a StepSuspension. The worker
7
7
  // catches it and re-queues the job for `resumeAt` instead of holding a process for three days.
8
+ // `waitForEvent` is `steps-wait.ts`; compensation, `steps-unwind.ts`; both over `steps-core.ts`.
8
9
 
9
10
  import type { Clock } from '@ultimat3/core';
10
11
  import { finiteOption, isUltimateError, logger, renderThrowable } from '@ultimat3/core';
@@ -12,114 +13,38 @@ import { expectedQueryLoop } from '@ultimat3/db';
12
13
  import type { DurationInput } from './clock';
13
14
  import { finiteDurationMs, nowMs } from './clock';
14
15
  import { JobAbortedError, JobTimeoutError, StepDuplicateError } from './errors';
15
- import { createRunSignal } from './run-signal';
16
16
  import {
17
- isStepSuspension,
18
- isWaitTimedOut,
19
- StepSuspension,
20
- WAIT_TIMED_OUT,
21
- } from './steps-suspension';
17
+ isReservedStepName,
18
+ StepCompensateUndeclaredError,
19
+ StepNameReservedError,
20
+ StepOutcomeUnknownError,
21
+ UNWINDING_STEP,
22
+ } from './errors-saga';
23
+ import { createRunSignal } from './run-signal';
24
+ import type { EventLookup, StepApi, StepRunOptions } from './steps-api';
25
+ import { DEFAULT_EVENT_POLL_MS, WAKING_EVENT_POLL_MS } from './steps-api';
26
+ import type { RunnerCore } from './steps-core';
27
+ import type { StepOutput } from './steps-output';
28
+ import { jsonForm } from './steps-output';
29
+ import type { StepFence, StepRecord, StepStore } from './steps-store';
30
+ import { isStepSuspension, StepSuspension, UnwindReached } from './steps-suspension';
22
31
  import { withStepTimeout } from './steps-timeout';
23
-
32
+ import { compensationStack } from './steps-unwind';
33
+ import { waitForEventStep } from './steps-wait';
34
+
35
+ export type {
36
+ CompensateContext,
37
+ EventLookup,
38
+ StepApi,
39
+ StepRunOptions,
40
+ WaitForEventOptions,
41
+ } from './steps-api';
42
+ export { DEFAULT_EVENT_POLL_MS, WAKING_EVENT_POLL_MS } from './steps-api';
43
+ export type { StepOutput } from './steps-output';
44
+ export type { StepFence, StepRecord, StepStatus, StepStore } from './steps-store';
45
+ export { isStepStatus, STEP_STATUSES } from './steps-store';
24
46
  export { isStepSuspension, StepSuspension } from './steps-suspension';
25
47
 
26
- /**
27
- * The runtime list is the declaration and `StepStatus` is derived from it, the shape
28
- * `BACKFILL_STATUSES` and `PRIMITIVE_KINDS` already have. A bare union cannot narrow a `text`
29
- * column, so `driver-pg-rows.ts` cast one instead — and a cast that lands an unknown status on a
30
- * record makes `stepRun`'s `existing?.status === 'completed'` false, which RE-EXECUTES a step
31
- * this file promises runs once.
32
- */
33
- export const STEP_STATUSES = ['completed', 'sleeping', 'waiting', 'failed'] as const;
34
-
35
- export type StepStatus = (typeof STEP_STATUSES)[number];
36
-
37
- /** Narrows a status read back out of a store. Never a cast — the list decides. */
38
- export const isStepStatus = (value: string): value is StepStatus =>
39
- (STEP_STATUSES as readonly string[]).includes(value);
40
-
41
- export interface StepRecord {
42
- readonly runId: string;
43
- readonly name: string;
44
- readonly status: StepStatus;
45
- /** The memoized return value. Present only when `status === 'completed'`. */
46
- readonly output?: unknown;
47
- readonly startedAt: number;
48
- readonly completedAt?: number;
49
- /** Epoch ms at which a sleeping/waiting step becomes runnable. */
50
- readonly wakeAt?: number;
51
- readonly event?: string;
52
- readonly correlationKey?: string;
53
- readonly attempts: number;
54
- readonly error?: string;
55
- }
56
-
57
- /** The claim a step write is made under: `claimOf(claimed)`, plus the row it names. */
58
- export interface StepFence {
59
- readonly job: string;
60
- readonly jobId: string;
61
- readonly workerId: string;
62
- readonly claim: number;
63
- }
64
-
65
- export interface StepStore {
66
- get(runId: string, name: string): Promise<StepRecord | undefined>;
67
- /**
68
- * With `by`, the write lands only while that claim still holds the row (`running`, same worker,
69
- * same claim ordinal — `SQL_ACK`'s fence) and is `X_JOB_LEASE_LOST` otherwise: the in-process
70
- * signal cannot stop a body in ANOTHER process whose lease lapsed. Without it the write is
71
- * unfenced — a transfer (`x jobs drain`) or a runner driven by hand, with no row to hold.
72
- */
73
- put(record: StepRecord, by?: StepFence): Promise<void>;
74
- list(runId: string): Promise<readonly StepRecord[]>;
75
- del(runId: string, name: string): Promise<void>;
76
- clear(runId: string): Promise<void>;
77
- }
78
-
79
- /** The slice of the event bus a waiting step needs. Implemented by `events.ts`. */
80
- export interface EventLookup {
81
- /**
82
- * The bus's OWN clock, in epoch ms: what a publish issued now would be stamped with. A consumer
83
- * that wants "published after I asked" takes its "asked at" from here and never from its own
84
- * process — `publishedAt` is this clock's, and two clocks compared is a skew between pods
85
- * deciding which answers count. `waitForEvent` stamps a NEW wait with it.
86
- */
87
- now(): Promise<number>;
88
- find(
89
- event: string,
90
- correlationKey: string | undefined,
91
- afterMs: number,
92
- ): Promise<{ readonly payload: unknown; readonly publishedAt: number } | undefined>;
93
- }
94
-
95
- export interface WaitForEventOptions {
96
- /** How long to wait before giving up. Default 24h. */
97
- readonly timeout?: DurationInput;
98
- /** Correlation key the published event must carry — usually an entity id. */
99
- readonly match?: string;
100
- /** Default false: a timeout resolves `undefined`. `true` fails the job with X_JOB_TIMEOUT. */
101
- readonly required?: boolean;
102
- }
103
-
104
- export interface StepApi {
105
- /**
106
- * Run once, ever. On replay the persisted output is returned and `fn` is not called.
107
- *
108
- * `fn` receives the step's `AbortSignal` — the run's cancellation and this step's own ceiling,
109
- * whichever fires first. Hand it to `fetch`, or read `.aborted` in a loop: past it, this step
110
- * may no longer write, because the attempt that replaced this one owns the run.
111
- */
112
- run<T>(name: string, fn: (signal: AbortSignal) => Promise<T> | T): Promise<T>;
113
- /** Suspend the run. `sleep(duration)` derives the step name from the duration. */
114
- sleep(name: string, duration: DurationInput): Promise<void>;
115
- sleep(duration: DurationInput): Promise<void>;
116
- waitForEvent<T>(
117
- name: string,
118
- event: string,
119
- options?: WaitForEventOptions,
120
- ): Promise<T | undefined>;
121
- }
122
-
123
48
  export interface StepRunnerOptions {
124
49
  readonly runId: string;
125
50
  readonly jobName: string;
@@ -128,8 +53,19 @@ export interface StepRunnerOptions {
128
53
  readonly fence?: StepFence;
129
54
  readonly clock?: Clock;
130
55
  readonly events?: EventLookup;
131
- /** How long a waiting step stays parked between event polls. Default 30s. */
56
+ /**
57
+ * How long a waiting step stays parked between event polls. Default `DEFAULT_EVENT_POLL_MS`
58
+ * (30 s), or `WAKING_EVENT_POLL_MS` (15 min) when `events.wakes`: a publish then brings the run
59
+ * forward, and the poll is only the backstop. A value set here wins either way.
60
+ */
132
61
  readonly eventPollMs?: number;
62
+ /** The job declares `compensates: true`: `step.run(…, { compensate })` is legal. */
63
+ readonly compensates?: boolean;
64
+ /**
65
+ * This attempt is past the policy's last (`isUnwindAttempt`): the run is unwinding whether or
66
+ * not `unwinding` was written — the final attempt crashed before it could be.
67
+ */
68
+ readonly unwind?: boolean;
133
69
  /** Per-step ceiling; the job-level timeout is enforced by the worker. */
134
70
  readonly stepTimeoutMs?: number;
135
71
  /**
@@ -157,6 +93,13 @@ export interface StepRunner {
157
93
  usedNames(): readonly string[];
158
94
  /** Names that were served from storage instead of executed. Bounded the same way. */
159
95
  replayedNames(): readonly string[];
96
+ /** Whether the run is unwinding: `options.unwind`, or `unwinding` found among its steps. */
97
+ unwinding(): Promise<boolean>;
98
+ /**
99
+ * Run this attempt's registered compensators, last-in first-out (`steps-unwind.ts`). Answers
100
+ * the failure the run unwinds from — the one already recorded, else `failure`.
101
+ */
102
+ unwind(failure: string): Promise<string>;
160
103
  }
161
104
 
162
105
  /** Stands in for an absent run signal, so the fence has one shape and no `undefined` branch. */
@@ -182,7 +125,12 @@ export function stepRunner(options: StepRunnerOptions): StepRunner {
182
125
  const used: string[] = [];
183
126
  const replayed: string[] = [];
184
127
  const clock = options.clock;
185
- const pollMs = finiteOption('step.waitForEvent', 'eventPollMs', options.eventPollMs ?? 30_000);
128
+ const pollMs = finiteOption(
129
+ 'step.waitForEvent',
130
+ 'eventPollMs',
131
+ options.eventPollMs ??
132
+ (options.events?.wakes === true ? WAKING_EVENT_POLL_MS : DEFAULT_EVENT_POLL_MS),
133
+ );
186
134
  // `job()` refuses a non-finite `stepTimeout` at declaration (`job.ts`); screened again here, where
187
135
  // it is read, because this runner is also built directly and `withStepTimeout` reads a `NaN`
188
136
  // ceiling as none at all — the backstop `pollMs` above already is.
@@ -203,23 +151,39 @@ export function stepRunner(options: StepRunnerOptions): StepRunner {
203
151
  * Sound because the fence in `put()` is: only the attempt that owns the run may write to it, so
204
152
  * within one attempt an absent name stays absent unless this runner writes it. Writes go into
205
153
  * the map as they go into the store, never the other way round — the store is still the record.
154
+ *
155
+ * The PROMISE is the memo, and the map is published only when the list has answered: a map
156
+ * assigned before the await let a second caller read it empty, so `Promise.all` over completed
157
+ * steps re-ran one on replay and answered a value the store did not hold. A rejected list clears
158
+ * the memo, so the next call asks again rather than this attempt failing on one blip for good.
206
159
  */
207
160
  let hydrated: Map<string, StepRecord> | undefined;
208
-
209
- const load = async (name: string): Promise<StepRecord | undefined> => {
210
- if (hydrated === undefined) {
211
- hydrated = new Map();
212
- for (const record of await store.list(runId)) hydrated.set(record.name, record);
213
- }
214
- return hydrated.get(name);
161
+ let hydrating: Promise<Map<string, StepRecord>> | undefined;
162
+
163
+ const hydrate = (): Promise<Map<string, StepRecord>> => {
164
+ if (hydrating !== undefined) return hydrating;
165
+ const pending = (async () => {
166
+ const records = await store.list(runId);
167
+ hydrated = new Map(records.map((record) => [record.name, record]));
168
+ return hydrated;
169
+ })();
170
+ hydrating = pending;
171
+ pending.catch(() => {
172
+ if (hydrating === pending) hydrating = undefined;
173
+ });
174
+ return pending;
215
175
  };
216
176
 
217
- /** Keep the hydrated view in step with what was just persisted. */
177
+ const load = async (name: string): Promise<StepRecord | undefined> => (await hydrate()).get(name);
178
+
179
+ /** Keep the hydrated view in step. Every write follows a `load`, so the list has answered. */
218
180
  const remember = (record: StepRecord): void => {
219
181
  hydrated?.set(record.name, record);
220
182
  };
221
183
 
222
184
  const claimName = (name: string): void => {
185
+ // Refused for EVERY job, so one that gains `compensates` later cannot meet a step it wrote.
186
+ if (isReservedStepName(name)) throw new StepNameReservedError({ job: jobName, step: name });
223
187
  // The Set decides, the array only reports. `Array.includes` made a long run quadratic —
224
188
  // `backfill({ batch: 50 })` over a million rows is 20,000 steps and ~200M string compares.
225
189
  if (claimed.has(name)) throw new StepDuplicateError({ job: jobName, step: name });
@@ -291,14 +255,25 @@ export function stepRunner(options: StepRunnerOptions): StepRunner {
291
255
 
292
256
  const now = (): number => nowMs(clock);
293
257
 
294
- async function run<T>(name: string, fn: (signal: AbortSignal) => Promise<T> | T): Promise<T> {
295
- claimName(name);
296
- const existing = await load(name);
297
- if (existing?.status === 'completed') {
298
- trace(replayed, name);
299
- return existing.output as T;
300
- }
258
+ /**
259
+ * The run is unwinding: this attempt is past the policy's last, or `unwinding` is among the
260
+ * steps the first `load` hydrated. Sticky — once true for an attempt it stays true.
261
+ */
262
+ let unwindMode = options.unwind === true;
263
+ const unwinding = async (): Promise<boolean> => {
264
+ if (!unwindMode) unwindMode = (await hydrate()).has(UNWINDING_STEP);
265
+ return unwindMode;
266
+ };
301
267
 
268
+ /**
269
+ * One step body, run under its ceiling and recorded: `completed` before it returns, `failed` on
270
+ * the way out. Shared by `step.run` and by every compensator (`steps-unwind.ts`).
271
+ */
272
+ const execute = async <T>(
273
+ name: string,
274
+ existing: StepRecord | undefined,
275
+ fn: (signal: AbortSignal) => Promise<T> | T,
276
+ ): Promise<StepOutput<T>> => {
302
277
  const startedAt = existing?.startedAt ?? now();
303
278
  const attempts = (existing?.attempts ?? 0) + 1;
304
279
  // The step's own ceiling, folded into the run's cancellation so the body reads ONE signal and
@@ -319,24 +294,31 @@ export function stepRunner(options: StepRunnerOptions): StepRunner {
319
294
  fn(signal),
320
295
  stepTimeoutMs,
321
296
  deadline,
322
- () => new JobTimeoutError({ job: jobName, step: name, timeoutMs: stepTimeoutMs ?? 0 }),
297
+ () =>
298
+ new JobTimeoutError({
299
+ job: jobName,
300
+ step: name,
301
+ timeoutMs: stepTimeoutMs ?? 0,
302
+ fix: `job({ name: '${jobName}', stepTimeout: '5m' }) # raises the ceiling; or split step.run('${name}', …) into smaller steps`,
303
+ }),
323
304
  );
305
+ // What goes in the store is what comes back, so both attempts see one value.
306
+ const stored = jsonForm(output as T, { job: jobName, step: name });
324
307
  // Persist BEFORE returning: a crash one line later must not re-run this step.
325
308
  await put({
326
309
  runId,
327
310
  name,
328
311
  status: 'completed',
329
- output,
312
+ output: stored,
330
313
  startedAt,
331
314
  completedAt: now(),
332
315
  attempts,
333
316
  });
334
- return output;
317
+ return stored;
335
318
  } catch (error) {
336
319
  if (isStepSuspension(error)) throw error;
337
320
  // The failure is this run's own history, so it is recorded on the way out — unless the
338
- // attempt was cancelled, in which case the history is no longer ours to write. The original
339
- // error is what the caller has to see, never one raised by the bookkeeping.
321
+ // attempt was cancelled, in which case the history is no longer ours to write.
340
322
  if (!cancelled()) {
341
323
  // Deliberately NOT through `put`: the caller has to see the original error, never one
342
324
  // raised by the bookkeeping. The hydrated view is updated by hand for the same reason.
@@ -365,6 +347,68 @@ export function stepRunner(options: StepRunnerOptions): StepRunner {
365
347
  // settled leaves its signal in whatever state it ended in.
366
348
  composed?.dispose();
367
349
  }
350
+ };
351
+
352
+ const core: RunnerCore = {
353
+ runId,
354
+ jobName,
355
+ events: options.events,
356
+ pollMs,
357
+ claim: claimName,
358
+ load,
359
+ put,
360
+ now,
361
+ replayed: (name) => trace(replayed, name),
362
+ forwardOnly: async (name) => {
363
+ if (await unwinding()) throw new UnwindReached(name);
364
+ },
365
+ };
366
+ const compensations = compensationStack(core, execute);
367
+
368
+ async function run<T>(
369
+ name: string,
370
+ fn: (signal: AbortSignal) => Promise<T> | T,
371
+ runOptions: StepRunOptions<T> = {},
372
+ ): Promise<StepOutput<T>> {
373
+ const { compensate, once = false } = runOptions;
374
+ if (compensate !== undefined && options.compensates !== true) {
375
+ throw new StepCompensateUndeclaredError({ job: jobName, step: name });
376
+ }
377
+ claimName(name);
378
+ const undo =
379
+ compensate === undefined
380
+ ? undefined
381
+ : (output: StepOutput<T>) => async (signal: AbortSignal) => {
382
+ // A compensator's answer is not the run's to keep: `null` is what its step records.
383
+ await compensate(output, { signal });
384
+ return null;
385
+ };
386
+ const existing = await load(name);
387
+ if (existing?.status === 'completed') {
388
+ trace(replayed, name);
389
+ const output = existing.output as StepOutput<T>;
390
+ if (undo !== undefined) compensations.done(name, undo(output));
391
+ return output;
392
+ }
393
+ // `started` or `failed` and never `completed`: a crash or a throw, and nobody knows which side
394
+ // of the side effect it stopped on. `cleared` is an operator's "it did not run".
395
+ const unknown = once && (existing?.status === 'started' || existing?.status === 'failed');
396
+ if (unknown && undo !== undefined) compensations.unknown(name);
397
+ await core.forwardOnly(name);
398
+ if (unknown) throw new StepOutcomeUnknownError({ job: jobName, step: name });
399
+ if (once) {
400
+ // BEFORE `fn`: from here on, a replay cannot tell whether `fn` ran, and so never runs it.
401
+ await put({
402
+ runId,
403
+ name,
404
+ status: 'started',
405
+ startedAt: existing?.startedAt ?? now(),
406
+ attempts: existing?.attempts ?? 0,
407
+ });
408
+ }
409
+ const output = await execute(name, once ? await load(name) : existing, fn);
410
+ if (undo !== undefined) compensations.done(name, undo(output));
411
+ return output;
368
412
  }
369
413
 
370
414
  async function sleep(a: string | DurationInput, b?: DurationInput): Promise<void> {
@@ -378,6 +422,7 @@ export function stepRunner(options: StepRunnerOptions): StepRunner {
378
422
  trace(replayed, name);
379
423
  return;
380
424
  }
425
+ await core.forwardOnly(name);
381
426
 
382
427
  const at = now();
383
428
  if (existing?.status === 'sleeping' && existing.wakeAt !== undefined) {
@@ -401,98 +446,17 @@ export function stepRunner(options: StepRunnerOptions): StepRunner {
401
446
  throw new StepSuspension({ step: name, resumeAt: wakeAt, reason: 'sleep' });
402
447
  }
403
448
 
404
- async function waitForEvent<T>(
405
- name: string,
406
- event: string,
407
- waitOptions: WaitForEventOptions = {},
408
- ): Promise<T | undefined> {
409
- claimName(name);
410
- const existing = await load(name);
411
- if (existing?.status === 'completed') {
412
- trace(replayed, name);
413
- return isWaitTimedOut(existing.output) ? undefined : (existing.output as T | undefined);
414
- }
415
-
416
- const at = now();
417
- // A NEW wait is stamped by the bus, whose clock stamps every `publishedAt` it is compared
418
- // with; the runner's own only when there is no bus to ask. A re-poll keeps what it persisted.
419
- const startedAt = existing?.startedAt ?? (await options.events?.now()) ?? at;
420
- const deadline =
421
- startedAt +
422
- finiteDurationMs(waitOptions.timeout ?? 86_400_000, 'step.waitForEvent', 'timeout');
423
- const correlationKey = waitOptions.match;
424
-
425
- const hit = await options.events?.find(event, correlationKey, startedAt);
426
- if (hit !== undefined) {
427
- await put({
428
- runId,
429
- name,
430
- status: 'completed',
431
- output: hit.payload,
432
- startedAt,
433
- completedAt: at,
434
- event,
435
- attempts: (existing?.attempts ?? 0) + 1,
436
- ...(correlationKey === undefined ? {} : { correlationKey }),
437
- });
438
- return hit.payload as T;
439
- }
440
-
441
- // `deadline` is on the BUS's clock, so the bus calls the timeout: asked only once this runner
442
- // believes the time is up, which costs a wait one statement at its end. A runner AHEAD of the
443
- // bus used to give up early by the skew — on an answer still inside its window. (A runner
444
- // BEHIND waits the skew longer; knowing that sooner would cost a round trip on every poll.)
445
- let remaining = deadline - at;
446
- if (remaining <= 0) remaining = deadline - ((await options.events?.now()) ?? at);
447
- if (remaining <= 0) {
448
- if (waitOptions.required === true) {
449
- throw new JobTimeoutError({
450
- job: jobName,
451
- step: name,
452
- timeoutMs: finiteDurationMs(
453
- waitOptions.timeout ?? 86_400_000,
454
- 'step.waitForEvent',
455
- 'timeout',
456
- ),
457
- });
458
- }
459
- logger.warn('jobs.step.wait-timeout', { job: jobName, step: name, event });
460
- await put({
461
- runId,
462
- name,
463
- status: 'completed',
464
- output: WAIT_TIMED_OUT,
465
- startedAt,
466
- completedAt: at,
467
- event,
468
- attempts: (existing?.attempts ?? 0) + 1,
469
- });
470
- return undefined;
471
- }
472
-
473
- const resumeAt = at + Math.min(remaining, pollMs);
474
- await put({
475
- runId,
476
- name,
477
- status: 'waiting',
478
- startedAt,
479
- wakeAt: resumeAt,
480
- event,
481
- attempts: (existing?.attempts ?? 0) + 1,
482
- ...(correlationKey === undefined ? {} : { correlationKey }),
483
- });
484
- throw new StepSuspension({ step: name, resumeAt, reason: 'event' });
485
- }
486
-
487
449
  const step: StepApi = {
488
450
  run,
489
451
  sleep: sleep as StepApi['sleep'],
490
- waitForEvent,
452
+ waitForEvent: waitForEventStep(core),
491
453
  };
492
454
 
493
455
  return {
494
456
  step,
495
457
  usedNames: () => [...used],
496
458
  replayedNames: () => [...replayed],
459
+ unwinding,
460
+ unwind: (failure) => compensations.unwind(failure),
497
461
  };
498
462
  }
@@ -0,0 +1,134 @@
1
+ // One `subjectErase()` pass: the framework targets first (they read the subject's row), then every
2
+ // loop-closing key cleared, then each table children first, the subject last. Every page of
3
+ // parent keys is its own step, so a killed run resumes at the page it stopped on; each statement
4
+ // is an equality on one key, so a replayed step deletes or overwrites nothing twice.
5
+
6
+ import type { StepApi } from './steps-api';
7
+ import type { SubjectRef, SubjectTarget, SubjectTargetEntry } from './subject';
8
+ import { isRetainedTarget } from './subject';
9
+ import type { SubjectStep } from './subject-plan';
10
+ import type { SubjectReader } from './subject-select';
11
+ import { rowIdentity } from './subject-select';
12
+
13
+ /** What one table or target went through. Bounded and JSON-safe: it is the job's result. */
14
+ export interface SubjectTally {
15
+ readonly deleted: number;
16
+ readonly anonymized: number;
17
+ readonly retained: number;
18
+ }
19
+
20
+ export interface SubjectEraseReport {
21
+ readonly tables: Readonly<Record<string, SubjectTally>>;
22
+ }
23
+
24
+ const tally = (deleted: number, anonymized: number, retained: number): SubjectTally => ({
25
+ deleted,
26
+ anonymized,
27
+ retained,
28
+ });
29
+
30
+ /** The subject as a target reads it, fresh: its row is never written into a checkpoint. */
31
+ const refOf = async (reader: SubjectReader): Promise<SubjectRef> => ({
32
+ key: reader.key,
33
+ row: (await reader.rootRow()) ?? {},
34
+ });
35
+
36
+ async function eraseTarget(step: StepApi, reader: SubjectReader, target: SubjectTarget) {
37
+ let total = 0;
38
+ for (let n = 0; ; n += 1) {
39
+ const removed = await step.run(`target:${target.name}:${n}`, async () =>
40
+ target.erase(await refOf(reader), reader.size),
41
+ );
42
+ total += removed;
43
+ if (removed < reader.size) return total;
44
+ }
45
+ }
46
+
47
+ /** One write per parent key in the page: `deleteWhere`/`updateWhere`, the repository's own. */
48
+ async function treatPage(
49
+ reader: SubjectReader,
50
+ step: SubjectStep,
51
+ property: string,
52
+ values: readonly unknown[],
53
+ ): Promise<number> {
54
+ const table = reader.table(step.entity);
55
+ let count = 0;
56
+ for (const value of values) {
57
+ const filter = { [property]: value };
58
+ count +=
59
+ step.treatment === 'delete'
60
+ ? await table.deleteWhere(filter)
61
+ : typeof step.treatment === 'object' && 'anonymize' in step.treatment
62
+ ? await table.updateWhere(filter, step.treatment.anonymize)
63
+ : 0;
64
+ }
65
+ return count;
66
+ }
67
+
68
+ async function countRows(reader: SubjectReader, entity: string): Promise<number> {
69
+ const core = reader.core(entity);
70
+ const seen = new Set<string>();
71
+ for await (const page of reader.pages(entity)) {
72
+ for (const row of page) seen.add(rowIdentity(core, row));
73
+ }
74
+ return seen.size;
75
+ }
76
+
77
+ async function eraseTable(
78
+ stepApi: StepApi,
79
+ reader: SubjectReader,
80
+ step: SubjectStep,
81
+ ): Promise<SubjectTally> {
82
+ const { entity, treatment } = step;
83
+ if (typeof treatment === 'object' && 'retain' in treatment) {
84
+ return tally(0, 0, await stepApi.run(`retain:${entity}`, () => countRows(reader, entity)));
85
+ }
86
+ let count = 0;
87
+ if (entity === reader.plan.subject) {
88
+ count = await stepApi.run(`${entity}:subject`, () =>
89
+ treatPage(reader, step, reader.plan.key, [reader.key]),
90
+ );
91
+ }
92
+ for (const via of step.via) {
93
+ let n = 0;
94
+ for await (const values of reader.parentKeys(via)) {
95
+ count += await stepApi.run(`${entity}.${via.property}:${n}`, () =>
96
+ treatPage(reader, step, via.property, values),
97
+ );
98
+ n += 1;
99
+ }
100
+ }
101
+ return treatment === 'delete' ? tally(count, 0, 0) : tally(0, count, 0);
102
+ }
103
+
104
+ export async function subjectErasePass(
105
+ stepApi: StepApi,
106
+ reader: SubjectReader,
107
+ targets: readonly SubjectTargetEntry[],
108
+ ): Promise<SubjectEraseReport> {
109
+ const tables = new Map<string, SubjectTally>();
110
+ for (const entry of targets) {
111
+ if (isRetainedTarget(entry)) {
112
+ tables.set(entry.target.name, tally(0, 0, 0));
113
+ continue;
114
+ }
115
+ tables.set(entry.name, tally(await eraseTarget(stepApi, reader, entry), 0, 0));
116
+ }
117
+ for (const unlink of reader.plan.unlink) {
118
+ let n = 0;
119
+ for await (const values of reader.parentKeys(unlink)) {
120
+ await stepApi.run(`unlink:${unlink.entity}.${unlink.property}:${n}`, async () => {
121
+ const table = reader.table(unlink.entity);
122
+ for (const value of values) {
123
+ await table.updateWhere({ [unlink.property]: value }, { [unlink.property]: null });
124
+ }
125
+ return null;
126
+ });
127
+ n += 1;
128
+ }
129
+ }
130
+ for (const step of reader.plan.steps) {
131
+ tables.set(step.entity, await eraseTable(stepApi, reader, step));
132
+ }
133
+ return { tables: Object.fromEntries(tables) };
134
+ }
@@ -0,0 +1,48 @@
1
+ // The three `X_SUBJECT_*` refusals a subject's plan can end on — each a refusal of the
2
+ // DECLARATION, so a run raises the first and `x exercise` lists every one. The codes, titles and
3
+ // retry classes are `error-codes.ts`'s; the values printed are entity and column names, never data.
4
+
5
+ import { renderFixShellArg, UltimateError } from '@ultimat3/core';
6
+
7
+ /** A table reaches the subject and `tables:` gives it no treatment. `path` is how it reaches. */
8
+ export class SubjectTableUndeclaredError extends UltimateError {
9
+ constructor(input: { subject: string; table: string; path: readonly string[] }) {
10
+ super({
11
+ code: 'X_SUBJECT_TABLE_UNDECLARED',
12
+ cause: `${input.table} reaches the subject ${input.subject} (${input.path.join(' <- ')}) and defineSubject(${input.subject}) gives it no treatment, so an erase would leave its rows and an export would miss them`,
13
+ fix: `tables: { ${input.table}: 'delete' } # in defineSubject(${input.subject}, …) — or { anonymize: { <column>: <value> } }, or { retain: '<why it is kept>' }`,
14
+ meta: { subject: input.subject, table: input.table, path: [...input.path] },
15
+ });
16
+ }
17
+ }
18
+
19
+ /** A treatment that cannot be carried out. `field` names what to change. */
20
+ export class SubjectTreatmentInvalidError extends UltimateError {
21
+ constructor(input: {
22
+ subject: string;
23
+ table: string;
24
+ field: string;
25
+ reason: string;
26
+ /** The treatment the repair is: `'delete'` for a kept row a delete would block. */
27
+ instead?: string;
28
+ }) {
29
+ super({
30
+ code: 'X_SUBJECT_TREATMENT_INVALID',
31
+ cause: `defineSubject(${input.subject}) treats ${input.table}: ${input.field} ${input.reason}`,
32
+ fix: `tables: { ${input.table}: ${input.instead ?? "{ retain: '<why it is kept>' }"} } # in defineSubject(${input.subject}, …), or correct ${input.field} as the cause says`,
33
+ meta: { subject: input.subject, table: input.table, field: input.field },
34
+ });
35
+ }
36
+ }
37
+
38
+ /** Tables the erase deletes reference each other through keys that are all `not null`. */
39
+ export class SubjectCycleError extends UltimateError {
40
+ constructor(input: { subject: string; table: string; column: string; parent: string }) {
41
+ super({
42
+ code: 'X_SUBJECT_CYCLE',
43
+ cause: `${input.table}.${input.column} references ${input.parent}, which the erase of ${input.subject} also deletes, and closes a loop — the key is not nullable, so neither side can go first`,
44
+ fix: `x db gen ${renderFixShellArg(`nullable ${input.table} ${input.column}`, '<description>')} # after ${input.column}: uuid().nullable() in entity('${input.table}'): the erase clears it before deleting either side`,
45
+ meta: { subject: input.subject, table: input.table, column: input.column },
46
+ });
47
+ }
48
+ }