@ultimat3/jobs 30.9.0 → 31.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/CLAUDE.md +72 -107
  2. package/README.md +202 -55
  3. package/package.json +8 -7
  4. package/src/adapter.ts +37 -0
  5. package/src/backfill-ledger.ts +2 -2
  6. package/src/backfill-rate.ts +1 -0
  7. package/src/clock.ts +2 -2
  8. package/src/drain-wait.ts +1 -0
  9. package/src/driver-memory.ts +2 -2
  10. package/src/driver-pg-ddl.ts +19 -0
  11. package/src/driver-pg-event-wake-sql.ts +43 -22
  12. package/src/driver-pg-sql.ts +68 -27
  13. package/src/driver-pg.ts +11 -10
  14. package/src/driver-redis.ts +2 -2
  15. package/src/driver.ts +14 -8
  16. package/src/error-codes.ts +27 -0
  17. package/src/errors-concurrency.ts +1 -1
  18. package/src/errors.ts +1 -1
  19. package/src/events-cursor.ts +31 -0
  20. package/src/events-pg.ts +52 -6
  21. package/src/events.ts +72 -36
  22. package/src/export.ts +2 -2
  23. package/src/gdpr-audit.ts +75 -0
  24. package/src/gdpr-coverage.ts +94 -0
  25. package/src/gdpr-errors.ts +35 -0
  26. package/src/import.ts +2 -2
  27. package/src/index.ts +45 -206
  28. package/src/inspect-operator.ts +2 -2
  29. package/src/inspect.ts +1 -1
  30. package/src/job-instrument.ts +115 -0
  31. package/src/job.ts +41 -4
  32. package/src/jobs-retention.ts +23 -6
  33. package/src/leases.ts +13 -3
  34. package/src/ledger-book.ts +168 -0
  35. package/src/ledger-errors.ts +48 -0
  36. package/src/ledger.ts +145 -0
  37. package/src/limits.ts +57 -19
  38. package/src/lock-errors.ts +54 -0
  39. package/src/lock.ts +176 -0
  40. package/src/outbox-relay.ts +4 -0
  41. package/src/outbox.ts +3 -3
  42. package/src/partitions.ts +119 -0
  43. package/src/reindex-sweep.ts +94 -0
  44. package/src/reindex.ts +147 -0
  45. package/src/retry-classification.ts +3 -2
  46. package/src/retry-jitter.ts +32 -0
  47. package/src/retry.ts +13 -16
  48. package/src/scheduler.ts +10 -6
  49. package/src/steps-api.ts +44 -5
  50. package/src/steps-suspension.ts +34 -3
  51. package/src/steps-wait.ts +52 -4
  52. package/src/steps.ts +1 -0
  53. package/src/subject-jobs.ts +12 -4
  54. package/src/webhook.ts +2 -2
  55. package/src/worker-admit.ts +8 -5
  56. package/src/worker-drain-cutoff.ts +1 -0
  57. package/src/worker-loop.ts +28 -5
  58. package/src/worker-registry.ts +7 -5
  59. package/src/worker-run.ts +53 -9
  60. package/src/worker.ts +8 -8
package/src/retry.ts CHANGED
@@ -3,7 +3,7 @@
3
3
  // when attempt 5 lands without running the queue. There is no `x jobs schedule`: the subcommands
4
4
  // are `ls`, `show`, `retry`, `cancel`, `drain`.
5
5
 
6
- import type { BackoffCurve, Random } from '@ultimat3/core';
6
+ import type { BackoffCurve, JitterMode, Random } from '@ultimat3/core';
7
7
  import { backoffDelay } from '@ultimat3/core';
8
8
  import type { DurationInput } from './clock';
9
9
  import { finiteDurationMs } from './clock';
@@ -24,10 +24,11 @@ export interface RetryPolicy {
24
24
  /** Ceiling for any single delay. Default 1h. */
25
25
  readonly maxDelay?: DurationInput;
26
26
  /**
27
- * Equal jitter (half fixed, half random) by default. Without it, a burst of failures
28
- * retries in lockstep and re-creates the thundering herd that killed the dependency.
27
+ * Core's `JitterMode`. `'equal'` (half fixed, half random) by default: a job that has already
28
+ * failed twice keeps a floor. `'none'` retries a burst in lockstep — the thundering herd that
29
+ * killed the dependency — and is for tests and printable schedules.
29
30
  */
30
- readonly jitter?: boolean;
31
+ readonly jitter?: JitterMode;
31
32
  /** Default true: an exhausted job is parked, never dropped. */
32
33
  readonly deadLetter?: boolean;
33
34
  }
@@ -37,7 +38,7 @@ export const DEFAULT_RETRY = {
37
38
  backoff: 'exponential',
38
39
  delay: 1_000,
39
40
  maxDelay: 3_600_000,
40
- jitter: true,
41
+ jitter: 'equal',
41
42
  deadLetter: true,
42
43
  } satisfies RetryPolicy;
43
44
 
@@ -48,14 +49,12 @@ export type { Random };
48
49
  *
49
50
  * The arithmetic is core's — one curve-and-jitter function for the framework, because four
50
51
  * packages shipped four of them. What stays here is the part that is this package's and not the
51
- * framework's: `DurationInput` (`'30s'` as well as a number), the `DEFAULT_RETRY` fallbacks, and
52
- * the `jitter: boolean` this package's public `RetryPolicy` has always spelled as a flag.
52
+ * framework's: `DurationInput` (`'30s'` as well as a number) and the `DEFAULT_RETRY` fallbacks.
53
+ * `jitter` is core's `JitterMode` itself since 31.0.0, never a flag mapped onto two of its modes.
53
54
  *
54
- * `?? DEFAULT_RETRY.jitter`, like every other option. It read `!== true` before, so an omitted
55
- * `jitter` meant OFF while the field's own doc says "Equal jitter … by default" — a burst of
56
- * failures then retried in lockstep, which is the thundering herd the default exists to break.
57
- * Masked for jobs declared through `job()` (it merges the defaults) and live for every direct
58
- * caller of this exported function, `retrySchedule` included.
55
+ * `?? DEFAULT_RETRY.jitter`, like every other option: an omitted `jitter` is the documented
56
+ * `'equal'`, never off — masked for jobs declared through `job()` (it merges the defaults) and live
57
+ * for every direct caller of this exported function, `retrySchedule` included.
59
58
  */
60
59
  export function backoffDelayMs(policy: RetryPolicy, attempt: number, random?: Random): number {
61
60
  return backoffDelay({
@@ -63,9 +62,7 @@ export function backoffDelayMs(policy: RetryPolicy, attempt: number, random?: Ra
63
62
  base: finiteDurationMs(policy.delay ?? DEFAULT_RETRY.delay, 'retry', 'delay'),
64
63
  max: finiteDurationMs(policy.maxDelay ?? DEFAULT_RETRY.maxDelay, 'retry', 'maxDelay'),
65
64
  curve: policy.backoff ?? DEFAULT_RETRY.backoff,
66
- // EQUAL, not full: a job that has already failed twice must not be handed a near-zero wait,
67
- // and this package's `jitter: true` has meant "half fixed, half random" since it shipped.
68
- jitter: (policy.jitter ?? DEFAULT_RETRY.jitter) === true ? 'equal' : 'none',
65
+ jitter: policy.jitter ?? DEFAULT_RETRY.jitter,
69
66
  random,
70
67
  });
71
68
  }
@@ -132,7 +129,7 @@ export function nextRetry(policy: RetryPolicy, attempt: number, random?: Random)
132
129
 
133
130
  /** Every delay in the policy, jitter off, for docs and `--json` output. */
134
131
  export function retrySchedule(policy: RetryPolicy): readonly number[] {
135
- const deterministic: RetryPolicy = { ...policy, jitter: false };
132
+ const deterministic: RetryPolicy = { ...policy, jitter: 'none' };
136
133
  const out: number[] = [];
137
134
  for (let attempt = 1; attempt < policy.attempts; attempt += 1) {
138
135
  out.push(backoffDelayMs(deterministic, attempt));
package/src/scheduler.ts CHANGED
@@ -27,6 +27,7 @@ import type { DrainBudget } from './drain-wait';
27
27
  import { createDrainBudget, settleAllBy } from './drain-wait';
28
28
  import type { JobDriver } from './driver';
29
29
  import { signalEnqueued } from './enqueue-signal';
30
+ import { instrumentedFire } from './job-instrument';
30
31
  import { enqueuedAttempts } from './retry';
31
32
  import type { LeaderElection } from './scheduler-leader';
32
33
  import { soleLeader } from './scheduler-leader';
@@ -165,12 +166,15 @@ export function jobScheduler(options: SchedulerOptions): Scheduler {
165
166
  });
166
167
  // Jobs and watermark in ONE step (`SchedulerState.fire`). `undefined` is an occurrence some
167
168
  // other dispatcher already fired: nothing was queued, and nothing is reported as dispatched.
168
- const results = await schedulerState.fire(options.driver, {
169
- task: handle.name,
170
- occurrenceMs,
171
- ...(watermarkMs === undefined ? {} : { watermarkMs }),
172
- jobs: requests,
173
- });
169
+ // Under the fire's producer span, which stamps each request's `traceparent` (`job-instrument.ts`).
170
+ const results = await instrumentedFire(handle.name, requests, (jobs) =>
171
+ schedulerState.fire(options.driver, {
172
+ task: handle.name,
173
+ occurrenceMs,
174
+ ...(watermarkMs === undefined ? {} : { watermarkMs }),
175
+ jobs,
176
+ }),
177
+ );
174
178
  if (results === undefined) {
175
179
  logger.info('jobs.scheduler.already-fired', {
176
180
  task: handle.name,
package/src/steps-api.ts CHANGED
@@ -14,6 +14,18 @@ export const DEFAULT_EVENT_POLL_MS = 30_000;
14
14
  */
15
15
  export const WAKING_EVENT_POLL_MS = 900_000;
16
16
 
17
+ /**
18
+ * Where a wait begins — its stamp, and what the bus could see then (`EventLookup.mark`) — and the
19
+ * events it refused BEHIND its cursor: one owed through the horizon sorts before the cursor, so
20
+ * the cursor stays and its id is excluded by name. Moving the cursor back to it re-opened every
21
+ * event between the two, the one refused before it included.
22
+ */
23
+ export interface EventSince {
24
+ readonly at: number;
25
+ readonly horizon?: string | undefined;
26
+ readonly refused?: readonly string[] | undefined;
27
+ }
28
+
17
29
  /** The slice of the event bus a waiting step needs. Implemented by `events.ts`. */
18
30
  export interface EventLookup {
19
31
  /**
@@ -31,18 +43,45 @@ export interface EventLookup {
31
43
  */
32
44
  now(): Promise<number>;
33
45
  /**
34
- * The earliest match published at or after `afterMs` — and, given `afterId`, strictly past THAT
35
- * event in the bus's own (published, id) order. The id is the cursor a refused event leaves: a
36
- * floor of `publishedAt + 1` stepped over every valid event stamped in the same millisecond.
37
- * An `afterId` the bus no longer holds narrows nothing: the event it named cannot match again.
46
+ * A NEW wait's starting line, read in ONE statement: the bus's `now()`, and the `horizon` — what
47
+ * the bus could see at that instant. Over a database an event is stamped at INSERT and visible
48
+ * at COMMIT, so a publish whose transaction was still open when the wait began carries a stamp
49
+ * BEFORE it: by `afterMs` alone every later search skips it and the wait runs to its timeout.
50
+ * `find` given the horizon also matches every event the horizon could not see. Absent on a bus
51
+ * with no transactions (the memory one): there every publish is visible as it is stamped.
52
+ */
53
+ mark?(): Promise<{ readonly at: number; readonly horizon?: string }>;
54
+ /**
55
+ * The earliest match past the cursor in the bus's own `(published, id)` order
56
+ * (`events-cursor.ts`): published at or after `afterMs` — and, given `afterId`, strictly past
57
+ * `(afterMs, afterId)`. The id is the cursor a refused event leaves: a floor of
58
+ * `publishedAt + 1` stepped over every valid event stamped in the same millisecond. Compared,
59
+ * never looked up, so a cursor whose event was purged still excludes what it excluded.
60
+ *
61
+ * Given a `horizon` (from `mark`, or a hit's own), ALSO every event that horizon could not see,
62
+ * whatever its stamp. A hit answers the horizon of the search that found it, which is the one a
63
+ * refusal's cursor must carry: anything that search could not see is still owed to the wait.
64
+ *
65
+ * `since` is the WAIT's own line, which a refusal's cursor never moves: only an event stamped at
66
+ * or after `since.at`, or unseen by `since.horizon` (the mark's), is the wait's at all. Absent,
67
+ * it is `{ at: afterMs, horizon }` — right for a search that never refused anything. A cursor
68
+ * alone let an event committed BEFORE the wait sort past a refused one and be handed out.
38
69
  */
39
70
  find(
40
71
  event: string,
41
72
  correlationKey: string | undefined,
42
73
  afterMs: number,
43
74
  afterId?: string,
75
+ horizon?: string,
76
+ since?: EventSince,
44
77
  ): Promise<
45
- { readonly id: string; readonly payload: unknown; readonly publishedAt: number } | undefined
78
+ | {
79
+ readonly id: string;
80
+ readonly payload: unknown;
81
+ readonly publishedAt: number;
82
+ readonly horizon?: string;
83
+ }
84
+ | undefined
46
85
  >;
47
86
  }
48
87
 
@@ -43,11 +43,22 @@ export function isWaitTimedOut(output: unknown): boolean {
43
43
 
44
44
  const WAIT_FLOOR_KEY = '~ultimate.waitFloor';
45
45
  const WAIT_FLOOR_ID_KEY = '~ultimate.waitFloorId';
46
+ const WAIT_HORIZON_KEY = '~ultimate.waitHorizon';
47
+ const WAIT_SINCE_HORIZON_KEY = '~ultimate.waitSinceHorizon';
48
+ const WAIT_REFUSED_KEY = '~ultimate.waitRefused';
46
49
 
47
- /** Where a wait's next search starts: at `afterMs`, and past the event `afterId` names. */
50
+ /**
51
+ * Where a wait's next search starts: at `afterMs`, past the event `afterId` names — and, given a
52
+ * `horizon` (`EventLookup.mark`), at every event that horizon could not see, whatever its stamp.
53
+ */
48
54
  export interface WaitFloor {
49
55
  readonly afterMs: number;
50
56
  readonly afterId?: string;
57
+ readonly horizon?: string;
58
+ /** The MARK's horizon: the wait's own line, kept by every refusal (`EventSince`). */
59
+ readonly sinceHorizon?: string;
60
+ /** Events refused BEHIND the cursor, excluded by id (`EventSince.refused`). */
61
+ readonly refused?: readonly string[];
51
62
  }
52
63
 
53
64
  /**
@@ -56,10 +67,19 @@ export interface WaitFloor {
56
67
  * event, so without this a repaired publisher's event sat behind the refused one until its TTL.
57
68
  * The refused event's id rides with its instant: a floor of `publishedAt + 1` skipped every valid
58
69
  * event stamped in the same millisecond. A record written before the id has `afterMs` alone.
70
+ * Over a bus that marks (`EventLookup.mark`), every parked wait holds one from its first park: the
71
+ * horizon rides here, since a re-poll must search under the line its wait began at.
59
72
  */
60
- export const waitFloor = (floor: WaitFloor): Record<string, number | string> => ({
73
+ export const waitFloor = (
74
+ floor: WaitFloor,
75
+ ): Record<string, number | string | readonly string[]> => ({
61
76
  [WAIT_FLOOR_KEY]: floor.afterMs,
62
77
  ...(floor.afterId === undefined ? {} : { [WAIT_FLOOR_ID_KEY]: floor.afterId }),
78
+ ...(floor.horizon === undefined ? {} : { [WAIT_HORIZON_KEY]: floor.horizon }),
79
+ ...(floor.sinceHorizon === undefined ? {} : { [WAIT_SINCE_HORIZON_KEY]: floor.sinceHorizon }),
80
+ ...(floor.refused === undefined || floor.refused.length === 0
81
+ ? {}
82
+ : { [WAIT_REFUSED_KEY]: floor.refused }),
63
83
  });
64
84
 
65
85
  export function waitFloorOf(output: unknown): WaitFloor | undefined {
@@ -68,7 +88,18 @@ export function waitFloorOf(output: unknown): WaitFloor | undefined {
68
88
  const afterMs = record[WAIT_FLOOR_KEY];
69
89
  if (typeof afterMs !== 'number') return undefined;
70
90
  const afterId = record[WAIT_FLOOR_ID_KEY];
71
- return typeof afterId === 'string' ? { afterMs, afterId } : { afterMs };
91
+ const horizon = record[WAIT_HORIZON_KEY];
92
+ const sinceHorizon = record[WAIT_SINCE_HORIZON_KEY];
93
+ const refused = record[WAIT_REFUSED_KEY];
94
+ return {
95
+ afterMs,
96
+ ...(typeof afterId === 'string' ? { afterId } : {}),
97
+ ...(typeof horizon === 'string' ? { horizon } : {}),
98
+ ...(typeof sinceHorizon === 'string' ? { sinceHorizon } : {}),
99
+ ...(Array.isArray(refused)
100
+ ? { refused: refused.filter((id): id is string => typeof id === 'string') }
101
+ : {}),
102
+ };
72
103
  }
73
104
 
74
105
  /**
package/src/steps-wait.ts CHANGED
@@ -8,10 +8,12 @@ import { formatIssues, validateAsync } from '@ultimat3/schema';
8
8
  import { finiteDurationMs } from './clock';
9
9
  import { JobTimeoutError } from './errors';
10
10
  import { JobEventInvalidError } from './errors-steps';
11
+ import { eventBefore } from './events-cursor';
11
12
  import type { WaitForEventOptions } from './steps-api';
12
13
  import type { RunnerCore } from './steps-core';
13
14
  import { jsonForm } from './steps-output';
14
15
  import type { StepRecord } from './steps-store';
16
+ import type { WaitFloor } from './steps-suspension';
15
17
  import {
16
18
  isWaitTimedOut,
17
19
  StepSuspension,
@@ -22,7 +24,12 @@ import {
22
24
 
23
25
  const DEFAULT_WAIT_MS = 86_400_000;
24
26
 
25
- type Hit = { readonly id: string; readonly payload: unknown; readonly publishedAt: number };
27
+ type Hit = {
28
+ readonly id: string;
29
+ readonly payload: unknown;
30
+ readonly publishedAt: number;
31
+ readonly horizon?: string;
32
+ };
26
33
 
27
34
  export function waitForEventStep(core: RunnerCore) {
28
35
  const { runId, jobName } = core;
@@ -50,7 +57,10 @@ export function waitForEventStep(core: RunnerCore) {
50
57
  const at = core.now();
51
58
  // A NEW wait is stamped by the bus, whose clock stamps every `publishedAt` it is compared
52
59
  // with; the runner's own only when there is no bus to ask. A re-poll keeps what it persisted.
53
- const startedAt = existing?.startedAt ?? (await core.events?.now()) ?? at;
60
+ // Its horizon is read in the same statement and persisted as the floor: a publish whose
61
+ // transaction was open at this instant is stamped before it, and is owed to this wait.
62
+ const mark = existing === undefined ? await core.events?.mark?.() : undefined;
63
+ const startedAt = existing?.startedAt ?? mark?.at ?? (await core.events?.now()) ?? at;
54
64
  const timeoutMs = finiteDurationMs(
55
65
  waitOptions.timeout ?? DEFAULT_WAIT_MS,
56
66
  'step.waitForEvent',
@@ -61,7 +71,41 @@ export function waitForEventStep(core: RunnerCore) {
61
71
  const attempts = (existing?.attempts ?? 0) + 1;
62
72
  const keyed = correlationKey === undefined ? {} : { correlationKey };
63
73
 
64
- const floor = waitFloorOf(existing?.output);
74
+ const floor =
75
+ waitFloorOf(existing?.output) ??
76
+ (mark?.horizon === undefined
77
+ ? undefined
78
+ : { afterMs: startedAt, horizon: mark.horizon, sinceHorizon: mark.horizon });
79
+ // The wait's own line, which a refusal's cursor never moves: `startedAt` and the mark's horizon.
80
+ const since = { at: startedAt, horizon: floor?.sinceHorizon, refused: floor?.refused };
81
+
82
+ /**
83
+ * The floor after `hit` is refused — monotone, so no refused event can match again. A hit past
84
+ * the cursor moves it there, under the horizon of the search that found it: what that search
85
+ * could not see is still owed, and everything it saw up to the hit is spent. A hit BEHIND the
86
+ * cursor (owed through a horizon) leaves the cursor where it is and is excluded by id: moving
87
+ * the cursor back re-opened every event between the two, the one refused before it included.
88
+ */
89
+ const refusedFloor = (hit: Hit): WaitFloor => {
90
+ const cursor = { publishedAt: floor?.afterMs ?? startedAt, id: floor?.afterId ?? '' };
91
+ const sinceHorizon =
92
+ floor?.sinceHorizon === undefined ? {} : { sinceHorizon: floor.sinceHorizon };
93
+ if (eventBefore(cursor, hit)) {
94
+ return {
95
+ afterMs: hit.publishedAt,
96
+ afterId: hit.id,
97
+ ...(hit.horizon === undefined ? {} : { horizon: hit.horizon }),
98
+ ...sinceHorizon,
99
+ };
100
+ }
101
+ return {
102
+ afterMs: cursor.publishedAt,
103
+ ...(floor?.afterId === undefined ? {} : { afterId: floor.afterId }),
104
+ ...(floor?.horizon === undefined ? {} : { horizon: floor.horizon }),
105
+ ...sinceHorizon,
106
+ refused: [...(floor?.refused ?? []), hit.id],
107
+ };
108
+ };
65
109
  const parked = {
66
110
  runId,
67
111
  name,
@@ -81,7 +125,7 @@ export function waitForEventStep(core: RunnerCore) {
81
125
  await core
82
126
  .put({
83
127
  ...parked,
84
- output: waitFloor({ afterMs: hit.publishedAt, afterId: hit.id }),
128
+ output: waitFloor(refusedFloor(hit)),
85
129
  wakeAt: at,
86
130
  })
87
131
  .catch((unwritten: unknown) =>
@@ -108,6 +152,8 @@ export function waitForEventStep(core: RunnerCore) {
108
152
  correlationKey,
109
153
  floor?.afterMs ?? startedAt,
110
154
  floor?.afterId,
155
+ floor?.horizon,
156
+ since,
111
157
  );
112
158
  if (hit !== undefined) return consume(hit);
113
159
 
@@ -155,6 +201,8 @@ export function waitForEventStep(core: RunnerCore) {
155
201
  correlationKey,
156
202
  floor?.afterMs ?? startedAt,
157
203
  floor?.afterId,
204
+ floor?.horizon,
205
+ since,
158
206
  );
159
207
  if (late !== undefined) return consume(late);
160
208
  }
package/src/steps.ts CHANGED
@@ -36,6 +36,7 @@ import { waitForEventStep } from './steps-wait';
36
36
  export type {
37
37
  CompensateContext,
38
38
  EventLookup,
39
+ EventSince,
39
40
  StepApi,
40
41
  StepRunOptions,
41
42
  WaitForEventOptions,
@@ -3,13 +3,15 @@
3
3
  // table before a row is touched. Who may call them is the app's action: no route, no MCP tool.
4
4
 
5
5
  import type { Clock } from '@ultimat3/core';
6
- import { systemClock } from '@ultimat3/core';
6
+ import { liveClock } from '@ultimat3/core';
7
7
  import { registeredEntities } from '@ultimat3/entity';
8
8
  import type { StandardSchemaV1 } from '@ultimat3/schema';
9
9
  import type { DurationInput } from './clock';
10
10
  import { nowMs } from './clock';
11
11
  import type { ExportFormat } from './export-format';
12
12
  import type { ExportSink } from './export-sink';
13
+ import { auditedErase } from './gdpr-audit';
14
+ import { unerasedPersonalData } from './gdpr-coverage';
13
15
  import type { JobHandle } from './job';
14
16
  import { job } from './job';
15
17
  import type { RetryPolicy } from './retry';
@@ -57,6 +59,10 @@ export function plannedSubject<I>(subject: Subject<I>): SubjectPlan {
57
59
  });
58
60
  const [first] = problems;
59
61
  if (first !== undefined) throw first;
62
+ // A reached table whose treatment would leave a `.personalData()` column standing: refused before
63
+ // a row is touched, as an untreated table is (`gdpr-coverage.ts`).
64
+ const [leak] = unerasedPersonalData(plan);
65
+ if (leak !== undefined) throw leak;
60
66
  return plan;
61
67
  }
62
68
 
@@ -78,14 +84,16 @@ export function subjectErase<I>(definition: SubjectJobDefinition<I>): JobHandle<
78
84
  ...common(definition),
79
85
  // One live erase per person: a second request while one runs is the same run.
80
86
  idempotencyKey: (input) => `${definition.name}:${subject.init.key(input)}`,
81
- run: ({ step, input }): Promise<SubjectEraseReport> =>
82
- subjectErasePass(step, readerFor(subject, input), subject.init.targets ?? []),
87
+ run: ({ step, input, ctx, runId }): Promise<SubjectEraseReport> =>
88
+ auditedErase({ job: definition.name, subject: subject.entity, runId, ctx }, step, () =>
89
+ subjectErasePass(step, readerFor(subject, input), subject.init.targets ?? []),
90
+ ),
83
91
  });
84
92
  }
85
93
 
86
94
  export function subjectExport<I>(definition: SubjectExportDefinition<I>): JobHandle<I> {
87
95
  const { subject } = definition;
88
- const clock = definition.clock ?? systemClock;
96
+ const clock = definition.clock ?? liveClock;
89
97
  return job<I>({
90
98
  ...common(definition),
91
99
  // One live pass per artifact, derived as `exportRows()` derives it.
package/src/webhook.ts CHANGED
@@ -20,7 +20,7 @@ import {
20
20
  finiteOption,
21
21
  isCanonicalWebhookField,
22
22
  isUltimateError,
23
- systemClock,
23
+ liveClock,
24
24
  throwIfAborted,
25
25
  WEBHOOK_FIELD_MAX,
26
26
  } from '@ultimat3/core';
@@ -189,7 +189,7 @@ export interface WebhookDefinition {
189
189
  }
190
190
 
191
191
  export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliveryInput> {
192
- const clock = definition.clock ?? systemClock;
192
+ const clock = definition.clock ?? liveClock;
193
193
  const disableAfter = finiteOption(
194
194
  'webhook()',
195
195
  'disableAfter',
@@ -34,11 +34,14 @@ export type Admission =
34
34
  | { readonly kind: 'waiting' }
35
35
  | { readonly kind: 'refused'; readonly execution: JobExecution };
36
36
 
37
- /** `behind` is the rest of the claimed batch: handed back before a store failure is rethrown. */
37
+ /**
38
+ * `behind` is the rest of the claimed batch: handed back before a store failure is rethrown. A
39
+ * thunk, read only on that failure: a slice per admitted job made a batch O(batch²) to admit.
40
+ */
38
41
  export type Admit = (
39
42
  job: ClaimedJob,
40
43
  queue: string,
41
- behind: readonly ClaimedJob[],
44
+ behind: () => readonly ClaimedJob[],
42
45
  ) => Promise<Admission>;
43
46
 
44
47
  export function createAdmission(options: AdmissionOptions): Admit {
@@ -79,7 +82,7 @@ export function createAdmission(options: AdmissionOptions): Admit {
79
82
  // The nack itself failed, so THIS job went nowhere either: it goes back with the jobs
80
83
  // behind it, one more best-effort try, before the round reports. Left out, it sat
81
84
  // `running` until its lease lapsed — an attempt spent on a body that never started.
82
- await handBack(driver, [job, ...behind], back);
85
+ await handBack(driver, [job, ...behind()], back);
83
86
  throw error;
84
87
  }
85
88
  return { kind: 'waiting' };
@@ -101,7 +104,7 @@ export function createAdmission(options: AdmissionOptions): Admit {
101
104
  lease.abandon();
102
105
  // This job and every one behind it go BACK, unburned — rethrowing alone stranded them in
103
106
  // `running` with an attempt spent on work that never started.
104
- await handBack(driver, [job, ...behind], back);
107
+ await handBack(driver, [job, ...behind()], back);
105
108
  throw error;
106
109
  }
107
110
  if (grant.outcome === 'granted') return { kind: 'run', lease };
@@ -132,7 +135,7 @@ export function createAdmission(options: AdmissionOptions): Admit {
132
135
  } catch (error) {
133
136
  // This job too, for the reason above. A refusal that DID settle before throwing is safe to
134
137
  // repeat: every nack is fenced on `running` and the claim, so a settled row ignores it.
135
- await handBack(driver, [job, ...behind], back);
138
+ await handBack(driver, [job, ...behind()], back);
136
139
  throw error;
137
140
  }
138
141
  return { kind: 'waiting' };
@@ -51,6 +51,7 @@ export function armDrainCutoff(
51
51
  for (const timer of timers.splice(0)) clearTimeout(timer);
52
52
  };
53
53
  const at = (instant: number, fire: () => void): void => {
54
+ // Real time, not `liveClock`: this races a real `setTimeout`, and a frozen clock under it never fires.
54
55
  const timer = setTimeout(fire, Math.max(0, instant - systemClock.monotonic()));
55
56
  timer.unref?.();
56
57
  timers.push(timer);
@@ -42,19 +42,34 @@ export interface ClaimLoop {
42
42
  stop(): void;
43
43
  /** True once a pass has come back empty. */
44
44
  idle(): boolean;
45
- /** A pass ended: `found` decides the next wait. */
46
- passed(found: boolean): void;
45
+ /**
46
+ * A pass ended: `found` decides the next wait. `full` — some ask came back with every row it
47
+ * asked for — means a backlog is likely, so a slot freeing before the next pass claims at once.
48
+ */
49
+ passed(found: boolean, full?: boolean): void;
47
50
  /** The wait the loop will arm, or has armed. */
48
51
  delayMs(): number;
49
52
  /**
50
53
  * A slot came free, or a handed-back row is due: one pass NOW if the wait in hand is longer
51
- * than the floor — and no reset, so a pass that finds nothing costs one statement, not a ramp.
54
+ * than the floor, or the last pass came back full — and no reset, so a pass that finds nothing
55
+ * costs one statement, not a ramp.
52
56
  */
53
57
  kick(): void;
54
58
  /** A row this worker settled back onto the queue is claimable again in `ms`. */
55
59
  dueIn(ms: number): void;
56
60
  }
57
61
 
62
+ /**
63
+ * Whether one ask came back FULL — every row it asked for STARTED. Counted on what a pass admitted,
64
+ * never on what it claimed: rows a concurrency or rate cap sent back are not a backlog this worker
65
+ * can run, and kicking on them is a claim plus N nacks per completion.
66
+ */
67
+ export function fillsAsk(admitted: ReadonlyMap<string, number>, limit: number): boolean {
68
+ let ran = 0;
69
+ for (const count of admitted.values()) ran += count;
70
+ return ran > 0 && ran >= limit;
71
+ }
72
+
58
73
  /** A due time further out than this is the poll's to find: no timer is held for a 3-day sleep. */
59
74
  const DUE_HINT_MAX_MS = 60_000;
60
75
  /** After the due time, never on it: the row's `run_at` is the database's clock, not this one. */
@@ -74,6 +89,12 @@ export function createClaimLoop(options: ClaimLoopOptions): ClaimLoop {
74
89
  /** The timer's own pass is in flight. A wake landing now asks for ONE more, never a second chain. */
75
90
  let passing = false;
76
91
  let again = false;
92
+ /**
93
+ * The last pass came back full. A pass that found work waits the FLOOR, and a kick shorter than
94
+ * that did nothing — so a backlog drained at one batch per floor: 8 slots / 250 ms = 32 claims/s,
95
+ * which `scripts/bench/worker-throughput.bench.ts` measured exactly. Cleared by the next pass.
96
+ */
97
+ let backlog = false;
77
98
  let unsubscribe: (() => void) | undefined;
78
99
  /** One timer per floor-wide bucket of due times, so a burst of retries holds a bounded few. */
79
100
  const due = new Map<number, unknown>();
@@ -126,7 +147,7 @@ export function createClaimLoop(options: ClaimLoopOptions): ClaimLoop {
126
147
  };
127
148
 
128
149
  const kick = (): void => {
129
- if (delayMs > floor) now();
150
+ if (delayMs > floor || backlog) now();
130
151
  };
131
152
 
132
153
  return {
@@ -140,6 +161,7 @@ export function createClaimLoop(options: ClaimLoopOptions): ClaimLoop {
140
161
  stop() {
141
162
  active = false;
142
163
  again = false;
164
+ backlog = false;
143
165
  if (timer !== undefined) timers.clear(timer);
144
166
  timer = undefined;
145
167
  for (const hint of due.values()) timers.clear(hint);
@@ -148,7 +170,8 @@ export function createClaimLoop(options: ClaimLoopOptions): ClaimLoop {
148
170
  unsubscribe = undefined;
149
171
  },
150
172
  idle: () => backoff.idle(),
151
- passed(found) {
173
+ passed(found, full = false) {
174
+ backlog = full;
152
175
  delayMs = backoff.next(found);
153
176
  },
154
177
  delayMs: () => delayMs,
@@ -6,7 +6,7 @@
6
6
  import type { Clock } from '@ultimat3/core';
7
7
  import { logger, renderThrowable } from '@ultimat3/core';
8
8
  import { nowMs } from './clock';
9
- import type { JobDriver } from './driver';
9
+ import { DEFAULT_QUEUE, type JobDriver } from './driver';
10
10
  import { MAX_WORKER_IN_FLIGHT } from './introspection';
11
11
  import { registeredJobs } from './job';
12
12
  import { type IntervalScheduler, startRenewalTimer } from './renewal-timer';
@@ -122,13 +122,15 @@ export function startWorkerRegistry(options: WorkerRegistryOptions): WorkerRegis
122
122
 
123
123
  /**
124
124
  * A queue this worker polls that NO registered job names. Not refused: a per-call
125
- * `enqueue({ queue })` may still land work there, and an idle pass is one statement however many
126
- * queues it covers. Said once, at boot, because it is almost always a config leftover — `x new`
127
- * configures `['<app>-default']` while a `job()` naming no queue lands on `default`.
125
+ * `enqueue({ queue })` may still land work there — `jobs.queues` exists to name exactly those — and
126
+ * an idle pass is one statement however many queues it covers. Said once, at boot, because it is
127
+ * almost always a config leftover (`x new` configured `['<app>-default']` until 31.0.0).
128
+ * `DEFAULT_QUEUE` is never one: it is where every `job()` naming no queue lands and what a worker
129
+ * with no list serves, so an app with no such job yet logged this on every boot of every pod.
128
130
  */
129
131
  export function reportUnregisteredQueues(workerId: string, queues: readonly string[]): void {
130
132
  const named = new Set(registeredJobs().map((handle) => handle.queue));
131
- const unnamed = queues.filter((queue) => !named.has(queue));
133
+ const unnamed = queues.filter((queue) => queue !== DEFAULT_QUEUE && !named.has(queue));
132
134
  if (unnamed.length === 0) return;
133
135
  logger.info('jobs.worker.queue-unregistered', {
134
136
  workerId,