@ultimat3/jobs 24.0.0 → 25.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/CLAUDE.md +25 -27
  2. package/README.md +50 -34
  3. package/package.json +7 -7
  4. package/src/backfill-errors.ts +3 -2
  5. package/src/backfill-ledger.ts +1 -1
  6. package/src/backfill-rate.ts +2 -2
  7. package/src/backfill.ts +2 -2
  8. package/src/counters.ts +1 -1
  9. package/src/driver-memory-operator.ts +3 -3
  10. package/src/driver-memory.ts +14 -17
  11. package/src/driver-pg-settle-sql.ts +2 -3
  12. package/src/driver-pg-sql.ts +1 -1
  13. package/src/driver-pg.ts +18 -18
  14. package/src/driver.ts +9 -15
  15. package/src/errors-concurrency.ts +1 -1
  16. package/src/errors.ts +5 -5
  17. package/src/events-pg.ts +4 -4
  18. package/src/events.ts +6 -6
  19. package/src/execute.ts +46 -5
  20. package/src/export.ts +2 -2
  21. package/src/heartbeat.ts +7 -2
  22. package/src/index.ts +37 -97
  23. package/src/inspect-operator.ts +1 -1
  24. package/src/inspect.ts +1 -1
  25. package/src/job.ts +1 -1
  26. package/src/leases.ts +1 -1
  27. package/src/limits.ts +13 -8
  28. package/src/outbox-lease.ts +1 -1
  29. package/src/outbox-pg.ts +5 -5
  30. package/src/outbox-relay.ts +6 -6
  31. package/src/outbox.ts +13 -13
  32. package/src/renewal-timer.ts +1 -1
  33. package/src/retry-classification.ts +52 -15
  34. package/src/scheduler-leader.ts +2 -2
  35. package/src/scheduler-pg.ts +16 -14
  36. package/src/scheduler-state.ts +2 -2
  37. package/src/scheduler.ts +5 -5
  38. package/src/steps-memory.ts +2 -2
  39. package/src/steps.ts +35 -7
  40. package/src/task.ts +1 -1
  41. package/src/webhook-attempt.ts +206 -0
  42. package/src/webhook-ledger.ts +15 -4
  43. package/src/webhook.ts +64 -118
  44. package/src/worker-admit.ts +7 -3
  45. package/src/worker-drain-cutoff.ts +110 -0
  46. package/src/worker-fleet-slots.ts +5 -2
  47. package/src/worker-held.ts +100 -0
  48. package/src/worker-options.ts +19 -15
  49. package/src/worker-run.ts +38 -8
  50. package/src/worker-tally.ts +53 -0
  51. package/src/worker-types.ts +7 -4
  52. package/src/worker.ts +85 -77
  53. package/src/driver-nats.ts +0 -89
  54. package/src/driver-redis.ts +0 -92
package/src/index.ts CHANGED
@@ -7,23 +7,10 @@
7
7
  // bypasses `defineApi`'s own result — the ambiguity axiom 1 exists to refuse.
8
8
  import './register';
9
9
 
10
- // The wire format, RE-EXPORTED from `@ultimat3/core` and never re-declared: it is one module at
11
- // the tier both halves can reach, because this package signs a delivery, `@ultimat3/http` verifies
12
- // one, and neither may import the other. Re-exported here so a `job` file needs one import rather
13
- // than two — the same reason `t` is re-exported above.
14
- export type { PgExecutor, WebhookMacInput, WebhookSigningInput } from '@ultimat3/core';
15
- export {
16
- isCanonicalWebhookField,
17
- WEBHOOK_FIELD_MAX,
18
- WEBHOOK_ID_HEADER,
19
- WEBHOOK_SIGNATURE_HEADER,
20
- WEBHOOK_SIGNATURE_VERSION,
21
- WEBHOOK_TOPIC_HEADER,
22
- webhookHeaders,
23
- webhookMac,
24
- webhookSignature,
25
- webhookSigningString,
26
- } from '@ultimat3/core';
10
+ // The webhook wire format (`WEBHOOK_*`, `webhookSignature`, `webhookMac`, …) is `@ultimat3/core`'s
11
+ // and imported from there: one module at the tier this package (the signer) and `@ultimat3/http`
12
+ // (the verifier) both reach, with one import path — never re-exported here (`X_HELPER_COPY`).
13
+ export type { PgExecutor } from '@ultimat3/core';
27
14
  /** Re-exported so a `job`/`task` file needs one import, not two. Same object as schema's. */
28
15
  export type { Infer } from '@ultimat3/schema';
29
16
  export { t } from '@ultimat3/schema';
@@ -57,9 +44,9 @@ export type {
57
44
  export {
58
45
  BACKFILL_STATUSES,
59
46
  backfillChecksum,
60
- createMemoryBackfillLedger,
61
47
  decideBackfill,
62
48
  isBackfillStatus,
49
+ memoryBackfillLedger,
63
50
  } from './backfill-ledger';
64
51
  export type {
65
52
  BackfillPendingReport,
@@ -73,7 +60,7 @@ export {
73
60
  pendingBackfills,
74
61
  } from './backfill-pending';
75
62
  export type { Pacer, PacerOptions } from './backfill-rate';
76
- export { createPacer, DEFAULT_BACKFILL_RATE } from './backfill-rate';
63
+ export { backfillPacer, DEFAULT_BACKFILL_RATE } from './backfill-rate';
77
64
  export type { BackfillCount, BackfillDeclaration, BackfillOrigin } from './backfill-registry';
78
65
  // `stampBackfill` is deliberately absent, for the reason `registerJob` is: a second way to make a
79
66
  // handle claim it is a backfill would let a plain `job()` inherit the pending diff and the gate.
@@ -120,62 +107,14 @@ export {
120
107
  resetJobDriver,
121
108
  setJobDriver,
122
109
  } from './driver';
123
- export type { MemoryDriverOptions, MemoryJobDriver } from './driver-memory';
124
- export { createMemoryDriver } from './driver-memory';
125
- export type { NatsDriverOptions } from './driver-nats';
126
- export { createNatsDriver } from './driver-nats';
127
- export type { PgDriverOptions } from './driver-pg';
128
- export { createPgDriver, createPgLeader } from './driver-pg';
129
- export {
130
- SQL_COUNTER_DROP,
131
- SQL_COUNTER_FOLD,
132
- SQL_COUNTER_TOTALS,
133
- SQL_COUNTERS,
134
- SQL_JOB_PROGRESS,
135
- SQL_JOB_PROMOTE,
136
- SQL_JOB_REMOVE,
137
- SQL_JOB_REMOVE_MANY,
138
- SQL_JOB_REQUEUE_MANY,
139
- SQL_PAUSE,
140
- SQL_PAUSED,
141
- SQL_RESUME,
142
- SQL_SCHEDULER_FIRE,
143
- SQL_WORKER_ANNOUNCE,
144
- SQL_WORKER_FORGET,
145
- SQL_WORKERS,
146
- } from './driver-pg-operator-sql';
147
- export {
148
- SQL_ACK,
149
- SQL_ADVISORY_UNLOCK,
150
- SQL_BACKFILL_FINISH,
151
- SQL_BACKFILL_LIST,
152
- SQL_BACKFILL_PROGRESS,
153
- SQL_BACKFILL_START,
154
- SQL_CANCEL,
155
- SQL_CLAIM,
156
- SQL_ENQUEUE,
157
- SQL_HEARTBEAT,
158
- SQL_JOBS_TABLE,
159
- SQL_LEADER_ACQUIRE,
160
- SQL_LEADER_RELEASE,
161
- SQL_LEASE_ACQUIRE,
162
- SQL_LEASE_HOLDERS,
163
- SQL_LEASE_RELEASE,
164
- SQL_LEASE_RENEW,
165
- SQL_NACK,
166
- SQL_OUTBOX_CLAIM,
167
- SQL_OUTBOX_MARK_PUBLISHED,
168
- SQL_OUTBOX_RELEASE,
169
- SQL_OUTBOX_STAGE,
170
- SQL_SCHEDULER_STATE_GET,
171
- SQL_SCHEDULER_STATE_MARK,
172
- SQL_STATS,
173
- SQL_STEP_GET,
174
- SQL_STEP_PUT,
175
- SQL_TRY_ADVISORY_LOCK,
176
- } from './driver-pg-sql';
177
- export type { RedisDriverOptions } from './driver-redis';
178
- export { createRedisDriver } from './driver-redis';
110
+ export type { MemoryJobDriver, MemoryJobDriverOptions } from './driver-memory';
111
+ export { memoryJobDriver } from './driver-memory';
112
+ export type { PostgresJobDriverOptions } from './driver-pg';
113
+ export { postgresJobDriver, postgresLeader } from './driver-pg';
114
+ // Only the statement another workspace runs: the install (`cli/src/framework-schema.ts`, action,
115
+ // notify). The rest are this package's internals, read where they are defined, and tested there
116
+ // too — the array-parameter proofs are `driver-pg-array.live.test.ts` (25.0.0).
117
+ export { SQL_JOBS_TABLE } from './driver-pg-sql';
179
118
  export { signalEnqueued, signalStaged } from './enqueue-signal';
180
119
  export type { JobErrorCode } from './errors';
181
120
  export {
@@ -216,16 +155,16 @@ export {
216
155
  export { JobNotFoundError, JobNotRequeueableError } from './errors-requeue';
217
156
  export type { EventBus, JobEvent, MemoryEventBusOptions, PublishOptions } from './events';
218
157
  export {
219
- createMemoryEventBus,
220
158
  EVENTS_PURGE_TARGET,
221
159
  eventBus,
222
160
  eventsPurgeTarget,
161
+ memoryEventBus,
223
162
  publishEvent,
224
163
  resetEventBus,
225
164
  setEventBus,
226
165
  } from './events';
227
- export type { PgEventBusOptions } from './events-pg';
228
- export { createPgEventBus } from './events-pg';
166
+ export type { PostgresEventBusOptions } from './events-pg';
167
+ export { postgresEventBus } from './events-pg';
229
168
  export type { ExecuteJobOptions, JobExecution, JobOutcome } from './execute';
230
169
  export { executeJob } from './execute';
231
170
  export type { ExportDefinition, ExportReport } from './export';
@@ -296,7 +235,7 @@ export type {
296
235
  } from './job';
297
236
  export { describeJobs, getJob, isJobHandle, job, registeredJobs, resetJobs } from './job';
298
237
  export type { HeldLease, LeaseStore, MemoryLeaseStore, MemoryLeaseStoreOptions } from './leases';
299
- export { createMemoryLeaseStore, jobLeaseKey } from './leases';
238
+ export { jobLeaseKey, memoryLeaseStore } from './leases';
300
239
  export type {
301
240
  Lease,
302
241
  LimitConfig,
@@ -306,7 +245,7 @@ export type {
306
245
  LimitSnapshot,
307
246
  RateLimit,
308
247
  } from './limits';
309
- export { createLimiter, NO_TENANT, tenantKeyFrom } from './limits';
248
+ export { concurrencyLimiter, NO_TENANT, tenantKeyFrom } from './limits';
310
249
  export {
311
250
  queueDeadJobs,
312
251
  queueOldestReady,
@@ -316,27 +255,27 @@ export {
316
255
  export type {
317
256
  EnqueueOptions,
318
257
  JobsFacade,
319
- MemoryOutboxOptions,
320
258
  MemoryOutboxStore,
259
+ MemoryOutboxStoreOptions,
321
260
  OutboxDeps,
322
261
  OutboxRecord,
323
262
  OutboxStore,
324
263
  } from './outbox';
325
264
  export {
326
- createJobsFacade,
327
- createMemoryOutboxStore,
328
265
  enqueueInTx,
329
266
  jobsFacade,
267
+ memoryOutboxStore,
268
+ outboxJobsFacade,
330
269
  resetJobsFacade,
331
270
  setJobsFacade,
332
271
  } from './outbox';
333
272
  // One definition of the lease, consumed by both stores — a memory default and a pg default that
334
273
  // could drift are two answers to "how long is a claim mine for", and the shorter one duplicates.
335
274
  export { DEFAULT_OUTBOX_CLAIM_LEASE_MS } from './outbox-lease';
336
- export type { PgOutboxOptions } from './outbox-pg';
337
- export { createPgOutboxStore } from './outbox-pg';
275
+ export type { PostgresOutboxStoreOptions } from './outbox-pg';
276
+ export { postgresOutboxStore } from './outbox-pg';
338
277
  export type { OutboxRelay, RelayOptions } from './outbox-relay';
339
- export { createOutboxRelay } from './outbox-relay';
278
+ export { outboxRelay } from './outbox-relay';
340
279
  export type { ProgressFn } from './progress';
341
280
  export type {
342
281
  PurgeDefinition,
@@ -352,7 +291,7 @@ export type { IntervalScheduler } from './renewal-timer';
352
291
  export type { BackoffStrategy, Random, RetryDecision, RetryPolicy } from './retry';
353
292
  export { backoffDelayMs, DEFAULT_RETRY, isFinalAttempt, nextRetry, retrySchedule } from './retry';
354
293
  export type { JobRetryDecision, JobStopReason } from './retry-classification';
355
- export { classifyThrown, failureForRow, nextRetryForError } from './retry-classification';
294
+ export { failureForRow, nextRetryForError } from './retry-classification';
356
295
  export type {
357
296
  CronResolver,
358
297
  DispatchedOccurrence,
@@ -361,22 +300,22 @@ export type {
361
300
  } from './scheduler';
362
301
  export {
363
302
  COUNTER_ROLLUP_INTERVAL_MS,
364
- createScheduler,
303
+ jobScheduler,
365
304
  PAUSE_RECHECK_MS,
366
305
  } from './scheduler';
367
306
  export type { LeaderElection } from './scheduler-leader';
368
307
  export { soleLeader } from './scheduler-leader';
369
308
  export { nextTaskRun } from './scheduler-occurrences';
370
- export type { PgLeaseLeaderOptions } from './scheduler-pg';
309
+ export type { PostgresLeaseLeaderOptions } from './scheduler-pg';
371
310
  export {
372
- createPgLeaseLeader,
373
311
  currentLeader,
374
312
  DEFAULT_LEADER_TTL_MS,
375
313
  LEASE_RENEWALS_PER_TTL,
376
- pgSchedulerState,
314
+ postgresLeaseLeader,
315
+ postgresSchedulerState,
377
316
  } from './scheduler-pg';
378
317
  export type { ScheduledFire, SchedulerState } from './scheduler-state';
379
- export { createMemorySchedulerState, fireThroughDriver } from './scheduler-state';
318
+ export { fireThroughDriver, memorySchedulerState } from './scheduler-state';
380
319
  export type { JobCompleted, JobFailed, JobSettled } from './settled';
381
320
  export { ON_SETTLED_ATTEMPTS } from './settled';
382
321
  export type {
@@ -391,14 +330,14 @@ export type {
391
330
  WaitForEventOptions,
392
331
  } from './steps';
393
332
  export {
394
- createStepRunner,
395
333
  isStepStatus,
396
334
  isStepSuspension,
397
335
  MAX_TRACE_NAMES,
398
336
  STEP_STATUSES,
399
337
  StepSuspension,
338
+ stepRunner,
400
339
  } from './steps';
401
- export { createMemoryStepStore } from './steps-memory';
340
+ export { memoryStepStore } from './steps-memory';
402
341
  export type {
403
342
  CatchUpPolicy,
404
343
  TaskDefinition,
@@ -421,10 +360,11 @@ export type {
421
360
  WebhookDeliveryInput,
422
361
  WebhookEndpoint,
423
362
  WebhookEvent,
424
- WebhookFetch,
425
363
  WebhookReport,
426
364
  } from './webhook';
427
- export { DEFAULT_WEBHOOK_DISABLE_AFTER, WEBHOOK_CONTENT_TYPE, webhook } from './webhook';
365
+ export { DEFAULT_WEBHOOK_DISABLE_AFTER, webhook } from './webhook';
366
+ export type { WebhookFetch } from './webhook-attempt';
367
+ export { WEBHOOK_CONTENT_TYPE } from './webhook-attempt';
428
368
  export {
429
369
  WebhookDeliveryFailedError,
430
370
  WebhookDeliveryRejectedError,
@@ -438,4 +378,4 @@ export {
438
378
  export type { MemoryWebhookLedger, WebhookAttempt, WebhookLedger } from './webhook-ledger';
439
379
  export { DEFAULT_MAX_WEBHOOK_ATTEMPTS, memoryWebhookLedger } from './webhook-ledger';
440
380
  export type { Worker, WorkerOptions, WorkerStats } from './worker';
441
- export { createWorker } from './worker';
381
+ export { jobWorker } from './worker';
@@ -12,7 +12,7 @@ function operatorOf(driver: JobDriver): JobIntrospection {
12
12
  if (driver.introspect === undefined) {
13
13
  throw new NotImplementedError({
14
14
  cause: `the operator surface of the "${driver.name}" jobs driver is not implemented: the driver has no introspect`,
15
- fix: 'call setJobDriver(createPgDriver()) at boot — the pg driver implements introspect — then: x jobs ls --json',
15
+ fix: 'call setJobDriver(postgresJobDriver()) at boot — the pg driver implements introspect — then: x jobs ls --json',
16
16
  });
17
17
  }
18
18
  return driver.introspect;
package/src/inspect.ts CHANGED
@@ -114,7 +114,7 @@ function requireIntrospection(driver: JobDriver): NonNullable<JobDriver['introsp
114
114
  if (driver.introspect === undefined) {
115
115
  throw new NotImplementedError({
116
116
  cause: `introspection for the "${driver.name}" jobs driver is not implemented: the driver has no introspect`,
117
- fix: 'call setJobDriver(createPgDriver()) at boot — only the pg driver implements introspect — then: x jobs ls --json',
117
+ fix: 'call setJobDriver(postgresJobDriver()) at boot — only the pg driver implements introspect — then: x jobs ls --json',
118
118
  });
119
119
  }
120
120
  return driver.introspect;
package/src/job.ts CHANGED
@@ -101,7 +101,7 @@ export interface JobDefinition<I, R = unknown> {
101
101
  *
102
102
  * Enforced by `JobDriver.leases` — a row every replica can see — and NOT by `limits.ts`, which
103
103
  * counts one process's heap. A driver with no lease store cannot hold this cap, so
104
- * `createWorker().start()` refuses to boot rather than let it pass silently
104
+ * `jobWorker().start()` refuses to boot rather than let it pass silently
105
105
  * (`X_JOB_CONCURRENCY_UNENFORCEABLE`): this field was declared, documented and in the manifest
106
106
  * while nothing read it, which is exactly what axiom 3 exists to prevent.
107
107
  */
package/src/leases.ts CHANGED
@@ -57,7 +57,7 @@ export interface MemoryLeaseStore extends LeaseStore {
57
57
  * memory driver enforces `concurrency` with the same code path the pg driver does, and so the
58
58
  * "a second worker is refused" test is a real test rather than a pg-only one.
59
59
  */
60
- export function createMemoryLeaseStore(options: MemoryLeaseStoreOptions = {}): MemoryLeaseStore {
60
+ export function memoryLeaseStore(options: MemoryLeaseStoreOptions = {}): MemoryLeaseStore {
61
61
  const clock = options.clock ?? systemClock;
62
62
  type Slots = Map<number, { holder: string; expiresAt: number }>;
63
63
  const slots = new Map<string, Slots>();
package/src/limits.ts CHANGED
@@ -119,20 +119,20 @@ export function tenantKeyFrom(actor: { readonly orgId?: string } | undefined): s
119
119
  return actor?.orgId ?? NO_TENANT;
120
120
  }
121
121
 
122
- export function createLimiter(
122
+ export function concurrencyLimiter(
123
123
  config: LimitConfig,
124
124
  clock: Clock = systemClock,
125
125
  options: { readonly maxTenants?: number | undefined } = {},
126
126
  ): Limiter {
127
127
  const requested = options.maxTenants ?? DEFAULT_MAX_LIMIT_TENANTS;
128
- // Refused where it was written, for the reason `createPacer` refuses `rate: 0`: `Math.floor(NaN)`
128
+ // Refused where it was written, for the reason `backfillPacer` refuses `rate: 0`: `Math.floor(NaN)`
129
129
  // is `NaN` and `Math.floor(Infinity)` is `Infinity`, so BOTH cap comparisons below read false and
130
130
  // the option silently means "no cap at all" — the one setting this bound exists to make
131
131
  // unreachable. `Number(process.env.X)` is how a deployment writes the first of those.
132
132
  assert(
133
133
  Number.isFinite(requested),
134
134
  `job limiter maxTenants is ${String(requested)}, which caps nothing — the per-tenant maps would grow without a bound`,
135
- `pass a finite maxTenants to createLimiter(...), or omit it for the default ${String(DEFAULT_MAX_LIMIT_TENANTS)}`,
135
+ `pass a finite maxTenants to concurrencyLimiter(...), or omit it for the default ${String(DEFAULT_MAX_LIMIT_TENANTS)}`,
136
136
  );
137
137
  const maxTenants = Math.max(1, Math.floor(requested));
138
138
  const evictTo = Math.max(1, Math.floor(maxTenants * 0.9));
@@ -141,7 +141,7 @@ export function createLimiter(
141
141
  // `config.x !== undefined && count >= config.x`, so a `NaN` leaves the option PRESENT and the
142
142
  // comparison false forever. Measured: `global: Number(process.env.WORKER_GLOBAL_CONCURRENCY)`
143
143
  // with the variable unset granted 1000 of 1000 acquires while `snapshot().config` still reported
144
- // a configured ceiling. `ratePerTenant.windowMs` is on the list because it is half the same
144
+ // a configured ceiling. `ratePerTenant.windowMs` is screened too because it is half the same
145
145
  // ceiling: `stamp > at - NaN` is false for every stamp, so the window reads empty on every call.
146
146
  //
147
147
  // `min` is 0 on all five, deliberately: zero is a HARD STOP here and one this repo's own suite
@@ -153,10 +153,16 @@ export function createLimiter(
153
153
  ['perQueue', config.perQueue],
154
154
  ['global', config.global],
155
155
  ['ratePerTenant.limit', config.ratePerTenant?.limit],
156
- ['ratePerTenant.windowMs', config.ratePerTenant?.windowMs],
157
156
  ] as const) {
158
- if (value !== undefined) finiteCount('createLimiter', option, value);
157
+ if (value !== undefined) finiteCount('concurrencyLimiter', option, value);
159
158
  }
159
+ // Out of the loop because it is also READ, by `sweep`: the screened value is the one it reads.
160
+ // An absent rate window is 0, which `sweep` treats as "every stamp is spent".
161
+ const rateWindowMs = finiteCount(
162
+ 'concurrencyLimiter',
163
+ 'ratePerTenant.windowMs',
164
+ config.ratePerTenant?.windowMs ?? 0,
165
+ );
160
166
  const byQueue = new Map<string, number>();
161
167
  const byTenant = new Map<string, number>();
162
168
  // `{queue, tenantId}` is ONE key — `blockedBy` has always read it that way. Without this counter
@@ -196,9 +202,8 @@ export function createLimiter(
196
202
  */
197
203
  const sweep = (at: number): void => {
198
204
  lastSweepMs = at;
199
- const windowMs = config.ratePerTenant?.windowMs ?? 0;
200
205
  for (const [tenant, stamps] of starts) {
201
- if (stamps.length === 0 || (stamps.at(-1) ?? 0) <= at - windowMs) starts.delete(tenant);
206
+ if (stamps.length === 0 || (stamps.at(-1) ?? 0) <= at - rateWindowMs) starts.delete(tenant);
202
207
  }
203
208
  for (const [key, refusal] of refusals) {
204
209
  if (refusal.atMs <= at - REFUSAL_TTL_MS) refusals.delete(key);
@@ -23,7 +23,7 @@ export function resolveClaimLeaseMs(value: number | undefined): number {
23
23
  assert(
24
24
  Number.isInteger(value) && value > 0,
25
25
  `outbox claimLeaseMs is ${String(value)}, which is not a positive whole number of milliseconds`,
26
- 'pass a positive whole claimLeaseMs: 30_000 — createPgOutboxStore({ executor, txExecutor, claimLeaseMs: 30_000 }) — or omit the field for the 30s default',
26
+ 'pass a positive whole claimLeaseMs: 30_000 — postgresOutboxStore({ executor, txExecutor, claimLeaseMs: 30_000 }) — or omit the field for the 30s default',
27
27
  );
28
28
  return value;
29
29
  }
package/src/outbox-pg.ts CHANGED
@@ -4,13 +4,13 @@
4
4
  // memory store hangs rows off the `Tx` object and is correct only inside one process; this is
5
5
  // what a deployment installs.
6
6
  //
7
- // `txExecutor` is injected rather than resolved here for the reason `createPgDriver` takes a
7
+ // `txExecutor` is injected rather than resolved here for the reason `postgresJobDriver` takes a
8
8
  // `PgExecutor`: this package holds no `@ultimat3/db` dependency, and "which connection is this
9
9
  // `Tx` on" is a question only boot can answer. Boot has `currentTx()` — a `DbTx` IS a client on
10
10
  // the transaction's connection — so the wiring is one line there and no tier crossing here.
11
11
 
12
12
  import type { Clock, PgExecutor } from '@ultimat3/core';
13
- import { uuid } from '@ultimat3/core';
13
+ import { uuidV7 } from '@ultimat3/core';
14
14
  import type { Tx } from '@ultimat3/entity';
15
15
  import { nowMs } from './clock';
16
16
  import {
@@ -39,7 +39,7 @@ interface OutboxRow {
39
39
  readonly claimed_by?: string | null;
40
40
  }
41
41
 
42
- export interface PgOutboxOptions {
42
+ export interface PostgresOutboxStoreOptions {
43
43
  /**
44
44
  * The pooled executor the RELAY uses: `claim`, `markPublished` and `pendingCount` all run after
45
45
  * the caller's transaction is gone, so they must not be bound to it.
@@ -87,7 +87,7 @@ function toRecord(row: OutboxRow): OutboxRecord {
87
87
  };
88
88
  }
89
89
 
90
- export function createPgOutboxStore(options: PgOutboxOptions): OutboxStore {
90
+ export function postgresOutboxStore(options: PostgresOutboxStoreOptions): OutboxStore {
91
91
  // What each open transaction has staged, for `commit()`'s return value only. Never the source
92
92
  // of truth — that is the row, and the row's fate is the transaction's. A WeakMap so a `Tx` that
93
93
  // is neither committed nor rolled back (a process killed mid-request) leaves nothing behind.
@@ -96,7 +96,7 @@ export function createPgOutboxStore(options: PgOutboxOptions): OutboxStore {
96
96
  // One id per store, minted here rather than per claim: `claimed_by` is read by an operator
97
97
  // asking which relay is sitting on a batch, and a value that changed every tick answers nobody.
98
98
  // Per-store is also the granularity the fence needs — two relays are two processes, two stores.
99
- const relayId = options.relayId ?? `relay-${uuid()}`;
99
+ const relayId = options.relayId ?? `relay-${uuidV7()}`;
100
100
  // Resolved once, at construction, so a lease this store could never honour fails where it was
101
101
  // written instead of inside a relay tick whose only trace is a log line nobody reads.
102
102
  const claimLeaseMs = resolveClaimLeaseMs(options.claimLeaseMs);
@@ -52,23 +52,23 @@ export interface OutboxRelay {
52
52
  * At-least-once by construction: publish, THEN mark published. A crash between the two
53
53
  * re-publishes, which the idempotency key collapses — the opposite order would lose jobs.
54
54
  */
55
- export function createOutboxRelay(options: RelayOptions): OutboxRelay {
55
+ export function outboxRelay(options: RelayOptions): OutboxRelay {
56
56
  const batchSize = options.batchSize ?? 100;
57
57
  const intervalMs = options.intervalMs ?? 200;
58
58
  // Refused, never clamped — `worker-options.ts` carries the reason. `setInterval(fn, NaN)` reads
59
59
  // the delay as 0 and `claim(NaN)` slices `(0, NaN)`: a relay that spins and publishes nothing.
60
60
  assert(
61
61
  Number.isSafeInteger(batchSize) && batchSize >= 1,
62
- `createOutboxRelay batchSize is ${String(batchSize)} — a batch is a whole number of staged rows, at least one`,
63
- 'pass a finite batchSize to createOutboxRelay(...), or omit it for the default 100',
62
+ `outboxRelay batchSize is ${String(batchSize)} — a batch is a whole number of staged rows, at least one`,
63
+ 'pass a finite batchSize to outboxRelay(...), or omit it for the default 100',
64
64
  );
65
65
  assert(
66
66
  Number.isFinite(intervalMs) && intervalMs >= 0,
67
- `createOutboxRelay intervalMs is ${String(intervalMs)}, which a timer reads as 0 — the relay would spin, not poll`,
68
- 'pass a finite intervalMs to createOutboxRelay(...), or omit it for the default 200',
67
+ `outboxRelay intervalMs is ${String(intervalMs)}, which a timer reads as 0 — the relay would spin, not poll`,
68
+ 'pass a finite intervalMs to outboxRelay(...), or omit it for the default 200',
69
69
  );
70
70
  const backoff = createIdleBackoff({
71
- subject: 'createOutboxRelay',
71
+ subject: 'outboxRelay',
72
72
  floorMs: intervalMs,
73
73
  ...(options.idlePollMaxMs === undefined ? {} : { ceilingMs: options.idlePollMaxMs }),
74
74
  });
package/src/outbox.ts CHANGED
@@ -14,9 +14,9 @@
14
14
  // `jobsFacade()` is what happens when they are not:
15
15
  //
16
16
  // 1. `x_outbox` exists — it ships in `SQL_JOBS_TABLE` now, so applying the queue DDL is enough.
17
- // 2. `setJobsFacade(createJobsFacade({ store, driver }, currentTx))` ran at boot, with a store
18
- // from `createPgOutboxStore` and a REAL `currentTx` accessor.
19
- // 3. `createOutboxRelay({ store, driver }).start()` is running somewhere.
17
+ // 2. `setJobsFacade(outboxJobsFacade({ store, driver }, currentTx))` ran at boot, with a store
18
+ // from `postgresOutboxStore` and a REAL `currentTx` accessor.
19
+ // 3. `outboxRelay({ store, driver }).start()` is running somewhere.
20
20
  //
21
21
  // With none of them, `jobsFacade()` answers the fallback below, whose `currentTx` is
22
22
  // `() => undefined`: every enqueue publishes straight to the driver, outside the caller's
@@ -29,7 +29,7 @@ import {
29
29
  describeValue,
30
30
  traceparent,
31
31
  UltimateError,
32
- uuid,
32
+ uuidV7,
33
33
  } from '@ultimat3/core';
34
34
  import type { Tx } from '@ultimat3/entity';
35
35
  import { nowMs } from './clock';
@@ -104,7 +104,7 @@ function byClaimOrder(a: OutboxRecord, b: OutboxRecord): number {
104
104
  return a.id < b.id ? -1 : 1;
105
105
  }
106
106
 
107
- export interface MemoryOutboxOptions {
107
+ export interface MemoryOutboxStoreOptions {
108
108
  readonly clock?: Clock;
109
109
  readonly claimLeaseMs?: number;
110
110
  }
@@ -123,7 +123,7 @@ export interface MemoryOutboxStore extends OutboxStore {
123
123
  * transaction" guarantee needs no cooperation from the DB layer and rollback is a delete.
124
124
  * The pg store swaps this for a real `x_outbox` table written by the same connection.
125
125
  */
126
- export function createMemoryOutboxStore(options: MemoryOutboxOptions = {}): MemoryOutboxStore {
126
+ export function memoryOutboxStore(options: MemoryOutboxStoreOptions = {}): MemoryOutboxStore {
127
127
  const staged = new WeakMap<object, OutboxRecord[]>();
128
128
  const committed = new Map<string, OutboxRecord>();
129
129
  /** Each claimed row's lease: when it was taken and by whom. Absent is `claimed_at is null`. */
@@ -260,7 +260,7 @@ function namedRunId(job: string, runId: unknown): string | undefined {
260
260
  code: 'X_ID_INVALID',
261
261
  // `describeValue`, never the value: what arrives here wrong is as often a key as a typo.
262
262
  cause: `enqueue of job "${job}" was given runId ${describeValue(runId)}, and a run id is a lowercase uuid (8-4-4-4-12 hex) — the queue stores it in a uuid column`,
263
- fix: "pass { runId: uuid() } to enqueue(), with import { uuid } from '@ultimat3/core'",
263
+ fix: "pass { runId: uuidV7() } to enqueue(), with import { uuidV7 } from '@ultimat3/core'",
264
264
  meta: { job, option: 'runId' },
265
265
  });
266
266
  }
@@ -285,8 +285,8 @@ export async function enqueueInTx<I>(
285
285
  const at = nowMs(deps.clock);
286
286
  const trace = options.traceparent ?? ambientTraceparent();
287
287
  const record: OutboxRecord = {
288
- id: uuid(),
289
- runId: runId ?? uuid(),
288
+ id: uuidV7(),
289
+ runId: runId ?? uuidV7(),
290
290
  job: handle.name,
291
291
  queue: options.queue ?? handle.queue ?? DEFAULT_QUEUE,
292
292
  input,
@@ -314,7 +314,7 @@ export interface JobsFacade {
314
314
  enqueue<I>(handle: JobHandle<I>, input: I, options?: EnqueueOptions): Promise<EnqueueResult>;
315
315
  }
316
316
 
317
- export function createJobsFacade(deps: OutboxDeps, currentTx: () => Tx | undefined): JobsFacade {
317
+ export function outboxJobsFacade(deps: OutboxDeps, currentTx: () => Tx | undefined): JobsFacade {
318
318
  return {
319
319
  async enqueue<I>(
320
320
  handle: JobHandle<I>,
@@ -379,7 +379,7 @@ export function setJobsFacade(facade: JobsFacade | null): void {
379
379
  */
380
380
  export function jobsFacade(): JobsFacade {
381
381
  if (ambient !== undefined) return ambient;
382
- fallback ??= createJobsFacade(
382
+ fallback ??= outboxJobsFacade(
383
383
  {
384
384
  // A getter, not a snapshot: `setJobDriver()` after the first enqueue is honoured, and a
385
385
  // missing driver is an error at the call rather than at import time.
@@ -389,14 +389,14 @@ export function jobsFacade(): JobsFacade {
389
389
  throw new DriverUnavailableError({
390
390
  driver: 'none',
391
391
  cause: 'no queue driver is installed in this process',
392
- fix: 'call setJobDriver(createMemoryDriver()) before enqueuing, or setJobDriver(createPgDriver()) for a real queue',
392
+ fix: 'call setJobDriver(memoryJobDriver()) before enqueuing, or setJobDriver(postgresJobDriver()) for a real queue',
393
393
  });
394
394
  }
395
395
  return installed;
396
396
  },
397
397
  // Unreachable while `currentTx` is `() => undefined`; present because `OutboxDeps`
398
398
  // requires a store, and a real one is cheaper than an assertion that cannot fire.
399
- store: createMemoryOutboxStore(),
399
+ store: memoryOutboxStore(),
400
400
  },
401
401
  () => undefined,
402
402
  );
@@ -20,7 +20,7 @@ export interface RenewalTimer {
20
20
 
21
21
  /**
22
22
  * Calls `tick` every `intervalMs` until the canceller it returns runs. The seam a lease, a fleet
23
- * slot and a registry row renew through: `createWorker({ schedule })` hands one in, and the
23
+ * slot and a registry row renew through: `jobWorker({ schedule })` hands one in, and the
24
24
  * `runJobs` fixture's is driven by the frozen clock, so a test renews when time passes and never
25
25
  * on the wall clock.
26
26
  */
@@ -3,10 +3,16 @@
3
3
  // The backoff arithmetic stays in ./retry — nothing here recomputes a delay `nextRetry` owns.
4
4
 
5
5
  import type { ErrorRetry } from '@ultimat3/core';
6
- import { classifyThrown, isUltimateError, renderThrowable, statedDelayMs } from '@ultimat3/core';
6
+ import {
7
+ classifyThrown,
8
+ isUltimateError,
9
+ jitterStatedDelay,
10
+ renderThrowable,
11
+ statedDelayMs,
12
+ } from '@ultimat3/core';
7
13
  import { finiteDurationMs } from './clock';
8
14
  import type { Random, RetryDecision, RetryPolicy } from './retry';
9
- import { DEFAULT_RETRY, nextRetry } from './retry';
15
+ import { backoffDelayMs, DEFAULT_RETRY, nextRetry } from './retry';
10
16
 
11
17
  /** Why this attempt was the last one. Absent while the job is still being retried. */
12
18
  export type JobStopReason = 'terminal' | 'attempts-exhausted';
@@ -17,16 +23,10 @@ export interface JobRetryDecision extends RetryDecision {
17
23
  readonly classification: ErrorRetry | undefined;
18
24
  }
19
25
 
20
- /**
21
- * Core's, re-exported rather than copied — they are the readers of core's classification table and
22
- * every executor in the framework has to answer them the same way. Both were declared here first
23
- * and moved down a tier VERBATIM, the subtle rule included: an UNCLASSIFIED code carrying an
24
- * instance `retry: 'terminal'` reads as unclassified, because a per-instance `terminal` is
25
- * indistinguishable from the fail-closed default and honouring it would dead-letter the first
26
- * attempt of every job in every app whose codes nobody has classified. `retry-classification.test.ts`
27
- * pins that they are the same FUNCTION, not merely two functions that agree today.
28
- */
29
- export { classifyThrown, statedDelayMs };
26
+ // `classifyThrown` and `statedDelayMs` are core's, imported — the readers of core's classification
27
+ // table, which every executor answers the same way. An UNCLASSIFIED code carrying an instance
28
+ // `retry: 'terminal'` reads as unclassified (a per-instance `terminal` is indistinguishable from
29
+ // the fail-closed default); `retry-classification.test.ts` holds that rule against core's function.
30
30
 
31
31
  /**
32
32
  * Retry, dead-letter, and when. `terminal` stops here on the attempt that failed — the same code
@@ -65,10 +65,47 @@ export function nextRetryForError(
65
65
 
66
66
  const stated = statedDelayMs(error);
67
67
  if (stated === undefined) return { ...decision, stoppedBy: undefined, classification };
68
- // Clamped by the policy's own ceiling, which is what `maxDelay` is for: a responder naming a
69
- // day is still a responder this deployment has not agreed to wait a day for.
68
+ // The floor is clamped by the policy's own ceiling, which is what `maxDelay` is for: a responder
69
+ // naming a day is still a responder this deployment has not agreed to wait a day for. The spread
70
+ // on top is core's `jitterStatedDelay` — the one rule for a named delay — because an HTTP-date
71
+ // hands every delivery to one receiver the SAME instant, and waking them together is the herd.
72
+ const cap = finiteDurationMs(policy.maxDelay ?? DEFAULT_RETRY.maxDelay, 'retry', 'maxDelay');
73
+ // `jitter: false` is the policy's own decision and is honoured: the bare floor.
74
+ const floor = Math.min(stated, cap);
75
+ const jittered = (policy.jitter ?? DEFAULT_RETRY.jitter) === true;
76
+ const delayMs = jittered ? jitterStatedDelay(floor, cap, random) : floor;
77
+ return { ...decision, delayMs, stoppedBy: undefined, classification };
78
+ }
79
+
80
+ /**
81
+ * `@ultimat3/http`'s `rateLimited()` — the refusal a declared `rateLimit:` answers on every
82
+ * surface, a job's `.job()`/`llm()` run included. Named, not classified: `retry-after` also covers
83
+ * `X_OVERLOADED`, a shed that should still spend the policy's attempts.
84
+ */
85
+ const RATE_LIMITED = 'X_RATE_LIMITED';
86
+
87
+ /**
88
+ * How long to wait out a rate-limit refusal, or `undefined` when `error` is not one. A refusal is
89
+ * "not yet", never "failed": the bucket refills, so the run is rescheduled with its attempt
90
+ * UNCOUNTED — counted, a backlog of rate-limited jobs spent every attempt waiting and was
91
+ * dead-lettered for it.
92
+ *
93
+ * The stated `Retry-After` is the FLOOR, never clamped: a run woken before the refill is refused
94
+ * again, so clamping a 3600 s refusal to `maxDelay` re-ran the backlog every minute for an hour.
95
+ * With none stated, the policy's own backoff. On top, a spread in `[0, min(wait / 2, maxDelay))`:
96
+ * a backlog refused together and woken at one instant is refused together again, a claim storm
97
+ * that repeats forever. `maxDelay` bounds only that spread.
98
+ */
99
+ export function rateLimitDeferralMs(
100
+ policy: RetryPolicy,
101
+ attempt: number,
102
+ error: unknown,
103
+ random: Random = Math.random,
104
+ ): number | undefined {
105
+ if (!isUltimateError(error) || error.code !== RATE_LIMITED) return undefined;
70
106
  const cap = finiteDurationMs(policy.maxDelay ?? DEFAULT_RETRY.maxDelay, 'retry', 'maxDelay');
71
- return { ...decision, delayMs: Math.min(stated, cap), stoppedBy: undefined, classification };
107
+ const wait = statedDelayMs(error) ?? Math.min(backoffDelayMs(policy, attempt), cap);
108
+ return jitterStatedDelay(wait, cap, random);
72
109
  }
73
110
 
74
111
  /**