@ultimat3/jobs 24.0.0 → 25.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +25 -27
- package/README.md +50 -34
- package/package.json +7 -7
- package/src/backfill-errors.ts +3 -2
- package/src/backfill-ledger.ts +1 -1
- package/src/backfill-rate.ts +2 -2
- package/src/backfill.ts +2 -2
- package/src/counters.ts +1 -1
- package/src/driver-memory-operator.ts +3 -3
- package/src/driver-memory.ts +14 -17
- package/src/driver-pg-settle-sql.ts +2 -3
- package/src/driver-pg-sql.ts +1 -1
- package/src/driver-pg.ts +18 -18
- package/src/driver.ts +9 -15
- package/src/errors-concurrency.ts +1 -1
- package/src/errors.ts +5 -5
- package/src/events-pg.ts +4 -4
- package/src/events.ts +6 -6
- package/src/execute.ts +46 -5
- package/src/export.ts +2 -2
- package/src/heartbeat.ts +7 -2
- package/src/index.ts +37 -97
- package/src/inspect-operator.ts +1 -1
- package/src/inspect.ts +1 -1
- package/src/job.ts +1 -1
- package/src/leases.ts +1 -1
- package/src/limits.ts +13 -8
- package/src/outbox-lease.ts +1 -1
- package/src/outbox-pg.ts +5 -5
- package/src/outbox-relay.ts +6 -6
- package/src/outbox.ts +13 -13
- package/src/renewal-timer.ts +1 -1
- package/src/retry-classification.ts +52 -15
- package/src/scheduler-leader.ts +2 -2
- package/src/scheduler-pg.ts +16 -14
- package/src/scheduler-state.ts +2 -2
- package/src/scheduler.ts +5 -5
- package/src/steps-memory.ts +2 -2
- package/src/steps.ts +35 -7
- package/src/task.ts +1 -1
- package/src/webhook-attempt.ts +206 -0
- package/src/webhook-ledger.ts +15 -4
- package/src/webhook.ts +64 -118
- package/src/worker-admit.ts +7 -3
- package/src/worker-drain-cutoff.ts +110 -0
- package/src/worker-fleet-slots.ts +5 -2
- package/src/worker-held.ts +100 -0
- package/src/worker-options.ts +19 -15
- package/src/worker-run.ts +38 -8
- package/src/worker-tally.ts +53 -0
- package/src/worker-types.ts +7 -4
- package/src/worker.ts +85 -77
- package/src/driver-nats.ts +0 -89
- package/src/driver-redis.ts +0 -92
package/src/index.ts
CHANGED
|
@@ -7,23 +7,10 @@
|
|
|
7
7
|
// bypasses `defineApi`'s own result — the ambiguity axiom 1 exists to refuse.
|
|
8
8
|
import './register';
|
|
9
9
|
|
|
10
|
-
// The wire format
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
|
|
14
|
-
export type { PgExecutor, WebhookMacInput, WebhookSigningInput } from '@ultimat3/core';
|
|
15
|
-
export {
|
|
16
|
-
isCanonicalWebhookField,
|
|
17
|
-
WEBHOOK_FIELD_MAX,
|
|
18
|
-
WEBHOOK_ID_HEADER,
|
|
19
|
-
WEBHOOK_SIGNATURE_HEADER,
|
|
20
|
-
WEBHOOK_SIGNATURE_VERSION,
|
|
21
|
-
WEBHOOK_TOPIC_HEADER,
|
|
22
|
-
webhookHeaders,
|
|
23
|
-
webhookMac,
|
|
24
|
-
webhookSignature,
|
|
25
|
-
webhookSigningString,
|
|
26
|
-
} from '@ultimat3/core';
|
|
10
|
+
// The webhook wire format (`WEBHOOK_*`, `webhookSignature`, `webhookMac`, …) is `@ultimat3/core`'s
|
|
11
|
+
// and imported from there: one module at the tier this package (the signer) and `@ultimat3/http`
|
|
12
|
+
// (the verifier) both reach, with one import path — never re-exported here (`X_HELPER_COPY`).
|
|
13
|
+
export type { PgExecutor } from '@ultimat3/core';
|
|
27
14
|
/** Re-exported so a `job`/`task` file needs one import, not two. Same object as schema's. */
|
|
28
15
|
export type { Infer } from '@ultimat3/schema';
|
|
29
16
|
export { t } from '@ultimat3/schema';
|
|
@@ -57,9 +44,9 @@ export type {
|
|
|
57
44
|
export {
|
|
58
45
|
BACKFILL_STATUSES,
|
|
59
46
|
backfillChecksum,
|
|
60
|
-
createMemoryBackfillLedger,
|
|
61
47
|
decideBackfill,
|
|
62
48
|
isBackfillStatus,
|
|
49
|
+
memoryBackfillLedger,
|
|
63
50
|
} from './backfill-ledger';
|
|
64
51
|
export type {
|
|
65
52
|
BackfillPendingReport,
|
|
@@ -73,7 +60,7 @@ export {
|
|
|
73
60
|
pendingBackfills,
|
|
74
61
|
} from './backfill-pending';
|
|
75
62
|
export type { Pacer, PacerOptions } from './backfill-rate';
|
|
76
|
-
export {
|
|
63
|
+
export { backfillPacer, DEFAULT_BACKFILL_RATE } from './backfill-rate';
|
|
77
64
|
export type { BackfillCount, BackfillDeclaration, BackfillOrigin } from './backfill-registry';
|
|
78
65
|
// `stampBackfill` is deliberately absent, for the reason `registerJob` is: a second way to make a
|
|
79
66
|
// handle claim it is a backfill would let a plain `job()` inherit the pending diff and the gate.
|
|
@@ -120,62 +107,14 @@ export {
|
|
|
120
107
|
resetJobDriver,
|
|
121
108
|
setJobDriver,
|
|
122
109
|
} from './driver';
|
|
123
|
-
export type {
|
|
124
|
-
export {
|
|
125
|
-
export type {
|
|
126
|
-
export {
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
SQL_COUNTER_FOLD,
|
|
132
|
-
SQL_COUNTER_TOTALS,
|
|
133
|
-
SQL_COUNTERS,
|
|
134
|
-
SQL_JOB_PROGRESS,
|
|
135
|
-
SQL_JOB_PROMOTE,
|
|
136
|
-
SQL_JOB_REMOVE,
|
|
137
|
-
SQL_JOB_REMOVE_MANY,
|
|
138
|
-
SQL_JOB_REQUEUE_MANY,
|
|
139
|
-
SQL_PAUSE,
|
|
140
|
-
SQL_PAUSED,
|
|
141
|
-
SQL_RESUME,
|
|
142
|
-
SQL_SCHEDULER_FIRE,
|
|
143
|
-
SQL_WORKER_ANNOUNCE,
|
|
144
|
-
SQL_WORKER_FORGET,
|
|
145
|
-
SQL_WORKERS,
|
|
146
|
-
} from './driver-pg-operator-sql';
|
|
147
|
-
export {
|
|
148
|
-
SQL_ACK,
|
|
149
|
-
SQL_ADVISORY_UNLOCK,
|
|
150
|
-
SQL_BACKFILL_FINISH,
|
|
151
|
-
SQL_BACKFILL_LIST,
|
|
152
|
-
SQL_BACKFILL_PROGRESS,
|
|
153
|
-
SQL_BACKFILL_START,
|
|
154
|
-
SQL_CANCEL,
|
|
155
|
-
SQL_CLAIM,
|
|
156
|
-
SQL_ENQUEUE,
|
|
157
|
-
SQL_HEARTBEAT,
|
|
158
|
-
SQL_JOBS_TABLE,
|
|
159
|
-
SQL_LEADER_ACQUIRE,
|
|
160
|
-
SQL_LEADER_RELEASE,
|
|
161
|
-
SQL_LEASE_ACQUIRE,
|
|
162
|
-
SQL_LEASE_HOLDERS,
|
|
163
|
-
SQL_LEASE_RELEASE,
|
|
164
|
-
SQL_LEASE_RENEW,
|
|
165
|
-
SQL_NACK,
|
|
166
|
-
SQL_OUTBOX_CLAIM,
|
|
167
|
-
SQL_OUTBOX_MARK_PUBLISHED,
|
|
168
|
-
SQL_OUTBOX_RELEASE,
|
|
169
|
-
SQL_OUTBOX_STAGE,
|
|
170
|
-
SQL_SCHEDULER_STATE_GET,
|
|
171
|
-
SQL_SCHEDULER_STATE_MARK,
|
|
172
|
-
SQL_STATS,
|
|
173
|
-
SQL_STEP_GET,
|
|
174
|
-
SQL_STEP_PUT,
|
|
175
|
-
SQL_TRY_ADVISORY_LOCK,
|
|
176
|
-
} from './driver-pg-sql';
|
|
177
|
-
export type { RedisDriverOptions } from './driver-redis';
|
|
178
|
-
export { createRedisDriver } from './driver-redis';
|
|
110
|
+
export type { MemoryJobDriver, MemoryJobDriverOptions } from './driver-memory';
|
|
111
|
+
export { memoryJobDriver } from './driver-memory';
|
|
112
|
+
export type { PostgresJobDriverOptions } from './driver-pg';
|
|
113
|
+
export { postgresJobDriver, postgresLeader } from './driver-pg';
|
|
114
|
+
// Only the statement another workspace runs: the install (`cli/src/framework-schema.ts`, action,
|
|
115
|
+
// notify). The rest are this package's internals, read where they are defined, and tested there
|
|
116
|
+
// too — the array-parameter proofs are `driver-pg-array.live.test.ts` (25.0.0).
|
|
117
|
+
export { SQL_JOBS_TABLE } from './driver-pg-sql';
|
|
179
118
|
export { signalEnqueued, signalStaged } from './enqueue-signal';
|
|
180
119
|
export type { JobErrorCode } from './errors';
|
|
181
120
|
export {
|
|
@@ -216,16 +155,16 @@ export {
|
|
|
216
155
|
export { JobNotFoundError, JobNotRequeueableError } from './errors-requeue';
|
|
217
156
|
export type { EventBus, JobEvent, MemoryEventBusOptions, PublishOptions } from './events';
|
|
218
157
|
export {
|
|
219
|
-
createMemoryEventBus,
|
|
220
158
|
EVENTS_PURGE_TARGET,
|
|
221
159
|
eventBus,
|
|
222
160
|
eventsPurgeTarget,
|
|
161
|
+
memoryEventBus,
|
|
223
162
|
publishEvent,
|
|
224
163
|
resetEventBus,
|
|
225
164
|
setEventBus,
|
|
226
165
|
} from './events';
|
|
227
|
-
export type {
|
|
228
|
-
export {
|
|
166
|
+
export type { PostgresEventBusOptions } from './events-pg';
|
|
167
|
+
export { postgresEventBus } from './events-pg';
|
|
229
168
|
export type { ExecuteJobOptions, JobExecution, JobOutcome } from './execute';
|
|
230
169
|
export { executeJob } from './execute';
|
|
231
170
|
export type { ExportDefinition, ExportReport } from './export';
|
|
@@ -296,7 +235,7 @@ export type {
|
|
|
296
235
|
} from './job';
|
|
297
236
|
export { describeJobs, getJob, isJobHandle, job, registeredJobs, resetJobs } from './job';
|
|
298
237
|
export type { HeldLease, LeaseStore, MemoryLeaseStore, MemoryLeaseStoreOptions } from './leases';
|
|
299
|
-
export {
|
|
238
|
+
export { jobLeaseKey, memoryLeaseStore } from './leases';
|
|
300
239
|
export type {
|
|
301
240
|
Lease,
|
|
302
241
|
LimitConfig,
|
|
@@ -306,7 +245,7 @@ export type {
|
|
|
306
245
|
LimitSnapshot,
|
|
307
246
|
RateLimit,
|
|
308
247
|
} from './limits';
|
|
309
|
-
export {
|
|
248
|
+
export { concurrencyLimiter, NO_TENANT, tenantKeyFrom } from './limits';
|
|
310
249
|
export {
|
|
311
250
|
queueDeadJobs,
|
|
312
251
|
queueOldestReady,
|
|
@@ -316,27 +255,27 @@ export {
|
|
|
316
255
|
export type {
|
|
317
256
|
EnqueueOptions,
|
|
318
257
|
JobsFacade,
|
|
319
|
-
MemoryOutboxOptions,
|
|
320
258
|
MemoryOutboxStore,
|
|
259
|
+
MemoryOutboxStoreOptions,
|
|
321
260
|
OutboxDeps,
|
|
322
261
|
OutboxRecord,
|
|
323
262
|
OutboxStore,
|
|
324
263
|
} from './outbox';
|
|
325
264
|
export {
|
|
326
|
-
createJobsFacade,
|
|
327
|
-
createMemoryOutboxStore,
|
|
328
265
|
enqueueInTx,
|
|
329
266
|
jobsFacade,
|
|
267
|
+
memoryOutboxStore,
|
|
268
|
+
outboxJobsFacade,
|
|
330
269
|
resetJobsFacade,
|
|
331
270
|
setJobsFacade,
|
|
332
271
|
} from './outbox';
|
|
333
272
|
// One definition of the lease, consumed by both stores — a memory default and a pg default that
|
|
334
273
|
// could drift are two answers to "how long is a claim mine for", and the shorter one duplicates.
|
|
335
274
|
export { DEFAULT_OUTBOX_CLAIM_LEASE_MS } from './outbox-lease';
|
|
336
|
-
export type {
|
|
337
|
-
export {
|
|
275
|
+
export type { PostgresOutboxStoreOptions } from './outbox-pg';
|
|
276
|
+
export { postgresOutboxStore } from './outbox-pg';
|
|
338
277
|
export type { OutboxRelay, RelayOptions } from './outbox-relay';
|
|
339
|
-
export {
|
|
278
|
+
export { outboxRelay } from './outbox-relay';
|
|
340
279
|
export type { ProgressFn } from './progress';
|
|
341
280
|
export type {
|
|
342
281
|
PurgeDefinition,
|
|
@@ -352,7 +291,7 @@ export type { IntervalScheduler } from './renewal-timer';
|
|
|
352
291
|
export type { BackoffStrategy, Random, RetryDecision, RetryPolicy } from './retry';
|
|
353
292
|
export { backoffDelayMs, DEFAULT_RETRY, isFinalAttempt, nextRetry, retrySchedule } from './retry';
|
|
354
293
|
export type { JobRetryDecision, JobStopReason } from './retry-classification';
|
|
355
|
-
export {
|
|
294
|
+
export { failureForRow, nextRetryForError } from './retry-classification';
|
|
356
295
|
export type {
|
|
357
296
|
CronResolver,
|
|
358
297
|
DispatchedOccurrence,
|
|
@@ -361,22 +300,22 @@ export type {
|
|
|
361
300
|
} from './scheduler';
|
|
362
301
|
export {
|
|
363
302
|
COUNTER_ROLLUP_INTERVAL_MS,
|
|
364
|
-
|
|
303
|
+
jobScheduler,
|
|
365
304
|
PAUSE_RECHECK_MS,
|
|
366
305
|
} from './scheduler';
|
|
367
306
|
export type { LeaderElection } from './scheduler-leader';
|
|
368
307
|
export { soleLeader } from './scheduler-leader';
|
|
369
308
|
export { nextTaskRun } from './scheduler-occurrences';
|
|
370
|
-
export type {
|
|
309
|
+
export type { PostgresLeaseLeaderOptions } from './scheduler-pg';
|
|
371
310
|
export {
|
|
372
|
-
createPgLeaseLeader,
|
|
373
311
|
currentLeader,
|
|
374
312
|
DEFAULT_LEADER_TTL_MS,
|
|
375
313
|
LEASE_RENEWALS_PER_TTL,
|
|
376
|
-
|
|
314
|
+
postgresLeaseLeader,
|
|
315
|
+
postgresSchedulerState,
|
|
377
316
|
} from './scheduler-pg';
|
|
378
317
|
export type { ScheduledFire, SchedulerState } from './scheduler-state';
|
|
379
|
-
export {
|
|
318
|
+
export { fireThroughDriver, memorySchedulerState } from './scheduler-state';
|
|
380
319
|
export type { JobCompleted, JobFailed, JobSettled } from './settled';
|
|
381
320
|
export { ON_SETTLED_ATTEMPTS } from './settled';
|
|
382
321
|
export type {
|
|
@@ -391,14 +330,14 @@ export type {
|
|
|
391
330
|
WaitForEventOptions,
|
|
392
331
|
} from './steps';
|
|
393
332
|
export {
|
|
394
|
-
createStepRunner,
|
|
395
333
|
isStepStatus,
|
|
396
334
|
isStepSuspension,
|
|
397
335
|
MAX_TRACE_NAMES,
|
|
398
336
|
STEP_STATUSES,
|
|
399
337
|
StepSuspension,
|
|
338
|
+
stepRunner,
|
|
400
339
|
} from './steps';
|
|
401
|
-
export {
|
|
340
|
+
export { memoryStepStore } from './steps-memory';
|
|
402
341
|
export type {
|
|
403
342
|
CatchUpPolicy,
|
|
404
343
|
TaskDefinition,
|
|
@@ -421,10 +360,11 @@ export type {
|
|
|
421
360
|
WebhookDeliveryInput,
|
|
422
361
|
WebhookEndpoint,
|
|
423
362
|
WebhookEvent,
|
|
424
|
-
WebhookFetch,
|
|
425
363
|
WebhookReport,
|
|
426
364
|
} from './webhook';
|
|
427
|
-
export { DEFAULT_WEBHOOK_DISABLE_AFTER,
|
|
365
|
+
export { DEFAULT_WEBHOOK_DISABLE_AFTER, webhook } from './webhook';
|
|
366
|
+
export type { WebhookFetch } from './webhook-attempt';
|
|
367
|
+
export { WEBHOOK_CONTENT_TYPE } from './webhook-attempt';
|
|
428
368
|
export {
|
|
429
369
|
WebhookDeliveryFailedError,
|
|
430
370
|
WebhookDeliveryRejectedError,
|
|
@@ -438,4 +378,4 @@ export {
|
|
|
438
378
|
export type { MemoryWebhookLedger, WebhookAttempt, WebhookLedger } from './webhook-ledger';
|
|
439
379
|
export { DEFAULT_MAX_WEBHOOK_ATTEMPTS, memoryWebhookLedger } from './webhook-ledger';
|
|
440
380
|
export type { Worker, WorkerOptions, WorkerStats } from './worker';
|
|
441
|
-
export {
|
|
381
|
+
export { jobWorker } from './worker';
|
package/src/inspect-operator.ts
CHANGED
|
@@ -12,7 +12,7 @@ function operatorOf(driver: JobDriver): JobIntrospection {
|
|
|
12
12
|
if (driver.introspect === undefined) {
|
|
13
13
|
throw new NotImplementedError({
|
|
14
14
|
cause: `the operator surface of the "${driver.name}" jobs driver is not implemented: the driver has no introspect`,
|
|
15
|
-
fix: 'call setJobDriver(
|
|
15
|
+
fix: 'call setJobDriver(postgresJobDriver()) at boot — the pg driver implements introspect — then: x jobs ls --json',
|
|
16
16
|
});
|
|
17
17
|
}
|
|
18
18
|
return driver.introspect;
|
package/src/inspect.ts
CHANGED
|
@@ -114,7 +114,7 @@ function requireIntrospection(driver: JobDriver): NonNullable<JobDriver['introsp
|
|
|
114
114
|
if (driver.introspect === undefined) {
|
|
115
115
|
throw new NotImplementedError({
|
|
116
116
|
cause: `introspection for the "${driver.name}" jobs driver is not implemented: the driver has no introspect`,
|
|
117
|
-
fix: 'call setJobDriver(
|
|
117
|
+
fix: 'call setJobDriver(postgresJobDriver()) at boot — only the pg driver implements introspect — then: x jobs ls --json',
|
|
118
118
|
});
|
|
119
119
|
}
|
|
120
120
|
return driver.introspect;
|
package/src/job.ts
CHANGED
|
@@ -101,7 +101,7 @@ export interface JobDefinition<I, R = unknown> {
|
|
|
101
101
|
*
|
|
102
102
|
* Enforced by `JobDriver.leases` — a row every replica can see — and NOT by `limits.ts`, which
|
|
103
103
|
* counts one process's heap. A driver with no lease store cannot hold this cap, so
|
|
104
|
-
* `
|
|
104
|
+
* `jobWorker().start()` refuses to boot rather than let it pass silently
|
|
105
105
|
* (`X_JOB_CONCURRENCY_UNENFORCEABLE`): this field was declared, documented and in the manifest
|
|
106
106
|
* while nothing read it, which is exactly what axiom 3 exists to prevent.
|
|
107
107
|
*/
|
package/src/leases.ts
CHANGED
|
@@ -57,7 +57,7 @@ export interface MemoryLeaseStore extends LeaseStore {
|
|
|
57
57
|
* memory driver enforces `concurrency` with the same code path the pg driver does, and so the
|
|
58
58
|
* "a second worker is refused" test is a real test rather than a pg-only one.
|
|
59
59
|
*/
|
|
60
|
-
export function
|
|
60
|
+
export function memoryLeaseStore(options: MemoryLeaseStoreOptions = {}): MemoryLeaseStore {
|
|
61
61
|
const clock = options.clock ?? systemClock;
|
|
62
62
|
type Slots = Map<number, { holder: string; expiresAt: number }>;
|
|
63
63
|
const slots = new Map<string, Slots>();
|
package/src/limits.ts
CHANGED
|
@@ -119,20 +119,20 @@ export function tenantKeyFrom(actor: { readonly orgId?: string } | undefined): s
|
|
|
119
119
|
return actor?.orgId ?? NO_TENANT;
|
|
120
120
|
}
|
|
121
121
|
|
|
122
|
-
export function
|
|
122
|
+
export function concurrencyLimiter(
|
|
123
123
|
config: LimitConfig,
|
|
124
124
|
clock: Clock = systemClock,
|
|
125
125
|
options: { readonly maxTenants?: number | undefined } = {},
|
|
126
126
|
): Limiter {
|
|
127
127
|
const requested = options.maxTenants ?? DEFAULT_MAX_LIMIT_TENANTS;
|
|
128
|
-
// Refused where it was written, for the reason `
|
|
128
|
+
// Refused where it was written, for the reason `backfillPacer` refuses `rate: 0`: `Math.floor(NaN)`
|
|
129
129
|
// is `NaN` and `Math.floor(Infinity)` is `Infinity`, so BOTH cap comparisons below read false and
|
|
130
130
|
// the option silently means "no cap at all" — the one setting this bound exists to make
|
|
131
131
|
// unreachable. `Number(process.env.X)` is how a deployment writes the first of those.
|
|
132
132
|
assert(
|
|
133
133
|
Number.isFinite(requested),
|
|
134
134
|
`job limiter maxTenants is ${String(requested)}, which caps nothing — the per-tenant maps would grow without a bound`,
|
|
135
|
-
`pass a finite maxTenants to
|
|
135
|
+
`pass a finite maxTenants to concurrencyLimiter(...), or omit it for the default ${String(DEFAULT_MAX_LIMIT_TENANTS)}`,
|
|
136
136
|
);
|
|
137
137
|
const maxTenants = Math.max(1, Math.floor(requested));
|
|
138
138
|
const evictTo = Math.max(1, Math.floor(maxTenants * 0.9));
|
|
@@ -141,7 +141,7 @@ export function createLimiter(
|
|
|
141
141
|
// `config.x !== undefined && count >= config.x`, so a `NaN` leaves the option PRESENT and the
|
|
142
142
|
// comparison false forever. Measured: `global: Number(process.env.WORKER_GLOBAL_CONCURRENCY)`
|
|
143
143
|
// with the variable unset granted 1000 of 1000 acquires while `snapshot().config` still reported
|
|
144
|
-
// a configured ceiling. `ratePerTenant.windowMs` is
|
|
144
|
+
// a configured ceiling. `ratePerTenant.windowMs` is screened too because it is half the same
|
|
145
145
|
// ceiling: `stamp > at - NaN` is false for every stamp, so the window reads empty on every call.
|
|
146
146
|
//
|
|
147
147
|
// `min` is 0 on all five, deliberately: zero is a HARD STOP here and one this repo's own suite
|
|
@@ -153,10 +153,16 @@ export function createLimiter(
|
|
|
153
153
|
['perQueue', config.perQueue],
|
|
154
154
|
['global', config.global],
|
|
155
155
|
['ratePerTenant.limit', config.ratePerTenant?.limit],
|
|
156
|
-
['ratePerTenant.windowMs', config.ratePerTenant?.windowMs],
|
|
157
156
|
] as const) {
|
|
158
|
-
if (value !== undefined) finiteCount('
|
|
157
|
+
if (value !== undefined) finiteCount('concurrencyLimiter', option, value);
|
|
159
158
|
}
|
|
159
|
+
// Out of the loop because it is also READ, by `sweep`: the screened value is the one it reads.
|
|
160
|
+
// An absent rate window is 0, which `sweep` treats as "every stamp is spent".
|
|
161
|
+
const rateWindowMs = finiteCount(
|
|
162
|
+
'concurrencyLimiter',
|
|
163
|
+
'ratePerTenant.windowMs',
|
|
164
|
+
config.ratePerTenant?.windowMs ?? 0,
|
|
165
|
+
);
|
|
160
166
|
const byQueue = new Map<string, number>();
|
|
161
167
|
const byTenant = new Map<string, number>();
|
|
162
168
|
// `{queue, tenantId}` is ONE key — `blockedBy` has always read it that way. Without this counter
|
|
@@ -196,9 +202,8 @@ export function createLimiter(
|
|
|
196
202
|
*/
|
|
197
203
|
const sweep = (at: number): void => {
|
|
198
204
|
lastSweepMs = at;
|
|
199
|
-
const windowMs = config.ratePerTenant?.windowMs ?? 0;
|
|
200
205
|
for (const [tenant, stamps] of starts) {
|
|
201
|
-
if (stamps.length === 0 || (stamps.at(-1) ?? 0) <= at -
|
|
206
|
+
if (stamps.length === 0 || (stamps.at(-1) ?? 0) <= at - rateWindowMs) starts.delete(tenant);
|
|
202
207
|
}
|
|
203
208
|
for (const [key, refusal] of refusals) {
|
|
204
209
|
if (refusal.atMs <= at - REFUSAL_TTL_MS) refusals.delete(key);
|
package/src/outbox-lease.ts
CHANGED
|
@@ -23,7 +23,7 @@ export function resolveClaimLeaseMs(value: number | undefined): number {
|
|
|
23
23
|
assert(
|
|
24
24
|
Number.isInteger(value) && value > 0,
|
|
25
25
|
`outbox claimLeaseMs is ${String(value)}, which is not a positive whole number of milliseconds`,
|
|
26
|
-
'pass a positive whole claimLeaseMs: 30_000 —
|
|
26
|
+
'pass a positive whole claimLeaseMs: 30_000 — postgresOutboxStore({ executor, txExecutor, claimLeaseMs: 30_000 }) — or omit the field for the 30s default',
|
|
27
27
|
);
|
|
28
28
|
return value;
|
|
29
29
|
}
|
package/src/outbox-pg.ts
CHANGED
|
@@ -4,13 +4,13 @@
|
|
|
4
4
|
// memory store hangs rows off the `Tx` object and is correct only inside one process; this is
|
|
5
5
|
// what a deployment installs.
|
|
6
6
|
//
|
|
7
|
-
// `txExecutor` is injected rather than resolved here for the reason `
|
|
7
|
+
// `txExecutor` is injected rather than resolved here for the reason `postgresJobDriver` takes a
|
|
8
8
|
// `PgExecutor`: this package holds no `@ultimat3/db` dependency, and "which connection is this
|
|
9
9
|
// `Tx` on" is a question only boot can answer. Boot has `currentTx()` — a `DbTx` IS a client on
|
|
10
10
|
// the transaction's connection — so the wiring is one line there and no tier crossing here.
|
|
11
11
|
|
|
12
12
|
import type { Clock, PgExecutor } from '@ultimat3/core';
|
|
13
|
-
import {
|
|
13
|
+
import { uuidV7 } from '@ultimat3/core';
|
|
14
14
|
import type { Tx } from '@ultimat3/entity';
|
|
15
15
|
import { nowMs } from './clock';
|
|
16
16
|
import {
|
|
@@ -39,7 +39,7 @@ interface OutboxRow {
|
|
|
39
39
|
readonly claimed_by?: string | null;
|
|
40
40
|
}
|
|
41
41
|
|
|
42
|
-
export interface
|
|
42
|
+
export interface PostgresOutboxStoreOptions {
|
|
43
43
|
/**
|
|
44
44
|
* The pooled executor the RELAY uses: `claim`, `markPublished` and `pendingCount` all run after
|
|
45
45
|
* the caller's transaction is gone, so they must not be bound to it.
|
|
@@ -87,7 +87,7 @@ function toRecord(row: OutboxRow): OutboxRecord {
|
|
|
87
87
|
};
|
|
88
88
|
}
|
|
89
89
|
|
|
90
|
-
export function
|
|
90
|
+
export function postgresOutboxStore(options: PostgresOutboxStoreOptions): OutboxStore {
|
|
91
91
|
// What each open transaction has staged, for `commit()`'s return value only. Never the source
|
|
92
92
|
// of truth — that is the row, and the row's fate is the transaction's. A WeakMap so a `Tx` that
|
|
93
93
|
// is neither committed nor rolled back (a process killed mid-request) leaves nothing behind.
|
|
@@ -96,7 +96,7 @@ export function createPgOutboxStore(options: PgOutboxOptions): OutboxStore {
|
|
|
96
96
|
// One id per store, minted here rather than per claim: `claimed_by` is read by an operator
|
|
97
97
|
// asking which relay is sitting on a batch, and a value that changed every tick answers nobody.
|
|
98
98
|
// Per-store is also the granularity the fence needs — two relays are two processes, two stores.
|
|
99
|
-
const relayId = options.relayId ?? `relay-${
|
|
99
|
+
const relayId = options.relayId ?? `relay-${uuidV7()}`;
|
|
100
100
|
// Resolved once, at construction, so a lease this store could never honour fails where it was
|
|
101
101
|
// written instead of inside a relay tick whose only trace is a log line nobody reads.
|
|
102
102
|
const claimLeaseMs = resolveClaimLeaseMs(options.claimLeaseMs);
|
package/src/outbox-relay.ts
CHANGED
|
@@ -52,23 +52,23 @@ export interface OutboxRelay {
|
|
|
52
52
|
* At-least-once by construction: publish, THEN mark published. A crash between the two
|
|
53
53
|
* re-publishes, which the idempotency key collapses — the opposite order would lose jobs.
|
|
54
54
|
*/
|
|
55
|
-
export function
|
|
55
|
+
export function outboxRelay(options: RelayOptions): OutboxRelay {
|
|
56
56
|
const batchSize = options.batchSize ?? 100;
|
|
57
57
|
const intervalMs = options.intervalMs ?? 200;
|
|
58
58
|
// Refused, never clamped — `worker-options.ts` carries the reason. `setInterval(fn, NaN)` reads
|
|
59
59
|
// the delay as 0 and `claim(NaN)` slices `(0, NaN)`: a relay that spins and publishes nothing.
|
|
60
60
|
assert(
|
|
61
61
|
Number.isSafeInteger(batchSize) && batchSize >= 1,
|
|
62
|
-
`
|
|
63
|
-
'pass a finite batchSize to
|
|
62
|
+
`outboxRelay batchSize is ${String(batchSize)} — a batch is a whole number of staged rows, at least one`,
|
|
63
|
+
'pass a finite batchSize to outboxRelay(...), or omit it for the default 100',
|
|
64
64
|
);
|
|
65
65
|
assert(
|
|
66
66
|
Number.isFinite(intervalMs) && intervalMs >= 0,
|
|
67
|
-
`
|
|
68
|
-
'pass a finite intervalMs to
|
|
67
|
+
`outboxRelay intervalMs is ${String(intervalMs)}, which a timer reads as 0 — the relay would spin, not poll`,
|
|
68
|
+
'pass a finite intervalMs to outboxRelay(...), or omit it for the default 200',
|
|
69
69
|
);
|
|
70
70
|
const backoff = createIdleBackoff({
|
|
71
|
-
subject: '
|
|
71
|
+
subject: 'outboxRelay',
|
|
72
72
|
floorMs: intervalMs,
|
|
73
73
|
...(options.idlePollMaxMs === undefined ? {} : { ceilingMs: options.idlePollMaxMs }),
|
|
74
74
|
});
|
package/src/outbox.ts
CHANGED
|
@@ -14,9 +14,9 @@
|
|
|
14
14
|
// `jobsFacade()` is what happens when they are not:
|
|
15
15
|
//
|
|
16
16
|
// 1. `x_outbox` exists — it ships in `SQL_JOBS_TABLE` now, so applying the queue DDL is enough.
|
|
17
|
-
// 2. `setJobsFacade(
|
|
18
|
-
// from `
|
|
19
|
-
// 3. `
|
|
17
|
+
// 2. `setJobsFacade(outboxJobsFacade({ store, driver }, currentTx))` ran at boot, with a store
|
|
18
|
+
// from `postgresOutboxStore` and a REAL `currentTx` accessor.
|
|
19
|
+
// 3. `outboxRelay({ store, driver }).start()` is running somewhere.
|
|
20
20
|
//
|
|
21
21
|
// With none of them, `jobsFacade()` answers the fallback below, whose `currentTx` is
|
|
22
22
|
// `() => undefined`: every enqueue publishes straight to the driver, outside the caller's
|
|
@@ -29,7 +29,7 @@ import {
|
|
|
29
29
|
describeValue,
|
|
30
30
|
traceparent,
|
|
31
31
|
UltimateError,
|
|
32
|
-
|
|
32
|
+
uuidV7,
|
|
33
33
|
} from '@ultimat3/core';
|
|
34
34
|
import type { Tx } from '@ultimat3/entity';
|
|
35
35
|
import { nowMs } from './clock';
|
|
@@ -104,7 +104,7 @@ function byClaimOrder(a: OutboxRecord, b: OutboxRecord): number {
|
|
|
104
104
|
return a.id < b.id ? -1 : 1;
|
|
105
105
|
}
|
|
106
106
|
|
|
107
|
-
export interface
|
|
107
|
+
export interface MemoryOutboxStoreOptions {
|
|
108
108
|
readonly clock?: Clock;
|
|
109
109
|
readonly claimLeaseMs?: number;
|
|
110
110
|
}
|
|
@@ -123,7 +123,7 @@ export interface MemoryOutboxStore extends OutboxStore {
|
|
|
123
123
|
* transaction" guarantee needs no cooperation from the DB layer and rollback is a delete.
|
|
124
124
|
* The pg store swaps this for a real `x_outbox` table written by the same connection.
|
|
125
125
|
*/
|
|
126
|
-
export function
|
|
126
|
+
export function memoryOutboxStore(options: MemoryOutboxStoreOptions = {}): MemoryOutboxStore {
|
|
127
127
|
const staged = new WeakMap<object, OutboxRecord[]>();
|
|
128
128
|
const committed = new Map<string, OutboxRecord>();
|
|
129
129
|
/** Each claimed row's lease: when it was taken and by whom. Absent is `claimed_at is null`. */
|
|
@@ -260,7 +260,7 @@ function namedRunId(job: string, runId: unknown): string | undefined {
|
|
|
260
260
|
code: 'X_ID_INVALID',
|
|
261
261
|
// `describeValue`, never the value: what arrives here wrong is as often a key as a typo.
|
|
262
262
|
cause: `enqueue of job "${job}" was given runId ${describeValue(runId)}, and a run id is a lowercase uuid (8-4-4-4-12 hex) — the queue stores it in a uuid column`,
|
|
263
|
-
fix: "pass { runId:
|
|
263
|
+
fix: "pass { runId: uuidV7() } to enqueue(), with import { uuidV7 } from '@ultimat3/core'",
|
|
264
264
|
meta: { job, option: 'runId' },
|
|
265
265
|
});
|
|
266
266
|
}
|
|
@@ -285,8 +285,8 @@ export async function enqueueInTx<I>(
|
|
|
285
285
|
const at = nowMs(deps.clock);
|
|
286
286
|
const trace = options.traceparent ?? ambientTraceparent();
|
|
287
287
|
const record: OutboxRecord = {
|
|
288
|
-
id:
|
|
289
|
-
runId: runId ??
|
|
288
|
+
id: uuidV7(),
|
|
289
|
+
runId: runId ?? uuidV7(),
|
|
290
290
|
job: handle.name,
|
|
291
291
|
queue: options.queue ?? handle.queue ?? DEFAULT_QUEUE,
|
|
292
292
|
input,
|
|
@@ -314,7 +314,7 @@ export interface JobsFacade {
|
|
|
314
314
|
enqueue<I>(handle: JobHandle<I>, input: I, options?: EnqueueOptions): Promise<EnqueueResult>;
|
|
315
315
|
}
|
|
316
316
|
|
|
317
|
-
export function
|
|
317
|
+
export function outboxJobsFacade(deps: OutboxDeps, currentTx: () => Tx | undefined): JobsFacade {
|
|
318
318
|
return {
|
|
319
319
|
async enqueue<I>(
|
|
320
320
|
handle: JobHandle<I>,
|
|
@@ -379,7 +379,7 @@ export function setJobsFacade(facade: JobsFacade | null): void {
|
|
|
379
379
|
*/
|
|
380
380
|
export function jobsFacade(): JobsFacade {
|
|
381
381
|
if (ambient !== undefined) return ambient;
|
|
382
|
-
fallback ??=
|
|
382
|
+
fallback ??= outboxJobsFacade(
|
|
383
383
|
{
|
|
384
384
|
// A getter, not a snapshot: `setJobDriver()` after the first enqueue is honoured, and a
|
|
385
385
|
// missing driver is an error at the call rather than at import time.
|
|
@@ -389,14 +389,14 @@ export function jobsFacade(): JobsFacade {
|
|
|
389
389
|
throw new DriverUnavailableError({
|
|
390
390
|
driver: 'none',
|
|
391
391
|
cause: 'no queue driver is installed in this process',
|
|
392
|
-
fix: 'call setJobDriver(
|
|
392
|
+
fix: 'call setJobDriver(memoryJobDriver()) before enqueuing, or setJobDriver(postgresJobDriver()) for a real queue',
|
|
393
393
|
});
|
|
394
394
|
}
|
|
395
395
|
return installed;
|
|
396
396
|
},
|
|
397
397
|
// Unreachable while `currentTx` is `() => undefined`; present because `OutboxDeps`
|
|
398
398
|
// requires a store, and a real one is cheaper than an assertion that cannot fire.
|
|
399
|
-
store:
|
|
399
|
+
store: memoryOutboxStore(),
|
|
400
400
|
},
|
|
401
401
|
() => undefined,
|
|
402
402
|
);
|
package/src/renewal-timer.ts
CHANGED
|
@@ -20,7 +20,7 @@ export interface RenewalTimer {
|
|
|
20
20
|
|
|
21
21
|
/**
|
|
22
22
|
* Calls `tick` every `intervalMs` until the canceller it returns runs. The seam a lease, a fleet
|
|
23
|
-
* slot and a registry row renew through: `
|
|
23
|
+
* slot and a registry row renew through: `jobWorker({ schedule })` hands one in, and the
|
|
24
24
|
* `runJobs` fixture's is driven by the frozen clock, so a test renews when time passes and never
|
|
25
25
|
* on the wall clock.
|
|
26
26
|
*/
|
|
@@ -3,10 +3,16 @@
|
|
|
3
3
|
// The backoff arithmetic stays in ./retry — nothing here recomputes a delay `nextRetry` owns.
|
|
4
4
|
|
|
5
5
|
import type { ErrorRetry } from '@ultimat3/core';
|
|
6
|
-
import {
|
|
6
|
+
import {
|
|
7
|
+
classifyThrown,
|
|
8
|
+
isUltimateError,
|
|
9
|
+
jitterStatedDelay,
|
|
10
|
+
renderThrowable,
|
|
11
|
+
statedDelayMs,
|
|
12
|
+
} from '@ultimat3/core';
|
|
7
13
|
import { finiteDurationMs } from './clock';
|
|
8
14
|
import type { Random, RetryDecision, RetryPolicy } from './retry';
|
|
9
|
-
import { DEFAULT_RETRY, nextRetry } from './retry';
|
|
15
|
+
import { backoffDelayMs, DEFAULT_RETRY, nextRetry } from './retry';
|
|
10
16
|
|
|
11
17
|
/** Why this attempt was the last one. Absent while the job is still being retried. */
|
|
12
18
|
export type JobStopReason = 'terminal' | 'attempts-exhausted';
|
|
@@ -17,16 +23,10 @@ export interface JobRetryDecision extends RetryDecision {
|
|
|
17
23
|
readonly classification: ErrorRetry | undefined;
|
|
18
24
|
}
|
|
19
25
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
* instance `retry: 'terminal'` reads as unclassified, because a per-instance `terminal` is
|
|
25
|
-
* indistinguishable from the fail-closed default and honouring it would dead-letter the first
|
|
26
|
-
* attempt of every job in every app whose codes nobody has classified. `retry-classification.test.ts`
|
|
27
|
-
* pins that they are the same FUNCTION, not merely two functions that agree today.
|
|
28
|
-
*/
|
|
29
|
-
export { classifyThrown, statedDelayMs };
|
|
26
|
+
// `classifyThrown` and `statedDelayMs` are core's, imported — the readers of core's classification
|
|
27
|
+
// table, which every executor answers the same way. An UNCLASSIFIED code carrying an instance
|
|
28
|
+
// `retry: 'terminal'` reads as unclassified (a per-instance `terminal` is indistinguishable from
|
|
29
|
+
// the fail-closed default); `retry-classification.test.ts` holds that rule against core's function.
|
|
30
30
|
|
|
31
31
|
/**
|
|
32
32
|
* Retry, dead-letter, and when. `terminal` stops here on the attempt that failed — the same code
|
|
@@ -65,10 +65,47 @@ export function nextRetryForError(
|
|
|
65
65
|
|
|
66
66
|
const stated = statedDelayMs(error);
|
|
67
67
|
if (stated === undefined) return { ...decision, stoppedBy: undefined, classification };
|
|
68
|
-
//
|
|
69
|
-
// day is still a responder this deployment has not agreed to wait a day for.
|
|
68
|
+
// The floor is clamped by the policy's own ceiling, which is what `maxDelay` is for: a responder
|
|
69
|
+
// naming a day is still a responder this deployment has not agreed to wait a day for. The spread
|
|
70
|
+
// on top is core's `jitterStatedDelay` — the one rule for a named delay — because an HTTP-date
|
|
71
|
+
// hands every delivery to one receiver the SAME instant, and waking them together is the herd.
|
|
72
|
+
const cap = finiteDurationMs(policy.maxDelay ?? DEFAULT_RETRY.maxDelay, 'retry', 'maxDelay');
|
|
73
|
+
// `jitter: false` is the policy's own decision and is honoured: the bare floor.
|
|
74
|
+
const floor = Math.min(stated, cap);
|
|
75
|
+
const jittered = (policy.jitter ?? DEFAULT_RETRY.jitter) === true;
|
|
76
|
+
const delayMs = jittered ? jitterStatedDelay(floor, cap, random) : floor;
|
|
77
|
+
return { ...decision, delayMs, stoppedBy: undefined, classification };
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* `@ultimat3/http`'s `rateLimited()` — the refusal a declared `rateLimit:` answers on every
|
|
82
|
+
* surface, a job's `.job()`/`llm()` run included. Named, not classified: `retry-after` also covers
|
|
83
|
+
* `X_OVERLOADED`, a shed that should still spend the policy's attempts.
|
|
84
|
+
*/
|
|
85
|
+
const RATE_LIMITED = 'X_RATE_LIMITED';
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* How long to wait out a rate-limit refusal, or `undefined` when `error` is not one. A refusal is
|
|
89
|
+
* "not yet", never "failed": the bucket refills, so the run is rescheduled with its attempt
|
|
90
|
+
* UNCOUNTED — counted, a backlog of rate-limited jobs spent every attempt waiting and was
|
|
91
|
+
* dead-lettered for it.
|
|
92
|
+
*
|
|
93
|
+
* The stated `Retry-After` is the FLOOR, never clamped: a run woken before the refill is refused
|
|
94
|
+
* again, so clamping a 3600 s refusal to `maxDelay` re-ran the backlog every minute for an hour.
|
|
95
|
+
* With none stated, the policy's own backoff. On top, a spread in `[0, min(wait / 2, maxDelay))`:
|
|
96
|
+
* a backlog refused together and woken at one instant is refused together again, a claim storm
|
|
97
|
+
* that repeats forever. `maxDelay` bounds only that spread.
|
|
98
|
+
*/
|
|
99
|
+
export function rateLimitDeferralMs(
|
|
100
|
+
policy: RetryPolicy,
|
|
101
|
+
attempt: number,
|
|
102
|
+
error: unknown,
|
|
103
|
+
random: Random = Math.random,
|
|
104
|
+
): number | undefined {
|
|
105
|
+
if (!isUltimateError(error) || error.code !== RATE_LIMITED) return undefined;
|
|
70
106
|
const cap = finiteDurationMs(policy.maxDelay ?? DEFAULT_RETRY.maxDelay, 'retry', 'maxDelay');
|
|
71
|
-
|
|
107
|
+
const wait = statedDelayMs(error) ?? Math.min(backoffDelayMs(policy, attempt), cap);
|
|
108
|
+
return jitterStatedDelay(wait, cap, random);
|
|
72
109
|
}
|
|
73
110
|
|
|
74
111
|
/**
|