@ultimat3/jobs 1.2.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/CLAUDE.md +660 -0
  2. package/README.md +432 -17
  3. package/package.json +7 -5
  4. package/src/backfill-gate.ts +97 -0
  5. package/src/backfill-inspect.ts +73 -0
  6. package/src/backfill-ledger.ts +183 -0
  7. package/src/backfill-pass.ts +276 -0
  8. package/src/backfill-pending.ts +131 -0
  9. package/src/backfill-rate.ts +109 -0
  10. package/src/backfill-registry.ts +108 -0
  11. package/src/backfill-scope.ts +70 -0
  12. package/src/backfill.ts +213 -0
  13. package/src/driver-memory.ts +61 -9
  14. package/src/driver-nats.ts +2 -1
  15. package/src/driver-pg-ddl.ts +191 -0
  16. package/src/driver-pg-rows.ts +123 -0
  17. package/src/driver-pg-sql.ts +312 -55
  18. package/src/driver-pg.ts +138 -92
  19. package/src/driver-redis.ts +2 -1
  20. package/src/driver.ts +91 -7
  21. package/src/errors.ts +314 -5
  22. package/src/events-pg.ts +121 -0
  23. package/src/events.ts +7 -1
  24. package/src/execute.ts +308 -0
  25. package/src/heartbeat.ts +148 -0
  26. package/src/index.ts +128 -27
  27. package/src/inspect.ts +43 -2
  28. package/src/job.ts +127 -3
  29. package/src/leases.ts +90 -0
  30. package/src/limits.ts +0 -0
  31. package/src/metrics.ts +35 -0
  32. package/src/outbox-lease.ts +29 -0
  33. package/src/outbox-pg.ts +188 -0
  34. package/src/outbox.ts +204 -59
  35. package/src/register.ts +1 -1
  36. package/src/renewal-timer.ts +35 -0
  37. package/src/retry-classification.ts +112 -0
  38. package/src/retry.ts +6 -1
  39. package/src/run-signal.ts +50 -0
  40. package/src/scheduler-pg.ts +103 -0
  41. package/src/scheduler.ts +159 -245
  42. package/src/steps.ts +155 -31
  43. package/src/task.ts +239 -0
  44. package/src/tenant.ts +61 -0
  45. package/src/worker-fleet-slots.ts +129 -0
  46. package/src/worker-run.ts +132 -0
  47. package/src/worker.ts +207 -190
package/src/errors.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  // The X_* codes owned by @ultimat3/jobs. Every one names the command or code change that
2
2
  // fixes it — a job failure an agent cannot act on is a job failure that gets retried forever.
3
- import { registerErrorCodes, UltimateError } from '@ultimat3/core';
3
+ import { registerErrorCodes, registerErrorRetry, UltimateError } from '@ultimat3/core';
4
4
 
5
5
  /** Codes this package declares and owns. */
6
6
  export const JOB_OWNED_ERROR_CODES = [
@@ -10,15 +10,27 @@ export const JOB_OWNED_ERROR_CODES = [
10
10
  'X_JOB_MAX_ATTEMPTS',
11
11
  'X_DRIVER_UNAVAILABLE',
12
12
  'X_IDEMPOTENCY_REQUIRED',
13
+ 'X_JOB_TENANT_REQUIRED',
14
+ 'X_JOB_CONCURRENCY_UNENFORCEABLE',
15
+ 'X_JOB_LEASE_LOST',
16
+ 'X_JOB_SLOT_LOST',
17
+ 'X_JOB_NOT_CANCELLABLE',
13
18
  'X_OUTBOX_NO_TX',
19
+ 'X_BACKFILL_PENDING',
20
+ 'X_BACKFILL_APPLIED',
21
+ 'X_BACKFILL_ENVIRONMENT',
22
+ 'X_BACKFILL_MIGRATION_PENDING',
23
+ 'X_BACKFILL_RUNNING',
24
+ 'X_BACKFILL_STALLED',
25
+ 'X_BACKFILL_UNKNOWN',
14
26
  ] as const;
15
27
 
16
28
  /**
17
- * `X_NOT_IMPLEMENTED` is `@ultimat3/core`'s. `JobsNotImplementedError` below throws it; jobs keeps
18
- * no title for it, because the copy this file used to hold was a second title that nothing would
19
- * have failed on once core's changed.
29
+ * `X_NOT_IMPLEMENTED` and `X_ABORTED` are `@ultimat3/core`'s. `JobsNotImplementedError` and
30
+ * `JobAbortedError` below throw them; jobs keeps no title for either, because the copy this file
31
+ * used to hold was a second title that nothing would have failed on once core's changed.
20
32
  */
21
- export const JOB_BORROWED_ERROR_CODES = ['X_NOT_IMPLEMENTED'] as const;
33
+ export const JOB_BORROWED_ERROR_CODES = ['X_NOT_IMPLEMENTED', 'X_ABORTED'] as const;
22
34
 
23
35
  /** Every code jobs can throw: the ones it owns plus the one it borrows. */
24
36
  export const JOB_ERROR_CODES = [...JOB_OWNED_ERROR_CODES, ...JOB_BORROWED_ERROR_CODES] as const;
@@ -33,7 +45,19 @@ export const JOB_ERROR_TITLES: Readonly<Record<JobOwnedErrorCode, string>> = {
33
45
  X_JOB_MAX_ATTEMPTS: 'the job exhausted its retries',
34
46
  X_DRIVER_UNAVAILABLE: 'the queue driver is unreachable',
35
47
  X_IDEMPOTENCY_REQUIRED: 'the job has no idempotencyKey',
48
+ X_JOB_TENANT_REQUIRED: 'the job declares no tenant',
49
+ X_JOB_CONCURRENCY_UNENFORCEABLE: 'job.concurrency is declared and cannot be enforced',
50
+ X_JOB_LEASE_LOST: 'the queue took this job back mid-run',
51
+ X_JOB_SLOT_LOST: 'the fleet concurrency slot was taken by another worker',
52
+ X_JOB_NOT_CANCELLABLE: 'the job cannot be cancelled',
36
53
  X_OUTBOX_NO_TX: 'enqueue outside a transaction',
54
+ X_BACKFILL_PENDING: 'a declared backfill has never completed',
55
+ X_BACKFILL_APPLIED: 'the ledger already holds a completed pass',
56
+ X_BACKFILL_ENVIRONMENT: 'the backfill is not declared for this environment',
57
+ X_BACKFILL_MIGRATION_PENDING: 'the migration this backfill requires is not applied',
58
+ X_BACKFILL_RUNNING: 'a pass under this name is already live',
59
+ X_BACKFILL_STALLED: 'the sweep ended with rows its own count still matches',
60
+ X_BACKFILL_UNKNOWN: 'no declaration carries this backfill name',
37
61
  };
38
62
 
39
63
  // One unconditional call, so a second package claiming one of jobs' codes throws
@@ -42,6 +66,29 @@ registerErrorCodes(
42
66
  Object.fromEntries(Object.entries(JOB_ERROR_TITLES).map(([code, title]) => [code, { title }])),
43
67
  );
44
68
 
69
+ /**
70
+ * The codes of this package's that can be thrown INSIDE a job body, classified — `executeJob`
71
+ * reads this, so a `terminal` one dead-letters on the attempt it happened instead of spending the
72
+ * whole policy on an answer that cannot change. Same rule every package uses: retryable means the
73
+ * same code, run again, has a real chance of a different answer.
74
+ *
75
+ * Two are deliberately absent. `X_JOB_LEASE_LOST` and `X_JOB_SLOT_LOST` mean the row is somebody
76
+ * else's now, so this attempt's verdict is not this attempt's to give: dead-lettering would settle
77
+ * a job another worker is running. They keep the attempt-count path, which ends in the queue
78
+ * re-delivering — the honest outcome for "we stopped owning it".
79
+ */
80
+ registerErrorRetry({
81
+ X_JOB_TIMEOUT: 'retryable',
82
+ X_DRIVER_UNAVAILABLE: 'retryable',
83
+ // A second `step.run` under one name is a defect in the handler, replayed identically forever.
84
+ X_STEP_DUPLICATE: 'terminal',
85
+ // A sweep whose source ran dry while its own count still matches rows: the next attempt resumes
86
+ // at the cursor that just ran dry and diverges again.
87
+ X_BACKFILL_STALLED: 'terminal',
88
+ X_BACKFILL_ENVIRONMENT: 'terminal',
89
+ X_BACKFILL_APPLIED: 'terminal',
90
+ });
91
+
45
92
  const docsFor = (code: JobErrorCode): string => `https://ultimate.dev/errors/${code}`;
46
93
 
47
94
  /** An enqueue collided with a live job holding the same idempotency key under `onConflict: 'error'`. */
@@ -103,6 +150,30 @@ export class JobTimeoutError extends UltimateError {
103
150
  }
104
151
  }
105
152
 
153
+ /**
154
+ * This attempt was cancelled — its deadline passed, or the caller went away — and something in it
155
+ * tried to keep going. The run belongs to whoever claims it next, so a step write from here would
156
+ * land on their history.
157
+ *
158
+ * Core's `X_ABORTED` rather than a code of jobs' own: the framework already means exactly one
159
+ * thing by "the signal fired, stop work", and a second name for it would make an agent ask which
160
+ * one it is looking at. `X_JOB_TIMEOUT` stays the code the ATTEMPT fails with; this is the code
161
+ * the work inside it stops with.
162
+ */
163
+ export class JobAbortedError extends UltimateError {
164
+ constructor(input: { job: string; step?: string }) {
165
+ super({
166
+ code: 'X_ABORTED',
167
+ cause:
168
+ input.step === undefined
169
+ ? `job "${input.job}" was cancelled — this attempt no longer owns the run`
170
+ : `job "${input.job}" was cancelled before step "${input.step}" could be recorded`,
171
+ fix: 'add throwIfAborted(ctx) before expensive work, or pass fetch(url, { signal: ctx.signal }) — the queue re-runs the job, so stop at the deadline instead of running past it',
172
+ docs: docsFor('X_ABORTED'),
173
+ });
174
+ }
175
+ }
176
+
106
177
  /** Retries exhausted. The job is in the dead-letter queue, not lost. */
107
178
  export class JobMaxAttemptsError extends UltimateError {
108
179
  constructor(input: { job: string; jobId: string; attempts: number; lastError: string }) {
@@ -141,6 +212,116 @@ export class IdempotencyRequiredError extends UltimateError {
141
212
  }
142
213
  }
143
214
 
215
+ /**
216
+ * A job declared no `tenant`. The type already requires it; this is the runtime backstop for
217
+ * generated code and JS callers, so the guarantee holds at both ends — the shape
218
+ * `IdempotencyRequiredError` above already has.
219
+ *
220
+ * It is refused rather than defaulted because both defaults are wrong. `'none'` silently reopens
221
+ * the hole this field closes: the body would run with no org, and before this field existed that
222
+ * meant `@ultimat3/entity`'s guard read no actor at all and accepted a caller-named tenant
223
+ * unchecked. Inheriting the worker's org would make one identity serve every job, which is the
224
+ * cross-tenant read the declaration exists to prevent.
225
+ */
226
+ export class JobTenantRequiredError extends UltimateError {
227
+ constructor(input: { job: string }) {
228
+ super({
229
+ code: 'X_JOB_TENANT_REQUIRED',
230
+ cause: `job "${input.job}" declares no tenant — a job body runs with no request behind it, so every tenant-scoped read inside it would be unscoped`,
231
+ // `'none'` reads differently either side of `backfill()` and the fix has to say so: for a
232
+ // plain job it means "touches no tenant-scoped table", because the org is STRIPPED and any
233
+ // scoped read fails closed — but a backfill declaring it is how a sweep says it spans every
234
+ // tenant, and the pass opens the cross-tenant scope for exactly that declaration. Half the
235
+ // callers of this code arrive through `backfill()`, which forwards its `tenant` to `job()`.
236
+ fix: `add tenant: (input) => input.orgId to job("${input.job}") — or tenant: 'none', which declares NO org: right for a job that touches no tenant-scoped table, and the spelling a backfill() uses to sweep every tenant`,
237
+ docs: docsFor('X_JOB_TENANT_REQUIRED'),
238
+ });
239
+ }
240
+ }
241
+
242
+ /**
243
+ * The queue took this job back while it was still running: `x jobs cancel` wrote a terminal state,
244
+ * or this worker's lease lapsed and another one re-claimed the row. Its own code and not
245
+ * `X_ABORTED`, because the response is different — `X_ABORTED` is "your deadline passed, make the
246
+ * work smaller", this is "somebody else owns this run now, stop writing to it".
247
+ */
248
+ export class LeaseLostError extends UltimateError {
249
+ constructor(input: { job: string; jobId: string }) {
250
+ super({
251
+ code: 'X_JOB_LEASE_LOST',
252
+ cause: `job "${input.job}" (${input.jobId}) is no longer claimed by this worker — it was cancelled, or its visibility lease lapsed and the queue re-delivered it`,
253
+ fix: `x jobs show ${input.jobId} --json`,
254
+ docs: docsFor('X_JOB_LEASE_LOST'),
255
+ });
256
+ }
257
+ }
258
+
259
+ /**
260
+ * The fleet slot this run holds under `job.concurrency` is somebody else's now: renewal answered
261
+ * "not yours", which is the one thing `LeaseStore.renew` can say that a retry cannot fix. Its own
262
+ * code and not `X_JOB_LEASE_LOST`, because they are different rows on different clocks — the
263
+ * queue may still consider this worker the owner of the JOB while another worker is already
264
+ * running one under the same cap, which is precisely the guarantee `concurrency` sells.
265
+ */
266
+ export class JobSlotLostError extends UltimateError {
267
+ constructor(input: { job: string; jobId: string; slot: number }) {
268
+ super({
269
+ code: 'X_JOB_SLOT_LOST',
270
+ cause: `job "${input.job}" (${input.jobId}) no longer holds fleet concurrency slot ${input.slot} — its lease expired and another worker took it`,
271
+ fix: `x jobs show ${input.jobId} --json`,
272
+ docs: docsFor('X_JOB_SLOT_LOST'),
273
+ });
274
+ }
275
+ }
276
+
277
+ /**
278
+ * `x jobs cancel` reached a job that already finished. Not a failure of the command — the work is
279
+ * done — but never a silent success either: an operator cancelling a runaway pass has to know
280
+ * whether they stopped it or missed it.
281
+ */
282
+ export class JobNotCancellableError extends UltimateError {
283
+ constructor(input: { jobId: string; state: string }) {
284
+ super({
285
+ code: 'X_JOB_NOT_CANCELLABLE',
286
+ cause:
287
+ input.state === 'missing'
288
+ ? `no job ${input.jobId} exists in this queue`
289
+ : `job ${input.jobId} is "${input.state}" and only a job that has not finished can be cancelled`,
290
+ fix: `x jobs ls --state running --json`,
291
+ docs: docsFor('X_JOB_NOT_CANCELLABLE'),
292
+ });
293
+ }
294
+ }
295
+
296
+ /** The driver has no `introspect.cancel`. The redis/nats stubs, and any hand-rolled driver. */
297
+ export class CancelUnsupportedError extends UltimateError {
298
+ constructor(input: { driver: string }) {
299
+ super({
300
+ code: 'X_JOB_NOT_CANCELLABLE',
301
+ cause: `the "${input.driver}" jobs driver cannot cancel a single job`,
302
+ fix: "set jobs: { driver: 'postgres' } in app.config.ts, then: x jobs cancel <id> --json",
303
+ docs: docsFor('X_JOB_NOT_CANCELLABLE'),
304
+ });
305
+ }
306
+ }
307
+
308
+ /**
309
+ * `job.concurrency` is declared and this driver has no `leases`, so the cap is per PROCESS and the
310
+ * fleet runs `concurrency x replicas`. Thrown at worker start rather than logged, because a
311
+ * documented guarantee that silently does nothing is exactly what axiom 3 exists to refuse — the
312
+ * worker refuses to start instead of running with the wrong number.
313
+ */
314
+ export class ConcurrencyUnenforceableError extends UltimateError {
315
+ constructor(input: { driver: string; jobs: readonly string[] }) {
316
+ super({
317
+ code: 'X_JOB_CONCURRENCY_UNENFORCEABLE',
318
+ cause: `${input.jobs.join(', ')} declare concurrency and the "${input.driver}" jobs driver has no lease store, so the cap would hold per process and the fleet would run concurrency x replicas`,
319
+ fix: `remove concurrency from job("${input.jobs[0] ?? 'the job'}"), or set jobs: { driver: 'postgres' } in app.config.ts`,
320
+ docs: docsFor('X_JOB_CONCURRENCY_UNENFORCEABLE'),
321
+ });
322
+ }
323
+ }
324
+
144
325
  /** An outbox enqueue happened with no ambient transaction to join. */
145
326
  export class OutboxNoTxError extends UltimateError {
146
327
  constructor(input: { job: string }) {
@@ -153,6 +334,134 @@ export class OutboxNoTxError extends UltimateError {
153
334
  }
154
335
  }
155
336
 
337
+ /**
338
+ * The seven backfill codes below all answer one question — "why is this sweep not running?" — and
339
+ * each is here because it sends the reader somewhere different: run it, force it, change
340
+ * environment, migrate first, wait, fix the predicates, or fix the name. A code that shared a fix
341
+ * line with another one would be code inflation, which is why `X_BACKFILL_WRITE_UNCONFIRMED` was
342
+ * considered and rejected: a dry run that wrote nothing did exactly what it was asked to.
343
+ *
344
+ * Every `fix` below is ONE line something can execute — a shell command, or the edit to make.
345
+ * Never a command with prose appended: a `fix:` is copied and run verbatim, so a trailing clause
346
+ * turns a working command into a syntax error at the one moment the reader is following it
347
+ * literally. Explanations belong in `cause`, which is read and never run.
348
+ */
349
+
350
+ /**
351
+ * Declared and never completed. The alarm the framework did not have: an author could
352
+ * `x g backfill`, merge and deploy, and nothing anywhere said the pass had not run.
353
+ */
354
+ export class BackfillPendingError extends UltimateError {
355
+ constructor(input: { backfill: string; environment: string }) {
356
+ super({
357
+ code: 'X_BACKFILL_PENDING',
358
+ cause: `backfill "${input.backfill}" is declared and x_backfills holds no completed pass for it in ${input.environment}`,
359
+ fix: `x db backfill ${input.backfill} --write --json`,
360
+ docs: docsFor('X_BACKFILL_PENDING'),
361
+ });
362
+ }
363
+ }
364
+
365
+ /** Already swept. A rerun is legitimate, so this names the flag rather than refusing outright. */
366
+ export class BackfillAppliedError extends UltimateError {
367
+ constructor(input: { backfill: string; runId: string; completedAt: string }) {
368
+ super({
369
+ code: 'X_BACKFILL_APPLIED',
370
+ cause: `backfill "${input.backfill}" completed as run ${input.runId} at ${input.completedAt}; a forced rerun writes a NEW ledger row and never edits that one`,
371
+ fix: `x db backfill ${input.backfill} --write --force --json`,
372
+ docs: docsFor('X_BACKFILL_APPLIED'),
373
+ });
374
+ }
375
+ }
376
+
377
+ /**
378
+ * The declaration names the environments it belongs to and this is not one. Declared DATA, never a
379
+ * hardcoded "cleanups are production": a staging rehearsal is correct practice, so which
380
+ * environments a sweep belongs to is the app's convention and this is only the mechanism carrying
381
+ * it (axiom 8).
382
+ */
383
+ export class BackfillEnvironmentError extends UltimateError {
384
+ constructor(input: { backfill: string; environment: string; declared: readonly string[] }) {
385
+ // The first declared environment, because the fix has to be ONE runnable line and the list is
386
+ // ordered by the author. The empty case cannot arise from `checkBackfillEnvironment`, which
387
+ // treats an empty list as "every environment" — but this constructor is public, so it answers
388
+ // with the command that lists what IS declared rather than an `ULTIMATE_ENV=undefined`.
389
+ const target = input.declared[0];
390
+ super({
391
+ code: 'X_BACKFILL_ENVIRONMENT',
392
+ cause: `backfill "${input.backfill}" declares environments: ${input.declared.join(', ')} and this process resolved ${input.environment} — add "${input.environment}" to that list if this deploy should sweep too`,
393
+ fix:
394
+ target === undefined
395
+ ? 'x db backfill --pending --json'
396
+ : `ULTIMATE_ENV=${target} x db backfill ${input.backfill} --write --json`,
397
+ docs: docsFor('X_BACKFILL_ENVIRONMENT'),
398
+ });
399
+ }
400
+ }
401
+
402
+ /**
403
+ * `requires` names a migration the ledger has not applied. Checked where `x_migrations` is
404
+ * readable — this package holds no `@ultimat3/db` dependency and growing one to read a ledger
405
+ * would put the migration engine on the tier-3 queue's import graph.
406
+ */
407
+ export class BackfillMigrationPendingError extends UltimateError {
408
+ constructor(input: { backfill: string; migration: string }) {
409
+ super({
410
+ code: 'X_BACKFILL_MIGRATION_PENDING',
411
+ cause: `backfill "${input.backfill}" requires migration ${input.migration}, which x_migrations does not record as applied`,
412
+ fix: 'x db migrate --json',
413
+ docs: docsFor('X_BACKFILL_MIGRATION_PENDING'),
414
+ });
415
+ }
416
+ }
417
+
418
+ /**
419
+ * The enqueue deduped: one live pass per name, so this run is the one already going. Distinct from
420
+ * `X_BACKFILL_APPLIED`, which is a pass that finished — the response there is `--force`, and the
421
+ * response here is to look at the run that is holding the key.
422
+ */
423
+ export class BackfillRunningError extends UltimateError {
424
+ constructor(input: { backfill: string; jobId: string }) {
425
+ super({
426
+ code: 'X_BACKFILL_RUNNING',
427
+ cause: `backfill "${input.backfill}" already has a live pass queued as ${input.jobId}, and one name holds one live pass; its step trace names the batch it is on, and a pass that is not advancing is a worker that lost its lease`,
428
+ fix: `x jobs show ${input.jobId} --json`,
429
+ docs: docsFor('X_BACKFILL_RUNNING'),
430
+ });
431
+ }
432
+ }
433
+
434
+ /**
435
+ * The source ran out of rows and the declaration's own `count()` still matches some. Two
436
+ * predicates that disagree is an authoring bug in any business — the sweep reported success over
437
+ * rows it never visited — so the pass fails rather than writing a completed row nobody can trust.
438
+ */
439
+ export class BackfillStalledError extends UltimateError {
440
+ constructor(input: { backfill: string; remaining: number; swept: number }) {
441
+ super({
442
+ code: 'X_BACKFILL_STALLED',
443
+ cause: `backfill "${input.backfill}" swept ${input.swept} rows, exhausted its source, and count() still matches ${input.remaining} — a WHERE the sweep narrows and the count does not is what leaves rows behind`,
444
+ fix: `make count() select on exactly what source() selects on in backfill("${input.backfill}")`,
445
+ docs: docsFor('X_BACKFILL_STALLED'),
446
+ });
447
+ }
448
+ }
449
+
450
+ /** A name no declaration in this app carries — a typo, or a backfill whose module was deleted. */
451
+ export class BackfillUnknownError extends UltimateError {
452
+ constructor(input: { backfill: string; known: readonly string[] }) {
453
+ super({
454
+ code: 'X_BACKFILL_UNKNOWN',
455
+ cause:
456
+ input.known.length === 0
457
+ ? `no backfill named "${input.backfill}" is declared, and this app declares none at all`
458
+ : `no backfill named "${input.backfill}" is declared (declared: ${input.known.join(', ')})`,
459
+ fix: 'x db backfill --pending --json',
460
+ docs: docsFor('X_BACKFILL_UNKNOWN'),
461
+ });
462
+ }
463
+ }
464
+
156
465
  export class JobsNotImplementedError extends UltimateError {
157
466
  constructor(input: { feature: string; fix: string }) {
158
467
  super({
@@ -0,0 +1,121 @@
1
+ // `x_job_events`: the event bus `step.waitForEvent` needs in a real deployment. The memory bus is
2
+ // one process's heap, and in a deployment the publisher and the resumer are never the same pod —
3
+ // a Stripe webhook lands on web-3 and the worker that resumes the run is worker-7, so `find`
4
+ // answers `undefined` until the 24h timeout and the run dead-letters with nothing logged first.
5
+ //
6
+ // Stored and not broadcast, exactly as the memory bus is: a step that suspends at 12:00 and
7
+ // resumes at 12:00:30 must still see an event published at 12:00:10.
8
+
9
+ import type { Clock } from '@ultimat3/core';
10
+ import { logger, systemClock, uuid } from '@ultimat3/core';
11
+ import type { DurationInput } from './clock';
12
+ import { nowMs, toMs } from './clock';
13
+ import type { PgExecutor } from './driver-pg';
14
+ import {
15
+ SQL_EVENT_FIND,
16
+ SQL_EVENT_LIST,
17
+ SQL_EVENT_PUBLISH,
18
+ SQL_EVENT_PURGE,
19
+ } from './driver-pg-sql';
20
+ import type { EventBus, JobEvent } from './events';
21
+
22
+ interface EventRow {
23
+ readonly id: string;
24
+ readonly name: string;
25
+ readonly payload: unknown;
26
+ readonly correlation_key: string | null;
27
+ readonly published_at: number | string;
28
+ readonly expires_at: number | string;
29
+ }
30
+
31
+ export interface PgEventBusOptions {
32
+ readonly executor: PgExecutor;
33
+ readonly clock?: Clock;
34
+ /** How long an event stays matchable. Default 7d — longer than any sane wait. */
35
+ readonly defaultTtl?: DurationInput;
36
+ /** Rows returned by `list()`. Diagnostics only; `find()` is what a step uses. */
37
+ readonly listLimit?: number;
38
+ }
39
+
40
+ /**
41
+ * `purgeExpired()` is SYNCHRONOUS in `EventBus` because the memory bus can be — it walks a Map.
42
+ * Over SQL the delete is a round trip, so this fires it and answers 0: the count is a diagnostic
43
+ * the memory bus offers and this one cannot, and blocking a step's resume on a housekeeping
44
+ * DELETE would be a far worse trade than an unanswered number. The index on `(name, published_at)`
45
+ * plus `expires_at > now()` in `find` means an unpurged row costs a filter, never a wrong answer.
46
+ */
47
+ export function createPgEventBus(options: PgEventBusOptions): EventBus {
48
+ const clock = options.clock ?? systemClock;
49
+ const defaultTtl = options.defaultTtl ?? 604_800_000;
50
+ const listLimit = options.listLimit ?? 1_000;
51
+ const exec = options.executor;
52
+
53
+ const purgeExpired = (): number => {
54
+ void exec.query(SQL_EVENT_PURGE, []).catch((error: unknown) => {
55
+ // Housekeeping never costs a publish: an unpurged row is filtered out of every read.
56
+ logger.warn('jobs.event.purge-failed', {
57
+ error: error instanceof Error ? error.message : String(error),
58
+ });
59
+ });
60
+ return 0;
61
+ };
62
+
63
+ return {
64
+ async publish(name, payload, publishOptions = {}) {
65
+ const at = nowMs(clock);
66
+ const event: JobEvent = {
67
+ id: uuid(),
68
+ name,
69
+ payload,
70
+ publishedAt: at,
71
+ expiresAt: at + toMs(publishOptions.ttl ?? defaultTtl),
72
+ ...(publishOptions.correlationKey === undefined
73
+ ? {}
74
+ : { correlationKey: publishOptions.correlationKey }),
75
+ };
76
+ await exec.query(SQL_EVENT_PUBLISH, [
77
+ event.id,
78
+ event.name,
79
+ JSON.stringify(event.payload ?? null),
80
+ event.correlationKey ?? null,
81
+ event.publishedAt,
82
+ event.expiresAt,
83
+ ]);
84
+ logger.debug('jobs.event.published', {
85
+ event: name,
86
+ correlationKey: publishOptions.correlationKey ?? null,
87
+ });
88
+ return event;
89
+ },
90
+
91
+ /** Earliest match at or after `afterMs`, so a resumed step consumes events in order. */
92
+ async find(name, correlationKey, afterMs) {
93
+ const rows = await exec.query<{ payload: unknown; published_at: number | string }>(
94
+ SQL_EVENT_FIND,
95
+ [name, correlationKey ?? null, afterMs],
96
+ );
97
+ const row = rows[0];
98
+ return row === undefined
99
+ ? undefined
100
+ : { payload: row.payload, publishedAt: Number(row.published_at) };
101
+ },
102
+
103
+ async list(name) {
104
+ const rows = await exec.query<EventRow>(SQL_EVENT_LIST, [name ?? null, listLimit]);
105
+ return rows.map((row) => ({
106
+ id: row.id,
107
+ name: row.name,
108
+ payload: row.payload,
109
+ publishedAt: Number(row.published_at),
110
+ expiresAt: Number(row.expires_at),
111
+ ...(row.correlation_key === null ? {} : { correlationKey: row.correlation_key }),
112
+ }));
113
+ },
114
+
115
+ purgeExpired,
116
+ // The memory bus's `size()` is its Map's; over SQL a `count(*)` is a round trip and every
117
+ // caller of this is a test asserting on a bound the memory bus has. `-1` is the honest
118
+ // "not a number this bus keeps" — never `0`, which reads as an empty bus.
119
+ size: () => -1,
120
+ };
121
+ }
package/src/events.ts CHANGED
@@ -115,7 +115,13 @@ export function createMemoryEventBus(options: MemoryEventBusOptions = {}): Event
115
115
 
116
116
  let ambientBus: EventBus = createMemoryEventBus();
117
117
 
118
- /** Swapped at boot for the NATS/Redis-streams bus in a multi-node deployment. */
118
+ /**
119
+ * **Install `createPgEventBus({ executor })` here in any deployment with more than one process.**
120
+ * The default above is one process's heap: the pod that publishes and the pod that resumes are
121
+ * never the same one, so a webhook landing on web-3 strands a run on worker-7 until its 24h
122
+ * timeout dead-letters it, with nothing logged before then. This line used to promise a
123
+ * "NATS/Redis-streams bus swapped at boot"; no such bus existed and boot installed the memory one.
124
+ */
119
125
  export function setEventBus(bus: EventBus): void {
120
126
  ambientBus = bus;
121
127
  }