@ultimat3/jobs 30.9.0 → 31.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/CLAUDE.md +72 -107
  2. package/README.md +202 -55
  3. package/package.json +8 -7
  4. package/src/adapter.ts +37 -0
  5. package/src/backfill-ledger.ts +2 -2
  6. package/src/backfill-rate.ts +1 -0
  7. package/src/clock.ts +2 -2
  8. package/src/drain-wait.ts +1 -0
  9. package/src/driver-memory.ts +2 -2
  10. package/src/driver-pg-ddl.ts +19 -0
  11. package/src/driver-pg-event-wake-sql.ts +43 -22
  12. package/src/driver-pg-sql.ts +68 -27
  13. package/src/driver-pg.ts +11 -10
  14. package/src/driver-redis.ts +2 -2
  15. package/src/driver.ts +14 -8
  16. package/src/error-codes.ts +27 -0
  17. package/src/errors-concurrency.ts +1 -1
  18. package/src/errors.ts +1 -1
  19. package/src/events-cursor.ts +31 -0
  20. package/src/events-pg.ts +52 -6
  21. package/src/events.ts +72 -36
  22. package/src/export.ts +2 -2
  23. package/src/gdpr-audit.ts +75 -0
  24. package/src/gdpr-coverage.ts +94 -0
  25. package/src/gdpr-errors.ts +35 -0
  26. package/src/import.ts +2 -2
  27. package/src/index.ts +45 -206
  28. package/src/inspect-operator.ts +2 -2
  29. package/src/inspect.ts +1 -1
  30. package/src/job-instrument.ts +115 -0
  31. package/src/job.ts +41 -4
  32. package/src/jobs-retention.ts +23 -6
  33. package/src/leases.ts +13 -3
  34. package/src/ledger-book.ts +168 -0
  35. package/src/ledger-errors.ts +48 -0
  36. package/src/ledger.ts +145 -0
  37. package/src/limits.ts +57 -19
  38. package/src/lock-errors.ts +54 -0
  39. package/src/lock.ts +176 -0
  40. package/src/outbox-relay.ts +4 -0
  41. package/src/outbox.ts +3 -3
  42. package/src/partitions.ts +119 -0
  43. package/src/reindex-sweep.ts +94 -0
  44. package/src/reindex.ts +147 -0
  45. package/src/retry-classification.ts +3 -2
  46. package/src/retry-jitter.ts +32 -0
  47. package/src/retry.ts +13 -16
  48. package/src/scheduler.ts +10 -6
  49. package/src/steps-api.ts +44 -5
  50. package/src/steps-suspension.ts +34 -3
  51. package/src/steps-wait.ts +52 -4
  52. package/src/steps.ts +1 -0
  53. package/src/subject-jobs.ts +12 -4
  54. package/src/webhook.ts +2 -2
  55. package/src/worker-admit.ts +8 -5
  56. package/src/worker-drain-cutoff.ts +1 -0
  57. package/src/worker-loop.ts +28 -5
  58. package/src/worker-registry.ts +7 -5
  59. package/src/worker-run.ts +53 -9
  60. package/src/worker.ts +8 -8
package/src/index.ts CHANGED
@@ -20,36 +20,28 @@ export type {
20
20
  BackfillInput,
21
21
  BackfillPage,
22
22
  BackfillPages,
23
- BackfillReport,
24
23
  ScopedBackfillDefinition,
25
24
  ScopedBackfillInput,
26
25
  } from './backfill';
27
26
  export { backfill, DEFAULT_BACKFILL_BATCH } from './backfill';
28
27
  export {
29
- BackfillAppliedError,
30
- BackfillEnvironmentError,
31
- BackfillMigrationPendingError,
32
28
  BackfillPendingError,
33
29
  BackfillRunningError,
34
- BackfillSourceInvalidError,
35
- BackfillStalledError,
36
30
  BackfillUnknownError,
37
31
  } from './backfill-errors';
38
32
  export type { BackfillGate, BackfillGateInput } from './backfill-gate';
39
- export { checkBackfillEnvironment, gateBackfill } from './backfill-gate';
33
+ export { gateBackfill } from './backfill-gate';
40
34
  export type { BackfillProgress } from './backfill-inspect';
41
- export { backfillForRun, inspectBackfills, toBackfillProgress } from './backfill-inspect';
35
+ export { inspectBackfills, toBackfillProgress } from './backfill-inspect';
42
36
  export type {
43
37
  BackfillFilter,
44
38
  BackfillLedger,
45
39
  BackfillRun,
46
40
  BackfillStatus,
47
- BackfillVerdict,
48
41
  } from './backfill-ledger';
49
42
  export {
50
43
  BACKFILL_STATUSES,
51
44
  backfillChecksum,
52
- decideBackfill,
53
45
  isBackfillStatus,
54
46
  memoryBackfillLedger,
55
47
  } from './backfill-ledger';
@@ -78,48 +70,32 @@ export {
78
70
  } from './backfill-registry';
79
71
  export type { AnnounceExhaustedOptions, ExhaustedCounts } from './claim-exhausted';
80
72
  export { announceExhausted } from './claim-exhausted';
81
- export type { JobConcurrency, KeyedConcurrency, WhenBusy } from './concurrency';
82
- export { MAX_CONCURRENCY_KEY_LENGTH, WHEN_BUSY } from './concurrency';
73
+ export type { JobConcurrency, KeyedConcurrency } from './concurrency';
83
74
  export { consumer } from './consumer';
84
- export { ConsumerInvalidError, ConsumerSelfEnqueueError } from './consumer-errors';
85
- export {
86
- CONSUMER_LINGER_MS,
87
- CONSUMER_PROGRESS_NOTE,
88
- DEFAULT_CONSUMER_BATCH,
89
- } from './consumer-loop';
75
+ export { CONSUMER_LINGER_MS } from './consumer-loop';
90
76
  // `stampConsumer` is deliberately absent, for the reason `stampBackfill` is.
91
77
  export type { ConsumerDeclaration, ConsumerRow } from './consumer-registry';
92
- export {
93
- consumerDeclarationOf,
94
- consumerRow,
95
- isConsumer,
96
- registeredConsumers,
97
- } from './consumer-registry';
78
+ export { consumerRow, registeredConsumers } from './consumer-registry';
98
79
  export type {
99
- ConsumerBatch,
100
80
  ConsumerCursor,
101
81
  ConsumerDefinition,
102
82
  ConsumerHandle,
103
83
  ConsumerInput,
104
- ConsumerReport,
105
84
  } from './consumer-types';
106
85
  export type { JobConcurrencyDescriptor, JobDescriptor } from './describe';
107
86
  export type {
108
- AckOptions,
109
87
  ClaimedJob,
110
88
  ClaimIdentity,
111
89
  ClaimOptions,
112
90
  ConflictPolicy,
113
91
  EnqueueRequest,
114
92
  EnqueueResult,
115
- HeartbeatOptions,
116
93
  JobDriver,
117
94
  JobRecord,
118
95
  JobState,
119
96
  NackOptions,
120
97
  QueueDemand,
121
98
  QueueStats,
122
- SettleBy,
123
99
  } from './driver';
124
100
  export {
125
101
  assertClaimBounds,
@@ -133,8 +109,6 @@ export {
133
109
  LEASE_LAPSED_FINAL_ATTEMPT,
134
110
  LIVE_STATES,
135
111
  nackState,
136
- resetJobDriver,
137
- setJobDriver,
138
112
  } from './driver';
139
113
  export type { MemoryJobDriver, MemoryJobDriverOptions } from './driver-memory';
140
114
  export { memoryJobDriver } from './driver-memory';
@@ -148,50 +122,17 @@ export { signalEnqueued, signalStaged } from './enqueue-signal';
148
122
  export type { JobErrorCode } from './error-codes';
149
123
  export { JOB_ERROR_CODES, JOB_ERROR_TITLES } from './error-codes';
150
124
  export {
151
- ActionJobUnbridgedError,
152
- CancelUnsupportedError,
153
- ClaimQueuesEmptyError,
154
125
  DriverUnavailableError,
155
- IdempotencyRequiredError,
156
126
  JobAbortedError,
157
127
  JobDrainedError,
158
128
  JobDuplicateError,
159
- JobEnqueueBatchTooLargeError,
160
- JobMaxAttemptsError,
161
- JobNameTakenError,
162
129
  JobNotCancellableError,
163
- JobRowStatusUnknownError,
164
- JobSlotLostError,
165
- JobTenantRequiredError,
166
130
  JobTimeoutError,
167
131
  LeaseLostError,
168
- OutboxNoTxError,
169
- StepDuplicateError,
170
132
  } from './errors';
171
- export {
172
- ConcurrencyUnenforceableError,
173
- JobConcurrencyInvalidError,
174
- JobConcurrencyKeyInvalidError,
175
- JobKeyBusyError,
176
- } from './errors-concurrency';
177
- export { JobDeclarationInvalidError } from './errors-declaration';
178
- export {
179
- JobNotPromotableError,
180
- JobNotRemovableError,
181
- JobOnSettledFailedError,
182
- JobPageInvalidError,
183
- } from './errors-operator';
184
- export { JobNotFoundError, JobNotRequeueableError } from './errors-requeue';
185
- export {
186
- COMPENSATE_PREFIX,
187
- JobCompensationFailedError,
188
- StepCompensateUndeclaredError,
189
- StepNameReservedError,
190
- StepNotResolvableError,
191
- StepOutcomeUnknownError,
192
- UNWINDING_STEP,
193
- } from './errors-saga';
194
- export { JobEventInvalidError, StepOutputUnserializableError } from './errors-steps';
133
+ export { ConcurrencyUnenforceableError, JobKeyBusyError } from './errors-concurrency';
134
+ export { JobNotFoundError } from './errors-requeue';
135
+ export { COMPENSATE_PREFIX, StepNameReservedError, UNWINDING_STEP } from './errors-saga';
195
136
  export { JobTenantMismatchError } from './errors-tenant';
196
137
  export type { EventBus, JobEvent, MemoryEventBusOptions, PublishOptions } from './events';
197
138
  export {
@@ -200,47 +141,28 @@ export {
200
141
  eventsPurgeTarget,
201
142
  memoryEventBus,
202
143
  publishEvent,
203
- resetEventBus,
204
- setEventBus,
205
144
  } from './events';
206
145
  export type { PostgresEventBusOptions } from './events-pg';
207
146
  export { postgresEventBus } from './events-pg';
208
147
  export type { ExecuteJobOptions } from './execute';
209
148
  export { executeJob } from './execute';
210
149
  export type { JobExecution, JobOutcome } from './execute-result';
211
- export type { ExportDefinition, ExportReport } from './export';
212
- export { DEFAULT_EXPORT_BATCH, DEFAULT_EXPORT_MAX_PART_BYTES, exportRows } from './export';
213
- export { ExportPartTooLargeError, ExportRowInvalidError } from './export-errors';
214
- export type { EncodePageInput, ExportFormat, ExportRecord, ExportValue } from './export-format';
215
- export { csvHeader, EXPORT_EXTENSION, EXPORT_FORMATS, encodeExportPage } from './export-format';
216
- export type { ExportManifest, ExportSink, MemoryExportSink } from './export-sink';
217
- export {
218
- EXPORT_PART_DIGITS,
219
- exportManifestKey,
220
- exportPartKey,
221
- memoryExportSink,
222
- } from './export-sink';
150
+ export type { ExportDefinition } from './export';
151
+ export { exportRows } from './export';
152
+ export type { EncodePageInput, ExportValue } from './export-format';
153
+ export { EXPORT_FORMATS, encodeExportPage } from './export-format';
154
+ export type { ExportSink, MemoryExportSink } from './export-sink';
155
+ export { memoryExportSink } from './export-sink';
223
156
  export type { CallerFailure } from './failure-row';
224
157
  export { callerFailure } from './failure-row';
158
+ export { personalDataProblems, unerasedPersonalData, unreachedPersonalData } from './gdpr-coverage';
159
+ export { PersonalDataUnerasedError, PersonalDataUnreachedError } from './gdpr-errors';
225
160
  export { IDLE_POLL_CEILING_MS } from './idle-backoff';
226
- export type { ImportDefinition, ImportReport } from './import';
227
- export {
228
- DEFAULT_IMPORT_BATCH,
229
- DEFAULT_IMPORT_MAX_RECORD_BYTES,
230
- DEFAULT_IMPORT_MAX_ROW_ERRORS,
231
- DEFAULT_IMPORT_PAGE_BYTES,
232
- importRows,
233
- } from './import';
234
- export {
235
- ImportHeaderMismatchError,
236
- ImportRecordTooLargeError,
237
- ImportTooManyErrorsError,
238
- } from './import-errors';
239
- export type { ImportRejection } from './import-format';
161
+ export type { ImportDefinition } from './import';
162
+ export { importRows } from './import';
240
163
  export { IMPORT_REJECTIONS } from './import-format';
241
- export type { ImportRowError } from './import-pass';
242
164
  export { importErrorKey } from './import-pass';
243
- export type { ImportObject, ImportSource, MemoryImportSource } from './import-source';
165
+ export type { ImportSource, MemoryImportSource } from './import-source';
244
166
  export { memoryImportSource } from './import-source';
245
167
  export type {
246
168
  DeadLetterEntry,
@@ -269,7 +191,6 @@ export type {
269
191
  BulkFilter,
270
192
  BulkResult,
271
193
  CounterBucketMs,
272
- CounterOutcome,
273
194
  CounterTotals,
274
195
  JobCounter,
275
196
  JobFilter,
@@ -282,26 +203,15 @@ export type {
282
203
  WorkerRecord,
283
204
  } from './introspection';
284
205
  export {
285
- COUNTER_BUCKET_MS,
286
206
  COUNTER_TIERS,
287
207
  DEFAULT_JOB_PAGE,
288
208
  jobCursor,
289
209
  MAX_BULK_ROWS,
290
- MAX_ERROR_STACK_LENGTH,
291
210
  MAX_JOB_PAGE,
292
- MAX_PROGRESS_NOTE_LENGTH,
293
211
  MAX_TASK_FIRES,
294
- MAX_WORKER_IN_FLIGHT,
295
- PROGRESS_INTERVAL_MS,
296
212
  PROMOTABLE_STATES,
297
213
  } from './introspection';
298
- export type {
299
- AnyJobHandle,
300
- JobActor,
301
- JobDefinition,
302
- JobHandle,
303
- JobRunArgs,
304
- } from './job';
214
+ export type { AnyJobHandle, JobDefinition, JobHandle, JobRunArgs } from './job';
305
215
  export {
306
216
  describeJobs,
307
217
  getJob,
@@ -317,24 +227,15 @@ export type { JobsRetentionOptions } from './jobs-retention';
317
227
  export { jobsPurgeTarget } from './jobs-retention';
318
228
  export type { HeldLease, LeaseStore, MemoryLeaseStore, MemoryLeaseStoreOptions } from './leases';
319
229
  export { jobLeaseKey, memoryLeaseStore } from './leases';
320
- export type {
321
- Lease,
322
- LimitConfig,
323
- Limiter,
324
- LimitKey,
325
- LimitReason,
326
- LimitSnapshot,
327
- RateLimit,
328
- } from './limits';
329
- export { concurrencyLimiter, NO_TENANT, tenantKeyFrom } from './limits';
330
- export {
331
- queueDeadJobs,
332
- queueDemand,
333
- queueOldestReady,
334
- recordQueueDeadJobs,
335
- recordQueueDemand,
336
- recordQueueOldestReady,
337
- } from './metrics';
230
+ export type { LedgerDefinition, LedgerEntry, LedgerPosting } from './ledger';
231
+ export { assertBalancedPosting, ledger, ledgerLines } from './ledger';
232
+ export type { EntityLedgerBookOptions, LedgerBook, LedgerLine } from './ledger-book';
233
+ export { entityLedgerBook } from './ledger-book';
234
+ export type { Lease, LimitConfig, Limiter, LimitSnapshot, RateLimit } from './limits';
235
+ export { concurrencyLimiter, NO_TENANT } from './limits';
236
+ export type { HeldLock, WithLockOptions } from './lock';
237
+ export { withLock } from './lock';
238
+ export { queueDemand, recordQueueDemand } from './metrics';
338
239
  export type {
339
240
  EnqueueOptions,
340
241
  JobsFacade,
@@ -360,6 +261,8 @@ export type { OutboxRelay, RelayOptions } from './outbox-relay';
360
261
  export { outboxRelay } from './outbox-relay';
361
262
  export type { OutboxRetentionOptions } from './outbox-retention';
362
263
  export { outboxPurgeTarget } from './outbox-retention';
264
+ export type { PartitionsDefinition, PartitionsJob } from './partitions';
265
+ export { partitionedEntities, partitions } from './partitions';
363
266
  export type { ProgressFn } from './progress';
364
267
  export type {
365
268
  PurgeDefinition,
@@ -371,16 +274,13 @@ export type {
371
274
  export { DEFAULT_PURGE_CRON, purge } from './purge';
372
275
  export type { PurgeBatching } from './purge-batches';
373
276
  export type { PurgeEntities } from './purge-entities';
374
- export {
375
- resetPurgeCoverage,
376
- retainedEntities,
377
- retainTarget,
378
- unsweptEntities,
379
- } from './purge-entities';
277
+ export { resetPurgeCoverage, retainTarget, unsweptEntities } from './purge-entities';
380
278
  export type { PgListener, QueueWake, QueueWakeOptions } from './queue-wake';
381
279
  export { startQueueWake } from './queue-wake';
382
- export type { RefreshViewDefinition, RefreshViewReport } from './refresh-view';
280
+ export type { RefreshViewDefinition } from './refresh-view';
383
281
  export { refreshView } from './refresh-view';
282
+ export type { ReindexDefinition } from './reindex';
283
+ export { reindex } from './reindex';
384
284
  export type { IntervalScheduler } from './renewal-timer';
385
285
  export type { BackoffStrategy, Random, RetryDecision, RetryPolicy } from './retry';
386
286
  export {
@@ -394,52 +294,28 @@ export {
394
294
  UNWIND_MARGIN,
395
295
  } from './retry';
396
296
  export type { JobRetryDecision, JobStopReason } from './retry-classification';
397
- export { failureForRow, nextRetryForError } from './retry-classification';
398
- export type {
399
- RollupBucket,
400
- RollupDefinition,
401
- RollupInput,
402
- RollupReport,
403
- RollupTarget,
404
- } from './rollup';
297
+ export { nextRetryForError } from './retry-classification';
298
+ export type { RollupBucket, RollupDefinition, RollupInput, RollupTarget } from './rollup';
405
299
  export { rollup } from './rollup';
406
- export { RefreshViewInvalidError, RollupInvalidError } from './rollup-errors';
407
- export type {
408
- CronResolver,
409
- DispatchedOccurrence,
410
- Scheduler,
411
- SchedulerOptions,
412
- } from './scheduler';
413
- export {
414
- COUNTER_ROLLUP_INTERVAL_MS,
415
- jobScheduler,
416
- PAUSE_RECHECK_MS,
417
- } from './scheduler';
300
+ export type { CronResolver, Scheduler, SchedulerOptions } from './scheduler';
301
+ export { COUNTER_ROLLUP_INTERVAL_MS, jobScheduler } from './scheduler';
418
302
  export type { LeaderElection } from './scheduler-leader';
419
303
  export { soleLeader } from './scheduler-leader';
420
304
  export { nextTaskRun } from './scheduler-occurrences';
421
305
  export type { PostgresLeaseLeaderOptions } from './scheduler-pg';
422
- export {
423
- currentLeader,
424
- DEFAULT_LEADER_TTL_MS,
425
- LEASE_RENEWALS_PER_TTL,
426
- postgresLeaseLeader,
427
- postgresSchedulerState,
428
- } from './scheduler-pg';
306
+ export { DEFAULT_LEADER_TTL_MS, postgresLeaseLeader, postgresSchedulerState } from './scheduler-pg';
429
307
  export type { ScheduledFire, SchedulerState } from './scheduler-state';
430
- export { fireThroughDriver, memorySchedulerState } from './scheduler-state';
308
+ export { memorySchedulerState } from './scheduler-state';
431
309
  export type { JobCompleted, JobFailed, JobSettled } from './settled';
432
310
  export { ON_SETTLED_ATTEMPTS, ON_SETTLED_TIMEOUT_MS } from './settled';
433
311
  export { currentStep } from './step-scope';
434
312
  export type {
435
- CompensateContext,
436
313
  EventLookup,
437
314
  StepApi,
438
315
  StepFence,
439
316
  StepRecord,
440
317
  StepRunner,
441
318
  StepRunnerOptions,
442
- StepRunOptions,
443
319
  StepStatus,
444
320
  StepStore,
445
321
  WaitForEventOptions,
@@ -460,27 +336,14 @@ export type {
460
336
  RetainedTarget,
461
337
  Subject,
462
338
  SubjectInit,
463
- SubjectRef,
464
339
  SubjectTarget,
465
340
  SubjectTargetEntry,
466
- SubjectTargetPage,
467
341
  SubjectTreatment,
468
342
  } from './subject';
469
- export { DEFAULT_SUBJECT_BATCH, defineSubject, registeredSubjects, resetSubjects } from './subject';
343
+ export { defineSubject, registeredSubjects, resetSubjects } from './subject';
470
344
  export type { SubjectEraseReport, SubjectTally } from './subject-erase-pass';
471
- export {
472
- SubjectCycleError,
473
- SubjectTableUndeclaredError,
474
- SubjectTreatmentInvalidError,
475
- } from './subject-errors';
476
- export type { SubjectExportReport, SubjectIndexEntry } from './subject-export-pass';
477
345
  export type { SubjectExportDefinition, SubjectJobDefinition } from './subject-jobs';
478
- export {
479
- plannedSubject,
480
- subjectErase,
481
- subjectEraseTables,
482
- subjectExport,
483
- } from './subject-jobs';
346
+ export { subjectErase, subjectEraseTables, subjectExport } from './subject-jobs';
484
347
  export type {
485
348
  SubjectPlan,
486
349
  SubjectPlanInput,
@@ -495,7 +358,6 @@ export type {
495
358
  TaskDescriptor,
496
359
  TaskEnqueueEntry,
497
360
  TaskHandle,
498
- TaskJobResult,
499
361
  } from './task';
500
362
  export { getTask, isTaskHandle, registeredTasks, resetTasks, restoreTasks, task } from './task';
501
363
  /**
@@ -510,39 +372,16 @@ export type {
510
372
  WebhookDefinition,
511
373
  WebhookDeliveryInput,
512
374
  WebhookEndpoint,
513
- WebhookEvent,
514
- WebhookReport,
515
375
  WebhookTenant,
516
376
  } from './webhook';
517
377
  export { DEFAULT_WEBHOOK_DISABLE_AFTER, webhook } from './webhook';
518
378
  export type { WebhookFetch } from './webhook-attempt';
519
- export { WEBHOOK_CONTENT_TYPE } from './webhook-attempt';
520
- export {
521
- WebhookDeliveryFailedError,
522
- WebhookDeliveryRejectedError,
523
- WebhookDeliveryThrottledError,
524
- WebhookEndpointDisabledError,
525
- WebhookEndpointInvalidError,
526
- WebhookEndpointUnknownError,
527
- WebhookEventInvalidError,
528
- WebhookEventUnknownError,
529
- } from './webhook-errors';
530
379
  export type { MemoryWebhookLedger, WebhookAttempt, WebhookLedger } from './webhook-ledger';
531
- export { DEFAULT_MAX_WEBHOOK_ATTEMPTS, memoryWebhookLedger } from './webhook-ledger';
380
+ export { memoryWebhookLedger } from './webhook-ledger';
532
381
  export { webhookReceiver, webhookReceiverRoutes } from './webhook-receiver';
533
- export {
534
- WEBHOOK_EVENT_ID_PATTERN,
535
- WEBHOOK_RECEIVER_RETRY_AFTER_SECONDS,
536
- WebhookEventIdInvalidError,
537
- WebhookReceiverInvalidError,
538
- WebhookReceiverUnavailableError,
539
- } from './webhook-receiver-errors';
382
+ export { WEBHOOK_EVENT_ID_PATTERN } from './webhook-receiver-errors';
540
383
  export type {
541
- ReceivedEvent,
542
384
  ReceivedEventFacts,
543
- WebhookAnswer,
544
- WebhookHandleArgs,
545
- WebhookInbound,
546
385
  WebhookReceiver,
547
386
  WebhookReceiverDefinition,
548
387
  WebhookReceiverInput,
@@ -13,7 +13,7 @@ function operatorOf(driver: JobDriver): JobIntrospection {
13
13
  if (driver.introspect === undefined) {
14
14
  throw new NotImplementedError({
15
15
  cause: `the operator surface of the "${driver.name}" jobs driver is not implemented: the driver has no introspect`,
16
- fix: 'call setJobDriver(postgresJobDriver()) at boot — the pg driver implements introspect — then: x jobs list --json',
16
+ fix: 'postgresJobDriver({ executor }) # jobs: in apps/<app>/runtime.ts, or drop that field and the boot builds it over DATABASE_URL — the pg driver implements introspect — then: x jobs list --json',
17
17
  });
18
18
  }
19
19
  return driver.introspect;
@@ -63,7 +63,7 @@ export function resolveJobStep(
63
63
  if (operator.resolveStep === undefined) {
64
64
  throw new NotImplementedError({
65
65
  cause: `the "${driver.name}" jobs driver keeps no step records an operator can resolve: its introspect has no resolveStep`,
66
- fix: 'setJobDriver(postgresJobDriver()) # at boot: the pg and memory drivers implement resolveStep — then x jobs resolve-step <id> <step> --ran --output <json>',
66
+ fix: 'postgresJobDriver({ executor }) # jobs: in apps/<app>/runtime.ts, or drop that field and the boot builds it over DATABASE_URL — the pg and memory drivers implement resolveStep — then x jobs resolve-step <id> <step> --ran --output <json>',
67
67
  });
68
68
  }
69
69
  return operator.resolveStep(jobId, step, resolution);
package/src/inspect.ts CHANGED
@@ -117,7 +117,7 @@ function requireIntrospection(driver: JobDriver): NonNullable<JobDriver['introsp
117
117
  if (driver.introspect === undefined) {
118
118
  throw new NotImplementedError({
119
119
  cause: `introspection for the "${driver.name}" jobs driver is not implemented: the driver has no introspect`,
120
- fix: 'call setJobDriver(postgresJobDriver()) at boot — only the pg driver implements introspect — then: x jobs list --json',
120
+ fix: 'postgresJobDriver({ executor }) # jobs: in apps/<app>/runtime.ts, or drop that field and the boot builds it over DATABASE_URL — only the pg driver implements introspect — then: x jobs list --json',
121
121
  });
122
122
  }
123
123
  return driver.introspect;
@@ -0,0 +1,115 @@
1
+ // What the queue reports while it works: the `job.enqueue` producer span and event around every
2
+ // enqueue, and the stable messaging attributes both the producer and the consumer span carry. One
3
+ // file so `job.ts` (the enqueue door) and `worker-run.ts` (the run) spell the semconv names once.
4
+
5
+ import type { SpanAttributes } from '@ultimat3/core';
6
+ import { instrument, SEMCONV, SEMCONV_VALUES, subscribed, withSpan } from '@ultimat3/core';
7
+ import type { EnqueueRequest, EnqueueResult } from './driver';
8
+ import { ambientTraceparent } from './enqueue-record';
9
+
10
+ /** The attributes every job span carries, producer and consumer alike. */
11
+ export function messagingAttributes(
12
+ operation: 'send' | 'process',
13
+ queue: string,
14
+ job: string,
15
+ ): SpanAttributes {
16
+ return {
17
+ [SEMCONV.messagingSystem]: SEMCONV_VALUES.messagingSystemUltimate,
18
+ [SEMCONV.messagingOperationType]:
19
+ operation === 'send'
20
+ ? SEMCONV_VALUES.messagingOperationSend
21
+ : SEMCONV_VALUES.messagingOperationProcess,
22
+ [SEMCONV.messagingOperationName]: operation === 'send' ? 'enqueue' : 'perform',
23
+ [SEMCONV.messagingDestinationName]: queue,
24
+ 'ultimate.job.name': job,
25
+ };
26
+ }
27
+
28
+ /**
29
+ * Run an enqueue under its producer span and tell the bus about each job it created. The span is
30
+ * open while the row is written, so the `traceparent` the row carries (`ambientTraceparent()`) is
31
+ * THIS span's — the run's consumer span becomes its child, as OpenTelemetry's messaging model
32
+ * says. A deduped enqueue still reports: the caller asked, and the id names the live job.
33
+ */
34
+ export async function instrumentedEnqueue<R extends EnqueueResult | readonly EnqueueResult[]>(
35
+ job: string,
36
+ queue: string,
37
+ enqueue: () => Promise<R>,
38
+ ): Promise<R> {
39
+ const results = await withSpan(
40
+ `job.${job}.enqueue`,
41
+ async (span) => {
42
+ const answered = await enqueue();
43
+ const list = listOf(answered);
44
+ const [only] = list;
45
+ if (list.length === 1 && only !== undefined) {
46
+ span.setAttribute(SEMCONV.messagingMessageId, only.id);
47
+ } else {
48
+ span.setAttribute('ultimate.job.count', list.length);
49
+ }
50
+ return answered;
51
+ },
52
+ { kind: 'producer', attributes: messagingAttributes('send', queue, job) },
53
+ );
54
+ if (subscribed('job.enqueue')) {
55
+ for (const { id } of listOf(results)) {
56
+ instrument('job.enqueue', { job, queue, id });
57
+ }
58
+ }
59
+ return results;
60
+ }
61
+
62
+ /**
63
+ * A scheduler fire — one task's jobs and its watermark in ONE store write, so no per-job span can
64
+ * wrap it — under one producer span named for the TASK. Each request is stamped with that span's
65
+ * `traceparent` before the write, so every run it starts is the span's child; each job the write
66
+ * created is one `job.enqueue`. `undefined` (another dispatcher fired it) reports nothing.
67
+ */
68
+ export async function instrumentedFire(
69
+ task: string,
70
+ requests: readonly EnqueueRequest[],
71
+ fire: (stamped: readonly EnqueueRequest[]) => Promise<readonly EnqueueResult[] | undefined>,
72
+ ): Promise<readonly EnqueueResult[] | undefined> {
73
+ const [first] = requests;
74
+ const oneQueue =
75
+ first !== undefined && requests.every((request) => request.queue === first.queue);
76
+ const results = await withSpan(
77
+ `job.${task}.enqueue`,
78
+ (span) => {
79
+ const trace = ambientTraceparent();
80
+ span.setAttribute('ultimate.job.count', requests.length);
81
+ return fire(
82
+ trace === undefined
83
+ ? requests
84
+ : requests.map((request) => ({ ...request, traceparent: trace })),
85
+ );
86
+ },
87
+ {
88
+ kind: 'producer',
89
+ attributes: {
90
+ [SEMCONV.messagingSystem]: SEMCONV_VALUES.messagingSystemUltimate,
91
+ [SEMCONV.messagingOperationType]: SEMCONV_VALUES.messagingOperationSend,
92
+ [SEMCONV.messagingOperationName]: 'enqueue',
93
+ ...(oneQueue ? { [SEMCONV.messagingDestinationName]: first.queue } : {}),
94
+ 'ultimate.task.name': task,
95
+ },
96
+ },
97
+ );
98
+ if (results !== undefined && subscribed('job.enqueue')) {
99
+ requests.forEach((request, index) => {
100
+ const id = results[index]?.id;
101
+ if (id !== undefined && id !== '') {
102
+ instrument('job.enqueue', { job: request.name, queue: request.queue, id });
103
+ }
104
+ });
105
+ }
106
+ return results;
107
+ }
108
+
109
+ /** One enqueue's answer or many, as a list — a function, so a generic `R` narrows through it. */
110
+ const listOf = (value: EnqueueResult | readonly EnqueueResult[]): readonly EnqueueResult[] =>
111
+ isList(value) ? value : [value];
112
+
113
+ const isList = (
114
+ value: EnqueueResult | readonly EnqueueResult[],
115
+ ): value is readonly EnqueueResult[] => Array.isArray(value);
package/src/job.ts CHANGED
@@ -18,10 +18,12 @@ import { DEFAULT_QUEUE } from './driver';
18
18
  import { IdempotencyRequiredError, JobNameTakenError } from './errors';
19
19
  import { JobDeclarationInvalidError, missingJobFields } from './errors-declaration';
20
20
  import type { EnqueueManyOptions } from './job-enqueue-many';
21
+ import { instrumentedEnqueue } from './job-instrument';
21
22
  import { NO_TENANT, tenantKeyFrom } from './limits';
22
23
  import type { EnqueueOptions } from './outbox';
23
24
  import { jobsFacade } from './outbox';
24
25
  import { DEFAULT_RETRY } from './retry';
26
+ import { assertRetryJitter } from './retry-jitter';
25
27
  import type { JobSettled } from './settled';
26
28
  import { ON_SETTLED_TIMEOUT_MS } from './settled';
27
29
  import { assertJobTenant, jobTenantFor } from './tenant';
@@ -30,7 +32,36 @@ export type { AnyJobHandle, JobActor, JobDefinition, JobHandle, JobRunArgs } fro
30
32
 
31
33
  import type { AnyJobHandle, JobActor, JobDefinition, JobHandle, JobRunArgs } from './job-types';
32
34
 
33
- const registry = new Map<string, AnyJobHandle>();
35
+ /**
36
+ * The registry, and its name-sorted list cached until the next write. Every claim reads the list
37
+ * (`dropExhausted`), and a sort per claim bought nothing: the registry changes at boot. Every
38
+ * write goes through these three, so no path can change the map and keep a stale list.
39
+ */
40
+ class JobRegistry extends Map<string, AnyJobHandle> {
41
+ #sorted: readonly AnyJobHandle[] | undefined;
42
+
43
+ override set(name: string, handle: AnyJobHandle): this {
44
+ this.#sorted = undefined;
45
+ return super.set(name, handle);
46
+ }
47
+
48
+ override delete(name: string): boolean {
49
+ this.#sorted = undefined;
50
+ return super.delete(name);
51
+ }
52
+
53
+ override clear(): void {
54
+ this.#sorted = undefined;
55
+ super.clear();
56
+ }
57
+
58
+ sorted(): readonly AnyJobHandle[] {
59
+ this.#sorted ??= Object.freeze([...this.values()].sort((a, b) => byName(a.name, b.name)));
60
+ return this.#sorted;
61
+ }
62
+ }
63
+
64
+ const registry = new JobRegistry();
34
65
  /** Process-monotonic — `resetJobs()` deliberately leaves it alone. */
35
66
  let anonymous = 0;
36
67
 
@@ -67,6 +98,7 @@ export function job<I, R = unknown>(definition: JobDefinition<I, R>): JobHandle<
67
98
  // `1.5`, which the memory driver ran and Postgres refused at the first enqueue (`$7::int`).
68
99
  // Zero is not "never retries" — that is `attempts: 1` — it is a job that never executes at all.
69
100
  finiteCount(`job "${name}"`, 'retry.attempts', definition.retry.attempts, 1);
101
+ assertRetryJitter(name, definition.retry.jitter);
70
102
  // `concurrency: 0` is not "no cap" — it is a fleet slot table that grants nothing, and the job
71
103
  // is permanently unrunnable with no log line. Refused where it is written, plain or keyed.
72
104
  const concurrency = resolveConcurrency(name, definition.concurrency);
@@ -151,11 +183,16 @@ export function job<I, R = unknown>(definition: JobDefinition<I, R>): JobHandle<
151
183
  // the cast restores exactly the types the declaration wrote.
152
184
  return definition.onSettled?.(settled as JobSettled<I, R>) ?? Promise.resolve();
153
185
  },
186
+ // Under the producer span, so the row's `traceparent` is the enqueue's own (`job-instrument.ts`).
154
187
  enqueue(input: I, options?: EnqueueOptions): Promise<EnqueueResult> {
155
- return jobsFacade().enqueue(handle, input, options);
188
+ return instrumentedEnqueue(handle.name, options?.queue ?? handle.queue, () =>
189
+ jobsFacade().enqueue(handle, input, options),
190
+ );
156
191
  },
157
192
  enqueueMany(inputs: readonly I[], options?: EnqueueManyOptions) {
158
- return jobsFacade().enqueueMany(handle, inputs, options);
193
+ return instrumentedEnqueue(handle.name, options?.queue ?? handle.queue, () =>
194
+ jobsFacade().enqueueMany(handle, inputs, options),
195
+ );
159
196
  },
160
197
  as(actor: JobActor | null, input: I, options: EnqueueOptions = {}): Promise<EnqueueResult> {
161
198
  const tenantId = options.tenantId ?? tenantFor(actor);
@@ -267,7 +304,7 @@ export function getJob(name: string): AnyJobHandle | undefined {
267
304
  const byName = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0);
268
305
 
269
306
  export function registeredJobs(): readonly AnyJobHandle[] {
270
- return [...registry.values()].sort((a, b) => byName(a.name, b.name));
307
+ return registry.sorted();
271
308
  }
272
309
 
273
310
  /**