@ultimat3/jobs 30.9.0 → 31.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +72 -107
- package/README.md +202 -55
- package/package.json +8 -7
- package/src/adapter.ts +37 -0
- package/src/backfill-ledger.ts +2 -2
- package/src/backfill-rate.ts +1 -0
- package/src/clock.ts +2 -2
- package/src/drain-wait.ts +1 -0
- package/src/driver-memory.ts +2 -2
- package/src/driver-pg-ddl.ts +19 -0
- package/src/driver-pg-event-wake-sql.ts +43 -22
- package/src/driver-pg-sql.ts +68 -27
- package/src/driver-pg.ts +11 -10
- package/src/driver-redis.ts +2 -2
- package/src/driver.ts +14 -8
- package/src/error-codes.ts +27 -0
- package/src/errors-concurrency.ts +1 -1
- package/src/errors.ts +1 -1
- package/src/events-cursor.ts +31 -0
- package/src/events-pg.ts +52 -6
- package/src/events.ts +72 -36
- package/src/export.ts +2 -2
- package/src/gdpr-audit.ts +75 -0
- package/src/gdpr-coverage.ts +94 -0
- package/src/gdpr-errors.ts +35 -0
- package/src/import.ts +2 -2
- package/src/index.ts +45 -206
- package/src/inspect-operator.ts +2 -2
- package/src/inspect.ts +1 -1
- package/src/job-instrument.ts +115 -0
- package/src/job.ts +41 -4
- package/src/jobs-retention.ts +23 -6
- package/src/leases.ts +13 -3
- package/src/ledger-book.ts +168 -0
- package/src/ledger-errors.ts +48 -0
- package/src/ledger.ts +145 -0
- package/src/limits.ts +57 -19
- package/src/lock-errors.ts +54 -0
- package/src/lock.ts +176 -0
- package/src/outbox-relay.ts +4 -0
- package/src/outbox.ts +3 -3
- package/src/partitions.ts +119 -0
- package/src/reindex-sweep.ts +94 -0
- package/src/reindex.ts +147 -0
- package/src/retry-classification.ts +3 -2
- package/src/retry-jitter.ts +32 -0
- package/src/retry.ts +13 -16
- package/src/scheduler.ts +10 -6
- package/src/steps-api.ts +44 -5
- package/src/steps-suspension.ts +34 -3
- package/src/steps-wait.ts +52 -4
- package/src/steps.ts +1 -0
- package/src/subject-jobs.ts +12 -4
- package/src/webhook.ts +2 -2
- package/src/worker-admit.ts +8 -5
- package/src/worker-drain-cutoff.ts +1 -0
- package/src/worker-loop.ts +28 -5
- package/src/worker-registry.ts +7 -5
- package/src/worker-run.ts +53 -9
- package/src/worker.ts +8 -8
package/src/index.ts
CHANGED
|
@@ -20,36 +20,28 @@ export type {
|
|
|
20
20
|
BackfillInput,
|
|
21
21
|
BackfillPage,
|
|
22
22
|
BackfillPages,
|
|
23
|
-
BackfillReport,
|
|
24
23
|
ScopedBackfillDefinition,
|
|
25
24
|
ScopedBackfillInput,
|
|
26
25
|
} from './backfill';
|
|
27
26
|
export { backfill, DEFAULT_BACKFILL_BATCH } from './backfill';
|
|
28
27
|
export {
|
|
29
|
-
BackfillAppliedError,
|
|
30
|
-
BackfillEnvironmentError,
|
|
31
|
-
BackfillMigrationPendingError,
|
|
32
28
|
BackfillPendingError,
|
|
33
29
|
BackfillRunningError,
|
|
34
|
-
BackfillSourceInvalidError,
|
|
35
|
-
BackfillStalledError,
|
|
36
30
|
BackfillUnknownError,
|
|
37
31
|
} from './backfill-errors';
|
|
38
32
|
export type { BackfillGate, BackfillGateInput } from './backfill-gate';
|
|
39
|
-
export {
|
|
33
|
+
export { gateBackfill } from './backfill-gate';
|
|
40
34
|
export type { BackfillProgress } from './backfill-inspect';
|
|
41
|
-
export {
|
|
35
|
+
export { inspectBackfills, toBackfillProgress } from './backfill-inspect';
|
|
42
36
|
export type {
|
|
43
37
|
BackfillFilter,
|
|
44
38
|
BackfillLedger,
|
|
45
39
|
BackfillRun,
|
|
46
40
|
BackfillStatus,
|
|
47
|
-
BackfillVerdict,
|
|
48
41
|
} from './backfill-ledger';
|
|
49
42
|
export {
|
|
50
43
|
BACKFILL_STATUSES,
|
|
51
44
|
backfillChecksum,
|
|
52
|
-
decideBackfill,
|
|
53
45
|
isBackfillStatus,
|
|
54
46
|
memoryBackfillLedger,
|
|
55
47
|
} from './backfill-ledger';
|
|
@@ -78,48 +70,32 @@ export {
|
|
|
78
70
|
} from './backfill-registry';
|
|
79
71
|
export type { AnnounceExhaustedOptions, ExhaustedCounts } from './claim-exhausted';
|
|
80
72
|
export { announceExhausted } from './claim-exhausted';
|
|
81
|
-
export type { JobConcurrency, KeyedConcurrency
|
|
82
|
-
export { MAX_CONCURRENCY_KEY_LENGTH, WHEN_BUSY } from './concurrency';
|
|
73
|
+
export type { JobConcurrency, KeyedConcurrency } from './concurrency';
|
|
83
74
|
export { consumer } from './consumer';
|
|
84
|
-
export {
|
|
85
|
-
export {
|
|
86
|
-
CONSUMER_LINGER_MS,
|
|
87
|
-
CONSUMER_PROGRESS_NOTE,
|
|
88
|
-
DEFAULT_CONSUMER_BATCH,
|
|
89
|
-
} from './consumer-loop';
|
|
75
|
+
export { CONSUMER_LINGER_MS } from './consumer-loop';
|
|
90
76
|
// `stampConsumer` is deliberately absent, for the reason `stampBackfill` is.
|
|
91
77
|
export type { ConsumerDeclaration, ConsumerRow } from './consumer-registry';
|
|
92
|
-
export {
|
|
93
|
-
consumerDeclarationOf,
|
|
94
|
-
consumerRow,
|
|
95
|
-
isConsumer,
|
|
96
|
-
registeredConsumers,
|
|
97
|
-
} from './consumer-registry';
|
|
78
|
+
export { consumerRow, registeredConsumers } from './consumer-registry';
|
|
98
79
|
export type {
|
|
99
|
-
ConsumerBatch,
|
|
100
80
|
ConsumerCursor,
|
|
101
81
|
ConsumerDefinition,
|
|
102
82
|
ConsumerHandle,
|
|
103
83
|
ConsumerInput,
|
|
104
|
-
ConsumerReport,
|
|
105
84
|
} from './consumer-types';
|
|
106
85
|
export type { JobConcurrencyDescriptor, JobDescriptor } from './describe';
|
|
107
86
|
export type {
|
|
108
|
-
AckOptions,
|
|
109
87
|
ClaimedJob,
|
|
110
88
|
ClaimIdentity,
|
|
111
89
|
ClaimOptions,
|
|
112
90
|
ConflictPolicy,
|
|
113
91
|
EnqueueRequest,
|
|
114
92
|
EnqueueResult,
|
|
115
|
-
HeartbeatOptions,
|
|
116
93
|
JobDriver,
|
|
117
94
|
JobRecord,
|
|
118
95
|
JobState,
|
|
119
96
|
NackOptions,
|
|
120
97
|
QueueDemand,
|
|
121
98
|
QueueStats,
|
|
122
|
-
SettleBy,
|
|
123
99
|
} from './driver';
|
|
124
100
|
export {
|
|
125
101
|
assertClaimBounds,
|
|
@@ -133,8 +109,6 @@ export {
|
|
|
133
109
|
LEASE_LAPSED_FINAL_ATTEMPT,
|
|
134
110
|
LIVE_STATES,
|
|
135
111
|
nackState,
|
|
136
|
-
resetJobDriver,
|
|
137
|
-
setJobDriver,
|
|
138
112
|
} from './driver';
|
|
139
113
|
export type { MemoryJobDriver, MemoryJobDriverOptions } from './driver-memory';
|
|
140
114
|
export { memoryJobDriver } from './driver-memory';
|
|
@@ -148,50 +122,17 @@ export { signalEnqueued, signalStaged } from './enqueue-signal';
|
|
|
148
122
|
export type { JobErrorCode } from './error-codes';
|
|
149
123
|
export { JOB_ERROR_CODES, JOB_ERROR_TITLES } from './error-codes';
|
|
150
124
|
export {
|
|
151
|
-
ActionJobUnbridgedError,
|
|
152
|
-
CancelUnsupportedError,
|
|
153
|
-
ClaimQueuesEmptyError,
|
|
154
125
|
DriverUnavailableError,
|
|
155
|
-
IdempotencyRequiredError,
|
|
156
126
|
JobAbortedError,
|
|
157
127
|
JobDrainedError,
|
|
158
128
|
JobDuplicateError,
|
|
159
|
-
JobEnqueueBatchTooLargeError,
|
|
160
|
-
JobMaxAttemptsError,
|
|
161
|
-
JobNameTakenError,
|
|
162
129
|
JobNotCancellableError,
|
|
163
|
-
JobRowStatusUnknownError,
|
|
164
|
-
JobSlotLostError,
|
|
165
|
-
JobTenantRequiredError,
|
|
166
130
|
JobTimeoutError,
|
|
167
131
|
LeaseLostError,
|
|
168
|
-
OutboxNoTxError,
|
|
169
|
-
StepDuplicateError,
|
|
170
132
|
} from './errors';
|
|
171
|
-
export {
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
JobConcurrencyKeyInvalidError,
|
|
175
|
-
JobKeyBusyError,
|
|
176
|
-
} from './errors-concurrency';
|
|
177
|
-
export { JobDeclarationInvalidError } from './errors-declaration';
|
|
178
|
-
export {
|
|
179
|
-
JobNotPromotableError,
|
|
180
|
-
JobNotRemovableError,
|
|
181
|
-
JobOnSettledFailedError,
|
|
182
|
-
JobPageInvalidError,
|
|
183
|
-
} from './errors-operator';
|
|
184
|
-
export { JobNotFoundError, JobNotRequeueableError } from './errors-requeue';
|
|
185
|
-
export {
|
|
186
|
-
COMPENSATE_PREFIX,
|
|
187
|
-
JobCompensationFailedError,
|
|
188
|
-
StepCompensateUndeclaredError,
|
|
189
|
-
StepNameReservedError,
|
|
190
|
-
StepNotResolvableError,
|
|
191
|
-
StepOutcomeUnknownError,
|
|
192
|
-
UNWINDING_STEP,
|
|
193
|
-
} from './errors-saga';
|
|
194
|
-
export { JobEventInvalidError, StepOutputUnserializableError } from './errors-steps';
|
|
133
|
+
export { ConcurrencyUnenforceableError, JobKeyBusyError } from './errors-concurrency';
|
|
134
|
+
export { JobNotFoundError } from './errors-requeue';
|
|
135
|
+
export { COMPENSATE_PREFIX, StepNameReservedError, UNWINDING_STEP } from './errors-saga';
|
|
195
136
|
export { JobTenantMismatchError } from './errors-tenant';
|
|
196
137
|
export type { EventBus, JobEvent, MemoryEventBusOptions, PublishOptions } from './events';
|
|
197
138
|
export {
|
|
@@ -200,47 +141,28 @@ export {
|
|
|
200
141
|
eventsPurgeTarget,
|
|
201
142
|
memoryEventBus,
|
|
202
143
|
publishEvent,
|
|
203
|
-
resetEventBus,
|
|
204
|
-
setEventBus,
|
|
205
144
|
} from './events';
|
|
206
145
|
export type { PostgresEventBusOptions } from './events-pg';
|
|
207
146
|
export { postgresEventBus } from './events-pg';
|
|
208
147
|
export type { ExecuteJobOptions } from './execute';
|
|
209
148
|
export { executeJob } from './execute';
|
|
210
149
|
export type { JobExecution, JobOutcome } from './execute-result';
|
|
211
|
-
export type { ExportDefinition
|
|
212
|
-
export {
|
|
213
|
-
export {
|
|
214
|
-
export
|
|
215
|
-
export {
|
|
216
|
-
export
|
|
217
|
-
export {
|
|
218
|
-
EXPORT_PART_DIGITS,
|
|
219
|
-
exportManifestKey,
|
|
220
|
-
exportPartKey,
|
|
221
|
-
memoryExportSink,
|
|
222
|
-
} from './export-sink';
|
|
150
|
+
export type { ExportDefinition } from './export';
|
|
151
|
+
export { exportRows } from './export';
|
|
152
|
+
export type { EncodePageInput, ExportValue } from './export-format';
|
|
153
|
+
export { EXPORT_FORMATS, encodeExportPage } from './export-format';
|
|
154
|
+
export type { ExportSink, MemoryExportSink } from './export-sink';
|
|
155
|
+
export { memoryExportSink } from './export-sink';
|
|
223
156
|
export type { CallerFailure } from './failure-row';
|
|
224
157
|
export { callerFailure } from './failure-row';
|
|
158
|
+
export { personalDataProblems, unerasedPersonalData, unreachedPersonalData } from './gdpr-coverage';
|
|
159
|
+
export { PersonalDataUnerasedError, PersonalDataUnreachedError } from './gdpr-errors';
|
|
225
160
|
export { IDLE_POLL_CEILING_MS } from './idle-backoff';
|
|
226
|
-
export type { ImportDefinition
|
|
227
|
-
export {
|
|
228
|
-
DEFAULT_IMPORT_BATCH,
|
|
229
|
-
DEFAULT_IMPORT_MAX_RECORD_BYTES,
|
|
230
|
-
DEFAULT_IMPORT_MAX_ROW_ERRORS,
|
|
231
|
-
DEFAULT_IMPORT_PAGE_BYTES,
|
|
232
|
-
importRows,
|
|
233
|
-
} from './import';
|
|
234
|
-
export {
|
|
235
|
-
ImportHeaderMismatchError,
|
|
236
|
-
ImportRecordTooLargeError,
|
|
237
|
-
ImportTooManyErrorsError,
|
|
238
|
-
} from './import-errors';
|
|
239
|
-
export type { ImportRejection } from './import-format';
|
|
161
|
+
export type { ImportDefinition } from './import';
|
|
162
|
+
export { importRows } from './import';
|
|
240
163
|
export { IMPORT_REJECTIONS } from './import-format';
|
|
241
|
-
export type { ImportRowError } from './import-pass';
|
|
242
164
|
export { importErrorKey } from './import-pass';
|
|
243
|
-
export type {
|
|
165
|
+
export type { ImportSource, MemoryImportSource } from './import-source';
|
|
244
166
|
export { memoryImportSource } from './import-source';
|
|
245
167
|
export type {
|
|
246
168
|
DeadLetterEntry,
|
|
@@ -269,7 +191,6 @@ export type {
|
|
|
269
191
|
BulkFilter,
|
|
270
192
|
BulkResult,
|
|
271
193
|
CounterBucketMs,
|
|
272
|
-
CounterOutcome,
|
|
273
194
|
CounterTotals,
|
|
274
195
|
JobCounter,
|
|
275
196
|
JobFilter,
|
|
@@ -282,26 +203,15 @@ export type {
|
|
|
282
203
|
WorkerRecord,
|
|
283
204
|
} from './introspection';
|
|
284
205
|
export {
|
|
285
|
-
COUNTER_BUCKET_MS,
|
|
286
206
|
COUNTER_TIERS,
|
|
287
207
|
DEFAULT_JOB_PAGE,
|
|
288
208
|
jobCursor,
|
|
289
209
|
MAX_BULK_ROWS,
|
|
290
|
-
MAX_ERROR_STACK_LENGTH,
|
|
291
210
|
MAX_JOB_PAGE,
|
|
292
|
-
MAX_PROGRESS_NOTE_LENGTH,
|
|
293
211
|
MAX_TASK_FIRES,
|
|
294
|
-
MAX_WORKER_IN_FLIGHT,
|
|
295
|
-
PROGRESS_INTERVAL_MS,
|
|
296
212
|
PROMOTABLE_STATES,
|
|
297
213
|
} from './introspection';
|
|
298
|
-
export type {
|
|
299
|
-
AnyJobHandle,
|
|
300
|
-
JobActor,
|
|
301
|
-
JobDefinition,
|
|
302
|
-
JobHandle,
|
|
303
|
-
JobRunArgs,
|
|
304
|
-
} from './job';
|
|
214
|
+
export type { AnyJobHandle, JobDefinition, JobHandle, JobRunArgs } from './job';
|
|
305
215
|
export {
|
|
306
216
|
describeJobs,
|
|
307
217
|
getJob,
|
|
@@ -317,24 +227,15 @@ export type { JobsRetentionOptions } from './jobs-retention';
|
|
|
317
227
|
export { jobsPurgeTarget } from './jobs-retention';
|
|
318
228
|
export type { HeldLease, LeaseStore, MemoryLeaseStore, MemoryLeaseStoreOptions } from './leases';
|
|
319
229
|
export { jobLeaseKey, memoryLeaseStore } from './leases';
|
|
320
|
-
export type {
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
} from './
|
|
329
|
-
export { concurrencyLimiter, NO_TENANT, tenantKeyFrom } from './limits';
|
|
330
|
-
export {
|
|
331
|
-
queueDeadJobs,
|
|
332
|
-
queueDemand,
|
|
333
|
-
queueOldestReady,
|
|
334
|
-
recordQueueDeadJobs,
|
|
335
|
-
recordQueueDemand,
|
|
336
|
-
recordQueueOldestReady,
|
|
337
|
-
} from './metrics';
|
|
230
|
+
export type { LedgerDefinition, LedgerEntry, LedgerPosting } from './ledger';
|
|
231
|
+
export { assertBalancedPosting, ledger, ledgerLines } from './ledger';
|
|
232
|
+
export type { EntityLedgerBookOptions, LedgerBook, LedgerLine } from './ledger-book';
|
|
233
|
+
export { entityLedgerBook } from './ledger-book';
|
|
234
|
+
export type { Lease, LimitConfig, Limiter, LimitSnapshot, RateLimit } from './limits';
|
|
235
|
+
export { concurrencyLimiter, NO_TENANT } from './limits';
|
|
236
|
+
export type { HeldLock, WithLockOptions } from './lock';
|
|
237
|
+
export { withLock } from './lock';
|
|
238
|
+
export { queueDemand, recordQueueDemand } from './metrics';
|
|
338
239
|
export type {
|
|
339
240
|
EnqueueOptions,
|
|
340
241
|
JobsFacade,
|
|
@@ -360,6 +261,8 @@ export type { OutboxRelay, RelayOptions } from './outbox-relay';
|
|
|
360
261
|
export { outboxRelay } from './outbox-relay';
|
|
361
262
|
export type { OutboxRetentionOptions } from './outbox-retention';
|
|
362
263
|
export { outboxPurgeTarget } from './outbox-retention';
|
|
264
|
+
export type { PartitionsDefinition, PartitionsJob } from './partitions';
|
|
265
|
+
export { partitionedEntities, partitions } from './partitions';
|
|
363
266
|
export type { ProgressFn } from './progress';
|
|
364
267
|
export type {
|
|
365
268
|
PurgeDefinition,
|
|
@@ -371,16 +274,13 @@ export type {
|
|
|
371
274
|
export { DEFAULT_PURGE_CRON, purge } from './purge';
|
|
372
275
|
export type { PurgeBatching } from './purge-batches';
|
|
373
276
|
export type { PurgeEntities } from './purge-entities';
|
|
374
|
-
export {
|
|
375
|
-
resetPurgeCoverage,
|
|
376
|
-
retainedEntities,
|
|
377
|
-
retainTarget,
|
|
378
|
-
unsweptEntities,
|
|
379
|
-
} from './purge-entities';
|
|
277
|
+
export { resetPurgeCoverage, retainTarget, unsweptEntities } from './purge-entities';
|
|
380
278
|
export type { PgListener, QueueWake, QueueWakeOptions } from './queue-wake';
|
|
381
279
|
export { startQueueWake } from './queue-wake';
|
|
382
|
-
export type { RefreshViewDefinition
|
|
280
|
+
export type { RefreshViewDefinition } from './refresh-view';
|
|
383
281
|
export { refreshView } from './refresh-view';
|
|
282
|
+
export type { ReindexDefinition } from './reindex';
|
|
283
|
+
export { reindex } from './reindex';
|
|
384
284
|
export type { IntervalScheduler } from './renewal-timer';
|
|
385
285
|
export type { BackoffStrategy, Random, RetryDecision, RetryPolicy } from './retry';
|
|
386
286
|
export {
|
|
@@ -394,52 +294,28 @@ export {
|
|
|
394
294
|
UNWIND_MARGIN,
|
|
395
295
|
} from './retry';
|
|
396
296
|
export type { JobRetryDecision, JobStopReason } from './retry-classification';
|
|
397
|
-
export {
|
|
398
|
-
export type {
|
|
399
|
-
RollupBucket,
|
|
400
|
-
RollupDefinition,
|
|
401
|
-
RollupInput,
|
|
402
|
-
RollupReport,
|
|
403
|
-
RollupTarget,
|
|
404
|
-
} from './rollup';
|
|
297
|
+
export { nextRetryForError } from './retry-classification';
|
|
298
|
+
export type { RollupBucket, RollupDefinition, RollupInput, RollupTarget } from './rollup';
|
|
405
299
|
export { rollup } from './rollup';
|
|
406
|
-
export {
|
|
407
|
-
export
|
|
408
|
-
CronResolver,
|
|
409
|
-
DispatchedOccurrence,
|
|
410
|
-
Scheduler,
|
|
411
|
-
SchedulerOptions,
|
|
412
|
-
} from './scheduler';
|
|
413
|
-
export {
|
|
414
|
-
COUNTER_ROLLUP_INTERVAL_MS,
|
|
415
|
-
jobScheduler,
|
|
416
|
-
PAUSE_RECHECK_MS,
|
|
417
|
-
} from './scheduler';
|
|
300
|
+
export type { CronResolver, Scheduler, SchedulerOptions } from './scheduler';
|
|
301
|
+
export { COUNTER_ROLLUP_INTERVAL_MS, jobScheduler } from './scheduler';
|
|
418
302
|
export type { LeaderElection } from './scheduler-leader';
|
|
419
303
|
export { soleLeader } from './scheduler-leader';
|
|
420
304
|
export { nextTaskRun } from './scheduler-occurrences';
|
|
421
305
|
export type { PostgresLeaseLeaderOptions } from './scheduler-pg';
|
|
422
|
-
export {
|
|
423
|
-
currentLeader,
|
|
424
|
-
DEFAULT_LEADER_TTL_MS,
|
|
425
|
-
LEASE_RENEWALS_PER_TTL,
|
|
426
|
-
postgresLeaseLeader,
|
|
427
|
-
postgresSchedulerState,
|
|
428
|
-
} from './scheduler-pg';
|
|
306
|
+
export { DEFAULT_LEADER_TTL_MS, postgresLeaseLeader, postgresSchedulerState } from './scheduler-pg';
|
|
429
307
|
export type { ScheduledFire, SchedulerState } from './scheduler-state';
|
|
430
|
-
export {
|
|
308
|
+
export { memorySchedulerState } from './scheduler-state';
|
|
431
309
|
export type { JobCompleted, JobFailed, JobSettled } from './settled';
|
|
432
310
|
export { ON_SETTLED_ATTEMPTS, ON_SETTLED_TIMEOUT_MS } from './settled';
|
|
433
311
|
export { currentStep } from './step-scope';
|
|
434
312
|
export type {
|
|
435
|
-
CompensateContext,
|
|
436
313
|
EventLookup,
|
|
437
314
|
StepApi,
|
|
438
315
|
StepFence,
|
|
439
316
|
StepRecord,
|
|
440
317
|
StepRunner,
|
|
441
318
|
StepRunnerOptions,
|
|
442
|
-
StepRunOptions,
|
|
443
319
|
StepStatus,
|
|
444
320
|
StepStore,
|
|
445
321
|
WaitForEventOptions,
|
|
@@ -460,27 +336,14 @@ export type {
|
|
|
460
336
|
RetainedTarget,
|
|
461
337
|
Subject,
|
|
462
338
|
SubjectInit,
|
|
463
|
-
SubjectRef,
|
|
464
339
|
SubjectTarget,
|
|
465
340
|
SubjectTargetEntry,
|
|
466
|
-
SubjectTargetPage,
|
|
467
341
|
SubjectTreatment,
|
|
468
342
|
} from './subject';
|
|
469
|
-
export {
|
|
343
|
+
export { defineSubject, registeredSubjects, resetSubjects } from './subject';
|
|
470
344
|
export type { SubjectEraseReport, SubjectTally } from './subject-erase-pass';
|
|
471
|
-
export {
|
|
472
|
-
SubjectCycleError,
|
|
473
|
-
SubjectTableUndeclaredError,
|
|
474
|
-
SubjectTreatmentInvalidError,
|
|
475
|
-
} from './subject-errors';
|
|
476
|
-
export type { SubjectExportReport, SubjectIndexEntry } from './subject-export-pass';
|
|
477
345
|
export type { SubjectExportDefinition, SubjectJobDefinition } from './subject-jobs';
|
|
478
|
-
export {
|
|
479
|
-
plannedSubject,
|
|
480
|
-
subjectErase,
|
|
481
|
-
subjectEraseTables,
|
|
482
|
-
subjectExport,
|
|
483
|
-
} from './subject-jobs';
|
|
346
|
+
export { subjectErase, subjectEraseTables, subjectExport } from './subject-jobs';
|
|
484
347
|
export type {
|
|
485
348
|
SubjectPlan,
|
|
486
349
|
SubjectPlanInput,
|
|
@@ -495,7 +358,6 @@ export type {
|
|
|
495
358
|
TaskDescriptor,
|
|
496
359
|
TaskEnqueueEntry,
|
|
497
360
|
TaskHandle,
|
|
498
|
-
TaskJobResult,
|
|
499
361
|
} from './task';
|
|
500
362
|
export { getTask, isTaskHandle, registeredTasks, resetTasks, restoreTasks, task } from './task';
|
|
501
363
|
/**
|
|
@@ -510,39 +372,16 @@ export type {
|
|
|
510
372
|
WebhookDefinition,
|
|
511
373
|
WebhookDeliveryInput,
|
|
512
374
|
WebhookEndpoint,
|
|
513
|
-
WebhookEvent,
|
|
514
|
-
WebhookReport,
|
|
515
375
|
WebhookTenant,
|
|
516
376
|
} from './webhook';
|
|
517
377
|
export { DEFAULT_WEBHOOK_DISABLE_AFTER, webhook } from './webhook';
|
|
518
378
|
export type { WebhookFetch } from './webhook-attempt';
|
|
519
|
-
export { WEBHOOK_CONTENT_TYPE } from './webhook-attempt';
|
|
520
|
-
export {
|
|
521
|
-
WebhookDeliveryFailedError,
|
|
522
|
-
WebhookDeliveryRejectedError,
|
|
523
|
-
WebhookDeliveryThrottledError,
|
|
524
|
-
WebhookEndpointDisabledError,
|
|
525
|
-
WebhookEndpointInvalidError,
|
|
526
|
-
WebhookEndpointUnknownError,
|
|
527
|
-
WebhookEventInvalidError,
|
|
528
|
-
WebhookEventUnknownError,
|
|
529
|
-
} from './webhook-errors';
|
|
530
379
|
export type { MemoryWebhookLedger, WebhookAttempt, WebhookLedger } from './webhook-ledger';
|
|
531
|
-
export {
|
|
380
|
+
export { memoryWebhookLedger } from './webhook-ledger';
|
|
532
381
|
export { webhookReceiver, webhookReceiverRoutes } from './webhook-receiver';
|
|
533
|
-
export {
|
|
534
|
-
WEBHOOK_EVENT_ID_PATTERN,
|
|
535
|
-
WEBHOOK_RECEIVER_RETRY_AFTER_SECONDS,
|
|
536
|
-
WebhookEventIdInvalidError,
|
|
537
|
-
WebhookReceiverInvalidError,
|
|
538
|
-
WebhookReceiverUnavailableError,
|
|
539
|
-
} from './webhook-receiver-errors';
|
|
382
|
+
export { WEBHOOK_EVENT_ID_PATTERN } from './webhook-receiver-errors';
|
|
540
383
|
export type {
|
|
541
|
-
ReceivedEvent,
|
|
542
384
|
ReceivedEventFacts,
|
|
543
|
-
WebhookAnswer,
|
|
544
|
-
WebhookHandleArgs,
|
|
545
|
-
WebhookInbound,
|
|
546
385
|
WebhookReceiver,
|
|
547
386
|
WebhookReceiverDefinition,
|
|
548
387
|
WebhookReceiverInput,
|
package/src/inspect-operator.ts
CHANGED
|
@@ -13,7 +13,7 @@ function operatorOf(driver: JobDriver): JobIntrospection {
|
|
|
13
13
|
if (driver.introspect === undefined) {
|
|
14
14
|
throw new NotImplementedError({
|
|
15
15
|
cause: `the operator surface of the "${driver.name}" jobs driver is not implemented: the driver has no introspect`,
|
|
16
|
-
fix: '
|
|
16
|
+
fix: 'postgresJobDriver({ executor }) # jobs: in apps/<app>/runtime.ts, or drop that field and the boot builds it over DATABASE_URL — the pg driver implements introspect — then: x jobs list --json',
|
|
17
17
|
});
|
|
18
18
|
}
|
|
19
19
|
return driver.introspect;
|
|
@@ -63,7 +63,7 @@ export function resolveJobStep(
|
|
|
63
63
|
if (operator.resolveStep === undefined) {
|
|
64
64
|
throw new NotImplementedError({
|
|
65
65
|
cause: `the "${driver.name}" jobs driver keeps no step records an operator can resolve: its introspect has no resolveStep`,
|
|
66
|
-
fix: '
|
|
66
|
+
fix: 'postgresJobDriver({ executor }) # jobs: in apps/<app>/runtime.ts, or drop that field and the boot builds it over DATABASE_URL — the pg and memory drivers implement resolveStep — then x jobs resolve-step <id> <step> --ran --output <json>',
|
|
67
67
|
});
|
|
68
68
|
}
|
|
69
69
|
return operator.resolveStep(jobId, step, resolution);
|
package/src/inspect.ts
CHANGED
|
@@ -117,7 +117,7 @@ function requireIntrospection(driver: JobDriver): NonNullable<JobDriver['introsp
|
|
|
117
117
|
if (driver.introspect === undefined) {
|
|
118
118
|
throw new NotImplementedError({
|
|
119
119
|
cause: `introspection for the "${driver.name}" jobs driver is not implemented: the driver has no introspect`,
|
|
120
|
-
fix: '
|
|
120
|
+
fix: 'postgresJobDriver({ executor }) # jobs: in apps/<app>/runtime.ts, or drop that field and the boot builds it over DATABASE_URL — only the pg driver implements introspect — then: x jobs list --json',
|
|
121
121
|
});
|
|
122
122
|
}
|
|
123
123
|
return driver.introspect;
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
// What the queue reports while it works: the `job.enqueue` producer span and event around every
|
|
2
|
+
// enqueue, and the stable messaging attributes both the producer and the consumer span carry. One
|
|
3
|
+
// file so `job.ts` (the enqueue door) and `worker-run.ts` (the run) spell the semconv names once.
|
|
4
|
+
|
|
5
|
+
import type { SpanAttributes } from '@ultimat3/core';
|
|
6
|
+
import { instrument, SEMCONV, SEMCONV_VALUES, subscribed, withSpan } from '@ultimat3/core';
|
|
7
|
+
import type { EnqueueRequest, EnqueueResult } from './driver';
|
|
8
|
+
import { ambientTraceparent } from './enqueue-record';
|
|
9
|
+
|
|
10
|
+
/** The attributes every job span carries, producer and consumer alike. */
|
|
11
|
+
export function messagingAttributes(
|
|
12
|
+
operation: 'send' | 'process',
|
|
13
|
+
queue: string,
|
|
14
|
+
job: string,
|
|
15
|
+
): SpanAttributes {
|
|
16
|
+
return {
|
|
17
|
+
[SEMCONV.messagingSystem]: SEMCONV_VALUES.messagingSystemUltimate,
|
|
18
|
+
[SEMCONV.messagingOperationType]:
|
|
19
|
+
operation === 'send'
|
|
20
|
+
? SEMCONV_VALUES.messagingOperationSend
|
|
21
|
+
: SEMCONV_VALUES.messagingOperationProcess,
|
|
22
|
+
[SEMCONV.messagingOperationName]: operation === 'send' ? 'enqueue' : 'perform',
|
|
23
|
+
[SEMCONV.messagingDestinationName]: queue,
|
|
24
|
+
'ultimate.job.name': job,
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Run an enqueue under its producer span and tell the bus about each job it created. The span is
|
|
30
|
+
* open while the row is written, so the `traceparent` the row carries (`ambientTraceparent()`) is
|
|
31
|
+
* THIS span's — the run's consumer span becomes its child, as OpenTelemetry's messaging model
|
|
32
|
+
* says. A deduped enqueue still reports: the caller asked, and the id names the live job.
|
|
33
|
+
*/
|
|
34
|
+
export async function instrumentedEnqueue<R extends EnqueueResult | readonly EnqueueResult[]>(
|
|
35
|
+
job: string,
|
|
36
|
+
queue: string,
|
|
37
|
+
enqueue: () => Promise<R>,
|
|
38
|
+
): Promise<R> {
|
|
39
|
+
const results = await withSpan(
|
|
40
|
+
`job.${job}.enqueue`,
|
|
41
|
+
async (span) => {
|
|
42
|
+
const answered = await enqueue();
|
|
43
|
+
const list = listOf(answered);
|
|
44
|
+
const [only] = list;
|
|
45
|
+
if (list.length === 1 && only !== undefined) {
|
|
46
|
+
span.setAttribute(SEMCONV.messagingMessageId, only.id);
|
|
47
|
+
} else {
|
|
48
|
+
span.setAttribute('ultimate.job.count', list.length);
|
|
49
|
+
}
|
|
50
|
+
return answered;
|
|
51
|
+
},
|
|
52
|
+
{ kind: 'producer', attributes: messagingAttributes('send', queue, job) },
|
|
53
|
+
);
|
|
54
|
+
if (subscribed('job.enqueue')) {
|
|
55
|
+
for (const { id } of listOf(results)) {
|
|
56
|
+
instrument('job.enqueue', { job, queue, id });
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
return results;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* A scheduler fire — one task's jobs and its watermark in ONE store write, so no per-job span can
|
|
64
|
+
* wrap it — under one producer span named for the TASK. Each request is stamped with that span's
|
|
65
|
+
* `traceparent` before the write, so every run it starts is the span's child; each job the write
|
|
66
|
+
* created is one `job.enqueue`. `undefined` (another dispatcher fired it) reports nothing.
|
|
67
|
+
*/
|
|
68
|
+
export async function instrumentedFire(
|
|
69
|
+
task: string,
|
|
70
|
+
requests: readonly EnqueueRequest[],
|
|
71
|
+
fire: (stamped: readonly EnqueueRequest[]) => Promise<readonly EnqueueResult[] | undefined>,
|
|
72
|
+
): Promise<readonly EnqueueResult[] | undefined> {
|
|
73
|
+
const [first] = requests;
|
|
74
|
+
const oneQueue =
|
|
75
|
+
first !== undefined && requests.every((request) => request.queue === first.queue);
|
|
76
|
+
const results = await withSpan(
|
|
77
|
+
`job.${task}.enqueue`,
|
|
78
|
+
(span) => {
|
|
79
|
+
const trace = ambientTraceparent();
|
|
80
|
+
span.setAttribute('ultimate.job.count', requests.length);
|
|
81
|
+
return fire(
|
|
82
|
+
trace === undefined
|
|
83
|
+
? requests
|
|
84
|
+
: requests.map((request) => ({ ...request, traceparent: trace })),
|
|
85
|
+
);
|
|
86
|
+
},
|
|
87
|
+
{
|
|
88
|
+
kind: 'producer',
|
|
89
|
+
attributes: {
|
|
90
|
+
[SEMCONV.messagingSystem]: SEMCONV_VALUES.messagingSystemUltimate,
|
|
91
|
+
[SEMCONV.messagingOperationType]: SEMCONV_VALUES.messagingOperationSend,
|
|
92
|
+
[SEMCONV.messagingOperationName]: 'enqueue',
|
|
93
|
+
...(oneQueue ? { [SEMCONV.messagingDestinationName]: first.queue } : {}),
|
|
94
|
+
'ultimate.task.name': task,
|
|
95
|
+
},
|
|
96
|
+
},
|
|
97
|
+
);
|
|
98
|
+
if (results !== undefined && subscribed('job.enqueue')) {
|
|
99
|
+
requests.forEach((request, index) => {
|
|
100
|
+
const id = results[index]?.id;
|
|
101
|
+
if (id !== undefined && id !== '') {
|
|
102
|
+
instrument('job.enqueue', { job: request.name, queue: request.queue, id });
|
|
103
|
+
}
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
return results;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** One enqueue's answer or many, as a list — a function, so a generic `R` narrows through it. */
|
|
110
|
+
const listOf = (value: EnqueueResult | readonly EnqueueResult[]): readonly EnqueueResult[] =>
|
|
111
|
+
isList(value) ? value : [value];
|
|
112
|
+
|
|
113
|
+
const isList = (
|
|
114
|
+
value: EnqueueResult | readonly EnqueueResult[],
|
|
115
|
+
): value is readonly EnqueueResult[] => Array.isArray(value);
|
package/src/job.ts
CHANGED
|
@@ -18,10 +18,12 @@ import { DEFAULT_QUEUE } from './driver';
|
|
|
18
18
|
import { IdempotencyRequiredError, JobNameTakenError } from './errors';
|
|
19
19
|
import { JobDeclarationInvalidError, missingJobFields } from './errors-declaration';
|
|
20
20
|
import type { EnqueueManyOptions } from './job-enqueue-many';
|
|
21
|
+
import { instrumentedEnqueue } from './job-instrument';
|
|
21
22
|
import { NO_TENANT, tenantKeyFrom } from './limits';
|
|
22
23
|
import type { EnqueueOptions } from './outbox';
|
|
23
24
|
import { jobsFacade } from './outbox';
|
|
24
25
|
import { DEFAULT_RETRY } from './retry';
|
|
26
|
+
import { assertRetryJitter } from './retry-jitter';
|
|
25
27
|
import type { JobSettled } from './settled';
|
|
26
28
|
import { ON_SETTLED_TIMEOUT_MS } from './settled';
|
|
27
29
|
import { assertJobTenant, jobTenantFor } from './tenant';
|
|
@@ -30,7 +32,36 @@ export type { AnyJobHandle, JobActor, JobDefinition, JobHandle, JobRunArgs } fro
|
|
|
30
32
|
|
|
31
33
|
import type { AnyJobHandle, JobActor, JobDefinition, JobHandle, JobRunArgs } from './job-types';
|
|
32
34
|
|
|
33
|
-
|
|
35
|
+
/**
|
|
36
|
+
* The registry, and its name-sorted list cached until the next write. Every claim reads the list
|
|
37
|
+
* (`dropExhausted`), and a sort per claim bought nothing: the registry changes at boot. Every
|
|
38
|
+
* write goes through these three, so no path can change the map and keep a stale list.
|
|
39
|
+
*/
|
|
40
|
+
class JobRegistry extends Map<string, AnyJobHandle> {
|
|
41
|
+
#sorted: readonly AnyJobHandle[] | undefined;
|
|
42
|
+
|
|
43
|
+
override set(name: string, handle: AnyJobHandle): this {
|
|
44
|
+
this.#sorted = undefined;
|
|
45
|
+
return super.set(name, handle);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
override delete(name: string): boolean {
|
|
49
|
+
this.#sorted = undefined;
|
|
50
|
+
return super.delete(name);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
override clear(): void {
|
|
54
|
+
this.#sorted = undefined;
|
|
55
|
+
super.clear();
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
sorted(): readonly AnyJobHandle[] {
|
|
59
|
+
this.#sorted ??= Object.freeze([...this.values()].sort((a, b) => byName(a.name, b.name)));
|
|
60
|
+
return this.#sorted;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
const registry = new JobRegistry();
|
|
34
65
|
/** Process-monotonic — `resetJobs()` deliberately leaves it alone. */
|
|
35
66
|
let anonymous = 0;
|
|
36
67
|
|
|
@@ -67,6 +98,7 @@ export function job<I, R = unknown>(definition: JobDefinition<I, R>): JobHandle<
|
|
|
67
98
|
// `1.5`, which the memory driver ran and Postgres refused at the first enqueue (`$7::int`).
|
|
68
99
|
// Zero is not "never retries" — that is `attempts: 1` — it is a job that never executes at all.
|
|
69
100
|
finiteCount(`job "${name}"`, 'retry.attempts', definition.retry.attempts, 1);
|
|
101
|
+
assertRetryJitter(name, definition.retry.jitter);
|
|
70
102
|
// `concurrency: 0` is not "no cap" — it is a fleet slot table that grants nothing, and the job
|
|
71
103
|
// is permanently unrunnable with no log line. Refused where it is written, plain or keyed.
|
|
72
104
|
const concurrency = resolveConcurrency(name, definition.concurrency);
|
|
@@ -151,11 +183,16 @@ export function job<I, R = unknown>(definition: JobDefinition<I, R>): JobHandle<
|
|
|
151
183
|
// the cast restores exactly the types the declaration wrote.
|
|
152
184
|
return definition.onSettled?.(settled as JobSettled<I, R>) ?? Promise.resolve();
|
|
153
185
|
},
|
|
186
|
+
// Under the producer span, so the row's `traceparent` is the enqueue's own (`job-instrument.ts`).
|
|
154
187
|
enqueue(input: I, options?: EnqueueOptions): Promise<EnqueueResult> {
|
|
155
|
-
return
|
|
188
|
+
return instrumentedEnqueue(handle.name, options?.queue ?? handle.queue, () =>
|
|
189
|
+
jobsFacade().enqueue(handle, input, options),
|
|
190
|
+
);
|
|
156
191
|
},
|
|
157
192
|
enqueueMany(inputs: readonly I[], options?: EnqueueManyOptions) {
|
|
158
|
-
return
|
|
193
|
+
return instrumentedEnqueue(handle.name, options?.queue ?? handle.queue, () =>
|
|
194
|
+
jobsFacade().enqueueMany(handle, inputs, options),
|
|
195
|
+
);
|
|
159
196
|
},
|
|
160
197
|
as(actor: JobActor | null, input: I, options: EnqueueOptions = {}): Promise<EnqueueResult> {
|
|
161
198
|
const tenantId = options.tenantId ?? tenantFor(actor);
|
|
@@ -267,7 +304,7 @@ export function getJob(name: string): AnyJobHandle | undefined {
|
|
|
267
304
|
const byName = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0);
|
|
268
305
|
|
|
269
306
|
export function registeredJobs(): readonly AnyJobHandle[] {
|
|
270
|
-
return
|
|
307
|
+
return registry.sorted();
|
|
271
308
|
}
|
|
272
309
|
|
|
273
310
|
/**
|