@ultimat3/jobs 22.14.0 → 23.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/CLAUDE.md +129 -87
  2. package/README.md +329 -19
  3. package/package.json +6 -6
  4. package/src/concurrency.ts +144 -0
  5. package/src/counters.ts +105 -0
  6. package/src/describe.ts +29 -0
  7. package/src/driver-memory-operator.ts +297 -0
  8. package/src/driver-memory.ts +79 -37
  9. package/src/driver-nats.ts +2 -2
  10. package/src/driver-pg-ddl.ts +71 -0
  11. package/src/driver-pg-jobs-sql.ts +71 -8
  12. package/src/driver-pg-operator-sql.ts +303 -0
  13. package/src/driver-pg-operator.ts +300 -0
  14. package/src/driver-pg-outbox-sql.ts +110 -0
  15. package/src/driver-pg-rows.ts +31 -0
  16. package/src/driver-pg-settle-sql.ts +71 -0
  17. package/src/driver-pg-sql.ts +80 -129
  18. package/src/driver-pg-wake-sql.ts +76 -0
  19. package/src/driver-pg.ts +74 -34
  20. package/src/driver-redis.ts +2 -2
  21. package/src/driver.ts +96 -34
  22. package/src/enqueue-signal.ts +50 -0
  23. package/src/errors-concurrency.ts +91 -0
  24. package/src/errors-operator.ts +94 -0
  25. package/src/errors.ts +13 -17
  26. package/src/events-pg.ts +38 -28
  27. package/src/events.ts +25 -0
  28. package/src/execute.ts +160 -81
  29. package/src/export.ts +23 -4
  30. package/src/heartbeat.ts +9 -3
  31. package/src/idle-backoff.ts +74 -0
  32. package/src/index.ts +105 -12
  33. package/src/inspect-operator.ts +49 -0
  34. package/src/inspect.ts +42 -2
  35. package/src/introspection.ts +262 -0
  36. package/src/job.ts +78 -15
  37. package/src/leases.ts +49 -9
  38. package/src/metrics.ts +6 -1
  39. package/src/outbox-pg.ts +3 -0
  40. package/src/outbox-relay.ts +114 -39
  41. package/src/outbox.ts +29 -6
  42. package/src/progress.ts +78 -0
  43. package/src/queue-wake.ts +220 -0
  44. package/src/redact-input.ts +26 -0
  45. package/src/renewal-timer.ts +28 -10
  46. package/src/retry-classification.ts +13 -1
  47. package/src/retry.ts +10 -2
  48. package/src/run-deadline.ts +57 -0
  49. package/src/scheduler-leader.ts +26 -0
  50. package/src/scheduler-occurrences.ts +76 -0
  51. package/src/scheduler-pg.ts +55 -2
  52. package/src/scheduler-state.ts +66 -0
  53. package/src/scheduler.ts +143 -103
  54. package/src/settled.ts +161 -0
  55. package/src/worker-admit.ts +163 -0
  56. package/src/worker-fleet-slots.ts +107 -38
  57. package/src/worker-hand-back.ts +6 -1
  58. package/src/worker-key-busy.ts +79 -0
  59. package/src/worker-loop.ts +145 -0
  60. package/src/worker-registry.ts +103 -0
  61. package/src/worker-run.ts +9 -0
  62. package/src/worker-types.ts +23 -0
  63. package/src/worker.ts +103 -134
package/src/index.ts CHANGED
@@ -84,30 +84,35 @@ export {
84
84
  isBackfill,
85
85
  registeredBackfills,
86
86
  } from './backfill-registry';
87
- export type { JobDescriptor } from './describe';
87
+ export type { JobConcurrency, KeyedConcurrency, WhenBusy } from './concurrency';
88
+ export { MAX_CONCURRENCY_KEY_LENGTH, WHEN_BUSY } from './concurrency';
89
+ export type { JobConcurrencyDescriptor, JobDescriptor } from './describe';
88
90
  export type {
91
+ AckOptions,
89
92
  ClaimedJob,
93
+ ClaimIdentity,
90
94
  ClaimOptions,
91
95
  ConflictPolicy,
92
96
  EnqueueRequest,
93
97
  EnqueueResult,
94
98
  HeartbeatOptions,
95
99
  JobDriver,
96
- JobFilter,
97
- JobIntrospection,
98
100
  JobRecord,
99
101
  JobState,
100
102
  NackOptions,
101
103
  QueueStats,
104
+ SettleBy,
102
105
  } from './driver';
103
106
  export {
104
107
  assertClaimBounds,
105
108
  assertClaimQueues,
109
+ claimOf,
106
110
  DEFAULT_QUEUE,
107
111
  DEFAULT_VISIBILITY_TIMEOUT_MS,
108
112
  isJobState,
109
113
  JOB_STATES,
110
114
  jobDriver,
115
+ nackState,
111
116
  resetJobDriver,
112
117
  setJobDriver,
113
118
  } from './driver';
@@ -117,6 +122,24 @@ export type { NatsDriverOptions } from './driver-nats';
117
122
  export { createNatsDriver } from './driver-nats';
118
123
  export type { PgDriverOptions, PgExecutor } from './driver-pg';
119
124
  export { createPgDriver, createPgLeader } from './driver-pg';
125
+ export {
126
+ SQL_COUNTER_DROP,
127
+ SQL_COUNTER_FOLD,
128
+ SQL_COUNTER_TOTALS,
129
+ SQL_COUNTERS,
130
+ SQL_JOB_PROGRESS,
131
+ SQL_JOB_PROMOTE,
132
+ SQL_JOB_REMOVE,
133
+ SQL_JOB_REMOVE_MANY,
134
+ SQL_JOB_REQUEUE_MANY,
135
+ SQL_PAUSE,
136
+ SQL_PAUSED,
137
+ SQL_RESUME,
138
+ SQL_SCHEDULER_FIRE,
139
+ SQL_WORKER_ANNOUNCE,
140
+ SQL_WORKER_FORGET,
141
+ SQL_WORKERS,
142
+ } from './driver-pg-operator-sql';
120
143
  export {
121
144
  SQL_ACK,
122
145
  SQL_ADVISORY_UNLOCK,
@@ -132,6 +155,7 @@ export {
132
155
  SQL_LEADER_ACQUIRE,
133
156
  SQL_LEADER_RELEASE,
134
157
  SQL_LEASE_ACQUIRE,
158
+ SQL_LEASE_HOLDERS,
135
159
  SQL_LEASE_RELEASE,
136
160
  SQL_LEASE_RENEW,
137
161
  SQL_NACK,
@@ -148,12 +172,12 @@ export {
148
172
  } from './driver-pg-sql';
149
173
  export type { RedisDriverOptions } from './driver-redis';
150
174
  export { createRedisDriver } from './driver-redis';
175
+ export { signalEnqueued, signalStaged } from './enqueue-signal';
151
176
  export type { JobErrorCode } from './errors';
152
177
  export {
153
178
  ActionJobUnbridgedError,
154
179
  CancelUnsupportedError,
155
180
  ClaimQueuesEmptyError,
156
- ConcurrencyUnenforceableError,
157
181
  DriverUnavailableError,
158
182
  IdempotencyRequiredError,
159
183
  JOB_ERROR_CODES,
@@ -173,10 +197,28 @@ export {
173
197
  OutboxNoTxError,
174
198
  StepDuplicateError,
175
199
  } from './errors';
200
+ export {
201
+ ConcurrencyUnenforceableError,
202
+ JobConcurrencyInvalidError,
203
+ JobConcurrencyKeyInvalidError,
204
+ JobKeyBusyError,
205
+ } from './errors-concurrency';
176
206
  export { JobDeclarationInvalidError } from './errors-declaration';
207
+ export {
208
+ JobNotPromotableError,
209
+ JobNotRemovableError,
210
+ JobOnSettledFailedError,
211
+ JobPageInvalidError,
212
+ } from './errors-operator';
177
213
  export { JobNotRequeueableError } from './errors-requeue';
178
214
  export type { EventBus, JobEvent, MemoryEventBusOptions, PublishOptions } from './events';
179
- export { createMemoryEventBus, eventBus, publishEvent, setEventBus } from './events';
215
+ export {
216
+ createMemoryEventBus,
217
+ eventBus,
218
+ publishEvent,
219
+ resetEventBus,
220
+ setEventBus,
221
+ } from './events';
180
222
  export type { PgEventBusOptions } from './events-pg';
181
223
  export { createPgEventBus } from './events-pg';
182
224
  export type { ExecuteJobOptions, JobExecution, JobOutcome } from './execute';
@@ -193,6 +235,7 @@ export {
193
235
  exportPartKey,
194
236
  memoryExportSink,
195
237
  } from './export-sink';
238
+ export { IDLE_POLL_CEILING_MS } from './idle-backoff';
196
239
  export type {
197
240
  DeadLetterEntry,
198
241
  JobsManifest,
@@ -209,9 +252,45 @@ export {
209
252
  inspectQueues,
210
253
  retryFromStep,
211
254
  } from './inspect';
212
- export type { AnyJobHandle, JobActor, JobDefinition, JobHandle, JobRunArgs } from './job';
255
+ export { pauseQueue, promoteJob, removeJob, resumeQueue } from './inspect-operator';
256
+ export type {
257
+ BulkFilter,
258
+ BulkResult,
259
+ CounterBucketMs,
260
+ CounterOutcome,
261
+ CounterTotals,
262
+ JobCounter,
263
+ JobFilter,
264
+ JobIntrospection,
265
+ JobProgress,
266
+ PausedName,
267
+ TaskFire,
268
+ WorkerAnnouncement,
269
+ WorkerRecord,
270
+ } from './introspection';
271
+ export {
272
+ COUNTER_BUCKET_MS,
273
+ COUNTER_TIERS,
274
+ DEFAULT_JOB_PAGE,
275
+ jobCursor,
276
+ MAX_BULK_ROWS,
277
+ MAX_ERROR_STACK_LENGTH,
278
+ MAX_JOB_PAGE,
279
+ MAX_PROGRESS_NOTE_LENGTH,
280
+ MAX_TASK_FIRES,
281
+ MAX_WORKER_IN_FLIGHT,
282
+ PROGRESS_INTERVAL_MS,
283
+ PROMOTABLE_STATES,
284
+ } from './introspection';
285
+ export type {
286
+ AnyJobHandle,
287
+ JobActor,
288
+ JobDefinition,
289
+ JobHandle,
290
+ JobRunArgs,
291
+ } from './job';
213
292
  export { describeJobs, getJob, isJobHandle, job, registeredJobs, resetJobs } from './job';
214
- export type { HeldLease, LeaseStore, MemoryLeaseStoreOptions } from './leases';
293
+ export type { HeldLease, LeaseStore, MemoryLeaseStore, MemoryLeaseStoreOptions } from './leases';
215
294
  export { createMemoryLeaseStore, jobLeaseKey } from './leases';
216
295
  export type {
217
296
  Lease,
@@ -253,6 +332,7 @@ export type { PgOutboxOptions } from './outbox-pg';
253
332
  export { createPgOutboxStore } from './outbox-pg';
254
333
  export type { OutboxRelay, RelayOptions } from './outbox-relay';
255
334
  export { createOutboxRelay } from './outbox-relay';
335
+ export type { ProgressFn } from './progress';
256
336
  export type {
257
337
  PurgeDefinition,
258
338
  PurgeInput,
@@ -261,26 +341,39 @@ export type {
261
341
  PurgeTarget,
262
342
  } from './purge';
263
343
  export { DEFAULT_PURGE_CRON, purge } from './purge';
344
+ export type { PgListener, QueueWake, QueueWakeOptions } from './queue-wake';
345
+ export { startQueueWake } from './queue-wake';
346
+ export type { IntervalScheduler } from './renewal-timer';
264
347
  export type { BackoffStrategy, Random, RetryDecision, RetryPolicy } from './retry';
265
- export { backoffDelayMs, DEFAULT_RETRY, nextRetry, retrySchedule } from './retry';
348
+ export { backoffDelayMs, DEFAULT_RETRY, isFinalAttempt, nextRetry, retrySchedule } from './retry';
266
349
  export type { JobRetryDecision, JobStopReason } from './retry-classification';
267
- export { classifyThrown, nextRetryForError } from './retry-classification';
350
+ export { classifyThrown, failureForRow, nextRetryForError } from './retry-classification';
268
351
  export type {
269
352
  CronResolver,
270
353
  DispatchedOccurrence,
271
- LeaderElection,
272
354
  Scheduler,
273
355
  SchedulerOptions,
274
- SchedulerState,
275
356
  } from './scheduler';
276
- export { createMemorySchedulerState, createScheduler, soleLeader } from './scheduler';
357
+ export {
358
+ COUNTER_ROLLUP_INTERVAL_MS,
359
+ createScheduler,
360
+ PAUSE_RECHECK_MS,
361
+ } from './scheduler';
362
+ export type { LeaderElection } from './scheduler-leader';
363
+ export { soleLeader } from './scheduler-leader';
364
+ export { nextTaskRun } from './scheduler-occurrences';
277
365
  export type { PgLeaseLeaderOptions } from './scheduler-pg';
278
366
  export {
279
367
  createPgLeaseLeader,
280
368
  currentLeader,
281
369
  DEFAULT_LEADER_TTL_MS,
370
+ LEASE_RENEWALS_PER_TTL,
282
371
  pgSchedulerState,
283
372
  } from './scheduler-pg';
373
+ export type { ScheduledFire, SchedulerState } from './scheduler-state';
374
+ export { createMemorySchedulerState, fireThroughDriver } from './scheduler-state';
375
+ export type { JobCompleted, JobFailed, JobSettled } from './settled';
376
+ export { ON_SETTLED_ATTEMPTS } from './settled';
284
377
  export type {
285
378
  EventLookup,
286
379
  StepApi,
@@ -0,0 +1,49 @@
1
+ // The operator verbs `x jobs` and a dashboard bind to: remove, promote, pause, resume. Each one
2
+ // refuses LOUDLY rather than answering "nothing happened" — an operator clearing a queue has to
3
+ // know whether the job is gone or was never theirs to remove. Apart from `inspect.ts`, which is
4
+ // the read side.
5
+
6
+ import type { JobDriver, JobRecord } from './driver';
7
+ import { JobsNotImplementedError } from './errors';
8
+ import { JobNotPromotableError } from './errors-operator';
9
+ import type { JobIntrospection, PausedName } from './introspection';
10
+
11
+ function operatorOf(driver: JobDriver): JobIntrospection {
12
+ if (driver.introspect === undefined) {
13
+ throw new JobsNotImplementedError({
14
+ feature: `the operator surface of the "${driver.name}" jobs driver`,
15
+ fix: 'call setJobDriver(createPgDriver()) at boot — the pg driver implements introspect — then: x jobs ls --json',
16
+ });
17
+ }
18
+ return driver.introspect;
19
+ }
20
+
21
+ /** `undefined` for an id nobody queued; `X_JOB_NOT_REMOVABLE` for one a worker holds. */
22
+ export function removeJob(driver: JobDriver, jobId: string): Promise<JobRecord | undefined> {
23
+ return operatorOf(driver).remove(jobId);
24
+ }
25
+
26
+ /** The promoted row. A job not waiting on its run time is `X_JOB_NOT_PROMOTABLE`, state named. */
27
+ export async function promoteJob(driver: JobDriver, jobId: string): Promise<JobRecord> {
28
+ const operator = operatorOf(driver);
29
+ const promoted = await operator.promote(jobId);
30
+ if (promoted !== undefined) return promoted;
31
+ const current = await operator.job(jobId);
32
+ throw new JobNotPromotableError({ jobId, state: current?.state ?? 'missing' });
33
+ }
34
+
35
+ /** Pause, then read back: the answer is the list as the queue now holds it. */
36
+ export async function pauseQueue(driver: JobDriver, queue: string): Promise<readonly PausedName[]> {
37
+ const operator = operatorOf(driver);
38
+ await operator.pauseQueue(queue);
39
+ return operator.pausedQueues();
40
+ }
41
+
42
+ export async function resumeQueue(
43
+ driver: JobDriver,
44
+ queue: string,
45
+ ): Promise<readonly PausedName[]> {
46
+ const operator = operatorOf(driver);
47
+ await operator.resumeQueue(queue);
48
+ return operator.pausedQueues();
49
+ }
package/src/inspect.ts CHANGED
@@ -4,9 +4,13 @@
4
4
 
5
5
  import type { BackfillProgress } from './backfill-inspect';
6
6
  import { backfillForRun } from './backfill-inspect';
7
- import type { JobDriver, JobFilter, JobRecord, QueueStats } from './driver';
7
+ import type { WhenBusy } from './concurrency';
8
+ import type { JobDriver, JobRecord, QueueStats } from './driver';
8
9
  import { CancelUnsupportedError, JobNotCancellableError, JobsNotImplementedError } from './errors';
10
+ import type { JobFilter, JobProgress } from './introspection';
11
+ import type { AnyJobHandle } from './job';
9
12
  import { registeredJobs } from './job';
13
+ import { redactInput } from './redact-input';
10
14
  import { retrySchedule } from './retry';
11
15
  import type { Scheduler } from './scheduler';
12
16
  import type { StepRecord } from './steps';
@@ -20,6 +24,7 @@ export interface QueueDepthReport {
20
24
  readonly delayed: number;
21
25
  readonly running: number;
22
26
  readonly suspended: number;
27
+ readonly failed: number;
23
28
  readonly dead: number;
24
29
  };
25
30
  /** Oldest claimable job across all queues, in ms. The autoscaling signal. */
@@ -34,9 +39,10 @@ export async function inspectQueues(driver: JobDriver): Promise<QueueDepthReport
34
39
  delayed: acc.delayed + queue.delayed,
35
40
  running: acc.running + queue.running,
36
41
  suspended: acc.suspended + queue.suspended,
42
+ failed: acc.failed + queue.failed,
37
43
  dead: acc.dead + queue.dead,
38
44
  }),
39
- { ready: 0, delayed: 0, running: 0, suspended: 0, dead: 0 },
45
+ { ready: 0, delayed: 0, running: 0, suspended: 0, failed: 0, dead: 0 },
40
46
  );
41
47
  return {
42
48
  driver: driver.name,
@@ -68,6 +74,15 @@ export interface JobTrace {
68
74
  readonly runId: string;
69
75
  readonly runAt: string;
70
76
  readonly lastError: string | null;
77
+ /** The thrown value's stack for `lastError`, bounded. `null` when the failure carried none. */
78
+ readonly stack: string | null;
79
+ /**
80
+ * The stored payload, with every key the app declared secret (`redactKeys`) replaced. App data
81
+ * all the same: a surface that shows it gates it like any other read of a tenant's rows.
82
+ */
83
+ readonly input: unknown;
84
+ /** What the body last reported through `progress()`; `null` until it reports. */
85
+ readonly progress: JobProgress | null;
71
86
  readonly tenantId: string | null;
72
87
  /** W3C `traceparent` of the request that queued it — paste it into the trace viewer. */
73
88
  readonly traceparent: string | null;
@@ -76,6 +91,13 @@ export interface JobTrace {
76
91
  readonly steps: readonly StepTrace[];
77
92
  /** Remaining retry delays in ms, jitter excluded. */
78
93
  readonly retryDelaysMs: readonly number[];
94
+ /**
95
+ * The concurrency key THIS run counts under — `concurrency.key(input)` — when its job declares a
96
+ * keyed cap. `null` for every other job, for a job this process has not registered, and for a
97
+ * stored input the key cannot be derived from: a trace is read to debug a stuck queue, so it
98
+ * degrades rather than refusing.
99
+ */
100
+ readonly concurrencyKey: string | null;
79
101
  /**
80
102
  * The `x_backfills` row this run wrote, when the job is a `backfill()`. `null` for every other
81
103
  * job and for a driver with no ledger — a step trace says which batch is next, and this says how
@@ -97,6 +119,15 @@ function requireIntrospection(driver: JobDriver): NonNullable<JobDriver['introsp
97
119
  return driver.introspect;
98
120
  }
99
121
 
122
+ function concurrencyKeyOf(handle: AnyJobHandle | undefined, record: JobRecord): string | null {
123
+ if (handle === undefined || handle.whenBusy === undefined) return null;
124
+ try {
125
+ return handle.concurrencyKeyFor(handle.parse(record.input)) ?? null;
126
+ } catch {
127
+ return null;
128
+ }
129
+ }
130
+
100
131
  function toStepTrace(record: StepRecord): StepTrace {
101
132
  return {
102
133
  name: record.name,
@@ -127,11 +158,15 @@ export async function inspectJob(driver: JobDriver, jobId: string): Promise<JobT
127
158
  runId: record.runId,
128
159
  runAt: new Date(record.runAt).toISOString(),
129
160
  lastError: record.lastError ?? null,
161
+ stack: record.lastErrorStack ?? null,
162
+ input: redactInput(record.input),
163
+ progress: record.progress ?? null,
130
164
  tenantId: record.tenantId ?? null,
131
165
  traceparent: record.traceparent ?? null,
132
166
  enqueuedBy: record.enqueuedBy ?? null,
133
167
  steps: steps.map(toStepTrace),
134
168
  retryDelaysMs: handle === undefined ? [] : [...retrySchedule(handle.retry)],
169
+ concurrencyKey: concurrencyKeyOf(handle, record),
135
170
  backfill: backfill ?? null,
136
171
  };
137
172
  }
@@ -217,6 +252,9 @@ export interface JobsManifest {
217
252
  readonly attempts: number;
218
253
  readonly backoff: string;
219
254
  readonly concurrency: number | null;
255
+ /** Non-null exactly when `concurrency` holds per key — see `JobDescriptor.concurrency`. */
256
+ readonly whenBusy: WhenBusy | null;
257
+ readonly onSettled: boolean;
220
258
  readonly timeoutMs: number | null;
221
259
  readonly retryDelaysMs: readonly number[];
222
260
  }[];
@@ -239,6 +277,8 @@ export function inspectManifest(scheduler?: Scheduler): JobsManifest {
239
277
  attempts: handle.retry.attempts,
240
278
  backoff: handle.retry.backoff ?? 'exponential',
241
279
  concurrency: handle.concurrency ?? null,
280
+ whenBusy: handle.whenBusy ?? null,
281
+ onSettled: handle.declaresOnSettled,
242
282
  timeoutMs: handle.timeoutMs ?? null,
243
283
  retryDelaysMs: [...retrySchedule(handle.retry)],
244
284
  })),
@@ -0,0 +1,262 @@
1
+ // What an OPERATOR may ask of a queue, and the bounds on each answer. Every capability a
2
+ // dashboard, `x jobs` or an MCP tool has is a member of `JobIntrospection`, implemented by the pg
3
+ // and the memory driver alike, so a surface that reads only this file reads nothing a driver does
4
+ // not promise. Apart from `driver.ts` because that file is the six-method claim/settle contract;
5
+ // this one is everything said ABOUT the rows it moves.
6
+
7
+ import { finiteCount } from '@ultimat3/core';
8
+ import type { ClaimIdentity, JobRecord, JobState } from './driver';
9
+ import { JobPageInvalidError } from './errors-operator';
10
+
11
+ /** Rows one `list()` call may answer. A page, never a table: a dashboard walks with `after`. */
12
+ export const MAX_JOB_PAGE = 200;
13
+ export const DEFAULT_JOB_PAGE = 100;
14
+ /** Rows one `requeueMany` / `removeMany` call may touch. It answers how many remain. */
15
+ export const MAX_BULK_ROWS = 1_000;
16
+ /** Gap between two progress writes of ONE run. The last call before a settle is always written. */
17
+ export const PROGRESS_INTERVAL_MS = 1_000;
18
+ export const MAX_PROGRESS_NOTE_LENGTH = 200;
19
+ /** How much of a thrown value's stack a row keeps. A stack is a diagnosis, not an archive. */
20
+ export const MAX_ERROR_STACK_LENGTH = 8_000;
21
+ /** In-flight job ids one registry row lists. A worker holding more reports the first of them. */
22
+ export const MAX_WORKER_IN_FLIGHT = 100;
23
+ /** Rows `taskFires()` answers. A task renamed away leaves its row behind; a screen is not a table. */
24
+ export const MAX_TASK_FIRES = 1_000;
25
+
26
+ /**
27
+ * The counter tiers: a bucket width, and how long buckets of that width are kept. One-minute
28
+ * buckets for a day, five-minute for a week, hourly for thirty days — 1,440 + 2,016 + 720 rows per
29
+ * job name whatever the throughput, which is the whole point: history that costs the same at ten
30
+ * jobs an hour and at ten million.
31
+ */
32
+ export const COUNTER_TIERS = Object.freeze([
33
+ Object.freeze({ bucketMs: 60_000, keepMs: 86_400_000 }),
34
+ Object.freeze({ bucketMs: 300_000, keepMs: 604_800_000 }),
35
+ Object.freeze({ bucketMs: 3_600_000, keepMs: 2_592_000_000 }),
36
+ ] as const);
37
+
38
+ export type CounterBucketMs = (typeof COUNTER_TIERS)[number]['bucketMs'];
39
+
40
+ /** The tier every settle writes to. Coarser ones are filled by `rollupCounters` only. */
41
+ export const COUNTER_BUCKET_MS = COUNTER_TIERS[0].bucketMs;
42
+
43
+ export interface JobFilter {
44
+ readonly queue?: string;
45
+ readonly name?: string;
46
+ readonly state?: JobState;
47
+ /** Rows whose id STARTS with this — an operator pastes the first characters of one. */
48
+ readonly idPrefix?: string;
49
+ /** Epoch ms, inclusive. Rows created at or after it. */
50
+ readonly createdFrom?: number;
51
+ /** Epoch ms, exclusive. Rows created before it. */
52
+ readonly createdTo?: number;
53
+ /** 1 to `MAX_JOB_PAGE`. Default `DEFAULT_JOB_PAGE`. */
54
+ readonly limit?: number;
55
+ /**
56
+ * `jobCursor(lastRowOfThePreviousPage)`. Keyset, never an offset: the order is newest first by
57
+ * `(createdAt, id)`, so a row inserted while a walk is in flight sorts BEFORE every cursor and
58
+ * each row already there is visited exactly once.
59
+ */
60
+ readonly after?: string;
61
+ /**
62
+ * `jobCursor(firstRowOfThisPage)` — the page BEFORE it: the `limit` rows nearest the cursor on
63
+ * its newer side, answered newest first like every page. What a dashboard's "previous" reads.
64
+ * Never with `after`.
65
+ */
66
+ readonly before?: string;
67
+ /** Rows queued for this tenant only (`JobRecord.tenantId`). An org-scoped operator's list. */
68
+ readonly tenantId?: string;
69
+ }
70
+
71
+ const CURSOR = /^(\d{1,16}):([0-9A-Za-z-]{1,64})$/;
72
+
73
+ /** The cursor a page's LAST row yields. A page shorter than its limit is the last page. */
74
+ export function jobCursor(record: Pick<JobRecord, 'createdAt' | 'id'>): string {
75
+ return `${record.createdAt}:${record.id}`;
76
+ }
77
+
78
+ export function parseJobCursor(
79
+ after: string,
80
+ subject = 'the job list',
81
+ ): { readonly createdAt: number; readonly id: string } {
82
+ const match = CURSOR.exec(after);
83
+ if (match === null) throw new JobPageInvalidError({ subject, maxPage: MAX_JOB_PAGE });
84
+ return { createdAt: Number(match[1]), id: match[2] ?? '' };
85
+ }
86
+
87
+ /** `limit`, defaulted and refused past the bound — a caller never widens a page by asking. */
88
+ export function pageLimit(subject: string, limit: number | undefined): number {
89
+ const asked = finiteCount(subject, 'limit', limit ?? DEFAULT_JOB_PAGE);
90
+ if (asked > MAX_JOB_PAGE) {
91
+ throw new JobPageInvalidError({ subject, maxPage: MAX_JOB_PAGE, limit: asked });
92
+ }
93
+ return asked;
94
+ }
95
+
96
+ /** Which rows a bulk call addresses. `state` is REQUIRED: "every job" is never one call. */
97
+ export interface BulkFilter {
98
+ readonly state: JobState;
99
+ readonly queue?: string;
100
+ readonly name?: string;
101
+ /** One tenant's rows only — what an org-scoped operator's "all matching" may touch. */
102
+ readonly tenantId?: string;
103
+ }
104
+
105
+ /** The states `promoteMany` accepts: the two a row waiting on its `runAt` can be in. */
106
+ export const PROMOTABLE_STATES: ReadonlySet<JobState> = new Set<JobState>(['ready', 'delayed']);
107
+
108
+ export interface BulkResult {
109
+ /** Rows this call changed — at most `MAX_BULK_ROWS`. */
110
+ readonly affected: number;
111
+ /** Rows still matching the filter afterwards. Call again until it is zero. */
112
+ readonly remaining: number;
113
+ }
114
+
115
+ export interface PausedName {
116
+ readonly name: string;
117
+ /** Epoch ms. */
118
+ readonly pausedAt: number;
119
+ }
120
+
121
+ /** What a worker says about itself. The registry adds `heartbeatAt`. */
122
+ export interface WorkerAnnouncement {
123
+ readonly id: string;
124
+ readonly host: string;
125
+ /** Epoch ms this worker started. */
126
+ readonly startedAt: number;
127
+ readonly queues: readonly string[];
128
+ /** Slots across every queue this worker serves. */
129
+ readonly concurrency: number;
130
+ /** Job ids it holds right now — the first `MAX_WORKER_IN_FLIGHT` of them. */
131
+ readonly inFlight: readonly string[];
132
+ }
133
+
134
+ export interface WorkerRecord extends WorkerAnnouncement {
135
+ /** Epoch ms of the announcement that wrote this row. */
136
+ readonly heartbeatAt: number;
137
+ }
138
+
139
+ /**
140
+ * The last occurrence of a task that DISPATCHED — queued its jobs. Arming a task first seen, or
141
+ * skipping occurrences a `catchUp` policy drops, moves the scheduler's watermark and is not one.
142
+ */
143
+ export interface TaskFire {
144
+ readonly task: string;
145
+ /** Epoch ms the occurrence was scheduled for. */
146
+ readonly occurrenceMs: number;
147
+ /** Epoch ms it was dispatched, on the store's clock. */
148
+ readonly firedAt: number;
149
+ }
150
+
151
+ export interface JobProgress {
152
+ readonly done: number;
153
+ readonly total: number;
154
+ readonly note?: string;
155
+ /** Epoch ms the body reported it. */
156
+ readonly at: number;
157
+ }
158
+
159
+ /** What one settle adds to its job's current bucket. Exactly one of the four counts is 1. */
160
+ export type CounterOutcome = 'done' | 'retried' | 'failed' | 'dead';
161
+
162
+ export interface JobCounter {
163
+ readonly job: string;
164
+ /** Epoch ms the bucket opens. */
165
+ readonly bucketStart: number;
166
+ readonly bucketMs: CounterBucketMs;
167
+ readonly done: number;
168
+ /** Attempts that failed and were handed back for another. */
169
+ readonly retried: number;
170
+ /** Runs that ended `failed`: refused by a busy key, or exhausted on a `deadLetter: false` job. */
171
+ readonly failed: number;
172
+ readonly dead: number;
173
+ /** Sum of the attempts' durations, so `durationMs / (done + retried + failed + dead)` is a mean. */
174
+ readonly durationMs: number;
175
+ }
176
+
177
+ export type CounterTotals = Omit<JobCounter, 'bucketStart' | 'bucketMs'>;
178
+
179
+ /** A driver's operator surface. Every member is implemented by every driver that ships one. */
180
+ export interface JobIntrospection {
181
+ job(jobId: string): Promise<JobRecord | undefined>;
182
+ /** One page, newest first. See `JobFilter.after`. */
183
+ list(filter?: JobFilter): Promise<readonly JobRecord[]>;
184
+ deadLetters(limit?: number): Promise<readonly JobRecord[]>;
185
+ /**
186
+ * Re-queue a finished job (`REQUEUEABLE_STATES`); a live one is `X_JOB_NOT_REQUEUEABLE`, and a
187
+ * key a live job holds is `X_JOB_DUPLICATE`. `fromStep` drops that step and every step that
188
+ * started after it.
189
+ */
190
+ requeue(jobId: string, options?: { readonly fromStep?: string }): Promise<JobRecord>;
191
+ /**
192
+ * Stop a job from outside. Terminal for a queued row immediately; a RUNNING one stops at its
193
+ * next heartbeat, which no longer matches its own row and cancels the attempt. Answers
194
+ * `undefined` for a job id it does not hold or one that already finished. Optional, as it
195
+ * shipped: a driver may have no way to address a single running row.
196
+ */
197
+ cancel?(jobId: string, reason?: string): Promise<JobRecord | undefined>;
198
+ /**
199
+ * Delete one job and its step records. `undefined` for an id nobody queued. A RUNNING job is
200
+ * refused (`X_JOB_NOT_REMOVABLE`): a worker holds it, and deleting the row under a body does
201
+ * not stop the body — cancel it first.
202
+ */
203
+ remove(jobId: string): Promise<JobRecord | undefined>;
204
+ /**
205
+ * Re-queue up to `MAX_BULK_ROWS` finished jobs. `filter.state` must be a requeueable state. A
206
+ * row whose key a live job holds is left where it is and counted in `remaining`.
207
+ */
208
+ requeueMany(filter: BulkFilter): Promise<BulkResult>;
209
+ /** Delete up to `MAX_BULK_ROWS` jobs. `filter.state` may be any state but `running`. */
210
+ removeMany(filter: BulkFilter): Promise<BulkResult>;
211
+ /**
212
+ * Make a job waiting on its `runAt` — delayed at enqueue, or backing off before a retry — due
213
+ * now. `undefined` when the job is not waiting on a clock (missing, running, finished, or
214
+ * suspended in a `step.sleep`, whose wake time is the step's and not the row's).
215
+ */
216
+ promote(jobId: string): Promise<JobRecord | undefined>;
217
+ /**
218
+ * `promote`, over up to `MAX_BULK_ROWS` rows: every one in `filter` still waiting on its
219
+ * `runAt` is due now. `filter.state` must be `ready` (a retry backing off) or `delayed`.
220
+ */
221
+ promoteMany(filter: BulkFilter): Promise<BulkResult>;
222
+ /** A paused queue is never claimed, by any worker; enqueues still land. Idempotent. */
223
+ pauseQueue(queue: string): Promise<void>;
224
+ resumeQueue(queue: string): Promise<void>;
225
+ pausedQueues(): Promise<readonly PausedName[]>;
226
+ /**
227
+ * A paused task is not dispatched. On resume its own `catchUp` policy decides what the pause
228
+ * missed — exactly as if the scheduler had been down.
229
+ */
230
+ pauseTask(task: string): Promise<void>;
231
+ resumeTask(task: string): Promise<void>;
232
+ pausedTasks(): Promise<readonly PausedName[]>;
233
+ /**
234
+ * The scheduler's record that an occurrence dispatched; the store stamps `firedAt`. The pg
235
+ * driver's own fire writes it in the firing statement (`SQL_SCHEDULER_FIRE`), so only a fire
236
+ * that goes through `driver.enqueue` calls this.
237
+ */
238
+ recordTaskFire(fire: Pick<TaskFire, 'task' | 'occurrenceMs'>): Promise<void>;
239
+ /** Every task that has dispatched, by task name — the first `MAX_TASK_FIRES` of them. */
240
+ taskFires(): Promise<readonly TaskFire[]>;
241
+ /** A worker's heartbeat. The row expires `ttlMs` after the last one — never by cleanup. */
242
+ announceWorker(worker: WorkerAnnouncement, ttlMs: number): Promise<void>;
243
+ /** A clean stop hands the row back at once rather than leaving it to expire. */
244
+ forgetWorker(workerId: string): Promise<void>;
245
+ /** Every worker whose row has not expired, oldest start first. */
246
+ workers(): Promise<readonly WorkerRecord[]>;
247
+ /** Fenced on the CLAIM, as a settle is: progress from one the row no longer carries is dropped. */
248
+ recordProgress(jobId: string, by: ClaimIdentity, progress: JobProgress): Promise<void>;
249
+ /** One job's buckets from `sinceMs` on, oldest first, at whatever tier each still lives in. */
250
+ counters(query: {
251
+ readonly job: string;
252
+ readonly sinceMs: number;
253
+ }): Promise<readonly JobCounter[]>;
254
+ /** One row per job name: every bucket from `sinceMs` on, summed. Ordered by name. */
255
+ counterTotals(sinceMs: number): Promise<readonly CounterTotals[]>;
256
+ /**
257
+ * Fold buckets past their tier's `keepMs` into the next tier and drop the ones past the last.
258
+ * The scheduler leader calls it; it is idempotent, and safe if two nodes both do. Answers how
259
+ * many buckets it moved or dropped.
260
+ */
261
+ rollupCounters(): Promise<number>;
262
+ }