@ultimat3/jobs 22.14.0 → 23.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +129 -87
- package/README.md +329 -19
- package/package.json +6 -6
- package/src/concurrency.ts +144 -0
- package/src/counters.ts +105 -0
- package/src/describe.ts +29 -0
- package/src/driver-memory-operator.ts +297 -0
- package/src/driver-memory.ts +79 -37
- package/src/driver-nats.ts +2 -2
- package/src/driver-pg-ddl.ts +71 -0
- package/src/driver-pg-jobs-sql.ts +71 -8
- package/src/driver-pg-operator-sql.ts +303 -0
- package/src/driver-pg-operator.ts +300 -0
- package/src/driver-pg-outbox-sql.ts +110 -0
- package/src/driver-pg-rows.ts +31 -0
- package/src/driver-pg-settle-sql.ts +71 -0
- package/src/driver-pg-sql.ts +80 -129
- package/src/driver-pg-wake-sql.ts +76 -0
- package/src/driver-pg.ts +74 -34
- package/src/driver-redis.ts +2 -2
- package/src/driver.ts +96 -34
- package/src/enqueue-signal.ts +50 -0
- package/src/errors-concurrency.ts +91 -0
- package/src/errors-operator.ts +94 -0
- package/src/errors.ts +13 -17
- package/src/events-pg.ts +38 -28
- package/src/events.ts +25 -0
- package/src/execute.ts +160 -81
- package/src/export.ts +23 -4
- package/src/heartbeat.ts +9 -3
- package/src/idle-backoff.ts +74 -0
- package/src/index.ts +105 -12
- package/src/inspect-operator.ts +49 -0
- package/src/inspect.ts +42 -2
- package/src/introspection.ts +262 -0
- package/src/job.ts +78 -15
- package/src/leases.ts +49 -9
- package/src/metrics.ts +6 -1
- package/src/outbox-pg.ts +3 -0
- package/src/outbox-relay.ts +114 -39
- package/src/outbox.ts +29 -6
- package/src/progress.ts +78 -0
- package/src/queue-wake.ts +220 -0
- package/src/redact-input.ts +26 -0
- package/src/renewal-timer.ts +28 -10
- package/src/retry-classification.ts +13 -1
- package/src/retry.ts +10 -2
- package/src/run-deadline.ts +57 -0
- package/src/scheduler-leader.ts +26 -0
- package/src/scheduler-occurrences.ts +76 -0
- package/src/scheduler-pg.ts +55 -2
- package/src/scheduler-state.ts +66 -0
- package/src/scheduler.ts +143 -103
- package/src/settled.ts +161 -0
- package/src/worker-admit.ts +163 -0
- package/src/worker-fleet-slots.ts +107 -38
- package/src/worker-hand-back.ts +6 -1
- package/src/worker-key-busy.ts +79 -0
- package/src/worker-loop.ts +145 -0
- package/src/worker-registry.ts +103 -0
- package/src/worker-run.ts +9 -0
- package/src/worker-types.ts +23 -0
- package/src/worker.ts +103 -134
package/src/index.ts
CHANGED
|
@@ -84,30 +84,35 @@ export {
|
|
|
84
84
|
isBackfill,
|
|
85
85
|
registeredBackfills,
|
|
86
86
|
} from './backfill-registry';
|
|
87
|
-
export type {
|
|
87
|
+
export type { JobConcurrency, KeyedConcurrency, WhenBusy } from './concurrency';
|
|
88
|
+
export { MAX_CONCURRENCY_KEY_LENGTH, WHEN_BUSY } from './concurrency';
|
|
89
|
+
export type { JobConcurrencyDescriptor, JobDescriptor } from './describe';
|
|
88
90
|
export type {
|
|
91
|
+
AckOptions,
|
|
89
92
|
ClaimedJob,
|
|
93
|
+
ClaimIdentity,
|
|
90
94
|
ClaimOptions,
|
|
91
95
|
ConflictPolicy,
|
|
92
96
|
EnqueueRequest,
|
|
93
97
|
EnqueueResult,
|
|
94
98
|
HeartbeatOptions,
|
|
95
99
|
JobDriver,
|
|
96
|
-
JobFilter,
|
|
97
|
-
JobIntrospection,
|
|
98
100
|
JobRecord,
|
|
99
101
|
JobState,
|
|
100
102
|
NackOptions,
|
|
101
103
|
QueueStats,
|
|
104
|
+
SettleBy,
|
|
102
105
|
} from './driver';
|
|
103
106
|
export {
|
|
104
107
|
assertClaimBounds,
|
|
105
108
|
assertClaimQueues,
|
|
109
|
+
claimOf,
|
|
106
110
|
DEFAULT_QUEUE,
|
|
107
111
|
DEFAULT_VISIBILITY_TIMEOUT_MS,
|
|
108
112
|
isJobState,
|
|
109
113
|
JOB_STATES,
|
|
110
114
|
jobDriver,
|
|
115
|
+
nackState,
|
|
111
116
|
resetJobDriver,
|
|
112
117
|
setJobDriver,
|
|
113
118
|
} from './driver';
|
|
@@ -117,6 +122,24 @@ export type { NatsDriverOptions } from './driver-nats';
|
|
|
117
122
|
export { createNatsDriver } from './driver-nats';
|
|
118
123
|
export type { PgDriverOptions, PgExecutor } from './driver-pg';
|
|
119
124
|
export { createPgDriver, createPgLeader } from './driver-pg';
|
|
125
|
+
export {
|
|
126
|
+
SQL_COUNTER_DROP,
|
|
127
|
+
SQL_COUNTER_FOLD,
|
|
128
|
+
SQL_COUNTER_TOTALS,
|
|
129
|
+
SQL_COUNTERS,
|
|
130
|
+
SQL_JOB_PROGRESS,
|
|
131
|
+
SQL_JOB_PROMOTE,
|
|
132
|
+
SQL_JOB_REMOVE,
|
|
133
|
+
SQL_JOB_REMOVE_MANY,
|
|
134
|
+
SQL_JOB_REQUEUE_MANY,
|
|
135
|
+
SQL_PAUSE,
|
|
136
|
+
SQL_PAUSED,
|
|
137
|
+
SQL_RESUME,
|
|
138
|
+
SQL_SCHEDULER_FIRE,
|
|
139
|
+
SQL_WORKER_ANNOUNCE,
|
|
140
|
+
SQL_WORKER_FORGET,
|
|
141
|
+
SQL_WORKERS,
|
|
142
|
+
} from './driver-pg-operator-sql';
|
|
120
143
|
export {
|
|
121
144
|
SQL_ACK,
|
|
122
145
|
SQL_ADVISORY_UNLOCK,
|
|
@@ -132,6 +155,7 @@ export {
|
|
|
132
155
|
SQL_LEADER_ACQUIRE,
|
|
133
156
|
SQL_LEADER_RELEASE,
|
|
134
157
|
SQL_LEASE_ACQUIRE,
|
|
158
|
+
SQL_LEASE_HOLDERS,
|
|
135
159
|
SQL_LEASE_RELEASE,
|
|
136
160
|
SQL_LEASE_RENEW,
|
|
137
161
|
SQL_NACK,
|
|
@@ -148,12 +172,12 @@ export {
|
|
|
148
172
|
} from './driver-pg-sql';
|
|
149
173
|
export type { RedisDriverOptions } from './driver-redis';
|
|
150
174
|
export { createRedisDriver } from './driver-redis';
|
|
175
|
+
export { signalEnqueued, signalStaged } from './enqueue-signal';
|
|
151
176
|
export type { JobErrorCode } from './errors';
|
|
152
177
|
export {
|
|
153
178
|
ActionJobUnbridgedError,
|
|
154
179
|
CancelUnsupportedError,
|
|
155
180
|
ClaimQueuesEmptyError,
|
|
156
|
-
ConcurrencyUnenforceableError,
|
|
157
181
|
DriverUnavailableError,
|
|
158
182
|
IdempotencyRequiredError,
|
|
159
183
|
JOB_ERROR_CODES,
|
|
@@ -173,10 +197,28 @@ export {
|
|
|
173
197
|
OutboxNoTxError,
|
|
174
198
|
StepDuplicateError,
|
|
175
199
|
} from './errors';
|
|
200
|
+
export {
|
|
201
|
+
ConcurrencyUnenforceableError,
|
|
202
|
+
JobConcurrencyInvalidError,
|
|
203
|
+
JobConcurrencyKeyInvalidError,
|
|
204
|
+
JobKeyBusyError,
|
|
205
|
+
} from './errors-concurrency';
|
|
176
206
|
export { JobDeclarationInvalidError } from './errors-declaration';
|
|
207
|
+
export {
|
|
208
|
+
JobNotPromotableError,
|
|
209
|
+
JobNotRemovableError,
|
|
210
|
+
JobOnSettledFailedError,
|
|
211
|
+
JobPageInvalidError,
|
|
212
|
+
} from './errors-operator';
|
|
177
213
|
export { JobNotRequeueableError } from './errors-requeue';
|
|
178
214
|
export type { EventBus, JobEvent, MemoryEventBusOptions, PublishOptions } from './events';
|
|
179
|
-
export {
|
|
215
|
+
export {
|
|
216
|
+
createMemoryEventBus,
|
|
217
|
+
eventBus,
|
|
218
|
+
publishEvent,
|
|
219
|
+
resetEventBus,
|
|
220
|
+
setEventBus,
|
|
221
|
+
} from './events';
|
|
180
222
|
export type { PgEventBusOptions } from './events-pg';
|
|
181
223
|
export { createPgEventBus } from './events-pg';
|
|
182
224
|
export type { ExecuteJobOptions, JobExecution, JobOutcome } from './execute';
|
|
@@ -193,6 +235,7 @@ export {
|
|
|
193
235
|
exportPartKey,
|
|
194
236
|
memoryExportSink,
|
|
195
237
|
} from './export-sink';
|
|
238
|
+
export { IDLE_POLL_CEILING_MS } from './idle-backoff';
|
|
196
239
|
export type {
|
|
197
240
|
DeadLetterEntry,
|
|
198
241
|
JobsManifest,
|
|
@@ -209,9 +252,45 @@ export {
|
|
|
209
252
|
inspectQueues,
|
|
210
253
|
retryFromStep,
|
|
211
254
|
} from './inspect';
|
|
212
|
-
export
|
|
255
|
+
export { pauseQueue, promoteJob, removeJob, resumeQueue } from './inspect-operator';
|
|
256
|
+
export type {
|
|
257
|
+
BulkFilter,
|
|
258
|
+
BulkResult,
|
|
259
|
+
CounterBucketMs,
|
|
260
|
+
CounterOutcome,
|
|
261
|
+
CounterTotals,
|
|
262
|
+
JobCounter,
|
|
263
|
+
JobFilter,
|
|
264
|
+
JobIntrospection,
|
|
265
|
+
JobProgress,
|
|
266
|
+
PausedName,
|
|
267
|
+
TaskFire,
|
|
268
|
+
WorkerAnnouncement,
|
|
269
|
+
WorkerRecord,
|
|
270
|
+
} from './introspection';
|
|
271
|
+
export {
|
|
272
|
+
COUNTER_BUCKET_MS,
|
|
273
|
+
COUNTER_TIERS,
|
|
274
|
+
DEFAULT_JOB_PAGE,
|
|
275
|
+
jobCursor,
|
|
276
|
+
MAX_BULK_ROWS,
|
|
277
|
+
MAX_ERROR_STACK_LENGTH,
|
|
278
|
+
MAX_JOB_PAGE,
|
|
279
|
+
MAX_PROGRESS_NOTE_LENGTH,
|
|
280
|
+
MAX_TASK_FIRES,
|
|
281
|
+
MAX_WORKER_IN_FLIGHT,
|
|
282
|
+
PROGRESS_INTERVAL_MS,
|
|
283
|
+
PROMOTABLE_STATES,
|
|
284
|
+
} from './introspection';
|
|
285
|
+
export type {
|
|
286
|
+
AnyJobHandle,
|
|
287
|
+
JobActor,
|
|
288
|
+
JobDefinition,
|
|
289
|
+
JobHandle,
|
|
290
|
+
JobRunArgs,
|
|
291
|
+
} from './job';
|
|
213
292
|
export { describeJobs, getJob, isJobHandle, job, registeredJobs, resetJobs } from './job';
|
|
214
|
-
export type { HeldLease, LeaseStore, MemoryLeaseStoreOptions } from './leases';
|
|
293
|
+
export type { HeldLease, LeaseStore, MemoryLeaseStore, MemoryLeaseStoreOptions } from './leases';
|
|
215
294
|
export { createMemoryLeaseStore, jobLeaseKey } from './leases';
|
|
216
295
|
export type {
|
|
217
296
|
Lease,
|
|
@@ -253,6 +332,7 @@ export type { PgOutboxOptions } from './outbox-pg';
|
|
|
253
332
|
export { createPgOutboxStore } from './outbox-pg';
|
|
254
333
|
export type { OutboxRelay, RelayOptions } from './outbox-relay';
|
|
255
334
|
export { createOutboxRelay } from './outbox-relay';
|
|
335
|
+
export type { ProgressFn } from './progress';
|
|
256
336
|
export type {
|
|
257
337
|
PurgeDefinition,
|
|
258
338
|
PurgeInput,
|
|
@@ -261,26 +341,39 @@ export type {
|
|
|
261
341
|
PurgeTarget,
|
|
262
342
|
} from './purge';
|
|
263
343
|
export { DEFAULT_PURGE_CRON, purge } from './purge';
|
|
344
|
+
export type { PgListener, QueueWake, QueueWakeOptions } from './queue-wake';
|
|
345
|
+
export { startQueueWake } from './queue-wake';
|
|
346
|
+
export type { IntervalScheduler } from './renewal-timer';
|
|
264
347
|
export type { BackoffStrategy, Random, RetryDecision, RetryPolicy } from './retry';
|
|
265
|
-
export { backoffDelayMs, DEFAULT_RETRY, nextRetry, retrySchedule } from './retry';
|
|
348
|
+
export { backoffDelayMs, DEFAULT_RETRY, isFinalAttempt, nextRetry, retrySchedule } from './retry';
|
|
266
349
|
export type { JobRetryDecision, JobStopReason } from './retry-classification';
|
|
267
|
-
export { classifyThrown, nextRetryForError } from './retry-classification';
|
|
350
|
+
export { classifyThrown, failureForRow, nextRetryForError } from './retry-classification';
|
|
268
351
|
export type {
|
|
269
352
|
CronResolver,
|
|
270
353
|
DispatchedOccurrence,
|
|
271
|
-
LeaderElection,
|
|
272
354
|
Scheduler,
|
|
273
355
|
SchedulerOptions,
|
|
274
|
-
SchedulerState,
|
|
275
356
|
} from './scheduler';
|
|
276
|
-
export {
|
|
357
|
+
export {
|
|
358
|
+
COUNTER_ROLLUP_INTERVAL_MS,
|
|
359
|
+
createScheduler,
|
|
360
|
+
PAUSE_RECHECK_MS,
|
|
361
|
+
} from './scheduler';
|
|
362
|
+
export type { LeaderElection } from './scheduler-leader';
|
|
363
|
+
export { soleLeader } from './scheduler-leader';
|
|
364
|
+
export { nextTaskRun } from './scheduler-occurrences';
|
|
277
365
|
export type { PgLeaseLeaderOptions } from './scheduler-pg';
|
|
278
366
|
export {
|
|
279
367
|
createPgLeaseLeader,
|
|
280
368
|
currentLeader,
|
|
281
369
|
DEFAULT_LEADER_TTL_MS,
|
|
370
|
+
LEASE_RENEWALS_PER_TTL,
|
|
282
371
|
pgSchedulerState,
|
|
283
372
|
} from './scheduler-pg';
|
|
373
|
+
export type { ScheduledFire, SchedulerState } from './scheduler-state';
|
|
374
|
+
export { createMemorySchedulerState, fireThroughDriver } from './scheduler-state';
|
|
375
|
+
export type { JobCompleted, JobFailed, JobSettled } from './settled';
|
|
376
|
+
export { ON_SETTLED_ATTEMPTS } from './settled';
|
|
284
377
|
export type {
|
|
285
378
|
EventLookup,
|
|
286
379
|
StepApi,
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
// The operator verbs `x jobs` and a dashboard bind to: remove, promote, pause, resume. Each one
|
|
2
|
+
// refuses LOUDLY rather than answering "nothing happened" — an operator clearing a queue has to
|
|
3
|
+
// know whether the job is gone or was never theirs to remove. Apart from `inspect.ts`, which is
|
|
4
|
+
// the read side.
|
|
5
|
+
|
|
6
|
+
import type { JobDriver, JobRecord } from './driver';
|
|
7
|
+
import { JobsNotImplementedError } from './errors';
|
|
8
|
+
import { JobNotPromotableError } from './errors-operator';
|
|
9
|
+
import type { JobIntrospection, PausedName } from './introspection';
|
|
10
|
+
|
|
11
|
+
function operatorOf(driver: JobDriver): JobIntrospection {
|
|
12
|
+
if (driver.introspect === undefined) {
|
|
13
|
+
throw new JobsNotImplementedError({
|
|
14
|
+
feature: `the operator surface of the "${driver.name}" jobs driver`,
|
|
15
|
+
fix: 'call setJobDriver(createPgDriver()) at boot — the pg driver implements introspect — then: x jobs ls --json',
|
|
16
|
+
});
|
|
17
|
+
}
|
|
18
|
+
return driver.introspect;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** `undefined` for an id nobody queued; `X_JOB_NOT_REMOVABLE` for one a worker holds. */
|
|
22
|
+
export function removeJob(driver: JobDriver, jobId: string): Promise<JobRecord | undefined> {
|
|
23
|
+
return operatorOf(driver).remove(jobId);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** The promoted row. A job not waiting on its run time is `X_JOB_NOT_PROMOTABLE`, state named. */
|
|
27
|
+
export async function promoteJob(driver: JobDriver, jobId: string): Promise<JobRecord> {
|
|
28
|
+
const operator = operatorOf(driver);
|
|
29
|
+
const promoted = await operator.promote(jobId);
|
|
30
|
+
if (promoted !== undefined) return promoted;
|
|
31
|
+
const current = await operator.job(jobId);
|
|
32
|
+
throw new JobNotPromotableError({ jobId, state: current?.state ?? 'missing' });
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Pause, then read back: the answer is the list as the queue now holds it. */
|
|
36
|
+
export async function pauseQueue(driver: JobDriver, queue: string): Promise<readonly PausedName[]> {
|
|
37
|
+
const operator = operatorOf(driver);
|
|
38
|
+
await operator.pauseQueue(queue);
|
|
39
|
+
return operator.pausedQueues();
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export async function resumeQueue(
|
|
43
|
+
driver: JobDriver,
|
|
44
|
+
queue: string,
|
|
45
|
+
): Promise<readonly PausedName[]> {
|
|
46
|
+
const operator = operatorOf(driver);
|
|
47
|
+
await operator.resumeQueue(queue);
|
|
48
|
+
return operator.pausedQueues();
|
|
49
|
+
}
|
package/src/inspect.ts
CHANGED
|
@@ -4,9 +4,13 @@
|
|
|
4
4
|
|
|
5
5
|
import type { BackfillProgress } from './backfill-inspect';
|
|
6
6
|
import { backfillForRun } from './backfill-inspect';
|
|
7
|
-
import type {
|
|
7
|
+
import type { WhenBusy } from './concurrency';
|
|
8
|
+
import type { JobDriver, JobRecord, QueueStats } from './driver';
|
|
8
9
|
import { CancelUnsupportedError, JobNotCancellableError, JobsNotImplementedError } from './errors';
|
|
10
|
+
import type { JobFilter, JobProgress } from './introspection';
|
|
11
|
+
import type { AnyJobHandle } from './job';
|
|
9
12
|
import { registeredJobs } from './job';
|
|
13
|
+
import { redactInput } from './redact-input';
|
|
10
14
|
import { retrySchedule } from './retry';
|
|
11
15
|
import type { Scheduler } from './scheduler';
|
|
12
16
|
import type { StepRecord } from './steps';
|
|
@@ -20,6 +24,7 @@ export interface QueueDepthReport {
|
|
|
20
24
|
readonly delayed: number;
|
|
21
25
|
readonly running: number;
|
|
22
26
|
readonly suspended: number;
|
|
27
|
+
readonly failed: number;
|
|
23
28
|
readonly dead: number;
|
|
24
29
|
};
|
|
25
30
|
/** Oldest claimable job across all queues, in ms. The autoscaling signal. */
|
|
@@ -34,9 +39,10 @@ export async function inspectQueues(driver: JobDriver): Promise<QueueDepthReport
|
|
|
34
39
|
delayed: acc.delayed + queue.delayed,
|
|
35
40
|
running: acc.running + queue.running,
|
|
36
41
|
suspended: acc.suspended + queue.suspended,
|
|
42
|
+
failed: acc.failed + queue.failed,
|
|
37
43
|
dead: acc.dead + queue.dead,
|
|
38
44
|
}),
|
|
39
|
-
{ ready: 0, delayed: 0, running: 0, suspended: 0, dead: 0 },
|
|
45
|
+
{ ready: 0, delayed: 0, running: 0, suspended: 0, failed: 0, dead: 0 },
|
|
40
46
|
);
|
|
41
47
|
return {
|
|
42
48
|
driver: driver.name,
|
|
@@ -68,6 +74,15 @@ export interface JobTrace {
|
|
|
68
74
|
readonly runId: string;
|
|
69
75
|
readonly runAt: string;
|
|
70
76
|
readonly lastError: string | null;
|
|
77
|
+
/** The thrown value's stack for `lastError`, bounded. `null` when the failure carried none. */
|
|
78
|
+
readonly stack: string | null;
|
|
79
|
+
/**
|
|
80
|
+
* The stored payload, with every key the app declared secret (`redactKeys`) replaced. App data
|
|
81
|
+
* all the same: a surface that shows it gates it like any other read of a tenant's rows.
|
|
82
|
+
*/
|
|
83
|
+
readonly input: unknown;
|
|
84
|
+
/** What the body last reported through `progress()`; `null` until it reports. */
|
|
85
|
+
readonly progress: JobProgress | null;
|
|
71
86
|
readonly tenantId: string | null;
|
|
72
87
|
/** W3C `traceparent` of the request that queued it — paste it into the trace viewer. */
|
|
73
88
|
readonly traceparent: string | null;
|
|
@@ -76,6 +91,13 @@ export interface JobTrace {
|
|
|
76
91
|
readonly steps: readonly StepTrace[];
|
|
77
92
|
/** Remaining retry delays in ms, jitter excluded. */
|
|
78
93
|
readonly retryDelaysMs: readonly number[];
|
|
94
|
+
/**
|
|
95
|
+
* The concurrency key THIS run counts under — `concurrency.key(input)` — when its job declares a
|
|
96
|
+
* keyed cap. `null` for every other job, for a job this process has not registered, and for a
|
|
97
|
+
* stored input the key cannot be derived from: a trace is read to debug a stuck queue, so it
|
|
98
|
+
* degrades rather than refusing.
|
|
99
|
+
*/
|
|
100
|
+
readonly concurrencyKey: string | null;
|
|
79
101
|
/**
|
|
80
102
|
* The `x_backfills` row this run wrote, when the job is a `backfill()`. `null` for every other
|
|
81
103
|
* job and for a driver with no ledger — a step trace says which batch is next, and this says how
|
|
@@ -97,6 +119,15 @@ function requireIntrospection(driver: JobDriver): NonNullable<JobDriver['introsp
|
|
|
97
119
|
return driver.introspect;
|
|
98
120
|
}
|
|
99
121
|
|
|
122
|
+
function concurrencyKeyOf(handle: AnyJobHandle | undefined, record: JobRecord): string | null {
|
|
123
|
+
if (handle === undefined || handle.whenBusy === undefined) return null;
|
|
124
|
+
try {
|
|
125
|
+
return handle.concurrencyKeyFor(handle.parse(record.input)) ?? null;
|
|
126
|
+
} catch {
|
|
127
|
+
return null;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
100
131
|
function toStepTrace(record: StepRecord): StepTrace {
|
|
101
132
|
return {
|
|
102
133
|
name: record.name,
|
|
@@ -127,11 +158,15 @@ export async function inspectJob(driver: JobDriver, jobId: string): Promise<JobT
|
|
|
127
158
|
runId: record.runId,
|
|
128
159
|
runAt: new Date(record.runAt).toISOString(),
|
|
129
160
|
lastError: record.lastError ?? null,
|
|
161
|
+
stack: record.lastErrorStack ?? null,
|
|
162
|
+
input: redactInput(record.input),
|
|
163
|
+
progress: record.progress ?? null,
|
|
130
164
|
tenantId: record.tenantId ?? null,
|
|
131
165
|
traceparent: record.traceparent ?? null,
|
|
132
166
|
enqueuedBy: record.enqueuedBy ?? null,
|
|
133
167
|
steps: steps.map(toStepTrace),
|
|
134
168
|
retryDelaysMs: handle === undefined ? [] : [...retrySchedule(handle.retry)],
|
|
169
|
+
concurrencyKey: concurrencyKeyOf(handle, record),
|
|
135
170
|
backfill: backfill ?? null,
|
|
136
171
|
};
|
|
137
172
|
}
|
|
@@ -217,6 +252,9 @@ export interface JobsManifest {
|
|
|
217
252
|
readonly attempts: number;
|
|
218
253
|
readonly backoff: string;
|
|
219
254
|
readonly concurrency: number | null;
|
|
255
|
+
/** Non-null exactly when `concurrency` holds per key — see `JobDescriptor.concurrency`. */
|
|
256
|
+
readonly whenBusy: WhenBusy | null;
|
|
257
|
+
readonly onSettled: boolean;
|
|
220
258
|
readonly timeoutMs: number | null;
|
|
221
259
|
readonly retryDelaysMs: readonly number[];
|
|
222
260
|
}[];
|
|
@@ -239,6 +277,8 @@ export function inspectManifest(scheduler?: Scheduler): JobsManifest {
|
|
|
239
277
|
attempts: handle.retry.attempts,
|
|
240
278
|
backoff: handle.retry.backoff ?? 'exponential',
|
|
241
279
|
concurrency: handle.concurrency ?? null,
|
|
280
|
+
whenBusy: handle.whenBusy ?? null,
|
|
281
|
+
onSettled: handle.declaresOnSettled,
|
|
242
282
|
timeoutMs: handle.timeoutMs ?? null,
|
|
243
283
|
retryDelaysMs: [...retrySchedule(handle.retry)],
|
|
244
284
|
})),
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
// What an OPERATOR may ask of a queue, and the bounds on each answer. Every capability a
|
|
2
|
+
// dashboard, `x jobs` or an MCP tool has is a member of `JobIntrospection`, implemented by the pg
|
|
3
|
+
// and the memory driver alike, so a surface that reads only this file reads nothing a driver does
|
|
4
|
+
// not promise. Apart from `driver.ts` because that file is the six-method claim/settle contract;
|
|
5
|
+
// this one is everything said ABOUT the rows it moves.
|
|
6
|
+
|
|
7
|
+
import { finiteCount } from '@ultimat3/core';
|
|
8
|
+
import type { ClaimIdentity, JobRecord, JobState } from './driver';
|
|
9
|
+
import { JobPageInvalidError } from './errors-operator';
|
|
10
|
+
|
|
11
|
+
/** Rows one `list()` call may answer. A page, never a table: a dashboard walks with `after`. */
|
|
12
|
+
export const MAX_JOB_PAGE = 200;
|
|
13
|
+
export const DEFAULT_JOB_PAGE = 100;
|
|
14
|
+
/** Rows one `requeueMany` / `removeMany` call may touch. It answers how many remain. */
|
|
15
|
+
export const MAX_BULK_ROWS = 1_000;
|
|
16
|
+
/** Gap between two progress writes of ONE run. The last call before a settle is always written. */
|
|
17
|
+
export const PROGRESS_INTERVAL_MS = 1_000;
|
|
18
|
+
export const MAX_PROGRESS_NOTE_LENGTH = 200;
|
|
19
|
+
/** How much of a thrown value's stack a row keeps. A stack is a diagnosis, not an archive. */
|
|
20
|
+
export const MAX_ERROR_STACK_LENGTH = 8_000;
|
|
21
|
+
/** In-flight job ids one registry row lists. A worker holding more reports the first of them. */
|
|
22
|
+
export const MAX_WORKER_IN_FLIGHT = 100;
|
|
23
|
+
/** Rows `taskFires()` answers. A task renamed away leaves its row behind; a screen is not a table. */
|
|
24
|
+
export const MAX_TASK_FIRES = 1_000;
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* The counter tiers: a bucket width, and how long buckets of that width are kept. One-minute
|
|
28
|
+
* buckets for a day, five-minute for a week, hourly for thirty days — 1,440 + 2,016 + 720 rows per
|
|
29
|
+
* job name whatever the throughput, which is the whole point: history that costs the same at ten
|
|
30
|
+
* jobs an hour and at ten million.
|
|
31
|
+
*/
|
|
32
|
+
export const COUNTER_TIERS = Object.freeze([
|
|
33
|
+
Object.freeze({ bucketMs: 60_000, keepMs: 86_400_000 }),
|
|
34
|
+
Object.freeze({ bucketMs: 300_000, keepMs: 604_800_000 }),
|
|
35
|
+
Object.freeze({ bucketMs: 3_600_000, keepMs: 2_592_000_000 }),
|
|
36
|
+
] as const);
|
|
37
|
+
|
|
38
|
+
export type CounterBucketMs = (typeof COUNTER_TIERS)[number]['bucketMs'];
|
|
39
|
+
|
|
40
|
+
/** The tier every settle writes to. Coarser ones are filled by `rollupCounters` only. */
|
|
41
|
+
export const COUNTER_BUCKET_MS = COUNTER_TIERS[0].bucketMs;
|
|
42
|
+
|
|
43
|
+
export interface JobFilter {
|
|
44
|
+
readonly queue?: string;
|
|
45
|
+
readonly name?: string;
|
|
46
|
+
readonly state?: JobState;
|
|
47
|
+
/** Rows whose id STARTS with this — an operator pastes the first characters of one. */
|
|
48
|
+
readonly idPrefix?: string;
|
|
49
|
+
/** Epoch ms, inclusive. Rows created at or after it. */
|
|
50
|
+
readonly createdFrom?: number;
|
|
51
|
+
/** Epoch ms, exclusive. Rows created before it. */
|
|
52
|
+
readonly createdTo?: number;
|
|
53
|
+
/** 1 to `MAX_JOB_PAGE`. Default `DEFAULT_JOB_PAGE`. */
|
|
54
|
+
readonly limit?: number;
|
|
55
|
+
/**
|
|
56
|
+
* `jobCursor(lastRowOfThePreviousPage)`. Keyset, never an offset: the order is newest first by
|
|
57
|
+
* `(createdAt, id)`, so a row inserted while a walk is in flight sorts BEFORE every cursor and
|
|
58
|
+
* each row already there is visited exactly once.
|
|
59
|
+
*/
|
|
60
|
+
readonly after?: string;
|
|
61
|
+
/**
|
|
62
|
+
* `jobCursor(firstRowOfThisPage)` — the page BEFORE it: the `limit` rows nearest the cursor on
|
|
63
|
+
* its newer side, answered newest first like every page. What a dashboard's "previous" reads.
|
|
64
|
+
* Never with `after`.
|
|
65
|
+
*/
|
|
66
|
+
readonly before?: string;
|
|
67
|
+
/** Rows queued for this tenant only (`JobRecord.tenantId`). An org-scoped operator's list. */
|
|
68
|
+
readonly tenantId?: string;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const CURSOR = /^(\d{1,16}):([0-9A-Za-z-]{1,64})$/;
|
|
72
|
+
|
|
73
|
+
/** The cursor a page's LAST row yields. A page shorter than its limit is the last page. */
|
|
74
|
+
export function jobCursor(record: Pick<JobRecord, 'createdAt' | 'id'>): string {
|
|
75
|
+
return `${record.createdAt}:${record.id}`;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export function parseJobCursor(
|
|
79
|
+
after: string,
|
|
80
|
+
subject = 'the job list',
|
|
81
|
+
): { readonly createdAt: number; readonly id: string } {
|
|
82
|
+
const match = CURSOR.exec(after);
|
|
83
|
+
if (match === null) throw new JobPageInvalidError({ subject, maxPage: MAX_JOB_PAGE });
|
|
84
|
+
return { createdAt: Number(match[1]), id: match[2] ?? '' };
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** `limit`, defaulted and refused past the bound — a caller never widens a page by asking. */
|
|
88
|
+
export function pageLimit(subject: string, limit: number | undefined): number {
|
|
89
|
+
const asked = finiteCount(subject, 'limit', limit ?? DEFAULT_JOB_PAGE);
|
|
90
|
+
if (asked > MAX_JOB_PAGE) {
|
|
91
|
+
throw new JobPageInvalidError({ subject, maxPage: MAX_JOB_PAGE, limit: asked });
|
|
92
|
+
}
|
|
93
|
+
return asked;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Which rows a bulk call addresses. `state` is REQUIRED: "every job" is never one call. */
|
|
97
|
+
export interface BulkFilter {
|
|
98
|
+
readonly state: JobState;
|
|
99
|
+
readonly queue?: string;
|
|
100
|
+
readonly name?: string;
|
|
101
|
+
/** One tenant's rows only — what an org-scoped operator's "all matching" may touch. */
|
|
102
|
+
readonly tenantId?: string;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** The states `promoteMany` accepts: the two a row waiting on its `runAt` can be in. */
|
|
106
|
+
export const PROMOTABLE_STATES: ReadonlySet<JobState> = new Set<JobState>(['ready', 'delayed']);
|
|
107
|
+
|
|
108
|
+
export interface BulkResult {
|
|
109
|
+
/** Rows this call changed — at most `MAX_BULK_ROWS`. */
|
|
110
|
+
readonly affected: number;
|
|
111
|
+
/** Rows still matching the filter afterwards. Call again until it is zero. */
|
|
112
|
+
readonly remaining: number;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export interface PausedName {
|
|
116
|
+
readonly name: string;
|
|
117
|
+
/** Epoch ms. */
|
|
118
|
+
readonly pausedAt: number;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** What a worker says about itself. The registry adds `heartbeatAt`. */
|
|
122
|
+
export interface WorkerAnnouncement {
|
|
123
|
+
readonly id: string;
|
|
124
|
+
readonly host: string;
|
|
125
|
+
/** Epoch ms this worker started. */
|
|
126
|
+
readonly startedAt: number;
|
|
127
|
+
readonly queues: readonly string[];
|
|
128
|
+
/** Slots across every queue this worker serves. */
|
|
129
|
+
readonly concurrency: number;
|
|
130
|
+
/** Job ids it holds right now — the first `MAX_WORKER_IN_FLIGHT` of them. */
|
|
131
|
+
readonly inFlight: readonly string[];
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export interface WorkerRecord extends WorkerAnnouncement {
|
|
135
|
+
/** Epoch ms of the announcement that wrote this row. */
|
|
136
|
+
readonly heartbeatAt: number;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* The last occurrence of a task that DISPATCHED — queued its jobs. Arming a task first seen, or
|
|
141
|
+
* skipping occurrences a `catchUp` policy drops, moves the scheduler's watermark and is not one.
|
|
142
|
+
*/
|
|
143
|
+
export interface TaskFire {
|
|
144
|
+
readonly task: string;
|
|
145
|
+
/** Epoch ms the occurrence was scheduled for. */
|
|
146
|
+
readonly occurrenceMs: number;
|
|
147
|
+
/** Epoch ms it was dispatched, on the store's clock. */
|
|
148
|
+
readonly firedAt: number;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
export interface JobProgress {
|
|
152
|
+
readonly done: number;
|
|
153
|
+
readonly total: number;
|
|
154
|
+
readonly note?: string;
|
|
155
|
+
/** Epoch ms the body reported it. */
|
|
156
|
+
readonly at: number;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/** What one settle adds to its job's current bucket. Exactly one of the four counts is 1. */
|
|
160
|
+
export type CounterOutcome = 'done' | 'retried' | 'failed' | 'dead';
|
|
161
|
+
|
|
162
|
+
export interface JobCounter {
|
|
163
|
+
readonly job: string;
|
|
164
|
+
/** Epoch ms the bucket opens. */
|
|
165
|
+
readonly bucketStart: number;
|
|
166
|
+
readonly bucketMs: CounterBucketMs;
|
|
167
|
+
readonly done: number;
|
|
168
|
+
/** Attempts that failed and were handed back for another. */
|
|
169
|
+
readonly retried: number;
|
|
170
|
+
/** Runs that ended `failed`: refused by a busy key, or exhausted on a `deadLetter: false` job. */
|
|
171
|
+
readonly failed: number;
|
|
172
|
+
readonly dead: number;
|
|
173
|
+
/** Sum of the attempts' durations, so `durationMs / (done + retried + failed + dead)` is a mean. */
|
|
174
|
+
readonly durationMs: number;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
export type CounterTotals = Omit<JobCounter, 'bucketStart' | 'bucketMs'>;
|
|
178
|
+
|
|
179
|
+
/** A driver's operator surface. Every member is implemented by every driver that ships one. */
|
|
180
|
+
export interface JobIntrospection {
|
|
181
|
+
job(jobId: string): Promise<JobRecord | undefined>;
|
|
182
|
+
/** One page, newest first. See `JobFilter.after`. */
|
|
183
|
+
list(filter?: JobFilter): Promise<readonly JobRecord[]>;
|
|
184
|
+
deadLetters(limit?: number): Promise<readonly JobRecord[]>;
|
|
185
|
+
/**
|
|
186
|
+
* Re-queue a finished job (`REQUEUEABLE_STATES`); a live one is `X_JOB_NOT_REQUEUEABLE`, and a
|
|
187
|
+
* key a live job holds is `X_JOB_DUPLICATE`. `fromStep` drops that step and every step that
|
|
188
|
+
* started after it.
|
|
189
|
+
*/
|
|
190
|
+
requeue(jobId: string, options?: { readonly fromStep?: string }): Promise<JobRecord>;
|
|
191
|
+
/**
|
|
192
|
+
* Stop a job from outside. Terminal for a queued row immediately; a RUNNING one stops at its
|
|
193
|
+
* next heartbeat, which no longer matches its own row and cancels the attempt. Answers
|
|
194
|
+
* `undefined` for a job id it does not hold or one that already finished. Optional, as it
|
|
195
|
+
* shipped: a driver may have no way to address a single running row.
|
|
196
|
+
*/
|
|
197
|
+
cancel?(jobId: string, reason?: string): Promise<JobRecord | undefined>;
|
|
198
|
+
/**
|
|
199
|
+
* Delete one job and its step records. `undefined` for an id nobody queued. A RUNNING job is
|
|
200
|
+
* refused (`X_JOB_NOT_REMOVABLE`): a worker holds it, and deleting the row under a body does
|
|
201
|
+
* not stop the body — cancel it first.
|
|
202
|
+
*/
|
|
203
|
+
remove(jobId: string): Promise<JobRecord | undefined>;
|
|
204
|
+
/**
|
|
205
|
+
* Re-queue up to `MAX_BULK_ROWS` finished jobs. `filter.state` must be a requeueable state. A
|
|
206
|
+
* row whose key a live job holds is left where it is and counted in `remaining`.
|
|
207
|
+
*/
|
|
208
|
+
requeueMany(filter: BulkFilter): Promise<BulkResult>;
|
|
209
|
+
/** Delete up to `MAX_BULK_ROWS` jobs. `filter.state` may be any state but `running`. */
|
|
210
|
+
removeMany(filter: BulkFilter): Promise<BulkResult>;
|
|
211
|
+
/**
|
|
212
|
+
* Make a job waiting on its `runAt` — delayed at enqueue, or backing off before a retry — due
|
|
213
|
+
* now. `undefined` when the job is not waiting on a clock (missing, running, finished, or
|
|
214
|
+
* suspended in a `step.sleep`, whose wake time is the step's and not the row's).
|
|
215
|
+
*/
|
|
216
|
+
promote(jobId: string): Promise<JobRecord | undefined>;
|
|
217
|
+
/**
|
|
218
|
+
* `promote`, over up to `MAX_BULK_ROWS` rows: every one in `filter` still waiting on its
|
|
219
|
+
* `runAt` is due now. `filter.state` must be `ready` (a retry backing off) or `delayed`.
|
|
220
|
+
*/
|
|
221
|
+
promoteMany(filter: BulkFilter): Promise<BulkResult>;
|
|
222
|
+
/** A paused queue is never claimed, by any worker; enqueues still land. Idempotent. */
|
|
223
|
+
pauseQueue(queue: string): Promise<void>;
|
|
224
|
+
resumeQueue(queue: string): Promise<void>;
|
|
225
|
+
pausedQueues(): Promise<readonly PausedName[]>;
|
|
226
|
+
/**
|
|
227
|
+
* A paused task is not dispatched. On resume its own `catchUp` policy decides what the pause
|
|
228
|
+
* missed — exactly as if the scheduler had been down.
|
|
229
|
+
*/
|
|
230
|
+
pauseTask(task: string): Promise<void>;
|
|
231
|
+
resumeTask(task: string): Promise<void>;
|
|
232
|
+
pausedTasks(): Promise<readonly PausedName[]>;
|
|
233
|
+
/**
|
|
234
|
+
* The scheduler's record that an occurrence dispatched; the store stamps `firedAt`. The pg
|
|
235
|
+
* driver's own fire writes it in the firing statement (`SQL_SCHEDULER_FIRE`), so only a fire
|
|
236
|
+
* that goes through `driver.enqueue` calls this.
|
|
237
|
+
*/
|
|
238
|
+
recordTaskFire(fire: Pick<TaskFire, 'task' | 'occurrenceMs'>): Promise<void>;
|
|
239
|
+
/** Every task that has dispatched, by task name — the first `MAX_TASK_FIRES` of them. */
|
|
240
|
+
taskFires(): Promise<readonly TaskFire[]>;
|
|
241
|
+
/** A worker's heartbeat. The row expires `ttlMs` after the last one — never by cleanup. */
|
|
242
|
+
announceWorker(worker: WorkerAnnouncement, ttlMs: number): Promise<void>;
|
|
243
|
+
/** A clean stop hands the row back at once rather than leaving it to expire. */
|
|
244
|
+
forgetWorker(workerId: string): Promise<void>;
|
|
245
|
+
/** Every worker whose row has not expired, oldest start first. */
|
|
246
|
+
workers(): Promise<readonly WorkerRecord[]>;
|
|
247
|
+
/** Fenced on the CLAIM, as a settle is: progress from one the row no longer carries is dropped. */
|
|
248
|
+
recordProgress(jobId: string, by: ClaimIdentity, progress: JobProgress): Promise<void>;
|
|
249
|
+
/** One job's buckets from `sinceMs` on, oldest first, at whatever tier each still lives in. */
|
|
250
|
+
counters(query: {
|
|
251
|
+
readonly job: string;
|
|
252
|
+
readonly sinceMs: number;
|
|
253
|
+
}): Promise<readonly JobCounter[]>;
|
|
254
|
+
/** One row per job name: every bucket from `sinceMs` on, summed. Ordered by name. */
|
|
255
|
+
counterTotals(sinceMs: number): Promise<readonly CounterTotals[]>;
|
|
256
|
+
/**
|
|
257
|
+
* Fold buckets past their tier's `keepMs` into the next tier and drop the ones past the last.
|
|
258
|
+
* The scheduler leader calls it; it is idempotent, and safe if two nodes both do. Answers how
|
|
259
|
+
* many buckets it moved or dropped.
|
|
260
|
+
*/
|
|
261
|
+
rollupCounters(): Promise<number>;
|
|
262
|
+
}
|