@ultimat3/jobs 29.3.2 → 30.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +68 -89
- package/README.md +304 -10
- package/package.json +18 -6
- package/src/backfill-definition.ts +184 -0
- package/src/backfill-errors.ts +18 -2
- package/src/backfill-pass.ts +43 -40
- package/src/backfill-pending.ts +40 -17
- package/src/backfill-reader.ts +94 -0
- package/src/backfill-registry.ts +5 -0
- package/src/backfill-scope.ts +1 -2
- package/src/backfill-source.ts +113 -0
- package/src/backfill.ts +54 -135
- package/src/consumer-errors.ts +42 -0
- package/src/consumer-guard.ts +19 -0
- package/src/consumer-loop.ts +171 -0
- package/src/consumer-registry.ts +103 -0
- package/src/consumer-types.ts +81 -0
- package/src/consumer.ts +147 -0
- package/src/driver-memory-operator.ts +25 -2
- package/src/driver-memory.ts +35 -1
- package/src/driver-pg-claim-sql.ts +128 -0
- package/src/driver-pg-ddl.ts +11 -2
- package/src/driver-pg-demand-sql.ts +27 -0
- package/src/driver-pg-enqueue-many.ts +153 -0
- package/src/driver-pg-event-wake-sql.ts +51 -0
- package/src/driver-pg-operator-sql.ts +34 -6
- package/src/driver-pg-operator.ts +36 -2
- package/src/driver-pg-outbox-sql.ts +34 -0
- package/src/driver-pg-sql.ts +24 -91
- package/src/driver-pg.ts +17 -1
- package/src/driver-redis-refusal.ts +25 -0
- package/src/driver-redis-scripts.ts +79 -30
- package/src/driver-redis.ts +70 -42
- package/src/driver.ts +38 -1
- package/src/enqueue-record.ts +137 -0
- package/src/error-codes.ts +233 -0
- package/src/errors-concurrency.ts +2 -2
- package/src/errors-declaration.ts +15 -2
- package/src/errors-operator.ts +1 -1
- package/src/errors-requeue.ts +1 -1
- package/src/errors-saga.ts +87 -0
- package/src/errors-steps.ts +35 -0
- package/src/errors-tenant.ts +1 -1
- package/src/errors.ts +25 -165
- package/src/events-pg.ts +20 -8
- package/src/events.ts +14 -3
- package/src/execute-ctx.ts +51 -0
- package/src/execute-unwind.ts +191 -0
- package/src/execute.ts +50 -51
- package/src/export-errors.ts +1 -1
- package/src/failure-row.ts +43 -0
- package/src/import-errors.ts +102 -0
- package/src/import-format.ts +232 -0
- package/src/import-pass.ts +255 -0
- package/src/import-source.ts +49 -0
- package/src/import.ts +175 -0
- package/src/index.ts +163 -7
- package/src/inspect-operator.ts +24 -3
- package/src/inspect.ts +17 -4
- package/src/introspection.ts +24 -1
- package/src/job-enqueue-many.ts +108 -0
- package/src/job-types.ts +229 -0
- package/src/job.ts +19 -208
- package/src/metrics.ts +10 -0
- package/src/outbox-pg.ts +35 -0
- package/src/outbox-relay.ts +58 -19
- package/src/outbox.ts +65 -73
- package/src/purge-entities.ts +92 -0
- package/src/purge.ts +10 -2
- package/src/queue-wake.ts +1 -1
- package/src/refresh-view.ts +76 -0
- package/src/retry-classification.ts +27 -8
- package/src/retry.ts +26 -0
- package/src/rollup-errors.ts +27 -0
- package/src/rollup.ts +190 -0
- package/src/run-deadline.ts +11 -7
- package/src/scheduler.ts +2 -1
- package/src/settled.ts +56 -15
- package/src/step-resolve.ts +37 -0
- package/src/steps-api.ts +111 -0
- package/src/steps-core.ts +26 -0
- package/src/steps-output.ts +142 -0
- package/src/steps-store.ts +68 -0
- package/src/steps-suspension.ts +51 -0
- package/src/steps-timeout.ts +10 -5
- package/src/steps-unwind.ts +94 -0
- package/src/steps-wait.ts +163 -0
- package/src/steps.ts +174 -210
- package/src/subject-erase-pass.ts +134 -0
- package/src/subject-errors.ts +48 -0
- package/src/subject-export-pass.ts +228 -0
- package/src/subject-jobs.ts +121 -0
- package/src/subject-plan.ts +287 -0
- package/src/subject-select.ts +106 -0
- package/src/subject.ts +119 -0
- package/src/tenant.ts +69 -1
- package/src/type-pins.ts +90 -0
- package/src/webhook-errors.ts +1 -1
- package/src/webhook-receiver-errors.ts +59 -0
- package/src/webhook-receiver-route.ts +127 -0
- package/src/webhook-receiver-types.ts +97 -0
- package/src/webhook-receiver.ts +132 -0
- package/src/webhook-target.ts +20 -43
- package/src/webhook.ts +7 -4
- package/src/worker-queue-demand.ts +57 -0
- package/src/worker-queue-depth.ts +15 -3
- package/src/worker-registry.ts +4 -1
- package/src/worker-types.ts +2 -0
- package/src/worker.ts +2 -2
|
@@ -30,40 +30,70 @@ redis.call('ZREM', P .. ':run:' .. queue, id)
|
|
|
30
30
|
`;
|
|
31
31
|
|
|
32
32
|
/**
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
33
|
+
* One enqueue, as a Lua function both enqueue scripts call, so the single and the batch insert are
|
|
34
|
+
* ONE body. `a[o]` onwards, `ENQUEUE_ARITY` fields: id, name, queue, input, idempotency key,
|
|
35
|
+
* runId, maxAttempts, runAt, now, tenant, traceparent, enqueuedBy, namespace field. Answers
|
|
36
|
+
* `{ outcome, id, runId }`: `published` when the caller-allocated id already names a row
|
|
37
|
+
* (`SQL_ENQUEUE`'s `not exists`), `deduped` when a live row holds the key, `created` otherwise.
|
|
37
38
|
*/
|
|
38
|
-
|
|
39
|
+
const ENQUEUE_ONE = `
|
|
39
40
|
${LIVE}
|
|
40
|
-
local P
|
|
41
|
-
local id =
|
|
42
|
-
local job = P .. ':job:' .. id
|
|
43
|
-
if redis.call('EXISTS', job) == 1 then
|
|
44
|
-
|
|
41
|
+
local function enqueue_one(P, a, o)
|
|
42
|
+
local id = a[o]
|
|
43
|
+
local job = P .. ':job:' .. id
|
|
44
|
+
if redis.call('EXISTS', job) == 1 then
|
|
45
|
+
return { 'published', id, redis.call('HGET', job, 'runId') }
|
|
46
|
+
end
|
|
47
|
+
local nk = a[o + 12]
|
|
48
|
+
local holder = redis.call('HGET', P .. ':idem', nk)
|
|
49
|
+
if holder then
|
|
50
|
+
local row = redis.call('HMGET', P .. ':job:' .. holder, 'state', 'runId')
|
|
51
|
+
if row[1] and LIVE[row[1]] then return { 'deduped', holder, row[2] } end
|
|
52
|
+
end
|
|
53
|
+
local now = tonumber(a[o + 8])
|
|
54
|
+
local runAt = tonumber(a[o + 7])
|
|
55
|
+
local state = 'ready'
|
|
56
|
+
if runAt > now then state = 'delayed' end
|
|
57
|
+
local queue = a[o + 2]
|
|
58
|
+
redis.call('HSET', job,
|
|
59
|
+
'id', id, 'name', a[o + 1], 'queue', queue, 'input', a[o + 3], 'idempotencyKey', a[o + 4],
|
|
60
|
+
'runId', a[o + 5], 'attempt', '0', 'maxAttempts', a[o + 6], 'state', state,
|
|
61
|
+
'runAt', a[o + 7], 'createdAt', a[o + 8], 'updatedAt', a[o + 8], 'claim', '0', 'nk', nk)
|
|
62
|
+
if a[o + 9] ~= '' then redis.call('HSET', job, 'tenantId', a[o + 9]) end
|
|
63
|
+
if a[o + 10] ~= '' then redis.call('HSET', job, 'traceparent', a[o + 10]) end
|
|
64
|
+
if a[o + 11] ~= '' then redis.call('HSET', job, 'enqueuedBy', a[o + 11]) end
|
|
65
|
+
redis.call('ZADD', P .. ':wait:' .. queue, runAt, id)
|
|
66
|
+
redis.call('SADD', P .. ':queues', queue)
|
|
67
|
+
redis.call('HSET', P .. ':idem', nk, id)
|
|
68
|
+
return { 'created', id, a[o + 5] }
|
|
45
69
|
end
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
70
|
+
`;
|
|
71
|
+
|
|
72
|
+
/** Fields one request binds, after the namespace — what `ENQUEUE_ONE` reads from `a[o]` on. */
|
|
73
|
+
export const ENQUEUE_ARITY = 13;
|
|
74
|
+
|
|
75
|
+
/** `SQL_ENQUEUE`. `ARGV`: namespace, then one request's `ENQUEUE_ARITY` fields. */
|
|
76
|
+
export const REDIS_ENQUEUE = `
|
|
77
|
+
${ENQUEUE_ONE}
|
|
78
|
+
return enqueue_one(ARGV[1], ARGV, 2)
|
|
79
|
+
`.trim();
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* `SQL_ENQUEUE_MANY`: every request in ONE script, so the batch is atomic on the server and a key
|
|
83
|
+
* twice in it is one job — the second request meets the first's `idem` entry. `ARGV`: namespace,
|
|
84
|
+
* then `ENQUEUE_ARITY` fields per request, in request order. Answers the three fields
|
|
85
|
+
* `enqueue_one` answers, per request, flat and in order.
|
|
86
|
+
*/
|
|
87
|
+
export const REDIS_ENQUEUE_MANY = `
|
|
88
|
+
${ENQUEUE_ONE}
|
|
89
|
+
local out = {}
|
|
90
|
+
for o = 2, #ARGV, ${ENQUEUE_ARITY} do
|
|
91
|
+
local answer = enqueue_one(ARGV[1], ARGV, o)
|
|
92
|
+
table.insert(out, answer[1])
|
|
93
|
+
table.insert(out, answer[2])
|
|
94
|
+
table.insert(out, answer[3])
|
|
51
95
|
end
|
|
52
|
-
|
|
53
|
-
local runAt = tonumber(ARGV[9])
|
|
54
|
-
local state = 'ready'
|
|
55
|
-
if runAt > now then state = 'delayed' end
|
|
56
|
-
redis.call('HSET', job,
|
|
57
|
-
'id', id, 'name', ARGV[3], 'queue', ARGV[4], 'input', ARGV[5], 'idempotencyKey', ARGV[6],
|
|
58
|
-
'runId', ARGV[7], 'attempt', '0', 'maxAttempts', ARGV[8], 'state', state, 'runAt', ARGV[9],
|
|
59
|
-
'createdAt', ARGV[10], 'updatedAt', ARGV[10], 'claim', '0', 'nk', nk)
|
|
60
|
-
if ARGV[11] ~= '' then redis.call('HSET', job, 'tenantId', ARGV[11]) end
|
|
61
|
-
if ARGV[12] ~= '' then redis.call('HSET', job, 'traceparent', ARGV[12]) end
|
|
62
|
-
if ARGV[13] ~= '' then redis.call('HSET', job, 'enqueuedBy', ARGV[13]) end
|
|
63
|
-
redis.call('ZADD', P .. ':wait:' .. ARGV[4], runAt, id)
|
|
64
|
-
redis.call('SADD', P .. ':queues', ARGV[4])
|
|
65
|
-
redis.call('HSET', P .. ':idem', nk, id)
|
|
66
|
-
return { 'created', id, ARGV[7] }
|
|
96
|
+
return out
|
|
67
97
|
`.trim();
|
|
68
98
|
|
|
69
99
|
/**
|
|
@@ -221,6 +251,25 @@ end
|
|
|
221
251
|
return out
|
|
222
252
|
`.trim();
|
|
223
253
|
|
|
254
|
+
/**
|
|
255
|
+
* `SQL_QUEUE_DEMAND`, one consistent read. `ARGV`: namespace, now. Answers `{ queue, demand }` per
|
|
256
|
+
* queue with some: every running row plus every due wait or park row. This driver keeps no worker
|
|
257
|
+
* registry and no pause, so no claimer is known to drain and no queue is paused: the rule
|
|
258
|
+
* `SQL_QUEUE_DEMAND` applies to a claimer with no live row, which counts.
|
|
259
|
+
*/
|
|
260
|
+
export const REDIS_DEMAND = `
|
|
261
|
+
local P = ARGV[1]
|
|
262
|
+
local now = tonumber(ARGV[2])
|
|
263
|
+
local out = {}
|
|
264
|
+
for _, queue in ipairs(redis.call('SMEMBERS', P .. ':queues')) do
|
|
265
|
+
local demand = redis.call('ZCARD', P .. ':run:' .. queue)
|
|
266
|
+
+ redis.call('ZCOUNT', P .. ':wait:' .. queue, '-inf', now)
|
|
267
|
+
+ redis.call('ZCOUNT', P .. ':park:' .. queue, '-inf', now)
|
|
268
|
+
if demand > 0 then table.insert(out, { queue, demand }) end
|
|
269
|
+
end
|
|
270
|
+
return out
|
|
271
|
+
`.trim();
|
|
272
|
+
|
|
224
273
|
/**
|
|
225
274
|
* `SQL_STEP_PUT` under a claim: the write lands only while that claim still holds the row.
|
|
226
275
|
* `ARGV`: namespace, job id, worker, claim, runId, step name, the record as JSON.
|
package/src/driver-redis.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
// A Redis queue on `
|
|
2
|
-
// the pg driver, held by `@ultimat3/testing`'s `jobDriverConformance`: every operation
|
|
3
|
-
// script (`driver-redis-scripts.ts`), so a claim, a settle and a renewal are each atomic
|
|
4
|
-
// server the way a `driver-pg-sql.ts` statement is in one transaction. Its own entry,
|
|
1
|
+
// A Redis queue on core's `redisClient()` — no client dependency, the runtime ships one. The same
|
|
2
|
+
// contract as the pg driver, held by `@ultimat3/testing`'s `jobDriverConformance`: every operation
|
|
3
|
+
// is one Lua script (`driver-redis-scripts.ts`), so a claim, a settle and a renewal are each atomic
|
|
4
|
+
// on the server the way a `driver-pg-sql.ts` statement is in one transaction. Its own entry,
|
|
5
5
|
// `@ultimat3/jobs/redis`: the barrel is on every role's boot path, and an app on Postgres loads none
|
|
6
6
|
// of this.
|
|
7
7
|
//
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
|
|
15
15
|
import type { Clock } from '@ultimat3/core';
|
|
16
16
|
import { finiteCount, finiteOption, systemClock, uuidV7 } from '@ultimat3/core';
|
|
17
|
+
import { redisClient } from '@ultimat3/core/redis';
|
|
17
18
|
import { nowMs } from './clock';
|
|
18
19
|
import type {
|
|
19
20
|
ClaimedJob,
|
|
@@ -21,28 +22,33 @@ import type {
|
|
|
21
22
|
EnqueueResult,
|
|
22
23
|
JobDriver,
|
|
23
24
|
JobRecord,
|
|
25
|
+
QueueDemand,
|
|
24
26
|
QueueStats,
|
|
25
27
|
} from './driver';
|
|
26
28
|
import {
|
|
27
29
|
assertClaimBounds,
|
|
28
30
|
assertClaimQueues,
|
|
31
|
+
assertEnqueueMany,
|
|
29
32
|
DEFAULT_QUEUE,
|
|
30
33
|
isJobState,
|
|
31
34
|
nackState,
|
|
32
35
|
} from './driver';
|
|
36
|
+
import { typedRefusals } from './driver-redis-refusal';
|
|
33
37
|
import {
|
|
34
38
|
REDIS_ACK,
|
|
35
39
|
REDIS_CLAIM,
|
|
40
|
+
REDIS_DEMAND,
|
|
36
41
|
REDIS_ENQUEUE,
|
|
42
|
+
REDIS_ENQUEUE_MANY,
|
|
37
43
|
REDIS_HEARTBEAT,
|
|
38
44
|
REDIS_NACK,
|
|
39
45
|
REDIS_STATS,
|
|
40
46
|
} from './driver-redis-scripts';
|
|
41
47
|
import { redisStepStore } from './driver-redis-steps';
|
|
42
|
-
import {
|
|
48
|
+
import { JobDuplicateError } from './errors';
|
|
43
49
|
import { MAX_ERROR_STACK_LENGTH } from './introspection';
|
|
44
50
|
|
|
45
|
-
/** The slice of Bun's Redis client this driver uses — `
|
|
51
|
+
/** The slice of Bun's Redis client this driver uses — `redisClient()` and `new Bun.RedisClient()`. */
|
|
46
52
|
export interface RedisSender {
|
|
47
53
|
send(command: string, args: string[]): Promise<unknown>;
|
|
48
54
|
}
|
|
@@ -51,7 +57,11 @@ export interface RedisSender {
|
|
|
51
57
|
export const DEFAULT_REDIS_DONE_TTL_MS = 7 * 24 * 60 * 60 * 1000;
|
|
52
58
|
|
|
53
59
|
export interface RedisJobDriverOptions {
|
|
54
|
-
/**
|
|
60
|
+
/**
|
|
61
|
+
* Injected in tests and for a second server; production takes `redisClient()` from
|
|
62
|
+
* `@ultimat3/core/redis` (`REDIS_URL`): while it is down every command refuses at once, as
|
|
63
|
+
* `X_DRIVER_UNAVAILABLE`. No deadline here: a stuck command has the request's.
|
|
64
|
+
*/
|
|
55
65
|
readonly client?: RedisSender;
|
|
56
66
|
/** Key namespace, the hash tag every key carries. Default `x:jobs`. */
|
|
57
67
|
readonly prefix?: string;
|
|
@@ -64,19 +74,6 @@ export interface RedisJobDriverOptions {
|
|
|
64
74
|
readonly doneTtlMs?: number;
|
|
65
75
|
}
|
|
66
76
|
|
|
67
|
-
function resolveClient(injected: RedisSender | undefined): RedisSender {
|
|
68
|
-
if (injected !== undefined) return injected;
|
|
69
|
-
const ambient = (Bun as unknown as { redis?: RedisSender }).redis;
|
|
70
|
-
if (ambient === undefined || typeof ambient.send !== 'function') {
|
|
71
|
-
throw new DriverUnavailableError({
|
|
72
|
-
driver: 'redis',
|
|
73
|
-
cause: 'Bun.redis is not available, and no client was handed to redisJobDriver()',
|
|
74
|
-
fix: 'REDIS_URL=redis://127.0.0.1:6379 x dev — or hand one in: redisJobDriver({ client: new Bun.RedisClient(url) })',
|
|
75
|
-
});
|
|
76
|
-
}
|
|
77
|
-
return ambient;
|
|
78
|
-
}
|
|
79
|
-
|
|
80
77
|
/** A Lua reply as an array; anything else is the empty one. */
|
|
81
78
|
const list = (reply: unknown): readonly unknown[] => (Array.isArray(reply) ? reply : []);
|
|
82
79
|
|
|
@@ -134,8 +131,33 @@ function toRecord(fields: ReadonlyMap<string, string>): JobRecord {
|
|
|
134
131
|
const namespaceField = (request: EnqueueRequest): string =>
|
|
135
132
|
JSON.stringify([request.name, request.tenantId ?? '', request.idempotencyKey]);
|
|
136
133
|
|
|
134
|
+
/** One request as `ENQUEUE_ONE` reads it: `ENQUEUE_ARITY` fields, in its order. */
|
|
135
|
+
function enqueueArgs(request: EnqueueRequest, runId: string, at: number): (string | number)[] {
|
|
136
|
+
return [
|
|
137
|
+
request.id ?? uuidV7(),
|
|
138
|
+
request.name,
|
|
139
|
+
request.queue || DEFAULT_QUEUE,
|
|
140
|
+
JSON.stringify(request.input ?? null),
|
|
141
|
+
request.idempotencyKey,
|
|
142
|
+
runId,
|
|
143
|
+
request.maxAttempts,
|
|
144
|
+
request.runAt ?? at,
|
|
145
|
+
at,
|
|
146
|
+
request.tenantId ?? '',
|
|
147
|
+
request.traceparent ?? '',
|
|
148
|
+
request.enqueuedBy ?? '',
|
|
149
|
+
namespaceField(request),
|
|
150
|
+
];
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/** The `{ outcome, id, runId }` triple at `at` in a reply, as the result `enqueue` answers. */
|
|
154
|
+
function answerOf(reply: readonly unknown[], at: number, runId: string): EnqueueResult {
|
|
155
|
+
const [outcome, id, heldRunId] = reply.slice(at, at + 3).map(String);
|
|
156
|
+
return { id: id ?? '', runId: heldRunId ?? runId, deduped: outcome !== 'created' };
|
|
157
|
+
}
|
|
158
|
+
|
|
137
159
|
export function redisJobDriver(options: RedisJobDriverOptions = {}): JobDriver {
|
|
138
|
-
const client =
|
|
160
|
+
const client = typedRefusals(options.client ?? redisClient());
|
|
139
161
|
const clock = options.clock ?? systemClock;
|
|
140
162
|
const ns = `{${options.prefix ?? 'x:jobs'}}`;
|
|
141
163
|
const doneTtlMs = finiteCount(
|
|
@@ -152,33 +174,31 @@ export function redisJobDriver(options: RedisJobDriverOptions = {}): JobDriver {
|
|
|
152
174
|
steps: redisStepStore({ ns, run, client }),
|
|
153
175
|
|
|
154
176
|
async enqueue(request): Promise<EnqueueResult> {
|
|
155
|
-
const at = nowMs(clock);
|
|
156
177
|
const runId = request.runId ?? uuidV7();
|
|
157
|
-
const
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
request.name,
|
|
161
|
-
request.queue || DEFAULT_QUEUE,
|
|
162
|
-
JSON.stringify(request.input ?? null),
|
|
163
|
-
request.idempotencyKey,
|
|
164
|
-
runId,
|
|
165
|
-
request.maxAttempts,
|
|
166
|
-
request.runAt ?? at,
|
|
167
|
-
at,
|
|
168
|
-
request.tenantId ?? '',
|
|
169
|
-
request.traceparent ?? '',
|
|
170
|
-
request.enqueuedBy ?? '',
|
|
171
|
-
namespaceField(request),
|
|
172
|
-
]),
|
|
173
|
-
).map(String);
|
|
174
|
-
if (outcome === 'deduped' && request.onConflict === 'error') {
|
|
178
|
+
const reply = list(await run(REDIS_ENQUEUE, enqueueArgs(request, runId, nowMs(clock))));
|
|
179
|
+
const answer = answerOf(reply, 0, runId);
|
|
180
|
+
if (reply[0] === 'deduped' && request.onConflict === 'error') {
|
|
175
181
|
throw new JobDuplicateError({
|
|
176
182
|
job: request.name,
|
|
177
183
|
idempotencyKey: request.idempotencyKey,
|
|
178
|
-
existingId: id
|
|
184
|
+
existingId: answer.id,
|
|
179
185
|
});
|
|
180
186
|
}
|
|
181
|
-
return
|
|
187
|
+
return answer;
|
|
188
|
+
},
|
|
189
|
+
|
|
190
|
+
// ONE script for the batch (`REDIS_ENQUEUE_MANY`), the body `enqueue` runs once per request:
|
|
191
|
+
// atomic on the server, and a key twice in it meets its own first request.
|
|
192
|
+
async enqueueMany(requests): Promise<readonly EnqueueResult[]> {
|
|
193
|
+
assertEnqueueMany('redis', requests);
|
|
194
|
+
if (requests.length === 0) return [];
|
|
195
|
+
const at = nowMs(clock);
|
|
196
|
+
const runIds = requests.map((request) => request.runId ?? uuidV7());
|
|
197
|
+
const args = requests.flatMap((request, index) =>
|
|
198
|
+
enqueueArgs(request, runIds[index] ?? uuidV7(), at),
|
|
199
|
+
);
|
|
200
|
+
const reply = list(await run(REDIS_ENQUEUE_MANY, args));
|
|
201
|
+
return runIds.map((runId, index) => answerOf(reply, index * 3, runId));
|
|
182
202
|
},
|
|
183
203
|
|
|
184
204
|
async claim(claimOptions): Promise<readonly ClaimedJob[]> {
|
|
@@ -274,5 +294,13 @@ export function redisJobDriver(options: RedisJobDriverOptions = {}): JobDriver {
|
|
|
274
294
|
// Code units, as `collate "C"` orders `SQL_STATS` — never `localeCompare`.
|
|
275
295
|
return rows.sort((a, b) => (a.queue < b.queue ? -1 : a.queue > b.queue ? 1 : 0));
|
|
276
296
|
},
|
|
297
|
+
|
|
298
|
+
async demand(): Promise<readonly QueueDemand[]> {
|
|
299
|
+
const rows = list(await run(REDIS_DEMAND, [nowMs(clock)])).map((row): QueueDemand => {
|
|
300
|
+
const [queue, demand] = list(row);
|
|
301
|
+
return { queue: String(queue), demand: Number(demand) };
|
|
302
|
+
});
|
|
303
|
+
return rows.sort((a, b) => (a.queue < b.queue ? -1 : a.queue > b.queue ? 1 : 0));
|
|
304
|
+
},
|
|
277
305
|
};
|
|
278
306
|
}
|
package/src/driver.ts
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
// for real. Swapping the driver is `setJobDriver(other)` and ZERO job-code change, and that
|
|
10
10
|
// is what the interface buys.
|
|
11
11
|
|
|
12
|
-
import { finiteCount, finiteOption } from '@ultimat3/core';
|
|
12
|
+
import { assert, finiteCount, finiteOption } from '@ultimat3/core';
|
|
13
13
|
import type { BackfillLedger } from './backfill-ledger';
|
|
14
14
|
import { ClaimQueuesEmptyError } from './errors';
|
|
15
15
|
import type { JobIntrospection, JobProgress } from './introspection';
|
|
@@ -305,6 +305,12 @@ export interface QueueStats {
|
|
|
305
305
|
readonly oldestReadyMs: number;
|
|
306
306
|
}
|
|
307
307
|
|
|
308
|
+
/** What one queue's fleet must hold: `JobDriver.demand`. */
|
|
309
|
+
export interface QueueDemand {
|
|
310
|
+
readonly queue: string;
|
|
311
|
+
readonly demand: number;
|
|
312
|
+
}
|
|
313
|
+
|
|
308
314
|
export interface HeartbeatOptions {
|
|
309
315
|
readonly visibilityTimeoutMs: number;
|
|
310
316
|
/** Renew only if this worker is still the claimant. Omit and any claimant matches. */
|
|
@@ -318,6 +324,13 @@ export interface JobDriver {
|
|
|
318
324
|
/** Step persistence lives with the queue: one store, one transaction boundary. */
|
|
319
325
|
readonly steps: StepStore;
|
|
320
326
|
enqueue(request: EnqueueRequest): Promise<EnqueueResult>;
|
|
327
|
+
/**
|
|
328
|
+
* `enqueue` for every request in ONE round trip — a statement on Postgres, a script on Redis —
|
|
329
|
+
* answered in request order with what `enqueue` would have answered each: a repeat of an `id` or
|
|
330
|
+
* a live key is `deduped: true`, a key twice in the batch is one job. REQUIRED, the one batch
|
|
331
|
+
* path `handle.enqueueMany` and the outbox relay take. Opens with `assertEnqueueMany`.
|
|
332
|
+
*/
|
|
333
|
+
enqueueMany(requests: readonly EnqueueRequest[]): Promise<readonly EnqueueResult[]>;
|
|
321
334
|
claim(options: ClaimOptions): Promise<readonly ClaimedJob[]>;
|
|
322
335
|
/**
|
|
323
336
|
* Both settles are fenced on `state = 'running'` AND on the claimer, and both answer whether
|
|
@@ -336,6 +349,14 @@ export interface JobDriver {
|
|
|
336
349
|
*/
|
|
337
350
|
heartbeat(jobId: string, options: HeartbeatOptions): Promise<boolean>;
|
|
338
351
|
stats(): Promise<readonly QueueStats[]>;
|
|
352
|
+
/**
|
|
353
|
+
* The jobs a live worker of this fleet must hold, per queue: running on a worker that still
|
|
354
|
+
* serves, plus every due row of an unpaused queue. Only a queue with some is listed. What
|
|
355
|
+
* `queue_demand` publishes and an autoscaler sizes by. Required since 30.0.0: `ready + running`
|
|
356
|
+
* off `stats()` cannot tell a draining worker's runs from a serving one's, and the HPA sized on it.
|
|
357
|
+
* A claimer no live registry row says drains is counted, so a driver with no registry counts all.
|
|
358
|
+
*/
|
|
359
|
+
demand(): Promise<readonly QueueDemand[]>;
|
|
339
360
|
/**
|
|
340
361
|
* Optional, like `introspect`: `x_backfills` records what a `backfill()` pass has already swept,
|
|
341
362
|
* and a driver without one runs backfills with no bookkeeping rather than refusing them. It
|
|
@@ -395,6 +416,22 @@ export const assertClaimQueues = (driver: string, options: ClaimOptions): void =
|
|
|
395
416
|
if (options.queues.length === 0) throw new ClaimQueuesEmptyError(driver);
|
|
396
417
|
};
|
|
397
418
|
|
|
419
|
+
/**
|
|
420
|
+
* Every driver's `enqueueMany` opens with this, so a batch is refused ONE way. `onConflict:
|
|
421
|
+
* 'error'` has no batch meaning the drivers could share: Postgres inserts every other row of the
|
|
422
|
+
* statement before the refusal is known, a loop only the rows before it — two different sets of
|
|
423
|
+
* jobs left behind by one refused call. A request that must refuse a duplicate is a single
|
|
424
|
+
* `enqueue`; `handle.enqueueMany` never sets it.
|
|
425
|
+
*/
|
|
426
|
+
export const assertEnqueueMany = (driver: string, requests: readonly EnqueueRequest[]): void => {
|
|
427
|
+
const at = requests.findIndex((request) => request.onConflict === 'error');
|
|
428
|
+
assert(
|
|
429
|
+
at === -1,
|
|
430
|
+
`the ${driver} driver's enqueueMany was handed onConflict: 'error' on request ${at}, and a batch that refuses a duplicate would leave a different set of jobs behind on every driver`,
|
|
431
|
+
"drop onConflict from the batch — a duplicate is answered deduped: true — or send that request alone: driver.enqueue({ ...request, onConflict: 'error' })",
|
|
432
|
+
);
|
|
433
|
+
};
|
|
434
|
+
|
|
398
435
|
/**
|
|
399
436
|
* The two NUMBERS of a claim, screened for both drivers in one place for the reason above: a value
|
|
400
437
|
* neither of them refuses is answered two ways. `limit: -1` sliced every ready row but the newest
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
// What one enqueue WRITES, built once for every path: the staged `OutboxRecord` and the direct
|
|
2
|
+
// `EnqueueRequest`, for `enqueue` and `enqueueMany` alike. Split from `outbox.ts` so the batch path
|
|
3
|
+
// builds the identical row the single one does, instead of a second spelling of it.
|
|
4
|
+
|
|
5
|
+
import { currentSpanContext, describeValue, traceparent, UltimateError } from '@ultimat3/core';
|
|
6
|
+
import type { EnqueueRequest } from './driver';
|
|
7
|
+
import { DEFAULT_QUEUE } from './driver';
|
|
8
|
+
import type { JobHandle } from './job';
|
|
9
|
+
import type { EnqueueOptions, OutboxRecord } from './outbox';
|
|
10
|
+
import { enqueuedAttempts } from './retry';
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* The W3C `traceparent` of the span this enqueue is happening inside, or `undefined` outside a
|
|
14
|
+
* trace. This is the ONE place the link is minted: `docs/idea/04-jobs.md` promises a job trace
|
|
15
|
+
* linked to the enqueuing request, and before this there was no field to carry the link, so a
|
|
16
|
+
* checkout's `chargeCard` opened a fresh root two seconds later with nothing pointing back.
|
|
17
|
+
*
|
|
18
|
+
* A context recovered from a `Ctx` has an empty `spanId` (`currentSpanContext`), which renders as
|
|
19
|
+
* an all-zero parent that every collector rejects — so that case carries no header rather than a
|
|
20
|
+
* malformed one, and the job opens a root as it did before.
|
|
21
|
+
*/
|
|
22
|
+
export function ambientTraceparent(): string | undefined {
|
|
23
|
+
const context = currentSpanContext();
|
|
24
|
+
if (context === undefined || context.spanId === '') return undefined;
|
|
25
|
+
return traceparent(context);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** What Postgres prints for a `uuid`, and so the only spelling both drivers store identically. */
|
|
29
|
+
const RUN_ID_SHAPE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* `EnqueueOptions.runId`, refused HERE when it is not a uuid — the one screen both enqueue paths
|
|
33
|
+
* read. `run_id` is a `uuid` column: the memory driver took any string, Postgres raised a raw
|
|
34
|
+
* `22P02`, and on the staged path that was the caller's whole transaction failing at the outbox
|
|
35
|
+
* insert. Any version, lowercase: an uppercase one would come back from Postgres re-spelled.
|
|
36
|
+
*/
|
|
37
|
+
export function namedRunId(job: string, runId: unknown): string | undefined {
|
|
38
|
+
if (runId === undefined) return undefined;
|
|
39
|
+
if (typeof runId === 'string' && RUN_ID_SHAPE.test(runId)) return runId;
|
|
40
|
+
throw new UltimateError({
|
|
41
|
+
code: 'X_ID_INVALID',
|
|
42
|
+
// `describeValue`, never the value: what arrives here wrong is as often a key as a typo.
|
|
43
|
+
cause: `enqueue of job "${job}" was given runId ${describeValue(runId)}, and a run id is a lowercase uuid (8-4-4-4-12 hex) — the queue stores it in a uuid column`,
|
|
44
|
+
fix: "pass { runId: uuidV7() } to enqueue(), with import { uuidV7 } from '@ultimat3/core'",
|
|
45
|
+
meta: { job, option: 'runId' },
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** The parts of one row the caller of the builders decides: its key, its clock, its ids. */
|
|
50
|
+
export interface RowFacts {
|
|
51
|
+
readonly idempotencyKey: string;
|
|
52
|
+
readonly at: number;
|
|
53
|
+
readonly trace: string | undefined;
|
|
54
|
+
readonly id?: string;
|
|
55
|
+
readonly runId?: string;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* The org the row is queued for: the one stated, else the one the payload names. Without the
|
|
60
|
+
* default a plain `enqueue(input)` queued for nobody, so org B's `invoice:1` deduped onto org A's
|
|
61
|
+
* job (the namespace is `(name, tenant, key)`) and `perTenant` counted every such row as no org's.
|
|
62
|
+
* The facade's `refuseEnqueueTenant` has already refused the two disagreeing.
|
|
63
|
+
*/
|
|
64
|
+
function queuedFor<I>(handle: JobHandle<I>, input: I, options: EnqueueOptions): string | undefined {
|
|
65
|
+
return options.tenantId ?? handle.tenantFor(input);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const optional = (
|
|
69
|
+
tenantId: string | undefined,
|
|
70
|
+
options: EnqueueOptions,
|
|
71
|
+
trace: string | undefined,
|
|
72
|
+
) => ({
|
|
73
|
+
...(tenantId === undefined ? {} : { tenantId }),
|
|
74
|
+
...(trace === undefined ? {} : { traceparent: trace }),
|
|
75
|
+
...(options.enqueuedBy === undefined ? {} : { enqueuedBy: options.enqueuedBy }),
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
/** The row a staged enqueue writes to the outbox; its `id` and `runId` are the job's to be. */
|
|
79
|
+
export function stagedRecord<I>(
|
|
80
|
+
handle: JobHandle<I>,
|
|
81
|
+
input: I,
|
|
82
|
+
options: EnqueueOptions,
|
|
83
|
+
facts: RowFacts & { readonly id: string; readonly runId: string },
|
|
84
|
+
): OutboxRecord {
|
|
85
|
+
return {
|
|
86
|
+
id: facts.id,
|
|
87
|
+
runId: facts.runId,
|
|
88
|
+
job: handle.name,
|
|
89
|
+
queue: options.queue ?? handle.queue ?? DEFAULT_QUEUE,
|
|
90
|
+
input,
|
|
91
|
+
idempotencyKey: facts.idempotencyKey,
|
|
92
|
+
maxAttempts: enqueuedAttempts(handle.retry, handle.compensates),
|
|
93
|
+
runAt: options.runAt ?? facts.at,
|
|
94
|
+
stagedAt: facts.at,
|
|
95
|
+
// Stamped at STAGE time and not at publish time: the relay runs after commit, in its own
|
|
96
|
+
// timer, with no request span in scope — a trace read there would be nobody's.
|
|
97
|
+
...optional(queuedFor(handle, input, options), options, facts.trace),
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** The request a direct enqueue hands the driver. Ids only when the caller allocated them. */
|
|
102
|
+
export function directRequest<I>(
|
|
103
|
+
handle: JobHandle<I>,
|
|
104
|
+
input: I,
|
|
105
|
+
options: EnqueueOptions,
|
|
106
|
+
facts: RowFacts,
|
|
107
|
+
): EnqueueRequest {
|
|
108
|
+
return {
|
|
109
|
+
...(facts.id === undefined ? {} : { id: facts.id }),
|
|
110
|
+
name: handle.name,
|
|
111
|
+
queue: options.queue ?? handle.queue,
|
|
112
|
+
input,
|
|
113
|
+
idempotencyKey: facts.idempotencyKey,
|
|
114
|
+
maxAttempts: enqueuedAttempts(handle.retry, handle.compensates),
|
|
115
|
+
runAt: options.runAt ?? facts.at,
|
|
116
|
+
...(facts.runId === undefined ? {} : { runId: facts.runId }),
|
|
117
|
+
...optional(queuedFor(handle, input, options), options, facts.trace),
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** The request the relay publishes a staged row as — under the ids the stage answered. */
|
|
122
|
+
export function publishRequest(record: OutboxRecord): EnqueueRequest {
|
|
123
|
+
return {
|
|
124
|
+
// A repeat of this publish — a crash before the mark — meets the job this one made.
|
|
125
|
+
id: record.id,
|
|
126
|
+
runId: record.runId,
|
|
127
|
+
name: record.job,
|
|
128
|
+
queue: record.queue,
|
|
129
|
+
input: record.input,
|
|
130
|
+
idempotencyKey: record.idempotencyKey,
|
|
131
|
+
maxAttempts: record.maxAttempts,
|
|
132
|
+
runAt: record.runAt,
|
|
133
|
+
...(record.tenantId === undefined ? {} : { tenantId: record.tenantId }),
|
|
134
|
+
...(record.traceparent === undefined ? {} : { traceparent: record.traceparent }),
|
|
135
|
+
...(record.enqueuedBy === undefined ? {} : { enqueuedBy: record.enqueuedBy }),
|
|
136
|
+
};
|
|
137
|
+
}
|