@ultimat3/jobs 29.4.0 → 30.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/CLAUDE.md +68 -89
  2. package/README.md +304 -10
  3. package/package.json +18 -6
  4. package/src/backfill-definition.ts +184 -0
  5. package/src/backfill-errors.ts +18 -2
  6. package/src/backfill-pass.ts +43 -40
  7. package/src/backfill-pending.ts +40 -17
  8. package/src/backfill-reader.ts +94 -0
  9. package/src/backfill-registry.ts +5 -0
  10. package/src/backfill-scope.ts +1 -2
  11. package/src/backfill-source.ts +113 -0
  12. package/src/backfill.ts +54 -135
  13. package/src/consumer-errors.ts +42 -0
  14. package/src/consumer-guard.ts +19 -0
  15. package/src/consumer-loop.ts +171 -0
  16. package/src/consumer-registry.ts +103 -0
  17. package/src/consumer-types.ts +81 -0
  18. package/src/consumer.ts +147 -0
  19. package/src/driver-memory-operator.ts +25 -2
  20. package/src/driver-memory.ts +35 -1
  21. package/src/driver-pg-claim-sql.ts +128 -0
  22. package/src/driver-pg-ddl.ts +11 -2
  23. package/src/driver-pg-demand-sql.ts +27 -0
  24. package/src/driver-pg-enqueue-many.ts +153 -0
  25. package/src/driver-pg-event-wake-sql.ts +51 -0
  26. package/src/driver-pg-operator-sql.ts +34 -6
  27. package/src/driver-pg-operator.ts +36 -2
  28. package/src/driver-pg-outbox-sql.ts +34 -0
  29. package/src/driver-pg-sql.ts +24 -91
  30. package/src/driver-pg.ts +17 -1
  31. package/src/driver-redis-refusal.ts +25 -0
  32. package/src/driver-redis-scripts.ts +79 -30
  33. package/src/driver-redis.ts +70 -42
  34. package/src/driver.ts +38 -1
  35. package/src/enqueue-record.ts +137 -0
  36. package/src/error-codes.ts +233 -0
  37. package/src/errors-concurrency.ts +2 -2
  38. package/src/errors-declaration.ts +15 -2
  39. package/src/errors-operator.ts +1 -1
  40. package/src/errors-requeue.ts +1 -1
  41. package/src/errors-saga.ts +87 -0
  42. package/src/errors-steps.ts +35 -0
  43. package/src/errors-tenant.ts +1 -1
  44. package/src/errors.ts +25 -165
  45. package/src/events-pg.ts +20 -8
  46. package/src/events.ts +14 -3
  47. package/src/execute-ctx.ts +51 -0
  48. package/src/execute-unwind.ts +191 -0
  49. package/src/execute.ts +50 -51
  50. package/src/export-errors.ts +1 -1
  51. package/src/failure-row.ts +43 -0
  52. package/src/import-errors.ts +102 -0
  53. package/src/import-format.ts +232 -0
  54. package/src/import-pass.ts +255 -0
  55. package/src/import-source.ts +49 -0
  56. package/src/import.ts +175 -0
  57. package/src/index.ts +163 -7
  58. package/src/inspect-operator.ts +24 -3
  59. package/src/inspect.ts +17 -4
  60. package/src/introspection.ts +24 -1
  61. package/src/job-enqueue-many.ts +108 -0
  62. package/src/job-types.ts +229 -0
  63. package/src/job.ts +19 -208
  64. package/src/metrics.ts +10 -0
  65. package/src/outbox-pg.ts +35 -0
  66. package/src/outbox-relay.ts +58 -19
  67. package/src/outbox.ts +65 -73
  68. package/src/purge-entities.ts +92 -0
  69. package/src/purge.ts +10 -2
  70. package/src/queue-wake.ts +1 -1
  71. package/src/refresh-view.ts +76 -0
  72. package/src/retry-classification.ts +27 -8
  73. package/src/retry.ts +26 -0
  74. package/src/rollup-errors.ts +27 -0
  75. package/src/rollup.ts +190 -0
  76. package/src/run-deadline.ts +11 -7
  77. package/src/scheduler.ts +2 -1
  78. package/src/settled.ts +56 -15
  79. package/src/step-resolve.ts +37 -0
  80. package/src/steps-api.ts +111 -0
  81. package/src/steps-core.ts +26 -0
  82. package/src/steps-output.ts +142 -0
  83. package/src/steps-store.ts +68 -0
  84. package/src/steps-suspension.ts +51 -0
  85. package/src/steps-timeout.ts +10 -5
  86. package/src/steps-unwind.ts +94 -0
  87. package/src/steps-wait.ts +163 -0
  88. package/src/steps.ts +174 -210
  89. package/src/subject-erase-pass.ts +134 -0
  90. package/src/subject-errors.ts +48 -0
  91. package/src/subject-export-pass.ts +228 -0
  92. package/src/subject-jobs.ts +121 -0
  93. package/src/subject-plan.ts +287 -0
  94. package/src/subject-select.ts +106 -0
  95. package/src/subject.ts +119 -0
  96. package/src/tenant.ts +69 -1
  97. package/src/type-pins.ts +90 -0
  98. package/src/webhook-errors.ts +1 -1
  99. package/src/webhook-receiver-errors.ts +59 -0
  100. package/src/webhook-receiver-route.ts +127 -0
  101. package/src/webhook-receiver-types.ts +97 -0
  102. package/src/webhook-receiver.ts +132 -0
  103. package/src/webhook-target.ts +20 -43
  104. package/src/webhook.ts +7 -4
  105. package/src/worker-queue-demand.ts +57 -0
  106. package/src/worker-queue-depth.ts +15 -3
  107. package/src/worker-registry.ts +4 -1
  108. package/src/worker-types.ts +2 -0
  109. package/src/worker.ts +2 -2
@@ -30,40 +30,70 @@ redis.call('ZREM', P .. ':run:' .. queue, id)
30
30
  `;
31
31
 
32
32
  /**
33
- * `ARGV`: namespace, id, name, queue, input, idempotency key, runId, maxAttempts, runAt, now,
34
- * tenant, traceparent, enqueuedBy, namespace field. Answers `{ outcome, id, runId }`: `published`
35
- * when the caller-allocated id already names a row (`SQL_ENQUEUE`'s `not exists`), `deduped` when
36
- * a live row holds the key, `created` otherwise.
33
+ * One enqueue, as a Lua function both enqueue scripts call, so the single and the batch insert are
34
+ * ONE body. `a[o]` onwards, `ENQUEUE_ARITY` fields: id, name, queue, input, idempotency key,
35
+ * runId, maxAttempts, runAt, now, tenant, traceparent, enqueuedBy, namespace field. Answers
36
+ * `{ outcome, id, runId }`: `published` when the caller-allocated id already names a row
37
+ * (`SQL_ENQUEUE`'s `not exists`), `deduped` when a live row holds the key, `created` otherwise.
37
38
  */
38
- export const REDIS_ENQUEUE = `
39
+ const ENQUEUE_ONE = `
39
40
  ${LIVE}
40
- local P = ARGV[1]
41
- local id = ARGV[2]
42
- local job = P .. ':job:' .. id
43
- if redis.call('EXISTS', job) == 1 then
44
- return { 'published', id, redis.call('HGET', job, 'runId') }
41
+ local function enqueue_one(P, a, o)
42
+ local id = a[o]
43
+ local job = P .. ':job:' .. id
44
+ if redis.call('EXISTS', job) == 1 then
45
+ return { 'published', id, redis.call('HGET', job, 'runId') }
46
+ end
47
+ local nk = a[o + 12]
48
+ local holder = redis.call('HGET', P .. ':idem', nk)
49
+ if holder then
50
+ local row = redis.call('HMGET', P .. ':job:' .. holder, 'state', 'runId')
51
+ if row[1] and LIVE[row[1]] then return { 'deduped', holder, row[2] } end
52
+ end
53
+ local now = tonumber(a[o + 8])
54
+ local runAt = tonumber(a[o + 7])
55
+ local state = 'ready'
56
+ if runAt > now then state = 'delayed' end
57
+ local queue = a[o + 2]
58
+ redis.call('HSET', job,
59
+ 'id', id, 'name', a[o + 1], 'queue', queue, 'input', a[o + 3], 'idempotencyKey', a[o + 4],
60
+ 'runId', a[o + 5], 'attempt', '0', 'maxAttempts', a[o + 6], 'state', state,
61
+ 'runAt', a[o + 7], 'createdAt', a[o + 8], 'updatedAt', a[o + 8], 'claim', '0', 'nk', nk)
62
+ if a[o + 9] ~= '' then redis.call('HSET', job, 'tenantId', a[o + 9]) end
63
+ if a[o + 10] ~= '' then redis.call('HSET', job, 'traceparent', a[o + 10]) end
64
+ if a[o + 11] ~= '' then redis.call('HSET', job, 'enqueuedBy', a[o + 11]) end
65
+ redis.call('ZADD', P .. ':wait:' .. queue, runAt, id)
66
+ redis.call('SADD', P .. ':queues', queue)
67
+ redis.call('HSET', P .. ':idem', nk, id)
68
+ return { 'created', id, a[o + 5] }
45
69
  end
46
- local nk = ARGV[14]
47
- local holder = redis.call('HGET', P .. ':idem', nk)
48
- if holder then
49
- local row = redis.call('HMGET', P .. ':job:' .. holder, 'state', 'runId')
50
- if row[1] and LIVE[row[1]] then return { 'deduped', holder, row[2] } end
70
+ `;
71
+
72
+ /** Fields one request binds, after the namespace — what `ENQUEUE_ONE` reads from `a[o]` on. */
73
+ export const ENQUEUE_ARITY = 13;
74
+
75
+ /** `SQL_ENQUEUE`. `ARGV`: namespace, then one request's `ENQUEUE_ARITY` fields. */
76
+ export const REDIS_ENQUEUE = `
77
+ ${ENQUEUE_ONE}
78
+ return enqueue_one(ARGV[1], ARGV, 2)
79
+ `.trim();
80
+
81
+ /**
82
+ * `SQL_ENQUEUE_MANY`: every request in ONE script, so the batch is atomic on the server and a key
83
+ * twice in it is one job — the second request meets the first's `idem` entry. `ARGV`: namespace,
84
+ * then `ENQUEUE_ARITY` fields per request, in request order. Answers the three fields
85
+ * `enqueue_one` answers, per request, flat and in order.
86
+ */
87
+ export const REDIS_ENQUEUE_MANY = `
88
+ ${ENQUEUE_ONE}
89
+ local out = {}
90
+ for o = 2, #ARGV, ${ENQUEUE_ARITY} do
91
+ local answer = enqueue_one(ARGV[1], ARGV, o)
92
+ table.insert(out, answer[1])
93
+ table.insert(out, answer[2])
94
+ table.insert(out, answer[3])
51
95
  end
52
- local now = tonumber(ARGV[10])
53
- local runAt = tonumber(ARGV[9])
54
- local state = 'ready'
55
- if runAt > now then state = 'delayed' end
56
- redis.call('HSET', job,
57
- 'id', id, 'name', ARGV[3], 'queue', ARGV[4], 'input', ARGV[5], 'idempotencyKey', ARGV[6],
58
- 'runId', ARGV[7], 'attempt', '0', 'maxAttempts', ARGV[8], 'state', state, 'runAt', ARGV[9],
59
- 'createdAt', ARGV[10], 'updatedAt', ARGV[10], 'claim', '0', 'nk', nk)
60
- if ARGV[11] ~= '' then redis.call('HSET', job, 'tenantId', ARGV[11]) end
61
- if ARGV[12] ~= '' then redis.call('HSET', job, 'traceparent', ARGV[12]) end
62
- if ARGV[13] ~= '' then redis.call('HSET', job, 'enqueuedBy', ARGV[13]) end
63
- redis.call('ZADD', P .. ':wait:' .. ARGV[4], runAt, id)
64
- redis.call('SADD', P .. ':queues', ARGV[4])
65
- redis.call('HSET', P .. ':idem', nk, id)
66
- return { 'created', id, ARGV[7] }
96
+ return out
67
97
  `.trim();
68
98
 
69
99
  /**
@@ -221,6 +251,25 @@ end
221
251
  return out
222
252
  `.trim();
223
253
 
254
+ /**
255
+ * `SQL_QUEUE_DEMAND`, one consistent read. `ARGV`: namespace, now. Answers `{ queue, demand }` per
256
+ * queue with some: every running row plus every due wait or park row. This driver keeps no worker
257
+ * registry and no pause, so no claimer is known to drain and no queue is paused: the rule
258
+ * `SQL_QUEUE_DEMAND` applies to a claimer with no live row, which counts.
259
+ */
260
+ export const REDIS_DEMAND = `
261
+ local P = ARGV[1]
262
+ local now = tonumber(ARGV[2])
263
+ local out = {}
264
+ for _, queue in ipairs(redis.call('SMEMBERS', P .. ':queues')) do
265
+ local demand = redis.call('ZCARD', P .. ':run:' .. queue)
266
+ + redis.call('ZCOUNT', P .. ':wait:' .. queue, '-inf', now)
267
+ + redis.call('ZCOUNT', P .. ':park:' .. queue, '-inf', now)
268
+ if demand > 0 then table.insert(out, { queue, demand }) end
269
+ end
270
+ return out
271
+ `.trim();
272
+
224
273
  /**
225
274
  * `SQL_STEP_PUT` under a claim: the write lands only while that claim still holds the row.
226
275
  * `ARGV`: namespace, job id, worker, claim, runId, step name, the record as JSON.
@@ -1,7 +1,7 @@
1
- // A Redis queue on `Bun.redis` — no client dependency, the runtime ships one. The same contract as
2
- // the pg driver, held by `@ultimat3/testing`'s `jobDriverConformance`: every operation is one Lua
3
- // script (`driver-redis-scripts.ts`), so a claim, a settle and a renewal are each atomic on the
4
- // server the way a `driver-pg-sql.ts` statement is in one transaction. Its own entry,
1
+ // A Redis queue on core's `redisClient()` — no client dependency, the runtime ships one. The same
2
+ // contract as the pg driver, held by `@ultimat3/testing`'s `jobDriverConformance`: every operation
3
+ // is one Lua script (`driver-redis-scripts.ts`), so a claim, a settle and a renewal are each atomic
4
+ // on the server the way a `driver-pg-sql.ts` statement is in one transaction. Its own entry,
5
5
  // `@ultimat3/jobs/redis`: the barrel is on every role's boot path, and an app on Postgres loads none
6
6
  // of this.
7
7
  //
@@ -14,6 +14,7 @@
14
14
 
15
15
  import type { Clock } from '@ultimat3/core';
16
16
  import { finiteCount, finiteOption, systemClock, uuidV7 } from '@ultimat3/core';
17
+ import { redisClient } from '@ultimat3/core/redis';
17
18
  import { nowMs } from './clock';
18
19
  import type {
19
20
  ClaimedJob,
@@ -21,28 +22,33 @@ import type {
21
22
  EnqueueResult,
22
23
  JobDriver,
23
24
  JobRecord,
25
+ QueueDemand,
24
26
  QueueStats,
25
27
  } from './driver';
26
28
  import {
27
29
  assertClaimBounds,
28
30
  assertClaimQueues,
31
+ assertEnqueueMany,
29
32
  DEFAULT_QUEUE,
30
33
  isJobState,
31
34
  nackState,
32
35
  } from './driver';
36
+ import { typedRefusals } from './driver-redis-refusal';
33
37
  import {
34
38
  REDIS_ACK,
35
39
  REDIS_CLAIM,
40
+ REDIS_DEMAND,
36
41
  REDIS_ENQUEUE,
42
+ REDIS_ENQUEUE_MANY,
37
43
  REDIS_HEARTBEAT,
38
44
  REDIS_NACK,
39
45
  REDIS_STATS,
40
46
  } from './driver-redis-scripts';
41
47
  import { redisStepStore } from './driver-redis-steps';
42
- import { DriverUnavailableError, JobDuplicateError } from './errors';
48
+ import { JobDuplicateError } from './errors';
43
49
  import { MAX_ERROR_STACK_LENGTH } from './introspection';
44
50
 
45
- /** The slice of Bun's Redis client this driver uses — `Bun.redis` and `new Bun.RedisClient()`. */
51
+ /** The slice of Bun's Redis client this driver uses — `redisClient()` and `new Bun.RedisClient()`. */
46
52
  export interface RedisSender {
47
53
  send(command: string, args: string[]): Promise<unknown>;
48
54
  }
@@ -51,7 +57,11 @@ export interface RedisSender {
51
57
  export const DEFAULT_REDIS_DONE_TTL_MS = 7 * 24 * 60 * 60 * 1000;
52
58
 
53
59
  export interface RedisJobDriverOptions {
54
- /** Injected in tests and for a second server; production reads `Bun.redis` (`REDIS_URL`). */
60
+ /**
61
+ * Injected in tests and for a second server; production takes `redisClient()` from
62
+ * `@ultimat3/core/redis` (`REDIS_URL`): while it is down every command refuses at once, as
63
+ * `X_DRIVER_UNAVAILABLE`. No deadline here: a stuck command has the request's.
64
+ */
55
65
  readonly client?: RedisSender;
56
66
  /** Key namespace, the hash tag every key carries. Default `x:jobs`. */
57
67
  readonly prefix?: string;
@@ -64,19 +74,6 @@ export interface RedisJobDriverOptions {
64
74
  readonly doneTtlMs?: number;
65
75
  }
66
76
 
67
- function resolveClient(injected: RedisSender | undefined): RedisSender {
68
- if (injected !== undefined) return injected;
69
- const ambient = (Bun as unknown as { redis?: RedisSender }).redis;
70
- if (ambient === undefined || typeof ambient.send !== 'function') {
71
- throw new DriverUnavailableError({
72
- driver: 'redis',
73
- cause: 'Bun.redis is not available, and no client was handed to redisJobDriver()',
74
- fix: 'REDIS_URL=redis://127.0.0.1:6379 x dev — or hand one in: redisJobDriver({ client: new Bun.RedisClient(url) })',
75
- });
76
- }
77
- return ambient;
78
- }
79
-
80
77
  /** A Lua reply as an array; anything else is the empty one. */
81
78
  const list = (reply: unknown): readonly unknown[] => (Array.isArray(reply) ? reply : []);
82
79
 
@@ -134,8 +131,33 @@ function toRecord(fields: ReadonlyMap<string, string>): JobRecord {
134
131
  const namespaceField = (request: EnqueueRequest): string =>
135
132
  JSON.stringify([request.name, request.tenantId ?? '', request.idempotencyKey]);
136
133
 
134
+ /** One request as `ENQUEUE_ONE` reads it: `ENQUEUE_ARITY` fields, in its order. */
135
+ function enqueueArgs(request: EnqueueRequest, runId: string, at: number): (string | number)[] {
136
+ return [
137
+ request.id ?? uuidV7(),
138
+ request.name,
139
+ request.queue || DEFAULT_QUEUE,
140
+ JSON.stringify(request.input ?? null),
141
+ request.idempotencyKey,
142
+ runId,
143
+ request.maxAttempts,
144
+ request.runAt ?? at,
145
+ at,
146
+ request.tenantId ?? '',
147
+ request.traceparent ?? '',
148
+ request.enqueuedBy ?? '',
149
+ namespaceField(request),
150
+ ];
151
+ }
152
+
153
+ /** The `{ outcome, id, runId }` triple at `at` in a reply, as the result `enqueue` answers. */
154
+ function answerOf(reply: readonly unknown[], at: number, runId: string): EnqueueResult {
155
+ const [outcome, id, heldRunId] = reply.slice(at, at + 3).map(String);
156
+ return { id: id ?? '', runId: heldRunId ?? runId, deduped: outcome !== 'created' };
157
+ }
158
+
137
159
  export function redisJobDriver(options: RedisJobDriverOptions = {}): JobDriver {
138
- const client = resolveClient(options.client);
160
+ const client = typedRefusals(options.client ?? redisClient());
139
161
  const clock = options.clock ?? systemClock;
140
162
  const ns = `{${options.prefix ?? 'x:jobs'}}`;
141
163
  const doneTtlMs = finiteCount(
@@ -152,33 +174,31 @@ export function redisJobDriver(options: RedisJobDriverOptions = {}): JobDriver {
152
174
  steps: redisStepStore({ ns, run, client }),
153
175
 
154
176
  async enqueue(request): Promise<EnqueueResult> {
155
- const at = nowMs(clock);
156
177
  const runId = request.runId ?? uuidV7();
157
- const [outcome, id, heldRunId] = list(
158
- await run(REDIS_ENQUEUE, [
159
- request.id ?? uuidV7(),
160
- request.name,
161
- request.queue || DEFAULT_QUEUE,
162
- JSON.stringify(request.input ?? null),
163
- request.idempotencyKey,
164
- runId,
165
- request.maxAttempts,
166
- request.runAt ?? at,
167
- at,
168
- request.tenantId ?? '',
169
- request.traceparent ?? '',
170
- request.enqueuedBy ?? '',
171
- namespaceField(request),
172
- ]),
173
- ).map(String);
174
- if (outcome === 'deduped' && request.onConflict === 'error') {
178
+ const reply = list(await run(REDIS_ENQUEUE, enqueueArgs(request, runId, nowMs(clock))));
179
+ const answer = answerOf(reply, 0, runId);
180
+ if (reply[0] === 'deduped' && request.onConflict === 'error') {
175
181
  throw new JobDuplicateError({
176
182
  job: request.name,
177
183
  idempotencyKey: request.idempotencyKey,
178
- existingId: id ?? '',
184
+ existingId: answer.id,
179
185
  });
180
186
  }
181
- return { id: id ?? '', runId: heldRunId ?? runId, deduped: outcome !== 'created' };
187
+ return answer;
188
+ },
189
+
190
+ // ONE script for the batch (`REDIS_ENQUEUE_MANY`), the body `enqueue` runs once per request:
191
+ // atomic on the server, and a key twice in it meets its own first request.
192
+ async enqueueMany(requests): Promise<readonly EnqueueResult[]> {
193
+ assertEnqueueMany('redis', requests);
194
+ if (requests.length === 0) return [];
195
+ const at = nowMs(clock);
196
+ const runIds = requests.map((request) => request.runId ?? uuidV7());
197
+ const args = requests.flatMap((request, index) =>
198
+ enqueueArgs(request, runIds[index] ?? uuidV7(), at),
199
+ );
200
+ const reply = list(await run(REDIS_ENQUEUE_MANY, args));
201
+ return runIds.map((runId, index) => answerOf(reply, index * 3, runId));
182
202
  },
183
203
 
184
204
  async claim(claimOptions): Promise<readonly ClaimedJob[]> {
@@ -274,5 +294,13 @@ export function redisJobDriver(options: RedisJobDriverOptions = {}): JobDriver {
274
294
  // Code units, as `collate "C"` orders `SQL_STATS` — never `localeCompare`.
275
295
  return rows.sort((a, b) => (a.queue < b.queue ? -1 : a.queue > b.queue ? 1 : 0));
276
296
  },
297
+
298
+ async demand(): Promise<readonly QueueDemand[]> {
299
+ const rows = list(await run(REDIS_DEMAND, [nowMs(clock)])).map((row): QueueDemand => {
300
+ const [queue, demand] = list(row);
301
+ return { queue: String(queue), demand: Number(demand) };
302
+ });
303
+ return rows.sort((a, b) => (a.queue < b.queue ? -1 : a.queue > b.queue ? 1 : 0));
304
+ },
277
305
  };
278
306
  }
package/src/driver.ts CHANGED
@@ -9,7 +9,7 @@
9
9
  // for real. Swapping the driver is `setJobDriver(other)` and ZERO job-code change, and that
10
10
  // is what the interface buys.
11
11
 
12
- import { finiteCount, finiteOption } from '@ultimat3/core';
12
+ import { assert, finiteCount, finiteOption } from '@ultimat3/core';
13
13
  import type { BackfillLedger } from './backfill-ledger';
14
14
  import { ClaimQueuesEmptyError } from './errors';
15
15
  import type { JobIntrospection, JobProgress } from './introspection';
@@ -305,6 +305,12 @@ export interface QueueStats {
305
305
  readonly oldestReadyMs: number;
306
306
  }
307
307
 
308
+ /** What one queue's fleet must hold: `JobDriver.demand`. */
309
+ export interface QueueDemand {
310
+ readonly queue: string;
311
+ readonly demand: number;
312
+ }
313
+
308
314
  export interface HeartbeatOptions {
309
315
  readonly visibilityTimeoutMs: number;
310
316
  /** Renew only if this worker is still the claimant. Omit and any claimant matches. */
@@ -318,6 +324,13 @@ export interface JobDriver {
318
324
  /** Step persistence lives with the queue: one store, one transaction boundary. */
319
325
  readonly steps: StepStore;
320
326
  enqueue(request: EnqueueRequest): Promise<EnqueueResult>;
327
+ /**
328
+ * `enqueue` for every request in ONE round trip — a statement on Postgres, a script on Redis —
329
+ * answered in request order with what `enqueue` would have answered each: a repeat of an `id` or
330
+ * a live key is `deduped: true`, a key twice in the batch is one job. REQUIRED, the one batch
331
+ * path `handle.enqueueMany` and the outbox relay take. Opens with `assertEnqueueMany`.
332
+ */
333
+ enqueueMany(requests: readonly EnqueueRequest[]): Promise<readonly EnqueueResult[]>;
321
334
  claim(options: ClaimOptions): Promise<readonly ClaimedJob[]>;
322
335
  /**
323
336
  * Both settles are fenced on `state = 'running'` AND on the claimer, and both answer whether
@@ -336,6 +349,14 @@ export interface JobDriver {
336
349
  */
337
350
  heartbeat(jobId: string, options: HeartbeatOptions): Promise<boolean>;
338
351
  stats(): Promise<readonly QueueStats[]>;
352
+ /**
353
+ * The jobs a live worker of this fleet must hold, per queue: running on a worker that still
354
+ * serves, plus every due row of an unpaused queue. Only a queue with some is listed. What
355
+ * `queue_demand` publishes and an autoscaler sizes by. Required since 30.0.0: `ready + running`
356
+ * off `stats()` cannot tell a draining worker's runs from a serving one's, and the HPA sized on it.
357
+ * A claimer no live registry row says drains is counted, so a driver with no registry counts all.
358
+ */
359
+ demand(): Promise<readonly QueueDemand[]>;
339
360
  /**
340
361
  * Optional, like `introspect`: `x_backfills` records what a `backfill()` pass has already swept,
341
362
  * and a driver without one runs backfills with no bookkeeping rather than refusing them. It
@@ -395,6 +416,22 @@ export const assertClaimQueues = (driver: string, options: ClaimOptions): void =
395
416
  if (options.queues.length === 0) throw new ClaimQueuesEmptyError(driver);
396
417
  };
397
418
 
419
+ /**
420
+ * Every driver's `enqueueMany` opens with this, so a batch is refused ONE way. `onConflict:
421
+ * 'error'` has no batch meaning the drivers could share: Postgres inserts every other row of the
422
+ * statement before the refusal is known, a loop only the rows before it — two different sets of
423
+ * jobs left behind by one refused call. A request that must refuse a duplicate is a single
424
+ * `enqueue`; `handle.enqueueMany` never sets it.
425
+ */
426
+ export const assertEnqueueMany = (driver: string, requests: readonly EnqueueRequest[]): void => {
427
+ const at = requests.findIndex((request) => request.onConflict === 'error');
428
+ assert(
429
+ at === -1,
430
+ `the ${driver} driver's enqueueMany was handed onConflict: 'error' on request ${at}, and a batch that refuses a duplicate would leave a different set of jobs behind on every driver`,
431
+ "drop onConflict from the batch — a duplicate is answered deduped: true — or send that request alone: driver.enqueue({ ...request, onConflict: 'error' })",
432
+ );
433
+ };
434
+
398
435
  /**
399
436
  * The two NUMBERS of a claim, screened for both drivers in one place for the reason above: a value
400
437
  * neither of them refuses is answered two ways. `limit: -1` sliced every ready row but the newest
@@ -0,0 +1,137 @@
1
+ // What one enqueue WRITES, built once for every path: the staged `OutboxRecord` and the direct
2
+ // `EnqueueRequest`, for `enqueue` and `enqueueMany` alike. Split from `outbox.ts` so the batch path
3
+ // builds the identical row the single one does, instead of a second spelling of it.
4
+
5
+ import { currentSpanContext, describeValue, traceparent, UltimateError } from '@ultimat3/core';
6
+ import type { EnqueueRequest } from './driver';
7
+ import { DEFAULT_QUEUE } from './driver';
8
+ import type { JobHandle } from './job';
9
+ import type { EnqueueOptions, OutboxRecord } from './outbox';
10
+ import { enqueuedAttempts } from './retry';
11
+
12
+ /**
13
+ * The W3C `traceparent` of the span this enqueue is happening inside, or `undefined` outside a
14
+ * trace. This is the ONE place the link is minted: `docs/idea/04-jobs.md` promises a job trace
15
+ * linked to the enqueuing request, and before this there was no field to carry the link, so a
16
+ * checkout's `chargeCard` opened a fresh root two seconds later with nothing pointing back.
17
+ *
18
+ * A context recovered from a `Ctx` has an empty `spanId` (`currentSpanContext`), which renders as
19
+ * an all-zero parent that every collector rejects — so that case carries no header rather than a
20
+ * malformed one, and the job opens a root as it did before.
21
+ */
22
+ export function ambientTraceparent(): string | undefined {
23
+ const context = currentSpanContext();
24
+ if (context === undefined || context.spanId === '') return undefined;
25
+ return traceparent(context);
26
+ }
27
+
28
+ /** What Postgres prints for a `uuid`, and so the only spelling both drivers store identically. */
29
+ const RUN_ID_SHAPE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
30
+
31
+ /**
32
+ * `EnqueueOptions.runId`, refused HERE when it is not a uuid — the one screen both enqueue paths
33
+ * read. `run_id` is a `uuid` column: the memory driver took any string, Postgres raised a raw
34
+ * `22P02`, and on the staged path that was the caller's whole transaction failing at the outbox
35
+ * insert. Any version, lowercase: an uppercase one would come back from Postgres re-spelled.
36
+ */
37
+ export function namedRunId(job: string, runId: unknown): string | undefined {
38
+ if (runId === undefined) return undefined;
39
+ if (typeof runId === 'string' && RUN_ID_SHAPE.test(runId)) return runId;
40
+ throw new UltimateError({
41
+ code: 'X_ID_INVALID',
42
+ // `describeValue`, never the value: what arrives here wrong is as often a key as a typo.
43
+ cause: `enqueue of job "${job}" was given runId ${describeValue(runId)}, and a run id is a lowercase uuid (8-4-4-4-12 hex) — the queue stores it in a uuid column`,
44
+ fix: "pass { runId: uuidV7() } to enqueue(), with import { uuidV7 } from '@ultimat3/core'",
45
+ meta: { job, option: 'runId' },
46
+ });
47
+ }
48
+
49
+ /** The parts of one row the caller of the builders decides: its key, its clock, its ids. */
50
+ export interface RowFacts {
51
+ readonly idempotencyKey: string;
52
+ readonly at: number;
53
+ readonly trace: string | undefined;
54
+ readonly id?: string;
55
+ readonly runId?: string;
56
+ }
57
+
58
+ /**
59
+ * The org the row is queued for: the one stated, else the one the payload names. Without the
60
+ * default a plain `enqueue(input)` queued for nobody, so org B's `invoice:1` deduped onto org A's
61
+ * job (the namespace is `(name, tenant, key)`) and `perTenant` counted every such row as no org's.
62
+ * The facade's `refuseEnqueueTenant` has already refused the two disagreeing.
63
+ */
64
+ function queuedFor<I>(handle: JobHandle<I>, input: I, options: EnqueueOptions): string | undefined {
65
+ return options.tenantId ?? handle.tenantFor(input);
66
+ }
67
+
68
+ const optional = (
69
+ tenantId: string | undefined,
70
+ options: EnqueueOptions,
71
+ trace: string | undefined,
72
+ ) => ({
73
+ ...(tenantId === undefined ? {} : { tenantId }),
74
+ ...(trace === undefined ? {} : { traceparent: trace }),
75
+ ...(options.enqueuedBy === undefined ? {} : { enqueuedBy: options.enqueuedBy }),
76
+ });
77
+
78
+ /** The row a staged enqueue writes to the outbox; its `id` and `runId` are the job's to be. */
79
+ export function stagedRecord<I>(
80
+ handle: JobHandle<I>,
81
+ input: I,
82
+ options: EnqueueOptions,
83
+ facts: RowFacts & { readonly id: string; readonly runId: string },
84
+ ): OutboxRecord {
85
+ return {
86
+ id: facts.id,
87
+ runId: facts.runId,
88
+ job: handle.name,
89
+ queue: options.queue ?? handle.queue ?? DEFAULT_QUEUE,
90
+ input,
91
+ idempotencyKey: facts.idempotencyKey,
92
+ maxAttempts: enqueuedAttempts(handle.retry, handle.compensates),
93
+ runAt: options.runAt ?? facts.at,
94
+ stagedAt: facts.at,
95
+ // Stamped at STAGE time and not at publish time: the relay runs after commit, in its own
96
+ // timer, with no request span in scope — a trace read there would be nobody's.
97
+ ...optional(queuedFor(handle, input, options), options, facts.trace),
98
+ };
99
+ }
100
+
101
+ /** The request a direct enqueue hands the driver. Ids only when the caller allocated them. */
102
+ export function directRequest<I>(
103
+ handle: JobHandle<I>,
104
+ input: I,
105
+ options: EnqueueOptions,
106
+ facts: RowFacts,
107
+ ): EnqueueRequest {
108
+ return {
109
+ ...(facts.id === undefined ? {} : { id: facts.id }),
110
+ name: handle.name,
111
+ queue: options.queue ?? handle.queue,
112
+ input,
113
+ idempotencyKey: facts.idempotencyKey,
114
+ maxAttempts: enqueuedAttempts(handle.retry, handle.compensates),
115
+ runAt: options.runAt ?? facts.at,
116
+ ...(facts.runId === undefined ? {} : { runId: facts.runId }),
117
+ ...optional(queuedFor(handle, input, options), options, facts.trace),
118
+ };
119
+ }
120
+
121
+ /** The request the relay publishes a staged row as — under the ids the stage answered. */
122
+ export function publishRequest(record: OutboxRecord): EnqueueRequest {
123
+ return {
124
+ // A repeat of this publish — a crash before the mark — meets the job this one made.
125
+ id: record.id,
126
+ runId: record.runId,
127
+ name: record.job,
128
+ queue: record.queue,
129
+ input: record.input,
130
+ idempotencyKey: record.idempotencyKey,
131
+ maxAttempts: record.maxAttempts,
132
+ runAt: record.runAt,
133
+ ...(record.tenantId === undefined ? {} : { tenantId: record.tenantId }),
134
+ ...(record.traceparent === undefined ? {} : { traceparent: record.traceparent }),
135
+ ...(record.enqueuedBy === undefined ? {} : { enqueuedBy: record.enqueuedBy }),
136
+ };
137
+ }