@ultimat3/jobs 1.2.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +567 -0
- package/README.md +355 -16
- package/package.json +7 -5
- package/src/backfill-gate.ts +97 -0
- package/src/backfill-inspect.ts +73 -0
- package/src/backfill-ledger.ts +183 -0
- package/src/backfill-pass.ts +276 -0
- package/src/backfill-pending.ts +131 -0
- package/src/backfill-rate.ts +109 -0
- package/src/backfill-registry.ts +108 -0
- package/src/backfill-scope.ts +70 -0
- package/src/backfill.ts +213 -0
- package/src/driver-memory.ts +61 -9
- package/src/driver-nats.ts +2 -1
- package/src/driver-pg-ddl.ts +180 -0
- package/src/driver-pg-rows.ts +123 -0
- package/src/driver-pg-sql.ts +251 -55
- package/src/driver-pg.ts +138 -92
- package/src/driver-redis.ts +2 -1
- package/src/driver.ts +91 -7
- package/src/errors.ts +290 -4
- package/src/events-pg.ts +121 -0
- package/src/events.ts +7 -1
- package/src/execute.ts +289 -0
- package/src/heartbeat.ts +146 -0
- package/src/index.ts +121 -27
- package/src/inspect.ts +43 -2
- package/src/job.ts +72 -2
- package/src/leases.ts +90 -0
- package/src/limits.ts +0 -0
- package/src/metrics.ts +35 -0
- package/src/outbox-pg.ts +137 -0
- package/src/outbox.ts +115 -53
- package/src/register.ts +1 -1
- package/src/retry.ts +6 -1
- package/src/run-signal.ts +50 -0
- package/src/scheduler-pg.ts +103 -0
- package/src/scheduler.ts +159 -245
- package/src/steps.ts +155 -31
- package/src/task.ts +229 -0
- package/src/tenant.ts +61 -0
- package/src/worker-fleet-slots.ts +124 -0
- package/src/worker-run.ts +132 -0
- package/src/worker.ts +207 -190
package/src/steps.ts
CHANGED
|
@@ -10,7 +10,7 @@ import type { Clock } from '@ultimat3/core';
|
|
|
10
10
|
import { logger } from '@ultimat3/core';
|
|
11
11
|
import type { DurationInput } from './clock';
|
|
12
12
|
import { nowMs, toMs } from './clock';
|
|
13
|
-
import { JobTimeoutError, StepDuplicateError } from './errors';
|
|
13
|
+
import { JobAbortedError, JobTimeoutError, StepDuplicateError } from './errors';
|
|
14
14
|
|
|
15
15
|
export type StepStatus = 'completed' | 'sleeping' | 'waiting' | 'failed';
|
|
16
16
|
|
|
@@ -57,8 +57,14 @@ export interface WaitForEventOptions {
|
|
|
57
57
|
}
|
|
58
58
|
|
|
59
59
|
export interface StepApi {
|
|
60
|
-
/**
|
|
61
|
-
|
|
60
|
+
/**
|
|
61
|
+
* Run once, ever. On replay the persisted output is returned and `fn` is not called.
|
|
62
|
+
*
|
|
63
|
+
* `fn` receives the step's `AbortSignal` — the run's cancellation and this step's own ceiling,
|
|
64
|
+
* whichever fires first. Hand it to `fetch`, or read `.aborted` in a loop: past it, this step
|
|
65
|
+
* may no longer write, because the attempt that replaced this one owns the run.
|
|
66
|
+
*/
|
|
67
|
+
run<T>(name: string, fn: (signal: AbortSignal) => Promise<T> | T): Promise<T>;
|
|
62
68
|
/** Suspend the run. `sleep(duration)` derives the step name from the duration. */
|
|
63
69
|
sleep(name: string, duration: DurationInput): Promise<void>;
|
|
64
70
|
sleep(duration: DurationInput): Promise<void>;
|
|
@@ -136,49 +142,148 @@ export interface StepRunnerOptions {
|
|
|
136
142
|
readonly eventPollMs?: number;
|
|
137
143
|
/** Per-step ceiling; the job-level timeout is enforced by the worker. */
|
|
138
144
|
readonly stepTimeoutMs?: number;
|
|
145
|
+
/**
|
|
146
|
+
* The run's cancellation — `executeJob` aborts it at the job's deadline. Once aborted this
|
|
147
|
+
* runner writes nothing: the nack that follows a deadline makes the job claimable, so the
|
|
148
|
+
* store belongs to whichever attempt has it now.
|
|
149
|
+
*/
|
|
150
|
+
readonly signal?: AbortSignal;
|
|
139
151
|
}
|
|
140
152
|
|
|
141
153
|
export interface StepRunner {
|
|
142
154
|
readonly step: StepApi;
|
|
143
|
-
/**
|
|
155
|
+
/**
|
|
156
|
+
* Names used in THIS attempt, in order — the trace shown by `x jobs show`. Bounded at
|
|
157
|
+
* `MAX_TRACE_NAMES`, oldest dropped: duplicate detection reads its own set, so this is a
|
|
158
|
+
* window on a long run and never the run's record.
|
|
159
|
+
*/
|
|
144
160
|
usedNames(): readonly string[];
|
|
145
|
-
/** Names that were served from storage instead of executed. */
|
|
161
|
+
/** Names that were served from storage instead of executed. Bounded the same way. */
|
|
146
162
|
replayedNames(): readonly string[];
|
|
147
163
|
}
|
|
148
164
|
|
|
165
|
+
/** Stands in for an absent run signal, so the fence has one shape and no `undefined` branch. */
|
|
166
|
+
const NEVER_ABORTED = new AbortController().signal;
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* What one attempt's trace keeps. The trace is a diagnostic `x jobs show` renders, never the
|
|
170
|
+
* run's record — `driver.steps.list(runId)` is that — and a `backfill()` over a million rows
|
|
171
|
+
* claims 20,000 names in a single attempt, all of them carried to the end of the run.
|
|
172
|
+
*/
|
|
173
|
+
export const MAX_TRACE_NAMES = 200;
|
|
174
|
+
|
|
175
|
+
/** Most recent first out: the tail of a long run is the half an operator is reading. */
|
|
176
|
+
function trace(into: string[], name: string): void {
|
|
177
|
+
into.push(name);
|
|
178
|
+
if (into.length > MAX_TRACE_NAMES) into.shift();
|
|
179
|
+
}
|
|
180
|
+
|
|
149
181
|
export function createStepRunner(options: StepRunnerOptions): StepRunner {
|
|
150
182
|
const { runId, jobName, store } = options;
|
|
183
|
+
/** Every name this attempt has claimed. Membership only — the trace is `used`. */
|
|
184
|
+
const claimed = new Set<string>();
|
|
151
185
|
const used: string[] = [];
|
|
152
186
|
const replayed: string[] = [];
|
|
153
187
|
const clock = options.clock;
|
|
154
188
|
const pollMs = options.eventPollMs ?? 30_000;
|
|
189
|
+
const runSignal = options.signal ?? NEVER_ABORTED;
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* This attempt's view of the run's persisted steps, hydrated from ONE `store.list(runId)` the
|
|
193
|
+
* first time a step asks. Replay used to cost one `SQL_STEP_GET` per completed step: a
|
|
194
|
+
* `backfill()` over 5M rows at `batch: 1000` is 5,000 steps, so an attempt killed at 4,800
|
|
195
|
+
* issued 4,800 sequential round trips before reading a single new row — and re-paid them on
|
|
196
|
+
* every retry, often outrunning its own visibility timeout while the heartbeat was still
|
|
197
|
+
* renewing.
|
|
198
|
+
*
|
|
199
|
+
* Sound because the fence in `put()` is: only the attempt that owns the run may write to it, so
|
|
200
|
+
* within one attempt an absent name stays absent unless this runner writes it. Writes go into
|
|
201
|
+
* the map as they go into the store, never the other way round — the store is still the record.
|
|
202
|
+
*/
|
|
203
|
+
let hydrated: Map<string, StepRecord> | undefined;
|
|
204
|
+
|
|
205
|
+
const load = async (name: string): Promise<StepRecord | undefined> => {
|
|
206
|
+
if (hydrated === undefined) {
|
|
207
|
+
hydrated = new Map();
|
|
208
|
+
for (const record of await store.list(runId)) hydrated.set(record.name, record);
|
|
209
|
+
}
|
|
210
|
+
return hydrated.get(name);
|
|
211
|
+
};
|
|
212
|
+
|
|
213
|
+
/** Keep the hydrated view in step with what was just persisted. */
|
|
214
|
+
const remember = (record: StepRecord): void => {
|
|
215
|
+
hydrated?.set(record.name, record);
|
|
216
|
+
};
|
|
155
217
|
|
|
156
218
|
const claimName = (name: string): void => {
|
|
157
|
-
|
|
158
|
-
|
|
219
|
+
// The Set decides, the array only reports. `Array.includes` made a long run quadratic —
|
|
220
|
+
// `backfill({ batch: 50 })` over a million rows is 20,000 steps and ~200M string compares.
|
|
221
|
+
if (claimed.has(name)) throw new StepDuplicateError({ job: jobName, step: name });
|
|
222
|
+
claimed.add(name);
|
|
223
|
+
trace(used, name);
|
|
224
|
+
};
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* The one-argument `sleep('1h')` derives its step name from the duration, so a poll loop —
|
|
228
|
+
* `for (…) { await step.sleep('1h') }`, the form the docs show — minted `sleep:1h` twice and
|
|
229
|
+
* died with `X_STEP_DUPLICATE` on iteration 2. The occurrence ORDINAL disambiguates them, the
|
|
230
|
+
* way `backfill()`'s `batch:<index>` does: the body re-runs from the top on every replay, so
|
|
231
|
+
* the same sequence of calls mints the same sequence of names.
|
|
232
|
+
*
|
|
233
|
+
* The FIRST occurrence keeps the bare `sleep:1h`, so a run already suspended on one resumes
|
|
234
|
+
* across this change instead of re-sleeping under a new key.
|
|
235
|
+
*/
|
|
236
|
+
const sleepOrdinals = new Map<string, number>();
|
|
237
|
+
const derivedSleepName = (duration: string): string => {
|
|
238
|
+
const base = `sleep:${duration}`;
|
|
239
|
+
const occurrence = (sleepOrdinals.get(base) ?? 0) + 1;
|
|
240
|
+
sleepOrdinals.set(base, occurrence);
|
|
241
|
+
return occurrence === 1 ? base : `${base}#${occurrence}`;
|
|
242
|
+
};
|
|
243
|
+
|
|
244
|
+
const cancelled = (): boolean => runSignal.aborted;
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* EVERY write this runner makes, and the one place the cancellation is enforced. A step result
|
|
248
|
+
* from a cancelled attempt is a write onto the attempt that replaced it: the deadline nacked
|
|
249
|
+
* this job, another worker claimed the same `runId`, and a late `put` would hand it a step it
|
|
250
|
+
* never ran — or overwrite one it did. The write is refused, and refusing it unwinds the body.
|
|
251
|
+
*/
|
|
252
|
+
const put = async (record: StepRecord): Promise<void> => {
|
|
253
|
+
if (cancelled()) throw new JobAbortedError({ job: jobName, step: record.name });
|
|
254
|
+
await store.put(record);
|
|
255
|
+
remember(record);
|
|
159
256
|
};
|
|
160
257
|
|
|
161
258
|
const now = (): number => nowMs(clock);
|
|
162
259
|
|
|
163
|
-
async function run<T>(name: string, fn: () => Promise<T> | T): Promise<T> {
|
|
260
|
+
async function run<T>(name: string, fn: (signal: AbortSignal) => Promise<T> | T): Promise<T> {
|
|
164
261
|
claimName(name);
|
|
165
|
-
const existing = await
|
|
262
|
+
const existing = await load(name);
|
|
166
263
|
if (existing?.status === 'completed') {
|
|
167
|
-
replayed
|
|
264
|
+
trace(replayed, name);
|
|
168
265
|
return existing.output as T;
|
|
169
266
|
}
|
|
170
267
|
|
|
171
268
|
const startedAt = existing?.startedAt ?? now();
|
|
172
269
|
const attempts = (existing?.attempts ?? 0) + 1;
|
|
270
|
+
// The step's own ceiling, folded into the run's cancellation so the body reads ONE signal and
|
|
271
|
+
// sees whichever deadline lands first. Composed only when there is a second one to compose.
|
|
272
|
+
const deadline = new AbortController();
|
|
273
|
+
const signal =
|
|
274
|
+
options.stepTimeoutMs === undefined
|
|
275
|
+
? runSignal
|
|
276
|
+
: AbortSignal.any([runSignal, deadline.signal]);
|
|
173
277
|
try {
|
|
174
278
|
const output = await withStepTimeout(
|
|
175
|
-
fn(),
|
|
279
|
+
fn(signal),
|
|
176
280
|
options.stepTimeoutMs,
|
|
281
|
+
deadline,
|
|
177
282
|
() =>
|
|
178
283
|
new JobTimeoutError({ job: jobName, step: name, timeoutMs: options.stepTimeoutMs ?? 0 }),
|
|
179
284
|
);
|
|
180
285
|
// Persist BEFORE returning: a crash one line later must not re-run this step.
|
|
181
|
-
await
|
|
286
|
+
await put({
|
|
182
287
|
runId,
|
|
183
288
|
name,
|
|
184
289
|
status: 'completed',
|
|
@@ -190,41 +295,50 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
|
|
|
190
295
|
return output;
|
|
191
296
|
} catch (error) {
|
|
192
297
|
if (isStepSuspension(error)) throw error;
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
298
|
+
// The failure is this run's own history, so it is recorded on the way out — unless the
|
|
299
|
+
// attempt was cancelled, in which case the history is no longer ours to write. The original
|
|
300
|
+
// error is what the caller has to see, never one raised by the bookkeeping.
|
|
301
|
+
if (!cancelled()) {
|
|
302
|
+
// Deliberately NOT through `put`: the caller has to see the original error, never one
|
|
303
|
+
// raised by the bookkeeping. The hydrated view is updated by hand for the same reason.
|
|
304
|
+
const failure: StepRecord = {
|
|
305
|
+
runId,
|
|
306
|
+
name,
|
|
307
|
+
status: 'failed',
|
|
308
|
+
startedAt,
|
|
309
|
+
attempts,
|
|
310
|
+
error: error instanceof Error ? error.message : String(error),
|
|
311
|
+
};
|
|
312
|
+
await store.put(failure);
|
|
313
|
+
remember(failure);
|
|
314
|
+
}
|
|
201
315
|
throw error;
|
|
202
316
|
}
|
|
203
317
|
}
|
|
204
318
|
|
|
205
319
|
async function sleep(a: string | DurationInput, b?: DurationInput): Promise<void> {
|
|
206
320
|
// `sleep('3d')` — the duration doubles as the step name, which stays deterministic.
|
|
207
|
-
const name = b === undefined ?
|
|
321
|
+
const name = b === undefined ? derivedSleepName(String(a)) : String(a);
|
|
208
322
|
const duration = b === undefined ? (a as DurationInput) : b;
|
|
209
323
|
claimName(name);
|
|
210
324
|
|
|
211
|
-
const existing = await
|
|
325
|
+
const existing = await load(name);
|
|
212
326
|
if (existing?.status === 'completed') {
|
|
213
|
-
replayed
|
|
327
|
+
trace(replayed, name);
|
|
214
328
|
return;
|
|
215
329
|
}
|
|
216
330
|
|
|
217
331
|
const at = now();
|
|
218
332
|
if (existing?.status === 'sleeping' && existing.wakeAt !== undefined) {
|
|
219
333
|
if (existing.wakeAt <= at) {
|
|
220
|
-
await
|
|
334
|
+
await put({ ...existing, status: 'completed', completedAt: at });
|
|
221
335
|
return;
|
|
222
336
|
}
|
|
223
337
|
throw new StepSuspension({ step: name, resumeAt: existing.wakeAt, reason: 'sleep' });
|
|
224
338
|
}
|
|
225
339
|
|
|
226
340
|
const wakeAt = at + toMs(duration);
|
|
227
|
-
await
|
|
341
|
+
await put({
|
|
228
342
|
runId,
|
|
229
343
|
name,
|
|
230
344
|
status: 'sleeping',
|
|
@@ -241,9 +355,9 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
|
|
|
241
355
|
waitOptions: WaitForEventOptions = {},
|
|
242
356
|
): Promise<T | undefined> {
|
|
243
357
|
claimName(name);
|
|
244
|
-
const existing = await
|
|
358
|
+
const existing = await load(name);
|
|
245
359
|
if (existing?.status === 'completed') {
|
|
246
|
-
replayed
|
|
360
|
+
trace(replayed, name);
|
|
247
361
|
return existing.output as T | undefined;
|
|
248
362
|
}
|
|
249
363
|
|
|
@@ -254,7 +368,7 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
|
|
|
254
368
|
|
|
255
369
|
const hit = await options.events?.find(event, correlationKey, startedAt);
|
|
256
370
|
if (hit !== undefined) {
|
|
257
|
-
await
|
|
371
|
+
await put({
|
|
258
372
|
runId,
|
|
259
373
|
name,
|
|
260
374
|
status: 'completed',
|
|
@@ -277,7 +391,7 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
|
|
|
277
391
|
});
|
|
278
392
|
}
|
|
279
393
|
logger.warn('jobs.step.wait-timeout', { job: jobName, step: name, event });
|
|
280
|
-
await
|
|
394
|
+
await put({
|
|
281
395
|
runId,
|
|
282
396
|
name,
|
|
283
397
|
status: 'completed',
|
|
@@ -291,7 +405,7 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
|
|
|
291
405
|
}
|
|
292
406
|
|
|
293
407
|
const resumeAt = Math.min(deadline, at + pollMs);
|
|
294
|
-
await
|
|
408
|
+
await put({
|
|
295
409
|
runId,
|
|
296
410
|
name,
|
|
297
411
|
status: 'waiting',
|
|
@@ -317,14 +431,24 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
|
|
|
317
431
|
};
|
|
318
432
|
}
|
|
319
433
|
|
|
434
|
+
/**
|
|
435
|
+
* A step's own ceiling. It ABORTS before it rejects, in that order: `run()` fails the step on this
|
|
436
|
+
* rejection and the job retries, so a body still holding a socket open past the deadline would be
|
|
437
|
+
* racing the attempt that replaced it. Cancelling first is the only thing that can stop it.
|
|
438
|
+
*/
|
|
320
439
|
function withStepTimeout<T>(
|
|
321
440
|
work: Promise<T> | T,
|
|
322
441
|
timeoutMs: number | undefined,
|
|
442
|
+
deadline: AbortController,
|
|
323
443
|
error: () => Error,
|
|
324
444
|
): Promise<T> {
|
|
325
445
|
if (timeoutMs === undefined || timeoutMs <= 0) return Promise.resolve(work);
|
|
326
446
|
return new Promise<T>((resolve, reject) => {
|
|
327
|
-
const timer = setTimeout(() =>
|
|
447
|
+
const timer = setTimeout(() => {
|
|
448
|
+
const failure = error();
|
|
449
|
+
deadline.abort(failure);
|
|
450
|
+
reject(failure);
|
|
451
|
+
}, timeoutMs);
|
|
328
452
|
Promise.resolve(work).then(
|
|
329
453
|
(value) => {
|
|
330
454
|
clearTimeout(timer);
|
package/src/task.ts
ADDED
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
// The `task` primitive: a cron declaration and its registry. A task NEVER does work — it
|
|
2
|
+
// enqueues jobs, so retries, idempotency and observability all come from the job machinery
|
|
3
|
+
// instead of being re-invented per cron. `scheduler.ts` is what fires one.
|
|
4
|
+
//
|
|
5
|
+
// `tz` is required by the type. A cron without a timezone is a bug waiting for March: `0 3 *
|
|
6
|
+
// * *` in a DST-observing zone runs twice or zero times on the switch day, and "the nightly
|
|
7
|
+
// digest went out at 2am and again at 3am" is not a mystery anyone should have to debug.
|
|
8
|
+
//
|
|
9
|
+
// **The framework's DST answer, `As of 2026-08`: ONE occurrence per calendar day, at the first
|
|
10
|
+
// valid instant.** Pinned against a real zone in `scheduler-dst.test.ts`, both transitions:
|
|
11
|
+
//
|
|
12
|
+
// FALL BACK — the repeated wall-clock hour yields ONE occurrence, the first (CEST) instant. It
|
|
13
|
+
// has to: `scheduler.ts`'s `dispatch` keys the idempotency key on `occurrenceMs`, and the two
|
|
14
|
+
// instants of a repeated 02:00 genuinely differ, so two occurrences would be two keys and the
|
|
15
|
+
// nightly digest would go out twice — the exact failure a required `tz` exists to prevent.
|
|
16
|
+
// Nothing downstream could catch it.
|
|
17
|
+
// SPRING FORWARD — the missing hour SHIFTS forward (02:00 fires at 03:00), never skips. A
|
|
18
|
+
// skipped day is a billing run that silently never happened, which is worse than one an hour
|
|
19
|
+
// late, and `catchUp` cannot recover an occurrence that was never an occurrence.
|
|
20
|
+
|
|
21
|
+
import { assert } from '@ultimat3/core';
|
|
22
|
+
import { isValidTimeZone } from '@ultimat3/time';
|
|
23
|
+
import { nowMs } from './clock';
|
|
24
|
+
import type { EnqueueResult } from './driver';
|
|
25
|
+
import { JobNameTakenError } from './errors';
|
|
26
|
+
import type { AnyJobHandle } from './job';
|
|
27
|
+
import type { EnqueueOptions } from './outbox';
|
|
28
|
+
|
|
29
|
+
/** `[[sendDigest, {}]]` — a job handle plus its input. */
|
|
30
|
+
export type TaskEnqueueEntry = readonly [AnyJobHandle, unknown];
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* What to do when the scheduler was down across one or more occurrences.
|
|
34
|
+
* `skip` (default) collapses them into ONE dispatch for the LATEST missed occurrence — the
|
|
35
|
+
* older ones are dropped, never replayed; `run-once` fires a single catch-up, the EARLIEST
|
|
36
|
+
* missed one; `run-all` fires one per missed occurrence, bounded by `maxCatchUp`.
|
|
37
|
+
*/
|
|
38
|
+
export type CatchUpPolicy = 'skip' | 'run-once' | 'run-all';
|
|
39
|
+
|
|
40
|
+
export interface TaskDefinition {
|
|
41
|
+
/** Omit it: `defineApi({ tasks })` assigns the export name. Set it only to pin the name the
|
|
42
|
+
* scheduler's `lastFiredAt` and occurrence lock are already keyed by. */
|
|
43
|
+
readonly name?: string;
|
|
44
|
+
readonly cron: string;
|
|
45
|
+
/** REQUIRED IANA zone, e.g. `'UTC'`, `'America/New_York'`. */
|
|
46
|
+
readonly tz: string;
|
|
47
|
+
/**
|
|
48
|
+
* Builds the entries for ONE occurrence, given that occurrence's instant in epoch ms.
|
|
49
|
+
*
|
|
50
|
+
* The argument exists because catch-up does: a tick dispatched late, or replayed for a
|
|
51
|
+
* missed occurrence, has a wall clock that no longer matches the occurrence being fired.
|
|
52
|
+
* A payload derived from `Date.now()` there is silently for the wrong day — and the
|
|
53
|
+
* scheduler's own key is occurrence-scoped, so nothing downstream catches it.
|
|
54
|
+
*/
|
|
55
|
+
enqueue: (occurrenceMs: number) => readonly TaskEnqueueEntry[];
|
|
56
|
+
readonly catchUp?: CatchUpPolicy;
|
|
57
|
+
readonly maxCatchUp?: number;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** One entry's outcome, from a scheduled dispatch or a manual `task.enqueue()` alike. */
|
|
61
|
+
export interface TaskJobResult {
|
|
62
|
+
readonly job: string;
|
|
63
|
+
readonly result: EnqueueResult;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** JSON-safe view of a task for the manifest, `/_x` and the MCP dev server. */
|
|
67
|
+
export interface TaskDescriptor {
|
|
68
|
+
readonly kind: 'task';
|
|
69
|
+
readonly name: string;
|
|
70
|
+
readonly cron: string;
|
|
71
|
+
readonly tz: string;
|
|
72
|
+
readonly catchUp: CatchUpPolicy;
|
|
73
|
+
readonly maxCatchUp: number;
|
|
74
|
+
readonly jobs: readonly string[];
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export interface TaskHandle {
|
|
78
|
+
readonly kind: 'task';
|
|
79
|
+
readonly name: string;
|
|
80
|
+
readonly cron: string;
|
|
81
|
+
readonly tz: string;
|
|
82
|
+
readonly catchUp: CatchUpPolicy;
|
|
83
|
+
readonly maxCatchUp: number;
|
|
84
|
+
/**
|
|
85
|
+
* Entries for `occurrenceMs`. Defaults to now, which is the honest answer for the two
|
|
86
|
+
* callers that have no occurrence: a manual `task.enqueue()` and `describe()`, which only
|
|
87
|
+
* wants the job names.
|
|
88
|
+
*/
|
|
89
|
+
entries(occurrenceMs?: number): readonly TaskEnqueueEntry[];
|
|
90
|
+
/**
|
|
91
|
+
* Fire this task's declared entries now, through the same facade `JobHandle.enqueue` uses —
|
|
92
|
+
* the backfill and "run it again" path, with no scheduler and no leader involved.
|
|
93
|
+
*/
|
|
94
|
+
enqueue(options?: EnqueueOptions): Promise<readonly TaskJobResult[]>;
|
|
95
|
+
describe(): TaskDescriptor;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
const registry = new Map<string, TaskHandle>();
|
|
99
|
+
let anonymous = 0;
|
|
100
|
+
|
|
101
|
+
/** Job's store, for tasks: proof `task()` built the handle, plus whether it named itself. */
|
|
102
|
+
interface TaskOrigin {
|
|
103
|
+
readonly declaredName: boolean;
|
|
104
|
+
/** The export name already stamped, once one has been. `undefined` while still provisional. */
|
|
105
|
+
readonly exportName?: string;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
const origin = new WeakMap<object, TaskOrigin>();
|
|
109
|
+
|
|
110
|
+
export function task(definition: TaskDefinition): TaskHandle {
|
|
111
|
+
anonymous += 1;
|
|
112
|
+
const name = definition.name ?? `anonymous-task-${anonymous}`;
|
|
113
|
+
// Runtime backstop; the type already makes an omitted tz a build error.
|
|
114
|
+
assert(
|
|
115
|
+
typeof definition.tz === 'string' && definition.tz.length > 0,
|
|
116
|
+
`task "${name}" needs an explicit IANA tz — a cron without a timezone is a bug`,
|
|
117
|
+
`add tz to task("${name}"), e.g. tz: 'UTC' — an unzoned cron silently drifts by an hour at every DST transition`,
|
|
118
|
+
);
|
|
119
|
+
// A non-empty string is not a timezone. `tz: 'Bogota'` would otherwise resolve every
|
|
120
|
+
// occurrence in UTC and the cron would run five hours off, silently, forever. `time`'s
|
|
121
|
+
// validator and not a local `Intl` probe: ES2024 `Intl` ACCEPTS `'+02:00'`, and a fixed offset
|
|
122
|
+
// carries no DST rules — the one thing a cron's timezone exists to supply. One validator means
|
|
123
|
+
// a zone `task()` accepts is a zone `@ultimat3/time` can then do arithmetic in.
|
|
124
|
+
assert(
|
|
125
|
+
isValidTimeZone(definition.tz),
|
|
126
|
+
`task "${name}" has tz "${definition.tz}", which is not a zone in the IANA tz database`,
|
|
127
|
+
`use the full zone id on task("${name}"), e.g. tz: 'America/Bogota' — list the valid ones with: bun -e "console.log(Intl.supportedValuesOf('timeZone').join('\\n'))"`,
|
|
128
|
+
);
|
|
129
|
+
|
|
130
|
+
const handle: TaskHandle = {
|
|
131
|
+
kind: 'task',
|
|
132
|
+
name,
|
|
133
|
+
cron: definition.cron,
|
|
134
|
+
tz: definition.tz,
|
|
135
|
+
catchUp: definition.catchUp ?? 'skip',
|
|
136
|
+
maxCatchUp: definition.maxCatchUp ?? 10,
|
|
137
|
+
// `nowMs()` and not `Date.now()`: every reading of time in this package goes through a
|
|
138
|
+
// Clock so a frozen one cannot be bypassed.
|
|
139
|
+
entries: (occurrenceMs: number = nowMs()) => definition.enqueue(occurrenceMs),
|
|
140
|
+
async enqueue(options?: EnqueueOptions): Promise<readonly TaskJobResult[]> {
|
|
141
|
+
const fired: TaskJobResult[] = [];
|
|
142
|
+
for (const [handleForJob, input] of handle.entries()) {
|
|
143
|
+
// The job's PLAIN key, deliberately not `dispatch()`'s `task:occurrence:key`: that one
|
|
144
|
+
// is occurrence-scoped so two schedulers cannot double-fire the same tick, and reusing
|
|
145
|
+
// it here would make a manual run dedupe against whichever occurrence it landed in.
|
|
146
|
+
fired.push({ job: handleForJob.name, result: await handleForJob.enqueue(input, options) });
|
|
147
|
+
}
|
|
148
|
+
return fired;
|
|
149
|
+
},
|
|
150
|
+
// Reads `handle`, never the captured `name`: `nameTasks()` rebinds the property in place.
|
|
151
|
+
describe(): TaskDescriptor {
|
|
152
|
+
return {
|
|
153
|
+
kind: 'task',
|
|
154
|
+
name: handle.name,
|
|
155
|
+
cron: handle.cron,
|
|
156
|
+
tz: handle.tz,
|
|
157
|
+
catchUp: handle.catchUp,
|
|
158
|
+
maxCatchUp: handle.maxCatchUp,
|
|
159
|
+
// Declaration order: a task's entries are a sequence, not a set.
|
|
160
|
+
jobs: handle.entries().map(([entry]) => entry.name),
|
|
161
|
+
};
|
|
162
|
+
},
|
|
163
|
+
};
|
|
164
|
+
origin.set(handle, { declaredName: definition.name !== undefined });
|
|
165
|
+
// Refused here, not at `registerTask`: a second `task({ name: 'nightly' })` would otherwise
|
|
166
|
+
// replace the seated handle, and the scheduler's persisted `lastFiredAt` — keyed by that name —
|
|
167
|
+
// would silently start driving a different cron. The anonymous names cannot collide.
|
|
168
|
+
if (registry.has(name)) throw new JobNameTakenError({ kind: 'task', name });
|
|
169
|
+
registry.set(name, handle);
|
|
170
|
+
return handle;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/** Structural, exactly as `isJobHandle` is: only a handle `task()` built has a cron behind it. */
|
|
174
|
+
export function isTaskHandle(value: unknown): value is TaskHandle {
|
|
175
|
+
return (
|
|
176
|
+
typeof value === 'object' &&
|
|
177
|
+
value !== null &&
|
|
178
|
+
(value as { kind?: unknown }).kind === 'task' &&
|
|
179
|
+
origin.has(value)
|
|
180
|
+
);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* Register `target` under `name`, stamped onto the handle the module exported — the scheduler's
|
|
185
|
+
* occurrence key is `task:occurrenceMs:jobKey`, so the task's name is what stops two nodes
|
|
186
|
+
* double-firing a tick, and a copy under a second name would defeat it.
|
|
187
|
+
*
|
|
188
|
+
* A definition that supplied its own `name` keeps it, for the same reason a job's does: the
|
|
189
|
+
* scheduler's persisted `lastFiredAt` is keyed by that name.
|
|
190
|
+
*/
|
|
191
|
+
export function registerTask(name: string, target: TaskHandle): TaskHandle {
|
|
192
|
+
const source = origin.get(target);
|
|
193
|
+
const key = source?.declaredName === true ? target.name : name;
|
|
194
|
+
const seated = registry.get(key);
|
|
195
|
+
// The same handle under the same name is one registration seen twice — `defineApi` and the
|
|
196
|
+
// framework's module scan both reach the same declaration file. A DIFFERENT task under a taken
|
|
197
|
+
// name is the ambiguity to refuse.
|
|
198
|
+
if (seated !== undefined) {
|
|
199
|
+
if (seated !== target) throw new JobNameTakenError({ kind: 'task', name: key });
|
|
200
|
+
return target;
|
|
201
|
+
}
|
|
202
|
+
// One handle exported under two names: the rebind below is in place, so the second alias would
|
|
203
|
+
// move the occurrence key the scheduler dedupes ticks on.
|
|
204
|
+
if (source?.exportName !== undefined && source.exportName !== key)
|
|
205
|
+
throw new JobNameTakenError({ kind: 'task', name: key });
|
|
206
|
+
registry.delete(target.name);
|
|
207
|
+
Object.defineProperty(target, 'name', { value: key, configurable: true });
|
|
208
|
+
if (source !== undefined) origin.set(target, { ...source, exportName: key });
|
|
209
|
+
registry.set(key, target);
|
|
210
|
+
return target;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/** `registerTasks(module)` is the call app code makes; this is the same rules over a record. */
|
|
214
|
+
export function nameTasks(record: Readonly<Record<string, TaskHandle>>): void {
|
|
215
|
+
for (const [exportName, handle] of Object.entries(record)) registerTask(exportName, handle);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
export function registeredTasks(): readonly TaskHandle[] {
|
|
219
|
+
return [...registry.values()].sort((a, b) => a.name.localeCompare(b.name));
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
export function getTask(name: string): TaskHandle | undefined {
|
|
223
|
+
return registry.get(name);
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
export function resetTasks(): void {
|
|
227
|
+
registry.clear();
|
|
228
|
+
anonymous = 0;
|
|
229
|
+
}
|
package/src/tenant.ts
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
// What tenant a job's body runs as. A job is server-authoritative work with no request behind it,
|
|
2
|
+
// so the org cannot be read off a caller — it is DECLARED, per job, and derived from that job's
|
|
3
|
+
// OWN input. A boot-supplied service actor was the alternative and is rejected: one identity shared
|
|
4
|
+
// by every job is the cross-tenant read this declaration exists to make impossible.
|
|
5
|
+
|
|
6
|
+
import type { Actor } from '@ultimat3/core';
|
|
7
|
+
import { assert } from '@ultimat3/core';
|
|
8
|
+
import { JobTenantRequiredError } from './errors';
|
|
9
|
+
|
|
10
|
+
/** The literal a job spells when it belongs to no tenant. */
|
|
11
|
+
export const NO_JOB_TENANT = 'none';
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Either the org this job's input names, or the explicit statement that it names none. There is no
|
|
15
|
+
* third spelling and no default: a job that declared neither used to run with whatever ambient
|
|
16
|
+
* context the worker happened to have, which was none — so `@ultimat3/entity`'s tenant guard read
|
|
17
|
+
* no actor, derived no predicate and accepted a caller-named tenant unchecked.
|
|
18
|
+
*/
|
|
19
|
+
export type JobTenant<I> = ((input: I) => string) | typeof NO_JOB_TENANT;
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Runtime backstop for generated code and JS callers; TS already forbids omitting it. The mirror of
|
|
23
|
+
* `idempotencyKey`'s backstop, and refused at `job()` for the same reason — the earliest point at
|
|
24
|
+
* which the mistake is decidable is the declaration, not the first claim.
|
|
25
|
+
*/
|
|
26
|
+
export function assertJobTenant(job: string, tenant: unknown): void {
|
|
27
|
+
if (typeof tenant === 'function' || tenant === NO_JOB_TENANT) return;
|
|
28
|
+
throw new JobTenantRequiredError({ job });
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* The org one run acts under: `undefined` for `'none'`, which is what makes a tenant-scoped read
|
|
33
|
+
* inside such a job fail closed with `X_TENANCY_ACTOR_ORG_REQUIRED` rather than read somebody's
|
|
34
|
+
* rows by accident.
|
|
35
|
+
*
|
|
36
|
+
* An empty answer is refused here rather than carried: `''` is not a tenant, and an actor holding
|
|
37
|
+
* one would satisfy the guard's `orgId !== undefined` check while matching no row's `org_id`.
|
|
38
|
+
* `assert` and not a code of its own, exactly as `idempotencyKeyFor`'s empty-key refusal is — the
|
|
39
|
+
* declaration is wrong in both cases and the repair is the same edit.
|
|
40
|
+
*/
|
|
41
|
+
export function jobTenantFor<I>(job: string, tenant: JobTenant<I>, input: I): string | undefined {
|
|
42
|
+
if (tenant === NO_JOB_TENANT) return undefined;
|
|
43
|
+
const orgId = tenant(input);
|
|
44
|
+
assert(
|
|
45
|
+
typeof orgId === 'string' && orgId.length > 0,
|
|
46
|
+
`job "${job}" tenant() returned an empty tenant, so the run would carry an org no row can match`,
|
|
47
|
+
`return the org id from job("${job}").tenant — tenant: (input) => input.orgId — or declare tenant: 'none' if this job touches no tenant-scoped table`,
|
|
48
|
+
);
|
|
49
|
+
return orgId;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* The actor the run's ambient context carries. The identity itself is whoever the app wired into
|
|
54
|
+
* `WorkerOptions.context()` — this changes only the ORG, because the tenant is a fact about the
|
|
55
|
+
* WORK and not about which process claimed it. `'none'` strips the org rather than leaving one
|
|
56
|
+
* behind: a job that declared no tenant must not inherit the worker's.
|
|
57
|
+
*/
|
|
58
|
+
export function jobRunActor(actor: Actor, orgId: string | undefined): Actor {
|
|
59
|
+
if (actor.orgId === orgId) return actor;
|
|
60
|
+
return Object.freeze({ ...actor, orgId });
|
|
61
|
+
}
|