@ultimat3/jobs 22.15.0 → 24.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +165 -124
- package/README.md +360 -21
- package/package.json +6 -6
- package/src/backfill-ledger.ts +24 -2
- package/src/backfill-pass.ts +22 -5
- package/src/claim-exhausted.ts +76 -0
- package/src/concurrency.ts +144 -0
- package/src/counters.ts +105 -0
- package/src/describe.ts +29 -0
- package/src/driver-memory-operator.ts +319 -0
- package/src/driver-memory.ts +146 -59
- package/src/driver-nats.ts +7 -4
- package/src/driver-pg-ddl.ts +71 -0
- package/src/driver-pg-jobs-sql.ts +96 -25
- package/src/driver-pg-operator-sql.ts +323 -0
- package/src/driver-pg-operator.ts +316 -0
- package/src/driver-pg-outbox-sql.ts +110 -0
- package/src/driver-pg-rows.ts +34 -3
- package/src/driver-pg-settle-sql.ts +71 -0
- package/src/driver-pg-sql.ts +171 -151
- package/src/driver-pg-wake-sql.ts +76 -0
- package/src/driver-pg.ts +150 -105
- package/src/driver-redis.ts +7 -4
- package/src/driver.ts +133 -35
- package/src/enqueue-signal.ts +50 -0
- package/src/errors-concurrency.ts +91 -0
- package/src/errors-operator.ts +102 -0
- package/src/errors-requeue.ts +21 -3
- package/src/errors.ts +18 -30
- package/src/events-pg.ts +45 -45
- package/src/events.ts +38 -2
- package/src/execute.ts +161 -81
- package/src/export.ts +23 -4
- package/src/heartbeat.ts +9 -3
- package/src/idle-backoff.ts +74 -0
- package/src/index.ts +115 -16
- package/src/inspect-operator.ts +49 -0
- package/src/inspect.ts +46 -5
- package/src/introspection.ts +274 -0
- package/src/job.ts +83 -21
- package/src/leases.ts +49 -9
- package/src/limits.ts +33 -7
- package/src/metrics.ts +6 -1
- package/src/outbox-pg.ts +4 -2
- package/src/outbox-relay.ts +114 -39
- package/src/outbox.ts +59 -8
- package/src/progress.ts +78 -0
- package/src/purge.ts +1 -1
- package/src/queue-wake.ts +219 -0
- package/src/redact-input.ts +26 -0
- package/src/renewal-timer.ts +28 -10
- package/src/retry-classification.ts +13 -1
- package/src/retry.ts +10 -2
- package/src/run-deadline.ts +57 -0
- package/src/scheduler-leader.ts +26 -0
- package/src/scheduler-occurrences.ts +76 -0
- package/src/scheduler-pg.ts +58 -4
- package/src/scheduler-state.ts +73 -0
- package/src/scheduler.ts +150 -111
- package/src/settled.ts +161 -0
- package/src/steps-suspension.ts +46 -0
- package/src/steps-timeout.ts +33 -0
- package/src/steps.ts +55 -81
- package/src/worker-admit.ts +167 -0
- package/src/worker-fleet-slots.ts +157 -54
- package/src/worker-hand-back.ts +6 -1
- package/src/worker-key-busy.ts +107 -0
- package/src/worker-loop.ts +169 -0
- package/src/worker-registry.ts +118 -0
- package/src/worker-run.ts +9 -0
- package/src/worker-types.ts +23 -0
- package/src/worker.ts +142 -130
package/src/backfill-pass.ts
CHANGED
|
@@ -164,7 +164,13 @@ export async function backfillPass<Row>(
|
|
|
164
164
|
// refusing it — the same degradation `introspect` already has.
|
|
165
165
|
const ledger = jobDriver()?.backfills;
|
|
166
166
|
|
|
167
|
-
const
|
|
167
|
+
const completed = await completedRun(ledger, name);
|
|
168
|
+
// Its OWN row is not a previous pass: a worker that died between `ledger.finish` and the ack is
|
|
169
|
+
// handed the same run again, and skipping there reported `{ skipped: true, previousRunId:
|
|
170
|
+
// <itself> }` for a pass that swept. It falls through and replays — every batch is served from
|
|
171
|
+
// its checkpoint, nothing is read or handled twice, and the report is the one it first made.
|
|
172
|
+
const redelivered = completed?.runId === runId;
|
|
173
|
+
const previous = redelivered ? undefined : completed;
|
|
168
174
|
const verdict = decideBackfill(previous, checksum, args.input.force === true);
|
|
169
175
|
if (!verdict.run) {
|
|
170
176
|
if (verdict.changed) {
|
|
@@ -196,7 +202,14 @@ export async function backfillPass<Row>(
|
|
|
196
202
|
definition.tenant,
|
|
197
203
|
args.ctx,
|
|
198
204
|
async (ctx): Promise<BackfillReport> => {
|
|
199
|
-
|
|
205
|
+
// A redelivered pass leaves its completed row ALONE until a batch body actually runs (an
|
|
206
|
+
// operator's `retryFromStep` re-opens one): replaying checkpoints sweeps nothing, so there is
|
|
207
|
+
// nothing to record and no convergence to re-ask — rows that started matching since would
|
|
208
|
+
// otherwise turn a finished pass into a failed one.
|
|
209
|
+
let recording = !redelivered;
|
|
210
|
+
const open = (): Promise<void> | undefined =>
|
|
211
|
+
ledger?.start({ runId, name, checksum, appVersion: appVersion() });
|
|
212
|
+
if (recording) await open();
|
|
200
213
|
|
|
201
214
|
let cursor: string | null = null;
|
|
202
215
|
let rows = 0;
|
|
@@ -242,6 +255,10 @@ export async function backfillPass<Row>(
|
|
|
242
255
|
const iteration = await iterate();
|
|
243
256
|
const next = await iteration.pull.next();
|
|
244
257
|
if (next.done === true) return { cursor: null, rows: 0 };
|
|
258
|
+
if (!recording) {
|
|
259
|
+
recording = true;
|
|
260
|
+
await open();
|
|
261
|
+
}
|
|
245
262
|
await definition.handle({ rows: next.value, ctx, signal, index });
|
|
246
263
|
swept = true;
|
|
247
264
|
return { cursor: iteration.batches.cursor, rows: next.value.length };
|
|
@@ -271,10 +288,10 @@ export async function backfillPass<Row>(
|
|
|
271
288
|
// this converge", asked once, where the answer is finally decidable. Inside the `try` so the
|
|
272
289
|
// ledger records the attempt as `failed`: a pass that left rows behind must not write the
|
|
273
290
|
// completed row that stops the next deploy re-running it.
|
|
274
|
-
await assertConverged(definition, ctx, rows);
|
|
291
|
+
if (recording) await assertConverged(definition, ctx, rows);
|
|
275
292
|
} catch (error) {
|
|
276
293
|
// Control flow, not a failure: a suspended run is parked and will be back on this step.
|
|
277
|
-
if (!isStepSuspension(error)) await markFailed(ledger, runId, rows);
|
|
294
|
+
if (!isStepSuspension(error) && recording) await markFailed(ledger, runId, rows);
|
|
278
295
|
throw error;
|
|
279
296
|
} finally {
|
|
280
297
|
// Whatever the iteration holds belongs to this attempt, and an attempt that failed, was
|
|
@@ -282,7 +299,7 @@ export async function backfillPass<Row>(
|
|
|
282
299
|
await live?.batches.close();
|
|
283
300
|
}
|
|
284
301
|
|
|
285
|
-
await ledger?.finish(runId, { status: 'completed', rows });
|
|
302
|
+
if (recording) await ledger?.finish(runId, { status: 'completed', rows });
|
|
286
303
|
return { name, batches, rows, skipped: false };
|
|
287
304
|
},
|
|
288
305
|
);
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
// The claim round's half of a burial: a row the claim settled `dead` (`ClaimOptions.onExhausted`)
|
|
2
|
+
// has no body to end it and no settle to announce it, so the pass that buried it says so — one log
|
|
3
|
+
// line an operator can alert on, and the job's `onSettled`, exactly as a dead letter a worker
|
|
4
|
+
// settled itself would get.
|
|
5
|
+
|
|
6
|
+
import type { Ctx } from '@ultimat3/core';
|
|
7
|
+
import { logger, renderThrowable } from '@ultimat3/core';
|
|
8
|
+
import type { JobRecord } from './driver';
|
|
9
|
+
import { getJob } from './job';
|
|
10
|
+
import { announceSettled } from './settled';
|
|
11
|
+
|
|
12
|
+
export interface AnnounceExhaustedOptions {
|
|
13
|
+
/** What `onExhausted` handed this pass — every row the claim buried, as it now stands. */
|
|
14
|
+
readonly exhausted: readonly JobRecord[];
|
|
15
|
+
readonly workerId: string;
|
|
16
|
+
/** The worker's own context: what the job's `onSettled` is scoped from, as a body would be. */
|
|
17
|
+
readonly context: () => Ctx;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** What one pass buried, by how each row ended — what the worker's own counters take. */
|
|
21
|
+
export interface ExhaustedCounts {
|
|
22
|
+
readonly deadLettered: number;
|
|
23
|
+
readonly dropped: number;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Announce every row one claim pass buried, and answer how each ended. Once per row, because the
|
|
28
|
+
* claim buries a row once: the statement that settled it is the only one that hands it to
|
|
29
|
+
* `onExhausted`. A row buried `failed` is a job declaring `retry.deadLetter: false`
|
|
30
|
+
* (`ClaimOptions.dropExhausted`) — `dropped`, as its own nack would have ended it. Never rejects:
|
|
31
|
+
* `announceSettled` owns a hook's failure, and a `context()` that throws is logged.
|
|
32
|
+
*/
|
|
33
|
+
export async function announceExhausted(
|
|
34
|
+
options: AnnounceExhaustedOptions,
|
|
35
|
+
): Promise<ExhaustedCounts> {
|
|
36
|
+
let dropped = 0;
|
|
37
|
+
for (const record of options.exhausted) {
|
|
38
|
+
const error = record.lastError ?? '';
|
|
39
|
+
const outcome = record.state === 'failed' ? 'dropped' : 'dead-lettered';
|
|
40
|
+
if (outcome === 'dropped') dropped += 1;
|
|
41
|
+
// `error`, not `warn`: nobody recovered from this one, and a job that outlives every worker
|
|
42
|
+
// that takes it is a crash loop somebody has to look at.
|
|
43
|
+
logger.error('jobs.claim.exhausted', {
|
|
44
|
+
workerId: options.workerId,
|
|
45
|
+
job: record.name,
|
|
46
|
+
jobId: record.id,
|
|
47
|
+
attempt: record.attempt,
|
|
48
|
+
maxAttempts: record.maxAttempts,
|
|
49
|
+
outcome,
|
|
50
|
+
error,
|
|
51
|
+
});
|
|
52
|
+
const handle = getJob(record.name);
|
|
53
|
+
if (handle === undefined) continue;
|
|
54
|
+
let ctx: Ctx;
|
|
55
|
+
try {
|
|
56
|
+
ctx = options.context();
|
|
57
|
+
} catch (raised) {
|
|
58
|
+
// The app's own function, asked AFTER the row was settled: thrown, it failed the round.
|
|
59
|
+
logger.error('jobs.claim.exhausted-unannounced', {
|
|
60
|
+
workerId: options.workerId,
|
|
61
|
+
job: record.name,
|
|
62
|
+
jobId: record.id,
|
|
63
|
+
error: renderThrowable(raised),
|
|
64
|
+
});
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
await announceSettled({
|
|
68
|
+
handle,
|
|
69
|
+
claimed: record,
|
|
70
|
+
ctx,
|
|
71
|
+
// No code: nothing was thrown, and `JobFailed.code` is never invented.
|
|
72
|
+
settlement: { outcome, error, code: undefined },
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
return { deadLettered: options.exhausted.length - dropped, dropped };
|
|
76
|
+
}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
// `job.concurrency` as DECLARED, resolved once: a plain number is one cap for the whole job, and
|
|
2
|
+
// `{ key, limit, whenBusy }` is that cap per key — "at most N runs of this job for this account,
|
|
3
|
+
// fleet-wide". Apart from `job.ts` because that file is the primitive and its registry; this one
|
|
4
|
+
// is the single reader of the two spellings, so the worker and the projections never branch on
|
|
5
|
+
// which one an author wrote.
|
|
6
|
+
|
|
7
|
+
import type { ConcurrencyKeyDefect } from './errors-concurrency';
|
|
8
|
+
import { JobConcurrencyInvalidError, JobConcurrencyKeyInvalidError } from './errors-concurrency';
|
|
9
|
+
|
|
10
|
+
/** What a claim does when its key already holds `limit` runs. Closed: there is no third answer. */
|
|
11
|
+
export const WHEN_BUSY = ['wait', 'fail'] as const;
|
|
12
|
+
|
|
13
|
+
export type WhenBusy = (typeof WHEN_BUSY)[number];
|
|
14
|
+
|
|
15
|
+
export interface KeyedConcurrency<I> {
|
|
16
|
+
/**
|
|
17
|
+
* The lease this input's run counts under. Deterministic from `input` only — it is asked once
|
|
18
|
+
* at enqueue and again at every claim, and two answers for one input are two locks.
|
|
19
|
+
*/
|
|
20
|
+
readonly key: (input: I) => string;
|
|
21
|
+
/** Max in-flight runs sharing one key, across the fleet. A whole number, 1 or more. */
|
|
22
|
+
readonly limit: number;
|
|
23
|
+
/**
|
|
24
|
+
* `'wait'` (default): the run stays queued and is claimed again once a slot frees.
|
|
25
|
+
* `'fail'`: the run settles `failed` with `X_JOB_KEY_BUSY`, its body never runs, nothing retries it.
|
|
26
|
+
*
|
|
27
|
+
* There is no `duration`: the bound after which a stuck holder stops blocking its key is the
|
|
28
|
+
* lease TTL the heartbeat renews (`leases.ts`), and a second number that could disagree with it
|
|
29
|
+
* would be a second way to say one thing.
|
|
30
|
+
*/
|
|
31
|
+
readonly whenBusy?: WhenBusy;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export type JobConcurrency<I> = number | KeyedConcurrency<I>;
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* A lease row is keyed by this text and `x_job_leases` indexes it, so a key is an id, never a
|
|
38
|
+
* payload. Far under what a btree entry holds, and far over any account or connection id.
|
|
39
|
+
*/
|
|
40
|
+
export const MAX_CONCURRENCY_KEY_LENGTH = 200;
|
|
41
|
+
|
|
42
|
+
export interface ResolvedConcurrency<I> {
|
|
43
|
+
/** The cap — per key when `whenBusy` is set, for the whole job when it is not. */
|
|
44
|
+
readonly limit: number | undefined;
|
|
45
|
+
/** Set exactly when the cap is KEYED, so it doubles as that question's answer. */
|
|
46
|
+
readonly whenBusy: WhenBusy | undefined;
|
|
47
|
+
/**
|
|
48
|
+
* The key this input counts under, or `undefined` when the cap is not keyed. `name` is the
|
|
49
|
+
* job's CURRENT name — registration renames a handle in place, after this was resolved.
|
|
50
|
+
*/
|
|
51
|
+
keyFor(name: string, input: I): string | undefined;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const unkeyed = (): undefined => undefined;
|
|
55
|
+
|
|
56
|
+
/** A cap some worker can fill: whole, and at least one. Refuses `NaN`, `Infinity` and `1.5` too. */
|
|
57
|
+
const fillable = (limit: unknown): limit is number =>
|
|
58
|
+
typeof limit === 'number' && Number.isInteger(limit) && limit >= 1;
|
|
59
|
+
|
|
60
|
+
const isWhenBusy = (value: unknown): value is WhenBusy =>
|
|
61
|
+
(WHEN_BUSY as readonly unknown[]).includes(value);
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Refuses a declaration no worker can honour, where it is written. A `limit` of zero is a slot
|
|
65
|
+
* table that grants nothing — `leases.acquire(key, 0, …)` answers `undefined` forever with no log
|
|
66
|
+
* line — and `NaN` / `Infinity` / `1.5` are the same defect spelled three other ways.
|
|
67
|
+
*/
|
|
68
|
+
export function resolveConcurrency<I>(
|
|
69
|
+
job: string,
|
|
70
|
+
declared: JobConcurrency<I> | undefined,
|
|
71
|
+
): ResolvedConcurrency<I> {
|
|
72
|
+
if (declared === undefined) return { limit: undefined, whenBusy: undefined, keyFor: unkeyed };
|
|
73
|
+
if (typeof declared === 'number') {
|
|
74
|
+
// ONE refusal for both spellings: the plain number used to raise `X_INVARIANT`, which was a
|
|
75
|
+
// second code for the statement `concurrency.limit` makes below.
|
|
76
|
+
if (!fillable(declared)) {
|
|
77
|
+
throw new JobConcurrencyInvalidError({
|
|
78
|
+
job,
|
|
79
|
+
field: 'concurrency',
|
|
80
|
+
declared,
|
|
81
|
+
why: 'which no worker can ever fill',
|
|
82
|
+
edit: 'set a whole concurrency of 1 or more, or omit the field for no cap at all,',
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
return { limit: declared, whenBusy: undefined, keyFor: unkeyed };
|
|
86
|
+
}
|
|
87
|
+
// Read off the VALUE: generated code and JS callers reach here with what the type forbids.
|
|
88
|
+
const loose: unknown = declared;
|
|
89
|
+
if (typeof loose !== 'object' || loose === null) {
|
|
90
|
+
throw new JobConcurrencyInvalidError({
|
|
91
|
+
job,
|
|
92
|
+
field: 'concurrency',
|
|
93
|
+
declared: loose,
|
|
94
|
+
why: 'which is neither a whole number nor { key, limit }',
|
|
95
|
+
edit: 'set concurrency: { key: (input) => input.accountId, limit: 1 }',
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
const { key, limit, whenBusy } = declared;
|
|
99
|
+
if (typeof key !== 'function') {
|
|
100
|
+
throw new JobConcurrencyInvalidError({
|
|
101
|
+
job,
|
|
102
|
+
field: 'concurrency.key',
|
|
103
|
+
declared: key,
|
|
104
|
+
why: 'which is not a function of the input',
|
|
105
|
+
edit: 'set concurrency.key: (input) => input.accountId',
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
if (!fillable(limit)) {
|
|
109
|
+
throw new JobConcurrencyInvalidError({
|
|
110
|
+
job,
|
|
111
|
+
field: 'concurrency.limit',
|
|
112
|
+
declared: limit,
|
|
113
|
+
why: 'which no worker can ever fill',
|
|
114
|
+
edit: 'set a whole concurrency.limit of 1 or more',
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
if (whenBusy !== undefined && !isWhenBusy(whenBusy)) {
|
|
118
|
+
throw new JobConcurrencyInvalidError({
|
|
119
|
+
job,
|
|
120
|
+
field: 'concurrency.whenBusy',
|
|
121
|
+
declared: whenBusy,
|
|
122
|
+
why: `and only ${WHEN_BUSY.join(' and ')} exist`,
|
|
123
|
+
edit: "set concurrency.whenBusy to 'wait' (the run stays queued) or 'fail' (the run settles failed with X_JOB_KEY_BUSY)",
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
return {
|
|
128
|
+
limit,
|
|
129
|
+
whenBusy: whenBusy ?? 'wait',
|
|
130
|
+
keyFor(name: string, input: I): string {
|
|
131
|
+
const answered: unknown = key(input);
|
|
132
|
+
const refuse = (defect: ConcurrencyKeyDefect): JobConcurrencyKeyInvalidError =>
|
|
133
|
+
new JobConcurrencyKeyInvalidError({
|
|
134
|
+
job: name,
|
|
135
|
+
defect,
|
|
136
|
+
maxLength: MAX_CONCURRENCY_KEY_LENGTH,
|
|
137
|
+
});
|
|
138
|
+
if (typeof answered !== 'string') throw refuse('unreadable');
|
|
139
|
+
if (answered.length === 0) throw refuse('empty');
|
|
140
|
+
if (answered.length > MAX_CONCURRENCY_KEY_LENGTH) throw refuse('too-long');
|
|
141
|
+
return answered;
|
|
142
|
+
},
|
|
143
|
+
};
|
|
144
|
+
}
|
package/src/counters.ts
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
// The per-job history counters: which bucket a settle lands in, what it adds, and how buckets
|
|
2
|
+
// age from one tier to the next. The arithmetic lives here ONCE — the memory store below runs it
|
|
3
|
+
// directly and the pg statements (`driver-pg-operator-sql.ts`) spell the same floor in SQL, so a
|
|
4
|
+
// count read under `x dev` and one read in production are the same number for the same settles.
|
|
5
|
+
|
|
6
|
+
import type { NackOptions } from './driver';
|
|
7
|
+
import { nackState } from './driver';
|
|
8
|
+
import type { CounterBucketMs, CounterOutcome, CounterTotals, JobCounter } from './introspection';
|
|
9
|
+
import { COUNTER_BUCKET_MS, COUNTER_TIERS } from './introspection';
|
|
10
|
+
|
|
11
|
+
/** The instant a bucket of `bucketMs` containing `atMs` opens. */
|
|
12
|
+
export const counterBucketStart = (atMs: number, bucketMs: number): number =>
|
|
13
|
+
Math.floor(atMs / bucketMs) * bucketMs;
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* What a nack adds, or nothing. A shed, a suspension and a drained attempt are handed back
|
|
17
|
+
* UNCOUNTED and are not history: counting them would make every `step.sleep` a failure.
|
|
18
|
+
*/
|
|
19
|
+
export function nackOutcome(options: NackOptions): CounterOutcome | undefined {
|
|
20
|
+
const state = nackState(options);
|
|
21
|
+
if (state === 'dead') return 'dead';
|
|
22
|
+
if (state === 'failed') return 'failed';
|
|
23
|
+
return state === 'ready' && options.countsAsAttempt !== false ? 'retried' : undefined;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
type Counts = Omit<JobCounter, 'job' | 'bucketStart' | 'bucketMs'>;
|
|
27
|
+
|
|
28
|
+
const ZERO: Counts = Object.freeze({ done: 0, retried: 0, failed: 0, dead: 0, durationMs: 0 });
|
|
29
|
+
|
|
30
|
+
const sum = (a: Counts, b: Counts): Counts => ({
|
|
31
|
+
done: a.done + b.done,
|
|
32
|
+
retried: a.retried + b.retried,
|
|
33
|
+
failed: a.failed + b.failed,
|
|
34
|
+
dead: a.dead + b.dead,
|
|
35
|
+
durationMs: a.durationMs + b.durationMs,
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
const byName = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0);
|
|
39
|
+
|
|
40
|
+
export interface MemoryCounters {
|
|
41
|
+
add(job: string, outcome: CounterOutcome, durationMs: number, atMs: number): void;
|
|
42
|
+
list(query: { readonly job: string; readonly sinceMs: number }): readonly JobCounter[];
|
|
43
|
+
totals(sinceMs: number): readonly CounterTotals[];
|
|
44
|
+
rollup(atMs: number): number;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** The memory driver's counters. Same buckets, same tiers, same fold as `x_job_counters`. */
|
|
48
|
+
export function createMemoryCounters(): MemoryCounters {
|
|
49
|
+
const buckets = new Map<string, JobCounter>();
|
|
50
|
+
const keyOf = (job: string, bucketMs: number, bucketStart: number): string =>
|
|
51
|
+
`${bucketMs}\u0000${bucketStart}\u0000${job}`;
|
|
52
|
+
|
|
53
|
+
const merge = (
|
|
54
|
+
job: string,
|
|
55
|
+
bucketMs: CounterBucketMs,
|
|
56
|
+
bucketStart: number,
|
|
57
|
+
add: Counts,
|
|
58
|
+
): void => {
|
|
59
|
+
const key = keyOf(job, bucketMs, bucketStart);
|
|
60
|
+
const current = buckets.get(key);
|
|
61
|
+
buckets.set(key, { job, bucketMs, bucketStart, ...sum(current ?? ZERO, add) });
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
return {
|
|
65
|
+
add(job, outcome, durationMs, atMs) {
|
|
66
|
+
const start = counterBucketStart(atMs, COUNTER_BUCKET_MS);
|
|
67
|
+
merge(job, COUNTER_BUCKET_MS, start, { ...ZERO, [outcome]: 1, durationMs });
|
|
68
|
+
},
|
|
69
|
+
list({ job, sinceMs }) {
|
|
70
|
+
return [...buckets.values()]
|
|
71
|
+
.filter((bucket) => bucket.job === job && bucket.bucketStart >= sinceMs)
|
|
72
|
+
.sort((a, b) => a.bucketStart - b.bucketStart || a.bucketMs - b.bucketMs);
|
|
73
|
+
},
|
|
74
|
+
totals(sinceMs) {
|
|
75
|
+
const byJob = new Map<string, Counts>();
|
|
76
|
+
for (const bucket of buckets.values()) {
|
|
77
|
+
if (bucket.bucketStart < sinceMs) continue;
|
|
78
|
+
byJob.set(bucket.job, sum(byJob.get(bucket.job) ?? ZERO, bucket));
|
|
79
|
+
}
|
|
80
|
+
return [...byJob.entries()]
|
|
81
|
+
.sort(([a], [b]) => byName(a, b))
|
|
82
|
+
.map(([job, counts]) => ({ job, ...sum(ZERO, counts) }));
|
|
83
|
+
},
|
|
84
|
+
rollup(atMs) {
|
|
85
|
+
let moved = 0;
|
|
86
|
+
for (const [index, tier] of COUNTER_TIERS.entries()) {
|
|
87
|
+
const next = COUNTER_TIERS[index + 1];
|
|
88
|
+
for (const [key, bucket] of [...buckets]) {
|
|
89
|
+
if (bucket.bucketMs !== tier.bucketMs || bucket.bucketStart >= atMs - tier.keepMs)
|
|
90
|
+
continue;
|
|
91
|
+
buckets.delete(key);
|
|
92
|
+
moved += 1;
|
|
93
|
+
if (next === undefined) continue;
|
|
94
|
+
merge(
|
|
95
|
+
bucket.job,
|
|
96
|
+
next.bucketMs,
|
|
97
|
+
counterBucketStart(bucket.bucketStart, next.bucketMs),
|
|
98
|
+
bucket,
|
|
99
|
+
);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
return moved;
|
|
103
|
+
},
|
|
104
|
+
};
|
|
105
|
+
}
|
package/src/describe.ts
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
// clock- or iteration-order-dependent in this shape is a spurious change on every build.
|
|
4
4
|
|
|
5
5
|
import { toJsonSchema } from '@ultimat3/schema';
|
|
6
|
+
import type { WhenBusy } from './concurrency';
|
|
6
7
|
import type { BackoffStrategy, RetryPolicy } from './retry';
|
|
7
8
|
import { DEFAULT_RETRY } from './retry';
|
|
8
9
|
|
|
@@ -22,6 +23,21 @@ export interface JobDescriptor {
|
|
|
22
23
|
* is app data, so a descriptor carrying it would put customer ids in `x.manifest.json`.
|
|
23
24
|
*/
|
|
24
25
|
readonly idempotent: boolean;
|
|
26
|
+
/**
|
|
27
|
+
* The declared fleet-wide cap, or `null` for none. `keyed: true` says `limit` holds per
|
|
28
|
+
* `concurrency.key(input)` rather than for the whole job, and `whenBusy` is what a claim over
|
|
29
|
+
* it does — `null` for a plain number, which always waits. The KEY never crosses, for
|
|
30
|
+
* `idempotent`'s reason: it is computed from an input. `x jobs show <id>` reports one run's.
|
|
31
|
+
*/
|
|
32
|
+
readonly concurrency: JobConcurrencyDescriptor | null;
|
|
33
|
+
/** Whether the job declares an `onSettled` hook — what is told how each run ended. */
|
|
34
|
+
readonly onSettled: boolean;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export interface JobConcurrencyDescriptor {
|
|
38
|
+
readonly limit: number;
|
|
39
|
+
readonly keyed: boolean;
|
|
40
|
+
readonly whenBusy: WhenBusy | null;
|
|
25
41
|
}
|
|
26
42
|
|
|
27
43
|
/**
|
|
@@ -40,6 +56,10 @@ export interface DescribableJob {
|
|
|
40
56
|
* which is the exact wrong answer the `/_x` jobs panel used to give for every job.
|
|
41
57
|
*/
|
|
42
58
|
readonly idempotencyKeyFor: unknown;
|
|
59
|
+
/** `JobHandle.concurrency` and `.whenBusy` — the cap as `job()` resolved it. */
|
|
60
|
+
readonly concurrency: number | undefined;
|
|
61
|
+
readonly whenBusy: WhenBusy | undefined;
|
|
62
|
+
readonly declaresOnSettled: boolean;
|
|
43
63
|
}
|
|
44
64
|
|
|
45
65
|
export function describeJob(handle: DescribableJob): JobDescriptor {
|
|
@@ -55,6 +75,15 @@ export function describeJob(handle: DescribableJob): JobDescriptor {
|
|
|
55
75
|
// not statically knowable. `inspect(name)` reports the steps an actual run recorded.
|
|
56
76
|
steps: [],
|
|
57
77
|
idempotent: typeof handle.idempotencyKeyFor === 'function',
|
|
78
|
+
concurrency:
|
|
79
|
+
handle.concurrency === undefined
|
|
80
|
+
? null
|
|
81
|
+
: {
|
|
82
|
+
limit: handle.concurrency,
|
|
83
|
+
keyed: handle.whenBusy !== undefined,
|
|
84
|
+
whenBusy: handle.whenBusy ?? null,
|
|
85
|
+
},
|
|
86
|
+
onSettled: handle.declaresOnSettled,
|
|
58
87
|
};
|
|
59
88
|
}
|
|
60
89
|
|