@ultimat3/testing 30.5.0 → 30.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +5 -5
- package/package.json +17 -17
- package/src/define-exercise.ts +19 -7
- package/src/errors.ts +6 -6
- package/src/exercise-actor-user.ts +63 -0
- package/src/exercise-actor.ts +36 -7
- package/src/exercise-crash.ts +7 -3
- package/src/exercise-drift-hint.ts +28 -0
- package/src/exercise-input.ts +3 -0
- package/src/exercise-job-run.ts +68 -0
- package/src/exercise-job-tenant.ts +10 -5
- package/src/exercise-override-input.ts +26 -5
- package/src/exercise-report.ts +38 -2
- package/src/exercise-subjects.ts +6 -4
- package/src/exercise-sweeps.ts +49 -7
- package/src/exercise.ts +50 -34
- package/src/fixture-jobs.ts +17 -5
- package/src/index.ts +4 -0
- package/src/job-crash-points.ts +36 -0
- package/src/job-crash-proof.ts +26 -6
- package/src/job-sim-errors.ts +24 -0
- package/src/job-sim-world.ts +16 -5
- package/src/plan-conditions.ts +106 -0
- package/src/plan-oracle.ts +65 -49
- package/src/plan-sort-key.ts +65 -0
- package/src/plan-sweep-fix.ts +87 -21
- package/src/plan-sweep-report.ts +48 -4
- package/src/seed-graph-values.ts +20 -11
- package/src/seed-graph.ts +30 -1
- package/src/tenant-proof-report.ts +12 -3
- package/src/tenant-proof.ts +32 -4
- package/src/seed-pattern-sample.ts +0 -216
package/src/exercise-sweeps.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// What runs after every primitive's own call: the scale sweep (each reached one again, at 2 and at 8
|
|
2
2
|
// rows a table) and the plan sweep (every statement the pass sent, planned on the migrated
|
|
3
|
-
// database). Both verdicts land on the record of the primitive that sent the statements
|
|
3
|
+
// database). Both verdicts land on the record of the primitive that sent the statements, and each
|
|
4
|
+
// sweep's time on the pass.
|
|
4
5
|
|
|
5
6
|
import type { Actor } from '@ultimat3/core';
|
|
6
7
|
import type { ExerciseOverrides } from './define-exercise';
|
|
@@ -9,11 +10,11 @@ import { asSeededRow, chooseActor } from './exercise-actor';
|
|
|
9
10
|
import type { Sampler } from './exercise-input';
|
|
10
11
|
import { bindIds, drawInput, seededIds } from './exercise-input';
|
|
11
12
|
import { overrideInput } from './exercise-override-input';
|
|
12
|
-
import type { ExerciseRecord } from './exercise-report';
|
|
13
|
+
import type { ExercisePhaseMs, ExerciseRecord } from './exercise-report';
|
|
13
14
|
import type { Subject } from './exercise-subjects';
|
|
14
15
|
import { seatIn } from './exercise-subjects';
|
|
15
16
|
import { sweepPlans } from './plan-sweep';
|
|
16
|
-
import type { ScaleSweepOptions } from './scale-sweep-run';
|
|
17
|
+
import type { ScaleSubject, ScaleSweepOptions } from './scale-sweep-run';
|
|
17
18
|
import { scaleSweep } from './scale-sweep-run';
|
|
18
19
|
import type { SeedGraph } from './seed-graph-plan';
|
|
19
20
|
import type { SeedEntity } from './seed-graph-shape';
|
|
@@ -30,7 +31,12 @@ export const actorRowsOf = (options: CallOptions, graph: SeedGraph): ActorRows |
|
|
|
30
31
|
const name = options.overrides?.actorEntity;
|
|
31
32
|
const entity = options.entities.find((one) => one.$name === name);
|
|
32
33
|
if (entity === undefined) return undefined;
|
|
33
|
-
|
|
34
|
+
const column = options.overrides?.actorColumn;
|
|
35
|
+
return {
|
|
36
|
+
entity,
|
|
37
|
+
rows: graph.inserts.filter((row) => row.entity === entity.$name),
|
|
38
|
+
...(column === undefined ? {} : { column }),
|
|
39
|
+
};
|
|
34
40
|
};
|
|
35
41
|
|
|
36
42
|
/** The actor of `chosen`, seated as a row of the app's actor entity in `tenant` when it names one. */
|
|
@@ -66,6 +72,25 @@ export async function sizedCall(
|
|
|
66
72
|
return { actor, input: seatIn(subject, given, tenant) };
|
|
67
73
|
}
|
|
68
74
|
|
|
75
|
+
/** The records with both sweeps' verdicts and scale time on them, and each sweep's whole time. */
|
|
76
|
+
export interface Swept {
|
|
77
|
+
readonly records: readonly ExerciseRecord[];
|
|
78
|
+
readonly ms: Pick<ExercisePhaseMs, 'scale' | 'plans'>;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** `subject`'s calls, each timed into `spent` under its name: the sweep calls it once per size. */
|
|
82
|
+
const timed = (subject: ScaleSubject, spent: Map<string, number>): ScaleSubject => ({
|
|
83
|
+
...subject,
|
|
84
|
+
call: async (...args) => {
|
|
85
|
+
const started = performance.now();
|
|
86
|
+
try {
|
|
87
|
+
return await subject.call(...args);
|
|
88
|
+
} finally {
|
|
89
|
+
spent.set(subject.name, (spent.get(subject.name) ?? 0) + performance.now() - started);
|
|
90
|
+
}
|
|
91
|
+
},
|
|
92
|
+
});
|
|
93
|
+
|
|
69
94
|
/**
|
|
70
95
|
* The scale sweep inside the room, the room closed, then the plan sweep — on the installed client
|
|
71
96
|
* only: with none, the pass ran on a driver that sends no SQL and there is no plan to read.
|
|
@@ -73,24 +98,41 @@ export async function sizedCall(
|
|
|
73
98
|
export async function attachSweeps(
|
|
74
99
|
records: readonly ExerciseRecord[],
|
|
75
100
|
options: ScaleSweepOptions,
|
|
76
|
-
): Promise<
|
|
101
|
+
): Promise<Swept> {
|
|
102
|
+
const spent = new Map<string, number>();
|
|
77
103
|
let scale: Awaited<ReturnType<typeof scaleSweep>>;
|
|
104
|
+
const scaleStarted = performance.now();
|
|
78
105
|
try {
|
|
79
|
-
scale = await scaleSweep(
|
|
106
|
+
scale = await scaleSweep({
|
|
107
|
+
...options,
|
|
108
|
+
subjects: options.subjects.map((subject) => timed(subject, spent)),
|
|
109
|
+
});
|
|
80
110
|
} finally {
|
|
81
111
|
await options.room[Symbol.asyncDispose]();
|
|
82
112
|
}
|
|
113
|
+
const plansStarted = performance.now();
|
|
83
114
|
const db = await import('@ultimat3/db');
|
|
84
115
|
const plans = db.dbClientInstalled()
|
|
85
116
|
? await sweepPlans(db.db(), options.room.sent(), options.entities)
|
|
86
117
|
: new Map();
|
|
87
|
-
|
|
118
|
+
const ms = {
|
|
119
|
+
scale: Math.round(plansStarted - scaleStarted),
|
|
120
|
+
plans: Math.round(performance.now() - plansStarted),
|
|
121
|
+
};
|
|
122
|
+
const swept = records.map((record) => {
|
|
88
123
|
const proof = plans.get(record.name);
|
|
89
124
|
const sized = scale.get(record.name);
|
|
125
|
+
const scaleMs = spent.get(record.name);
|
|
126
|
+
const proofMs = {
|
|
127
|
+
...record.proofMs,
|
|
128
|
+
...(scaleMs === undefined ? {} : { scale: Math.round(scaleMs) }),
|
|
129
|
+
};
|
|
90
130
|
return {
|
|
91
131
|
...record,
|
|
92
132
|
...(proof === undefined ? {} : { plans: proof }),
|
|
93
133
|
...(sized === undefined ? {} : { scale: sized }),
|
|
134
|
+
...(Object.keys(proofMs).length === 0 ? {} : { proofMs }),
|
|
94
135
|
};
|
|
95
136
|
});
|
|
137
|
+
return { records: swept, ms };
|
|
96
138
|
}
|
package/src/exercise.ts
CHANGED
|
@@ -8,14 +8,15 @@
|
|
|
8
8
|
|
|
9
9
|
import type { Actor, ExerciseAttempt, ExerciseVerdict } from '@ultimat3/core';
|
|
10
10
|
import { isUltimateError, renderThrowable, runExercise } from '@ultimat3/core';
|
|
11
|
-
import type { JobExecution, JobRecord } from '@ultimat3/jobs';
|
|
12
11
|
import { registeredJobs } from '@ultimat3/jobs';
|
|
13
12
|
import type { ExerciseOverride, ExerciseOverrides } from './define-exercise';
|
|
14
13
|
import { assertOverridesKnown, unknownDeclaration } from './define-exercise';
|
|
15
14
|
import type { RoleGrants } from './exercise-actor';
|
|
16
15
|
import { chooseActor } from './exercise-actor';
|
|
16
|
+
import { seatActorUsers } from './exercise-actor-user';
|
|
17
17
|
import { proveCrash } from './exercise-crash';
|
|
18
18
|
import { defectDetail } from './exercise-defect';
|
|
19
|
+
import { driftOverrideLine } from './exercise-drift-hint';
|
|
19
20
|
import type { IdLookup, Sampler } from './exercise-input';
|
|
20
21
|
import {
|
|
21
22
|
bindIds,
|
|
@@ -25,11 +26,13 @@ import {
|
|
|
25
26
|
namesActorEntity,
|
|
26
27
|
seededIds,
|
|
27
28
|
} from './exercise-input';
|
|
29
|
+
import { ambientJobRows } from './exercise-job-run';
|
|
28
30
|
import type { OverrideInput } from './exercise-override-input';
|
|
29
31
|
import { overrideInput } from './exercise-override-input';
|
|
30
32
|
import type {
|
|
31
33
|
EffectPayload,
|
|
32
34
|
ExerciseEffect,
|
|
35
|
+
ExercisePhaseMs,
|
|
33
36
|
ExerciseRecord,
|
|
34
37
|
ExerciseRoom,
|
|
35
38
|
Observed,
|
|
@@ -61,6 +64,8 @@ export interface ExercisePassOptions {
|
|
|
61
64
|
/** One primitive by name; unknown is `X_DECLARATION_UNKNOWN`. */
|
|
62
65
|
readonly only?: string;
|
|
63
66
|
readonly roles: RoleGrants;
|
|
67
|
+
/** Told once, as the pass returns: how long each phase took, so a slow pass names its slow part. */
|
|
68
|
+
readonly onPhases?: (ms: ExercisePhaseMs) => void;
|
|
64
69
|
}
|
|
65
70
|
|
|
66
71
|
const isOff = (overrides: ExerciseOverrides | undefined, name: string): boolean => {
|
|
@@ -76,13 +81,15 @@ const codeOf = (error: unknown): string | undefined => {
|
|
|
76
81
|
const reasonOf = (error: unknown): string =>
|
|
77
82
|
isUltimateError(error) ? error.cause : typeof error === 'string' ? error : renderThrowable(error);
|
|
78
83
|
|
|
79
|
-
/**
|
|
80
|
-
|
|
84
|
+
/**
|
|
85
|
+
* Jobs the primitive queued, its own job excepted. Never run: a job's children are left queued as
|
|
86
|
+
* an action's are (`exercise-job-run.ts`), so their statements are theirs, never this record's.
|
|
87
|
+
*/
|
|
88
|
+
async function enqueued(subject: Subject, jobs: RunJobs) {
|
|
81
89
|
const effects: ExerciseEffect[] = [];
|
|
82
90
|
for (const handle of registeredJobs()) {
|
|
83
91
|
if (subject.kind === 'job' && handle.name === subject.name) continue;
|
|
84
|
-
const
|
|
85
|
-
const count = ran + (await jobs.depth(handle));
|
|
92
|
+
const count = await jobs.depth(handle);
|
|
86
93
|
if (count > 0) effects.push({ kind: 'enqueue', job: handle.name, count });
|
|
87
94
|
}
|
|
88
95
|
return effects;
|
|
@@ -93,21 +100,7 @@ async function enqueued(subject: Subject, jobs: RunJobs, runs: readonly JobExecu
|
|
|
93
100
|
* input is what B was handed. Read off the leg's own queue, page by page.
|
|
94
101
|
*/
|
|
95
102
|
async function queuedPayloads(subject: Subject): Promise<readonly EffectPayload[]> {
|
|
96
|
-
|
|
97
|
-
const introspect = jobs.jobDriver()?.introspect;
|
|
98
|
-
if (introspect === undefined) return [];
|
|
99
|
-
const records: JobRecord[] = [];
|
|
100
|
-
for (let after: string | undefined; ; ) {
|
|
101
|
-
const page = await introspect.list({
|
|
102
|
-
limit: jobs.MAX_JOB_PAGE,
|
|
103
|
-
...(after === undefined ? {} : { after }),
|
|
104
|
-
});
|
|
105
|
-
records.push(...page);
|
|
106
|
-
const last = page[page.length - 1];
|
|
107
|
-
if (last === undefined || page.length < jobs.MAX_JOB_PAGE) break;
|
|
108
|
-
after = jobs.jobCursor(last);
|
|
109
|
-
}
|
|
110
|
-
return records
|
|
103
|
+
return (await ambientJobRows())
|
|
111
104
|
.filter((record) => !(subject.kind === 'job' && record.name === subject.name))
|
|
112
105
|
.map((record, i) => ({
|
|
113
106
|
path: `effect.enqueue[${i}]`,
|
|
@@ -122,7 +115,7 @@ interface Run {
|
|
|
122
115
|
readonly lastInput: unknown;
|
|
123
116
|
}
|
|
124
117
|
|
|
125
|
-
function judged(subject: Subject, run: Run, observed: Observed) {
|
|
118
|
+
function judged(subject: Subject, run: Run, observed: Observed, sampler: Sampler) {
|
|
126
119
|
const idFields = (input: unknown): readonly string[] =>
|
|
127
120
|
typeof input === 'object' && input !== null
|
|
128
121
|
? Object.keys(input).filter((key) => idStem(key) !== undefined)
|
|
@@ -140,7 +133,12 @@ function judged(subject: Subject, run: Run, observed: Observed) {
|
|
|
140
133
|
}
|
|
141
134
|
if (run.verdict.kind === 'pass' && run.answered) return { outcome: 'reached' as const };
|
|
142
135
|
if (run.verdict.kind === 'drift') {
|
|
143
|
-
|
|
136
|
+
// The field no draw can fill, named: `...drawn` alone told the author nothing to write.
|
|
137
|
+
const fields = idFields(run.lastInput);
|
|
138
|
+
return {
|
|
139
|
+
...unreached('X_CONTRACT_DRIFT', 'its input schema refused every generated value'),
|
|
140
|
+
override: driftOverrideLine(subject.name, subject.input, sampler, fields),
|
|
141
|
+
};
|
|
144
142
|
}
|
|
145
143
|
if (run.verdict.kind === 'defect') {
|
|
146
144
|
return unreached(codeOf(run.verdict.error) ?? 'X_CONTRACT_DEFECT', run.verdict.defect);
|
|
@@ -174,7 +172,6 @@ async function control(
|
|
|
174
172
|
let lastError: unknown;
|
|
175
173
|
let lastInput: unknown;
|
|
176
174
|
let output: unknown;
|
|
177
|
-
let executions: readonly JobExecution[] = [];
|
|
178
175
|
room.begin(name);
|
|
179
176
|
const started = performance.now();
|
|
180
177
|
const verdict = await runExercise({
|
|
@@ -209,10 +206,9 @@ async function control(
|
|
|
209
206
|
statusOf: (await import('@ultimat3/http')).statusFor,
|
|
210
207
|
});
|
|
211
208
|
const ms = Math.round(performance.now() - started);
|
|
212
|
-
if (kind === 'job') executions = (await jobs.drain()).executions;
|
|
213
209
|
const observed = room.end();
|
|
214
|
-
const effects = [...observed.effects, ...(await enqueued(subject, jobs
|
|
215
|
-
const verdictOf = judged(subject, { verdict, answered, lastError, lastInput }, observed);
|
|
210
|
+
const effects = [...observed.effects, ...(await enqueued(subject, jobs))];
|
|
211
|
+
const verdictOf = judged(subject, { verdict, answered, lastError, lastInput }, observed, sampler);
|
|
216
212
|
// A 5xx's cause, statement and fields: the code alone sent the author to reproduce it by hand.
|
|
217
213
|
const defect =
|
|
218
214
|
verdict.kind === 'defect' && verdictOf.outcome === 'unreached'
|
|
@@ -286,10 +282,13 @@ async function exerciseOne(
|
|
|
286
282
|
const intruder = withoutCrossTenant(
|
|
287
283
|
theirs.kind === 'actor' ? seated(theirs.actor, other, rows) : null,
|
|
288
284
|
);
|
|
285
|
+
// Both are users, as a request's actor always is: their `x_users` rows, before any call.
|
|
286
|
+
await seatActorUsers([actor, intruder]);
|
|
289
287
|
|
|
290
288
|
const scope = { tenant, actorId: actor?.id, subject: name, actorEntity: rows?.entity.$name };
|
|
291
289
|
const bindInput = overrideInput(name, override);
|
|
292
|
-
|
|
290
|
+
// `off` returned above: any other entry's `runs` is declared, with or without an `actor`.
|
|
291
|
+
const declaredRuns = override === undefined || 'off' in override ? undefined : override.runs;
|
|
293
292
|
// A field naming the actor entity is tried as the actor's own row, then as somebody else's:
|
|
294
293
|
// both readings are legitimate and only the handler knows which it meant. Same minimal draw.
|
|
295
294
|
const twoWay =
|
|
@@ -325,12 +324,14 @@ async function exerciseOne(
|
|
|
325
324
|
}
|
|
326
325
|
const input = seat === undefined ? given : seat.input;
|
|
327
326
|
let reachedWith: unknown;
|
|
327
|
+
const tenantStarted = performance.now();
|
|
328
328
|
const record = await proveTenant({
|
|
329
329
|
taint: tenantTaint(graph, tenant),
|
|
330
330
|
input,
|
|
331
331
|
public: publicReason(subject.policy),
|
|
332
332
|
read: kind === 'query',
|
|
333
333
|
sweep: kind === 'job',
|
|
334
|
+
tenantless: seat?.kind === 'none',
|
|
334
335
|
attack: () => attack(subject, input, intruder, other, room),
|
|
335
336
|
control: async () => {
|
|
336
337
|
const done = await control(run, room, sampler);
|
|
@@ -339,10 +340,16 @@ async function exerciseOne(
|
|
|
339
340
|
},
|
|
340
341
|
snapshot: (seeded) => options.writer.snapshot(seeded),
|
|
341
342
|
});
|
|
343
|
+
// The control's own call is `ms`: the tenant proof's time is everything around it.
|
|
344
|
+
const tenantMs = Math.max(0, Math.round(performance.now() - tenantStarted) - record.ms);
|
|
342
345
|
// After the tenant proof, outside the room's begin/end: the crash runs' effects are their own.
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
+
if (subject.handle === undefined || record.outcome !== 'reached') {
|
|
347
|
+
return { ...record, proofMs: { tenant: tenantMs } };
|
|
348
|
+
}
|
|
349
|
+
const crashStarted = performance.now();
|
|
350
|
+
const crash = await proveCrash(subject.handle, reachedWith, actor);
|
|
351
|
+
const proofMs = { tenant: tenantMs, crash: Math.round(performance.now() - crashStarted) };
|
|
352
|
+
return { ...record, crash, proofMs };
|
|
346
353
|
}
|
|
347
354
|
|
|
348
355
|
export async function runExercisePass(
|
|
@@ -363,6 +370,8 @@ export async function runExercisePass(
|
|
|
363
370
|
const live = chosen.filter((subject) => !isOff(options.overrides, subject.name));
|
|
364
371
|
const records: ExerciseRecord[] = [];
|
|
365
372
|
let graph: SeedGraph;
|
|
373
|
+
const seedStarted = performance.now();
|
|
374
|
+
let primitivesStarted = seedStarted;
|
|
366
375
|
try {
|
|
367
376
|
// Seeded before the first primitive runs: a failure here is the pass's, never one primitive's.
|
|
368
377
|
graph = await seedPass(options.entities, options.writer, {
|
|
@@ -372,6 +381,7 @@ export async function runExercisePass(
|
|
|
372
381
|
? {}
|
|
373
382
|
: { factories: options.overrides.factories }),
|
|
374
383
|
});
|
|
384
|
+
primitivesStarted = performance.now();
|
|
375
385
|
// In order, one at a time: the seams the room installs are the process's, not the primitive's.
|
|
376
386
|
for (const subject of chosen) {
|
|
377
387
|
const index = live.indexOf(subject);
|
|
@@ -383,9 +393,11 @@ export async function runExercisePass(
|
|
|
383
393
|
await room[Symbol.asyncDispose]();
|
|
384
394
|
throw error;
|
|
385
395
|
}
|
|
396
|
+
const seed = Math.round(primitivesStarted - seedStarted);
|
|
397
|
+
const primitives = Math.round(performance.now() - primitivesStarted);
|
|
386
398
|
// The sweeps: every reached primitive again at two sizes, then every statement planned.
|
|
387
399
|
const reached = new Set(records.filter((r) => r.outcome === 'reached').map((r) => r.name));
|
|
388
|
-
|
|
400
|
+
const swept = await attachSweeps(records, {
|
|
389
401
|
subjects: chosen.filter((subject) => reached.has(subject.name)),
|
|
390
402
|
entities: options.entities,
|
|
391
403
|
// The sized graphs close their cycles as the pass's did: a refused deferred key stays null.
|
|
@@ -396,9 +408,13 @@ export async function runExercisePass(
|
|
|
396
408
|
factories: options.overrides?.factories,
|
|
397
409
|
setup: async (subject, sized, tenant, self) => {
|
|
398
410
|
const full = chosen.find((one) => one.name === subject.name);
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
411
|
+
if (full === undefined) return undefined;
|
|
412
|
+
const call = await sizedCall(full, options, sampler, { graph: sized, tenant, self });
|
|
413
|
+
// Before the sweep's room opens: the sized tenant's actor is a user too.
|
|
414
|
+
if (call !== undefined) await seatActorUsers([call.actor]);
|
|
415
|
+
return call;
|
|
402
416
|
},
|
|
403
417
|
});
|
|
418
|
+
options.onPhases?.({ seed, primitives, ...swept.ms });
|
|
419
|
+
return swept.records;
|
|
404
420
|
}
|
package/src/fixture-jobs.ts
CHANGED
|
@@ -69,6 +69,11 @@ export interface RunJobs extends AsyncDisposable {
|
|
|
69
69
|
enqueue<I>(handle: JobHandle<I>, input: I, options?: JobEnqueueAs): Promise<EnqueueResult>;
|
|
70
70
|
/** Claim and execute everything due at the current instant, until nothing is. */
|
|
71
71
|
drain(options?: JobDrainAs): Promise<JobRunTrace>;
|
|
72
|
+
/**
|
|
73
|
+
* ONE worker pass: what is due now runs, and what those runs enqueue waits for the next pass —
|
|
74
|
+
* "it queued one child per row" asserted without running a child.
|
|
75
|
+
*/
|
|
76
|
+
tick(options?: JobDrainAs): Promise<JobRunTrace>;
|
|
72
77
|
/** Live jobs — ready, delayed, running or suspended — optionally for one job only. */
|
|
73
78
|
depth(handle?: AnyJobHandle): Promise<number>;
|
|
74
79
|
/** Live jobs claimable right now. `clock.advance()` is what turns delayed into due. */
|
|
@@ -194,14 +199,20 @@ export async function testJobs(): Promise<RunJobs> {
|
|
|
194
199
|
/** Every execution this fixture has driven, because the trace is cumulative. */
|
|
195
200
|
const history: JobExecution[] = [];
|
|
196
201
|
|
|
202
|
+
const ctxFor = (options: JobDrainAs): Ctx =>
|
|
203
|
+
options.actor === undefined ? anonymousWorker : ctxOf({ role: 'worker', actor: options.actor });
|
|
204
|
+
const traced = (): JobRunTrace => ({ executions: [...history], steps: tallyOf(history) });
|
|
205
|
+
|
|
206
|
+
const tick = async (options: JobDrainAs = {}): Promise<JobRunTrace> => {
|
|
207
|
+
history.push(...(await round(ctxFor(options))));
|
|
208
|
+
return traced();
|
|
209
|
+
};
|
|
210
|
+
|
|
197
211
|
const drain = async (options: JobDrainAs = {}): Promise<JobRunTrace> => {
|
|
198
|
-
const ctx =
|
|
199
|
-
options.actor === undefined
|
|
200
|
-
? anonymousWorker
|
|
201
|
-
: ctxOf({ role: 'worker', actor: options.actor });
|
|
212
|
+
const ctx = ctxFor(options);
|
|
202
213
|
for (let rounds = 0; rounds < MAX_ROUNDS; rounds += 1) {
|
|
203
214
|
const batch = await round(ctx);
|
|
204
|
-
if (batch.length === 0) return
|
|
215
|
+
if (batch.length === 0) return traced();
|
|
205
216
|
history.push(...batch);
|
|
206
217
|
}
|
|
207
218
|
assert(
|
|
@@ -239,6 +250,7 @@ export async function testJobs(): Promise<RunJobs> {
|
|
|
239
250
|
return Object.assign(enqueueThenDrain, {
|
|
240
251
|
enqueue,
|
|
241
252
|
drain,
|
|
253
|
+
tick,
|
|
242
254
|
depth: async (handle?: AnyJobHandle) => (await live(handle?.name)).length,
|
|
243
255
|
due: async () =>
|
|
244
256
|
(await live()).filter(
|
package/src/index.ts
CHANGED
|
@@ -207,7 +207,9 @@ export type {
|
|
|
207
207
|
EffectPayload,
|
|
208
208
|
ExerciseEffect,
|
|
209
209
|
ExerciseOutcome,
|
|
210
|
+
ExercisePhaseMs,
|
|
210
211
|
ExerciseRecord,
|
|
212
|
+
ProofMs,
|
|
211
213
|
TenantFinding,
|
|
212
214
|
TenantLeg,
|
|
213
215
|
TenantProof,
|
|
@@ -366,6 +368,7 @@ export {
|
|
|
366
368
|
normalizeIslandName,
|
|
367
369
|
} from './island-states-resolve';
|
|
368
370
|
export { ISOLATED_ENV, releasePluginsAfterIsolatedFile } from './isolated-plugins';
|
|
371
|
+
export { CRASH_POINT_BOUND } from './job-crash-points';
|
|
369
372
|
export type {
|
|
370
373
|
CrashBoundary,
|
|
371
374
|
CrashDivergence,
|
|
@@ -379,6 +382,7 @@ export type { SimEffect } from './job-sim-effects';
|
|
|
379
382
|
export {
|
|
380
383
|
crashEffectPlace,
|
|
381
384
|
JobCrashDivergedError,
|
|
385
|
+
JobCrashUnboundedError,
|
|
382
386
|
JobSimInvariantError,
|
|
383
387
|
JobSimNondeterministicError,
|
|
384
388
|
} from './job-sim-errors';
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
// Which boundaries a crash proof kills at. A loop's steps named by row id are one point of the
|
|
2
|
+
// code, not one per row: each boundary shape is killed at its first and its last occurrence, so the
|
|
3
|
+
// proof costs what the job's code has, never what the database holds. Pure: labels in, indexes out.
|
|
4
|
+
|
|
5
|
+
/** Kill points one mode of a proof may cost; past it the proof refuses, `X_JOB_CRASH_UNBOUNDED`. */
|
|
6
|
+
export const CRASH_POINT_BOUND = 32;
|
|
7
|
+
|
|
8
|
+
const UUID = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi;
|
|
9
|
+
/** A ULID, a hex digest, a cuid: eight or more letters and digits with a digit among them. */
|
|
10
|
+
const TOKEN = /\b(?=[A-Za-z]*\d)[A-Za-z0-9]{8,}\b/g;
|
|
11
|
+
const DIGITS = /\d+/g;
|
|
12
|
+
|
|
13
|
+
/** `fanout:<id>`, `page-<n>`, `sleep:<n>h (sleeping)`: a boundary label with its ids taken out. */
|
|
14
|
+
export const boundaryShape = (label: string): string =>
|
|
15
|
+
label.replace(UUID, '<id>').replace(TOKEN, '<id>').replace(DIGITS, '<n>');
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* The 1-based positions to kill at, in order: the first and the last boundary of every shape.
|
|
19
|
+
* The first proves the code before the loop and the loop's own body; the last, every row before it
|
|
20
|
+
* replayed and the code after it — an effect between any two rows is made again at the last.
|
|
21
|
+
*/
|
|
22
|
+
export function crashPoints(labels: readonly string[]): readonly number[] {
|
|
23
|
+
const first = new Map<string, number>();
|
|
24
|
+
const last = new Map<string, number>();
|
|
25
|
+
labels.forEach((label, i) => {
|
|
26
|
+
const shape = boundaryShape(label);
|
|
27
|
+
if (!first.has(shape)) first.set(shape, i + 1);
|
|
28
|
+
last.set(shape, i + 1);
|
|
29
|
+
});
|
|
30
|
+
return [...new Set([...first.values(), ...last.values()])].sort((a, b) => a - b);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** The distinct shapes of `labels`, first seen first — what an unbounded refusal names. */
|
|
34
|
+
export const boundaryShapes = (labels: readonly string[]): readonly string[] => [
|
|
35
|
+
...new Set(labels.map(boundaryShape)),
|
|
36
|
+
];
|
package/src/job-crash-proof.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
// `crashProof(handle, input)`: one clean run, then one run killed at each boundary the
|
|
2
|
-
// crossed —
|
|
3
|
-
// lost. Judged by the effect log: a killed run that makes an effect more often than the clean one
|
|
1
|
+
// `crashProof(handle, input)`: one clean run, then one run killed at each distinct boundary the
|
|
2
|
+
// clean run crossed — a step write landing, a settle; a loop's rows one point (`job-crash-points.ts`)
|
|
3
|
+
// — and one more where that write's answer is lost. Judged by the effect log: a killed run that makes an effect more often than the clean one
|
|
4
4
|
// is `X_JOB_CRASH_DIVERGED`; a lost answer rerunning a step is at-least-once, reported, never red.
|
|
5
5
|
|
|
6
6
|
import type { Ctx } from '@ultimat3/core';
|
|
@@ -8,11 +8,12 @@ import { finiteCount } from '@ultimat3/core';
|
|
|
8
8
|
import type { JobHandle } from '@ultimat3/jobs';
|
|
9
9
|
import { COMPENSATE_PREFIX, UNWINDING_STEP } from '@ultimat3/jobs';
|
|
10
10
|
import { advanceClock, frozenNow } from './determinism';
|
|
11
|
+
import { boundaryShapes, CRASH_POINT_BOUND, crashPoints } from './job-crash-points';
|
|
11
12
|
import type { SimDecision, SimOp } from './job-sim-driver';
|
|
12
13
|
import { SEND_AND_ANSWER } from './job-sim-driver';
|
|
13
14
|
import type { SimEffect } from './job-sim-effects';
|
|
14
15
|
import { effectCounts } from './job-sim-effects';
|
|
15
|
-
import { JobCrashDivergedError } from './job-sim-errors';
|
|
16
|
+
import { JobCrashDivergedError, JobCrashUnboundedError } from './job-sim-errors';
|
|
16
17
|
import type { SimWorld } from './job-sim-world';
|
|
17
18
|
import { openSimWorld } from './job-sim-world';
|
|
18
19
|
|
|
@@ -47,7 +48,10 @@ export interface CrashRerun {
|
|
|
47
48
|
}
|
|
48
49
|
|
|
49
50
|
export interface CrashReport {
|
|
51
|
+
/** The boundaries killed at: each distinct one the clean run crossed, a loop's rows as one. */
|
|
50
52
|
readonly boundaries: readonly CrashBoundary[];
|
|
53
|
+
/** Every boundary the clean run crossed, the forced-fail run's included: what the kills stand for. */
|
|
54
|
+
readonly crossed: number;
|
|
51
55
|
readonly divergences: readonly CrashDivergence[];
|
|
52
56
|
readonly reruns: readonly CrashRerun[];
|
|
53
57
|
}
|
|
@@ -134,7 +138,10 @@ async function runOnce<I>(
|
|
|
134
138
|
if (op.step.startsWith(COMPENSATE_PREFIX)) compensated.push(op.step);
|
|
135
139
|
else if (op.step !== UNWINDING_STEP) forward.add(op.step);
|
|
136
140
|
};
|
|
141
|
+
// Alone: a child the job queues is its enqueue effect, never run — the clean run that ran it
|
|
142
|
+
// first wrote its rows, and every killed run after found them there (0× against 6×).
|
|
137
143
|
await using world = await openSimWorld({
|
|
144
|
+
alone: true,
|
|
138
145
|
seed: finiteCount('crashProof', 'seed', options.seed ?? 1),
|
|
139
146
|
decide,
|
|
140
147
|
landed,
|
|
@@ -209,6 +216,7 @@ export async function crashProof<I>(
|
|
|
209
216
|
const boundaries: CrashBoundary[] = [];
|
|
210
217
|
const divergences: CrashDivergence[] = [];
|
|
211
218
|
const reruns: CrashRerun[] = [];
|
|
219
|
+
let crossed = 0;
|
|
212
220
|
const modes: readonly CrashBoundary['mode'][] = handle.compensates
|
|
213
221
|
? ['clean', 'forced-fail']
|
|
214
222
|
: ['clean'];
|
|
@@ -225,7 +233,19 @@ export async function crashProof<I>(
|
|
|
225
233
|
: await runOnce(handle, input, { forceFail }, options);
|
|
226
234
|
const afterOf = (key: string): string | undefined =>
|
|
227
235
|
clean.effects.find((effect) => effect.key === key)?.after;
|
|
228
|
-
|
|
236
|
+
// Judged before the first kill: a proof past its bound costs one clean run, then refuses.
|
|
237
|
+
const points = crashPoints(clean.boundaries);
|
|
238
|
+
crossed += clean.boundaries.length;
|
|
239
|
+
if (points.length > CRASH_POINT_BOUND) {
|
|
240
|
+
throw new JobCrashUnboundedError({
|
|
241
|
+
job: handle.name,
|
|
242
|
+
points: points.length,
|
|
243
|
+
bound: CRASH_POINT_BOUND,
|
|
244
|
+
crossed: clean.boundaries.length,
|
|
245
|
+
shapes: boundaryShapes(clean.boundaries),
|
|
246
|
+
});
|
|
247
|
+
}
|
|
248
|
+
for (const at of points) {
|
|
229
249
|
const label = clean.boundaries[at - 1] ?? '?';
|
|
230
250
|
const killed = await runOnce(handle, input, { at, fault: 'kill', forceFail }, options);
|
|
231
251
|
boundaries.push({
|
|
@@ -269,5 +289,5 @@ export async function crashProof<I>(
|
|
|
269
289
|
}
|
|
270
290
|
}
|
|
271
291
|
}
|
|
272
|
-
return { boundaries, divergences, reruns };
|
|
292
|
+
return { boundaries, crossed, divergences, reruns };
|
|
273
293
|
}
|
package/src/job-sim-errors.ts
CHANGED
|
@@ -101,3 +101,27 @@ export class JobSimNondeterministicError extends UltimateError {
|
|
|
101
101
|
});
|
|
102
102
|
}
|
|
103
103
|
}
|
|
104
|
+
|
|
105
|
+
export interface CrashUnboundedInput {
|
|
106
|
+
readonly job: string;
|
|
107
|
+
/** Kill points the clean run's boundaries asked for, after a loop's rows were taken as one. */
|
|
108
|
+
readonly points: number;
|
|
109
|
+
readonly bound: number;
|
|
110
|
+
/** Boundaries the clean run crossed. */
|
|
111
|
+
readonly crossed: number;
|
|
112
|
+
/** The first distinct boundary shapes, in order: where the names that did not collapse are. */
|
|
113
|
+
readonly shapes: readonly string[];
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** Refused before the first kill: a proof that grows with the rows is one no gate can wait for. */
|
|
117
|
+
export class JobCrashUnboundedError extends UltimateError {
|
|
118
|
+
constructor(input: CrashUnboundedInput) {
|
|
119
|
+
const shown = input.shapes.slice(0, 5).join(', ');
|
|
120
|
+
super({
|
|
121
|
+
code: 'X_JOB_CRASH_UNBOUNDED',
|
|
122
|
+
cause: `job ${renderCauseValue(input.job)} crossed ${input.crossed} step boundaries, ${input.points} distinct kill points against a bound of ${input.bound} — its steps are named so a loop's rows do not read as one point (first: ${renderCauseValue(shown)})`,
|
|
123
|
+
fix: `step.run(\`<name>:\${row.id}\`, async () => …) # in job(${renderFixLiteral(input.job, '<job>')}): name a per-row step by the row's id, so its loop is one kill point; then: x test job --filter <the test file>`,
|
|
124
|
+
meta: { ...input, shapes: [...input.shapes] },
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
}
|
package/src/job-sim-world.ts
CHANGED
|
@@ -31,6 +31,8 @@ export const SIM_VISIBILITY_MS = 30_000;
|
|
|
31
31
|
const RENEW_MS = 1;
|
|
32
32
|
const CLAIM_LIMIT = 64;
|
|
33
33
|
const LIVE: ReadonlySet<string> = new Set(['ready', 'delayed', 'running', 'suspended']);
|
|
34
|
+
/** Where an `alone` world files the harness's row: no body enqueues here, so no worker claims theirs. */
|
|
35
|
+
const HARNESS_QUEUE = 'x-sim-harness';
|
|
34
36
|
|
|
35
37
|
export interface SimWorldOptions {
|
|
36
38
|
/** Seeds `Math.random` for the run: a body's jitter and draws are the seed's. */
|
|
@@ -40,6 +42,11 @@ export interface SimWorldOptions {
|
|
|
40
42
|
decide(op: SimOp): SimDecision | Promise<SimDecision>;
|
|
41
43
|
/** Told every landed call, after the world has read it. */
|
|
42
44
|
landed?(op: SimOp, answer: unknown): void;
|
|
45
|
+
/**
|
|
46
|
+
* Only the row the harness filed is ever claimed: what a body enqueues lands, dedupes and is an
|
|
47
|
+
* `enqueue:` effect, and never runs — its effects are its own job's, not this run's.
|
|
48
|
+
*/
|
|
49
|
+
readonly alone?: boolean;
|
|
43
50
|
}
|
|
44
51
|
|
|
45
52
|
export interface SimWorld extends AsyncDisposable {
|
|
@@ -115,9 +122,11 @@ export async function openSimWorld(options: SimWorldOptions): Promise<SimWorld>
|
|
|
115
122
|
const anonymous = ctxOf({ role: 'worker' });
|
|
116
123
|
const context = options.context ?? (() => anonymous);
|
|
117
124
|
|
|
118
|
-
const queues = (): readonly string[] =>
|
|
119
|
-
|
|
120
|
-
|
|
125
|
+
const queues = (): readonly string[] =>
|
|
126
|
+
options.alone === true
|
|
127
|
+
? [HARNESS_QUEUE]
|
|
128
|
+
: [...new Set([jobs.DEFAULT_QUEUE, ...jobs.registeredJobs().map((handle) => handle.queue)])];
|
|
129
|
+
const claimable = new Set(queues());
|
|
121
130
|
|
|
122
131
|
const rows = async (): Promise<readonly JobRecord[]> => {
|
|
123
132
|
const all: JobRecord[] = [];
|
|
@@ -146,7 +155,7 @@ export async function openSimWorld(options: SimWorldOptions): Promise<SimWorld>
|
|
|
146
155
|
enqueue: async (handle, input) => {
|
|
147
156
|
const result = await base.enqueue({
|
|
148
157
|
name: handle.name,
|
|
149
|
-
queue: handle.queue,
|
|
158
|
+
queue: options.alone === true ? HARNESS_QUEUE : handle.queue,
|
|
150
159
|
input,
|
|
151
160
|
idempotencyKey: handle.idempotencyKeyFor(input),
|
|
152
161
|
maxAttempts: jobs.enqueuedAttempts(handle.retry, handle.compensates),
|
|
@@ -179,7 +188,9 @@ export async function openSimWorld(options: SimWorldOptions): Promise<SimWorld>
|
|
|
179
188
|
rows,
|
|
180
189
|
nextDue: async () => {
|
|
181
190
|
const due = (await rows())
|
|
182
|
-
.filter(
|
|
191
|
+
.filter(
|
|
192
|
+
(row) => LIVE.has(row.state) && (options.alone !== true || claimable.has(row.queue)),
|
|
193
|
+
)
|
|
183
194
|
.map((row) => (row.state === 'running' ? (row.visibleAt ?? row.runAt) : row.runAt));
|
|
184
195
|
return due.length === 0 ? undefined : Math.min(...due);
|
|
185
196
|
},
|