@ultimat3/testing 30.5.0 → 30.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,7 @@
1
1
  // What runs after every primitive's own call: the scale sweep (each reached one again, at 2 and at 8
2
2
  // rows a table) and the plan sweep (every statement the pass sent, planned on the migrated
3
- // database). Both verdicts land on the record of the primitive that sent the statements.
3
+ // database). Both verdicts land on the record of the primitive that sent the statements, and each
4
+ // sweep's time on the pass.
4
5
 
5
6
  import type { Actor } from '@ultimat3/core';
6
7
  import type { ExerciseOverrides } from './define-exercise';
@@ -9,11 +10,11 @@ import { asSeededRow, chooseActor } from './exercise-actor';
9
10
  import type { Sampler } from './exercise-input';
10
11
  import { bindIds, drawInput, seededIds } from './exercise-input';
11
12
  import { overrideInput } from './exercise-override-input';
12
- import type { ExerciseRecord } from './exercise-report';
13
+ import type { ExercisePhaseMs, ExerciseRecord } from './exercise-report';
13
14
  import type { Subject } from './exercise-subjects';
14
15
  import { seatIn } from './exercise-subjects';
15
16
  import { sweepPlans } from './plan-sweep';
16
- import type { ScaleSweepOptions } from './scale-sweep-run';
17
+ import type { ScaleSubject, ScaleSweepOptions } from './scale-sweep-run';
17
18
  import { scaleSweep } from './scale-sweep-run';
18
19
  import type { SeedGraph } from './seed-graph-plan';
19
20
  import type { SeedEntity } from './seed-graph-shape';
@@ -30,7 +31,12 @@ export const actorRowsOf = (options: CallOptions, graph: SeedGraph): ActorRows |
30
31
  const name = options.overrides?.actorEntity;
31
32
  const entity = options.entities.find((one) => one.$name === name);
32
33
  if (entity === undefined) return undefined;
33
- return { entity, rows: graph.inserts.filter((row) => row.entity === entity.$name) };
34
+ const column = options.overrides?.actorColumn;
35
+ return {
36
+ entity,
37
+ rows: graph.inserts.filter((row) => row.entity === entity.$name),
38
+ ...(column === undefined ? {} : { column }),
39
+ };
34
40
  };
35
41
 
36
42
  /** The actor of `chosen`, seated as a row of the app's actor entity in `tenant` when it names one. */
@@ -66,6 +72,25 @@ export async function sizedCall(
66
72
  return { actor, input: seatIn(subject, given, tenant) };
67
73
  }
68
74
 
75
+ /** The records with both sweeps' verdicts and scale time on them, and each sweep's whole time. */
76
+ export interface Swept {
77
+ readonly records: readonly ExerciseRecord[];
78
+ readonly ms: Pick<ExercisePhaseMs, 'scale' | 'plans'>;
79
+ }
80
+
81
+ /** `subject`'s calls, each timed into `spent` under its name: the sweep calls it once per size. */
82
+ const timed = (subject: ScaleSubject, spent: Map<string, number>): ScaleSubject => ({
83
+ ...subject,
84
+ call: async (...args) => {
85
+ const started = performance.now();
86
+ try {
87
+ return await subject.call(...args);
88
+ } finally {
89
+ spent.set(subject.name, (spent.get(subject.name) ?? 0) + performance.now() - started);
90
+ }
91
+ },
92
+ });
93
+
69
94
  /**
70
95
  * The scale sweep inside the room, the room closed, then the plan sweep — on the installed client
71
96
  * only: with none, the pass ran on a driver that sends no SQL and there is no plan to read.
@@ -73,24 +98,41 @@ export async function sizedCall(
73
98
  export async function attachSweeps(
74
99
  records: readonly ExerciseRecord[],
75
100
  options: ScaleSweepOptions,
76
- ): Promise<readonly ExerciseRecord[]> {
101
+ ): Promise<Swept> {
102
+ const spent = new Map<string, number>();
77
103
  let scale: Awaited<ReturnType<typeof scaleSweep>>;
104
+ const scaleStarted = performance.now();
78
105
  try {
79
- scale = await scaleSweep(options);
106
+ scale = await scaleSweep({
107
+ ...options,
108
+ subjects: options.subjects.map((subject) => timed(subject, spent)),
109
+ });
80
110
  } finally {
81
111
  await options.room[Symbol.asyncDispose]();
82
112
  }
113
+ const plansStarted = performance.now();
83
114
  const db = await import('@ultimat3/db');
84
115
  const plans = db.dbClientInstalled()
85
116
  ? await sweepPlans(db.db(), options.room.sent(), options.entities)
86
117
  : new Map();
87
- return records.map((record) => {
118
+ const ms = {
119
+ scale: Math.round(plansStarted - scaleStarted),
120
+ plans: Math.round(performance.now() - plansStarted),
121
+ };
122
+ const swept = records.map((record) => {
88
123
  const proof = plans.get(record.name);
89
124
  const sized = scale.get(record.name);
125
+ const scaleMs = spent.get(record.name);
126
+ const proofMs = {
127
+ ...record.proofMs,
128
+ ...(scaleMs === undefined ? {} : { scale: Math.round(scaleMs) }),
129
+ };
90
130
  return {
91
131
  ...record,
92
132
  ...(proof === undefined ? {} : { plans: proof }),
93
133
  ...(sized === undefined ? {} : { scale: sized }),
134
+ ...(Object.keys(proofMs).length === 0 ? {} : { proofMs }),
94
135
  };
95
136
  });
137
+ return { records: swept, ms };
96
138
  }
package/src/exercise.ts CHANGED
@@ -8,14 +8,15 @@
8
8
 
9
9
  import type { Actor, ExerciseAttempt, ExerciseVerdict } from '@ultimat3/core';
10
10
  import { isUltimateError, renderThrowable, runExercise } from '@ultimat3/core';
11
- import type { JobExecution, JobRecord } from '@ultimat3/jobs';
12
11
  import { registeredJobs } from '@ultimat3/jobs';
13
12
  import type { ExerciseOverride, ExerciseOverrides } from './define-exercise';
14
13
  import { assertOverridesKnown, unknownDeclaration } from './define-exercise';
15
14
  import type { RoleGrants } from './exercise-actor';
16
15
  import { chooseActor } from './exercise-actor';
16
+ import { seatActorUsers } from './exercise-actor-user';
17
17
  import { proveCrash } from './exercise-crash';
18
18
  import { defectDetail } from './exercise-defect';
19
+ import { driftOverrideLine } from './exercise-drift-hint';
19
20
  import type { IdLookup, Sampler } from './exercise-input';
20
21
  import {
21
22
  bindIds,
@@ -25,11 +26,13 @@ import {
25
26
  namesActorEntity,
26
27
  seededIds,
27
28
  } from './exercise-input';
29
+ import { ambientJobRows } from './exercise-job-run';
28
30
  import type { OverrideInput } from './exercise-override-input';
29
31
  import { overrideInput } from './exercise-override-input';
30
32
  import type {
31
33
  EffectPayload,
32
34
  ExerciseEffect,
35
+ ExercisePhaseMs,
33
36
  ExerciseRecord,
34
37
  ExerciseRoom,
35
38
  Observed,
@@ -61,6 +64,8 @@ export interface ExercisePassOptions {
61
64
  /** One primitive by name; unknown is `X_DECLARATION_UNKNOWN`. */
62
65
  readonly only?: string;
63
66
  readonly roles: RoleGrants;
67
+ /** Told once, as the pass returns: how long each phase took, so a slow pass names its slow part. */
68
+ readonly onPhases?: (ms: ExercisePhaseMs) => void;
64
69
  }
65
70
 
66
71
  const isOff = (overrides: ExerciseOverrides | undefined, name: string): boolean => {
@@ -76,13 +81,15 @@ const codeOf = (error: unknown): string | undefined => {
76
81
  const reasonOf = (error: unknown): string =>
77
82
  isUltimateError(error) ? error.cause : typeof error === 'string' ? error : renderThrowable(error);
78
83
 
79
- /** Jobs queued or run by the primitive, the primitive's own job excepted. */
80
- async function enqueued(subject: Subject, jobs: RunJobs, runs: readonly JobExecution[]) {
84
+ /**
85
+ * Jobs the primitive queued, its own job excepted. Never run: a job's children are left queued as
86
+ * an action's are (`exercise-job-run.ts`), so their statements are theirs, never this record's.
87
+ */
88
+ async function enqueued(subject: Subject, jobs: RunJobs) {
81
89
  const effects: ExerciseEffect[] = [];
82
90
  for (const handle of registeredJobs()) {
83
91
  if (subject.kind === 'job' && handle.name === subject.name) continue;
84
- const ran = new Set(runs.filter((run) => run.job === handle.name).map((r) => r.jobId)).size;
85
- const count = ran + (await jobs.depth(handle));
92
+ const count = await jobs.depth(handle);
86
93
  if (count > 0) effects.push({ kind: 'enqueue', job: handle.name, count });
87
94
  }
88
95
  return effects;
@@ -93,21 +100,7 @@ async function enqueued(subject: Subject, jobs: RunJobs, runs: readonly JobExecu
93
100
  * input is what B was handed. Read off the leg's own queue, page by page.
94
101
  */
95
102
  async function queuedPayloads(subject: Subject): Promise<readonly EffectPayload[]> {
96
- const jobs = await import('@ultimat3/jobs');
97
- const introspect = jobs.jobDriver()?.introspect;
98
- if (introspect === undefined) return [];
99
- const records: JobRecord[] = [];
100
- for (let after: string | undefined; ; ) {
101
- const page = await introspect.list({
102
- limit: jobs.MAX_JOB_PAGE,
103
- ...(after === undefined ? {} : { after }),
104
- });
105
- records.push(...page);
106
- const last = page[page.length - 1];
107
- if (last === undefined || page.length < jobs.MAX_JOB_PAGE) break;
108
- after = jobs.jobCursor(last);
109
- }
110
- return records
103
+ return (await ambientJobRows())
111
104
  .filter((record) => !(subject.kind === 'job' && record.name === subject.name))
112
105
  .map((record, i) => ({
113
106
  path: `effect.enqueue[${i}]`,
@@ -122,7 +115,7 @@ interface Run {
122
115
  readonly lastInput: unknown;
123
116
  }
124
117
 
125
- function judged(subject: Subject, run: Run, observed: Observed) {
118
+ function judged(subject: Subject, run: Run, observed: Observed, sampler: Sampler) {
126
119
  const idFields = (input: unknown): readonly string[] =>
127
120
  typeof input === 'object' && input !== null
128
121
  ? Object.keys(input).filter((key) => idStem(key) !== undefined)
@@ -140,7 +133,12 @@ function judged(subject: Subject, run: Run, observed: Observed) {
140
133
  }
141
134
  if (run.verdict.kind === 'pass' && run.answered) return { outcome: 'reached' as const };
142
135
  if (run.verdict.kind === 'drift') {
143
- return unreached('X_CONTRACT_DRIFT', 'its input schema refused every generated value');
136
+ // The field no draw can fill, named: `...drawn` alone told the author nothing to write.
137
+ const fields = idFields(run.lastInput);
138
+ return {
139
+ ...unreached('X_CONTRACT_DRIFT', 'its input schema refused every generated value'),
140
+ override: driftOverrideLine(subject.name, subject.input, sampler, fields),
141
+ };
144
142
  }
145
143
  if (run.verdict.kind === 'defect') {
146
144
  return unreached(codeOf(run.verdict.error) ?? 'X_CONTRACT_DEFECT', run.verdict.defect);
@@ -174,7 +172,6 @@ async function control(
174
172
  let lastError: unknown;
175
173
  let lastInput: unknown;
176
174
  let output: unknown;
177
- let executions: readonly JobExecution[] = [];
178
175
  room.begin(name);
179
176
  const started = performance.now();
180
177
  const verdict = await runExercise({
@@ -209,10 +206,9 @@ async function control(
209
206
  statusOf: (await import('@ultimat3/http')).statusFor,
210
207
  });
211
208
  const ms = Math.round(performance.now() - started);
212
- if (kind === 'job') executions = (await jobs.drain()).executions;
213
209
  const observed = room.end();
214
- const effects = [...observed.effects, ...(await enqueued(subject, jobs, executions))];
215
- const verdictOf = judged(subject, { verdict, answered, lastError, lastInput }, observed);
210
+ const effects = [...observed.effects, ...(await enqueued(subject, jobs))];
211
+ const verdictOf = judged(subject, { verdict, answered, lastError, lastInput }, observed, sampler);
216
212
  // A 5xx's cause, statement and fields: the code alone sent the author to reproduce it by hand.
217
213
  const defect =
218
214
  verdict.kind === 'defect' && verdictOf.outcome === 'unreached'
@@ -286,10 +282,13 @@ async function exerciseOne(
286
282
  const intruder = withoutCrossTenant(
287
283
  theirs.kind === 'actor' ? seated(theirs.actor, other, rows) : null,
288
284
  );
285
+ // Both are users, as a request's actor always is: their `x_users` rows, before any call.
286
+ await seatActorUsers([actor, intruder]);
289
287
 
290
288
  const scope = { tenant, actorId: actor?.id, subject: name, actorEntity: rows?.entity.$name };
291
289
  const bindInput = overrideInput(name, override);
292
- const declaredRuns = override !== undefined && 'actor' in override ? override.runs : undefined;
290
+ // `off` returned above: any other entry's `runs` is declared, with or without an `actor`.
291
+ const declaredRuns = override === undefined || 'off' in override ? undefined : override.runs;
293
292
  // A field naming the actor entity is tried as the actor's own row, then as somebody else's:
294
293
  // both readings are legitimate and only the handler knows which it meant. Same minimal draw.
295
294
  const twoWay =
@@ -325,12 +324,14 @@ async function exerciseOne(
325
324
  }
326
325
  const input = seat === undefined ? given : seat.input;
327
326
  let reachedWith: unknown;
327
+ const tenantStarted = performance.now();
328
328
  const record = await proveTenant({
329
329
  taint: tenantTaint(graph, tenant),
330
330
  input,
331
331
  public: publicReason(subject.policy),
332
332
  read: kind === 'query',
333
333
  sweep: kind === 'job',
334
+ tenantless: seat?.kind === 'none',
334
335
  attack: () => attack(subject, input, intruder, other, room),
335
336
  control: async () => {
336
337
  const done = await control(run, room, sampler);
@@ -339,10 +340,16 @@ async function exerciseOne(
339
340
  },
340
341
  snapshot: (seeded) => options.writer.snapshot(seeded),
341
342
  });
343
+ // The control's own call is `ms`: the tenant proof's time is everything around it.
344
+ const tenantMs = Math.max(0, Math.round(performance.now() - tenantStarted) - record.ms);
342
345
  // After the tenant proof, outside the room's begin/end: the crash runs' effects are their own.
343
- return subject.handle === undefined || record.outcome !== 'reached'
344
- ? record
345
- : { ...record, crash: await proveCrash(subject.handle, reachedWith, actor) };
346
+ if (subject.handle === undefined || record.outcome !== 'reached') {
347
+ return { ...record, proofMs: { tenant: tenantMs } };
348
+ }
349
+ const crashStarted = performance.now();
350
+ const crash = await proveCrash(subject.handle, reachedWith, actor);
351
+ const proofMs = { tenant: tenantMs, crash: Math.round(performance.now() - crashStarted) };
352
+ return { ...record, crash, proofMs };
346
353
  }
347
354
 
348
355
  export async function runExercisePass(
@@ -363,6 +370,8 @@ export async function runExercisePass(
363
370
  const live = chosen.filter((subject) => !isOff(options.overrides, subject.name));
364
371
  const records: ExerciseRecord[] = [];
365
372
  let graph: SeedGraph;
373
+ const seedStarted = performance.now();
374
+ let primitivesStarted = seedStarted;
366
375
  try {
367
376
  // Seeded before the first primitive runs: a failure here is the pass's, never one primitive's.
368
377
  graph = await seedPass(options.entities, options.writer, {
@@ -372,6 +381,7 @@ export async function runExercisePass(
372
381
  ? {}
373
382
  : { factories: options.overrides.factories }),
374
383
  });
384
+ primitivesStarted = performance.now();
375
385
  // In order, one at a time: the seams the room installs are the process's, not the primitive's.
376
386
  for (const subject of chosen) {
377
387
  const index = live.indexOf(subject);
@@ -383,9 +393,11 @@ export async function runExercisePass(
383
393
  await room[Symbol.asyncDispose]();
384
394
  throw error;
385
395
  }
396
+ const seed = Math.round(primitivesStarted - seedStarted);
397
+ const primitives = Math.round(performance.now() - primitivesStarted);
386
398
  // The sweeps: every reached primitive again at two sizes, then every statement planned.
387
399
  const reached = new Set(records.filter((r) => r.outcome === 'reached').map((r) => r.name));
388
- return attachSweeps(records, {
400
+ const swept = await attachSweeps(records, {
389
401
  subjects: chosen.filter((subject) => reached.has(subject.name)),
390
402
  entities: options.entities,
391
403
  // The sized graphs close their cycles as the pass's did: a refused deferred key stays null.
@@ -396,9 +408,13 @@ export async function runExercisePass(
396
408
  factories: options.overrides?.factories,
397
409
  setup: async (subject, sized, tenant, self) => {
398
410
  const full = chosen.find((one) => one.name === subject.name);
399
- return full === undefined
400
- ? undefined
401
- : await sizedCall(full, options, sampler, { graph: sized, tenant, self });
411
+ if (full === undefined) return undefined;
412
+ const call = await sizedCall(full, options, sampler, { graph: sized, tenant, self });
413
+ // Before the sweep's room opens: the sized tenant's actor is a user too.
414
+ if (call !== undefined) await seatActorUsers([call.actor]);
415
+ return call;
402
416
  },
403
417
  });
418
+ options.onPhases?.({ seed, primitives, ...swept.ms });
419
+ return swept.records;
404
420
  }
@@ -69,6 +69,11 @@ export interface RunJobs extends AsyncDisposable {
69
69
  enqueue<I>(handle: JobHandle<I>, input: I, options?: JobEnqueueAs): Promise<EnqueueResult>;
70
70
  /** Claim and execute everything due at the current instant, until nothing is. */
71
71
  drain(options?: JobDrainAs): Promise<JobRunTrace>;
72
+ /**
73
+ * ONE worker pass: what is due now runs, and what those runs enqueue waits for the next pass —
74
+ * "it queued one child per row" asserted without running a child.
75
+ */
76
+ tick(options?: JobDrainAs): Promise<JobRunTrace>;
72
77
  /** Live jobs — ready, delayed, running or suspended — optionally for one job only. */
73
78
  depth(handle?: AnyJobHandle): Promise<number>;
74
79
  /** Live jobs claimable right now. `clock.advance()` is what turns delayed into due. */
@@ -194,14 +199,20 @@ export async function testJobs(): Promise<RunJobs> {
194
199
  /** Every execution this fixture has driven, because the trace is cumulative. */
195
200
  const history: JobExecution[] = [];
196
201
 
202
+ const ctxFor = (options: JobDrainAs): Ctx =>
203
+ options.actor === undefined ? anonymousWorker : ctxOf({ role: 'worker', actor: options.actor });
204
+ const traced = (): JobRunTrace => ({ executions: [...history], steps: tallyOf(history) });
205
+
206
+ const tick = async (options: JobDrainAs = {}): Promise<JobRunTrace> => {
207
+ history.push(...(await round(ctxFor(options))));
208
+ return traced();
209
+ };
210
+
197
211
  const drain = async (options: JobDrainAs = {}): Promise<JobRunTrace> => {
198
- const ctx =
199
- options.actor === undefined
200
- ? anonymousWorker
201
- : ctxOf({ role: 'worker', actor: options.actor });
212
+ const ctx = ctxFor(options);
202
213
  for (let rounds = 0; rounds < MAX_ROUNDS; rounds += 1) {
203
214
  const batch = await round(ctx);
204
- if (batch.length === 0) return { executions: [...history], steps: tallyOf(history) };
215
+ if (batch.length === 0) return traced();
205
216
  history.push(...batch);
206
217
  }
207
218
  assert(
@@ -239,6 +250,7 @@ export async function testJobs(): Promise<RunJobs> {
239
250
  return Object.assign(enqueueThenDrain, {
240
251
  enqueue,
241
252
  drain,
253
+ tick,
242
254
  depth: async (handle?: AnyJobHandle) => (await live(handle?.name)).length,
243
255
  due: async () =>
244
256
  (await live()).filter(
package/src/index.ts CHANGED
@@ -207,7 +207,9 @@ export type {
207
207
  EffectPayload,
208
208
  ExerciseEffect,
209
209
  ExerciseOutcome,
210
+ ExercisePhaseMs,
210
211
  ExerciseRecord,
212
+ ProofMs,
211
213
  TenantFinding,
212
214
  TenantLeg,
213
215
  TenantProof,
@@ -366,6 +368,7 @@ export {
366
368
  normalizeIslandName,
367
369
  } from './island-states-resolve';
368
370
  export { ISOLATED_ENV, releasePluginsAfterIsolatedFile } from './isolated-plugins';
371
+ export { CRASH_POINT_BOUND } from './job-crash-points';
369
372
  export type {
370
373
  CrashBoundary,
371
374
  CrashDivergence,
@@ -379,6 +382,7 @@ export type { SimEffect } from './job-sim-effects';
379
382
  export {
380
383
  crashEffectPlace,
381
384
  JobCrashDivergedError,
385
+ JobCrashUnboundedError,
382
386
  JobSimInvariantError,
383
387
  JobSimNondeterministicError,
384
388
  } from './job-sim-errors';
@@ -0,0 +1,36 @@
1
+ // Which boundaries a crash proof kills at. A loop's steps named by row id are one point of the
2
+ // code, not one per row: each boundary shape is killed at its first and its last occurrence, so the
3
+ // proof costs what the job's code has, never what the database holds. Pure: labels in, indexes out.
4
+
5
+ /** Kill points one mode of a proof may cost; past it the proof refuses, `X_JOB_CRASH_UNBOUNDED`. */
6
+ export const CRASH_POINT_BOUND = 32;
7
+
8
+ const UUID = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi;
9
+ /** A ULID, a hex digest, a cuid: eight or more letters and digits with a digit among them. */
10
+ const TOKEN = /\b(?=[A-Za-z]*\d)[A-Za-z0-9]{8,}\b/g;
11
+ const DIGITS = /\d+/g;
12
+
13
+ /** `fanout:<id>`, `page-<n>`, `sleep:<n>h (sleeping)`: a boundary label with its ids taken out. */
14
+ export const boundaryShape = (label: string): string =>
15
+ label.replace(UUID, '<id>').replace(TOKEN, '<id>').replace(DIGITS, '<n>');
16
+
17
+ /**
18
+ * The 1-based positions to kill at, in order: the first and the last boundary of every shape.
19
+ * The first proves the code before the loop and the loop's own body; the last, every row before it
20
+ * replayed and the code after it — an effect between any two rows is made again at the last.
21
+ */
22
+ export function crashPoints(labels: readonly string[]): readonly number[] {
23
+ const first = new Map<string, number>();
24
+ const last = new Map<string, number>();
25
+ labels.forEach((label, i) => {
26
+ const shape = boundaryShape(label);
27
+ if (!first.has(shape)) first.set(shape, i + 1);
28
+ last.set(shape, i + 1);
29
+ });
30
+ return [...new Set([...first.values(), ...last.values()])].sort((a, b) => a - b);
31
+ }
32
+
33
+ /** The distinct shapes of `labels`, first seen first — what an unbounded refusal names. */
34
+ export const boundaryShapes = (labels: readonly string[]): readonly string[] => [
35
+ ...new Set(labels.map(boundaryShape)),
36
+ ];
@@ -1,6 +1,6 @@
1
- // `crashProof(handle, input)`: one clean run, then one run killed at each boundary the clean run
2
- // crossed — every step write landing, every settle — and one more where that write's answer is
3
- // lost. Judged by the effect log: a killed run that makes an effect more often than the clean one
1
+ // `crashProof(handle, input)`: one clean run, then one run killed at each distinct boundary the
2
+ // clean run crossed — a step write landing, a settle; a loop's rows one point (`job-crash-points.ts`)
3
+ // — and one more where that write's answer is lost. Judged by the effect log: a killed run that makes an effect more often than the clean one
4
4
  // is `X_JOB_CRASH_DIVERGED`; a lost answer rerunning a step is at-least-once, reported, never red.
5
5
 
6
6
  import type { Ctx } from '@ultimat3/core';
@@ -8,11 +8,12 @@ import { finiteCount } from '@ultimat3/core';
8
8
  import type { JobHandle } from '@ultimat3/jobs';
9
9
  import { COMPENSATE_PREFIX, UNWINDING_STEP } from '@ultimat3/jobs';
10
10
  import { advanceClock, frozenNow } from './determinism';
11
+ import { boundaryShapes, CRASH_POINT_BOUND, crashPoints } from './job-crash-points';
11
12
  import type { SimDecision, SimOp } from './job-sim-driver';
12
13
  import { SEND_AND_ANSWER } from './job-sim-driver';
13
14
  import type { SimEffect } from './job-sim-effects';
14
15
  import { effectCounts } from './job-sim-effects';
15
- import { JobCrashDivergedError } from './job-sim-errors';
16
+ import { JobCrashDivergedError, JobCrashUnboundedError } from './job-sim-errors';
16
17
  import type { SimWorld } from './job-sim-world';
17
18
  import { openSimWorld } from './job-sim-world';
18
19
 
@@ -47,7 +48,10 @@ export interface CrashRerun {
47
48
  }
48
49
 
49
50
  export interface CrashReport {
51
+ /** The boundaries killed at: each distinct one the clean run crossed, a loop's rows as one. */
50
52
  readonly boundaries: readonly CrashBoundary[];
53
+ /** Every boundary the clean run crossed, the forced-fail run's included: what the kills stand for. */
54
+ readonly crossed: number;
51
55
  readonly divergences: readonly CrashDivergence[];
52
56
  readonly reruns: readonly CrashRerun[];
53
57
  }
@@ -134,7 +138,10 @@ async function runOnce<I>(
134
138
  if (op.step.startsWith(COMPENSATE_PREFIX)) compensated.push(op.step);
135
139
  else if (op.step !== UNWINDING_STEP) forward.add(op.step);
136
140
  };
141
+ // Alone: a child the job queues is its enqueue effect, never run — the clean run that ran it
142
+ // first wrote its rows, and every killed run after found them there (0× against 6×).
137
143
  await using world = await openSimWorld({
144
+ alone: true,
138
145
  seed: finiteCount('crashProof', 'seed', options.seed ?? 1),
139
146
  decide,
140
147
  landed,
@@ -209,6 +216,7 @@ export async function crashProof<I>(
209
216
  const boundaries: CrashBoundary[] = [];
210
217
  const divergences: CrashDivergence[] = [];
211
218
  const reruns: CrashRerun[] = [];
219
+ let crossed = 0;
212
220
  const modes: readonly CrashBoundary['mode'][] = handle.compensates
213
221
  ? ['clean', 'forced-fail']
214
222
  : ['clean'];
@@ -225,7 +233,19 @@ export async function crashProof<I>(
225
233
  : await runOnce(handle, input, { forceFail }, options);
226
234
  const afterOf = (key: string): string | undefined =>
227
235
  clean.effects.find((effect) => effect.key === key)?.after;
228
- for (let at = 1; at <= clean.boundaries.length; at += 1) {
236
+ // Judged before the first kill: a proof past its bound costs one clean run, then refuses.
237
+ const points = crashPoints(clean.boundaries);
238
+ crossed += clean.boundaries.length;
239
+ if (points.length > CRASH_POINT_BOUND) {
240
+ throw new JobCrashUnboundedError({
241
+ job: handle.name,
242
+ points: points.length,
243
+ bound: CRASH_POINT_BOUND,
244
+ crossed: clean.boundaries.length,
245
+ shapes: boundaryShapes(clean.boundaries),
246
+ });
247
+ }
248
+ for (const at of points) {
229
249
  const label = clean.boundaries[at - 1] ?? '?';
230
250
  const killed = await runOnce(handle, input, { at, fault: 'kill', forceFail }, options);
231
251
  boundaries.push({
@@ -269,5 +289,5 @@ export async function crashProof<I>(
269
289
  }
270
290
  }
271
291
  }
272
- return { boundaries, divergences, reruns };
292
+ return { boundaries, crossed, divergences, reruns };
273
293
  }
@@ -101,3 +101,27 @@ export class JobSimNondeterministicError extends UltimateError {
101
101
  });
102
102
  }
103
103
  }
104
+
105
+ export interface CrashUnboundedInput {
106
+ readonly job: string;
107
+ /** Kill points the clean run's boundaries asked for, after a loop's rows were taken as one. */
108
+ readonly points: number;
109
+ readonly bound: number;
110
+ /** Boundaries the clean run crossed. */
111
+ readonly crossed: number;
112
+ /** The first distinct boundary shapes, in order: where the names that did not collapse are. */
113
+ readonly shapes: readonly string[];
114
+ }
115
+
116
+ /** Refused before the first kill: a proof that grows with the rows is one no gate can wait for. */
117
+ export class JobCrashUnboundedError extends UltimateError {
118
+ constructor(input: CrashUnboundedInput) {
119
+ const shown = input.shapes.slice(0, 5).join(', ');
120
+ super({
121
+ code: 'X_JOB_CRASH_UNBOUNDED',
122
+ cause: `job ${renderCauseValue(input.job)} crossed ${input.crossed} step boundaries, ${input.points} distinct kill points against a bound of ${input.bound} — its steps are named so a loop's rows do not read as one point (first: ${renderCauseValue(shown)})`,
123
+ fix: `step.run(\`<name>:\${row.id}\`, async () => …) # in job(${renderFixLiteral(input.job, '<job>')}): name a per-row step by the row's id, so its loop is one kill point; then: x test job --filter <the test file>`,
124
+ meta: { ...input, shapes: [...input.shapes] },
125
+ });
126
+ }
127
+ }
@@ -31,6 +31,8 @@ export const SIM_VISIBILITY_MS = 30_000;
31
31
  const RENEW_MS = 1;
32
32
  const CLAIM_LIMIT = 64;
33
33
  const LIVE: ReadonlySet<string> = new Set(['ready', 'delayed', 'running', 'suspended']);
34
+ /** Where an `alone` world files the harness's row: no body enqueues here, so no worker claims theirs. */
35
+ const HARNESS_QUEUE = 'x-sim-harness';
34
36
 
35
37
  export interface SimWorldOptions {
36
38
  /** Seeds `Math.random` for the run: a body's jitter and draws are the seed's. */
@@ -40,6 +42,11 @@ export interface SimWorldOptions {
40
42
  decide(op: SimOp): SimDecision | Promise<SimDecision>;
41
43
  /** Told every landed call, after the world has read it. */
42
44
  landed?(op: SimOp, answer: unknown): void;
45
+ /**
46
+ * Only the row the harness filed is ever claimed: what a body enqueues lands, dedupes and is an
47
+ * `enqueue:` effect, and never runs — its effects are its own job's, not this run's.
48
+ */
49
+ readonly alone?: boolean;
43
50
  }
44
51
 
45
52
  export interface SimWorld extends AsyncDisposable {
@@ -115,9 +122,11 @@ export async function openSimWorld(options: SimWorldOptions): Promise<SimWorld>
115
122
  const anonymous = ctxOf({ role: 'worker' });
116
123
  const context = options.context ?? (() => anonymous);
117
124
 
118
- const queues = (): readonly string[] => [
119
- ...new Set([jobs.DEFAULT_QUEUE, ...jobs.registeredJobs().map((handle) => handle.queue)]),
120
- ];
125
+ const queues = (): readonly string[] =>
126
+ options.alone === true
127
+ ? [HARNESS_QUEUE]
128
+ : [...new Set([jobs.DEFAULT_QUEUE, ...jobs.registeredJobs().map((handle) => handle.queue)])];
129
+ const claimable = new Set(queues());
121
130
 
122
131
  const rows = async (): Promise<readonly JobRecord[]> => {
123
132
  const all: JobRecord[] = [];
@@ -146,7 +155,7 @@ export async function openSimWorld(options: SimWorldOptions): Promise<SimWorld>
146
155
  enqueue: async (handle, input) => {
147
156
  const result = await base.enqueue({
148
157
  name: handle.name,
149
- queue: handle.queue,
158
+ queue: options.alone === true ? HARNESS_QUEUE : handle.queue,
150
159
  input,
151
160
  idempotencyKey: handle.idempotencyKeyFor(input),
152
161
  maxAttempts: jobs.enqueuedAttempts(handle.retry, handle.compensates),
@@ -179,7 +188,9 @@ export async function openSimWorld(options: SimWorldOptions): Promise<SimWorld>
179
188
  rows,
180
189
  nextDue: async () => {
181
190
  const due = (await rows())
182
- .filter((row) => LIVE.has(row.state))
191
+ .filter(
192
+ (row) => LIVE.has(row.state) && (options.alone !== true || claimable.has(row.queue)),
193
+ )
183
194
  .map((row) => (row.state === 'running' ? (row.visibleAt ?? row.runAt) : row.runAt));
184
195
  return due.length === 0 ? undefined : Math.min(...due);
185
196
  },