@rulvar/store-conformance 1.69.0 → 1.70.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -3
- package/dist/index.d.ts +190 -1
- package/dist/index.js +632 -17
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -2,9 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
The executable conformance kit for Rulvar store adapters: append
|
|
4
4
|
atomicity, total per-run order, read-your-writes, payload opacity, lease
|
|
5
|
-
fencing,
|
|
6
|
-
|
|
7
|
-
|
|
5
|
+
fencing, golden fold-state fixtures, the adversarial multi-process soak,
|
|
6
|
+
and the engine-level kill-point suite (a child process SIGKILLed around
|
|
7
|
+
each durable write, resumed from another process, with the documented
|
|
8
|
+
re-pay counts asserted). If you implement a custom store, this suite is
|
|
9
|
+
the contract your implementation must pass. Exports
|
|
10
|
+
`journalStoreConformance`, `leasableStoreConformance`,
|
|
11
|
+
`runMultiProcessSoak`, `killPointConformance`, and
|
|
8
12
|
`registerConformance`.
|
|
9
13
|
|
|
10
14
|
Part of [Rulvar](https://rulvar.com), an embeddable TypeScript engine
|
package/dist/index.d.ts
CHANGED
|
@@ -278,6 +278,195 @@ declare function verifySoakHistory(fixture: FencedTranscriptsFixture, events: re
|
|
|
278
278
|
*/
|
|
279
279
|
declare function runMultiProcessSoak(options: MultiProcessSoakOptions): Promise<MultiProcessSoakResult>;
|
|
280
280
|
//#endregion
|
|
281
|
+
//#region src/kill-points.d.ts
|
|
282
|
+
/** The five durable writes a scenario kills around. */
|
|
283
|
+
type KillPointName = "running" | "ok-terminal" | "limit-terminal" | "settle" | "meta";
|
|
284
|
+
/** `before` = the write is lost; `after` = everything past it is lost. */
|
|
285
|
+
type KillPointPhase = "before" | "after";
|
|
286
|
+
/** The two scripted runs: two plain steps, or one tool-capped agent. */
|
|
287
|
+
type KillPointWorkflowKind = "happy" | "limit";
|
|
288
|
+
/** The pinned recovery semantics a scenario asserts. */
|
|
289
|
+
interface KillPointExpectation {
|
|
290
|
+
/** Provider calls the child paid before dying. */
|
|
291
|
+
childCalls: number;
|
|
292
|
+
/** Tool executions the child performed before dying. */
|
|
293
|
+
childToolExecutions: number;
|
|
294
|
+
/** Provider calls the resume pays (the bracket's documented re-pay). */
|
|
295
|
+
resumeCalls: number;
|
|
296
|
+
/** Tool executions during the resume. */
|
|
297
|
+
resumeToolExecutions: number;
|
|
298
|
+
/** `agent` terminals with status `limit` in the final journal. */
|
|
299
|
+
limitTerminals: number;
|
|
300
|
+
/** The workflow value after recovery. */
|
|
301
|
+
value: unknown;
|
|
302
|
+
}
|
|
303
|
+
interface KillPointScenario {
|
|
304
|
+
/** Stable scenario id (`<workflow>-<point>-<phase>`). */
|
|
305
|
+
id: string;
|
|
306
|
+
workflow: KillPointWorkflowKind;
|
|
307
|
+
point: KillPointName;
|
|
308
|
+
phase: KillPointPhase;
|
|
309
|
+
/** Which matching write dies (1-based; step two of the happy run is 2). */
|
|
310
|
+
occurrence: number;
|
|
311
|
+
expected: KillPointExpectation;
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* The full table: both brackets of all five write points. The expected
|
|
315
|
+
* counts ARE the engine's documented recovery semantics; a count moving
|
|
316
|
+
* here means the durability contract moved and the change must be
|
|
317
|
+
* deliberate.
|
|
318
|
+
*/
|
|
319
|
+
declare const KILL_POINT_SCENARIOS: readonly KillPointScenario[];
|
|
320
|
+
/**
|
|
321
|
+
* The per-scenario contract, serialized as JSON into the
|
|
322
|
+
* `RULVAR_KILL_POINT_CONFIG` environment variable of the spawned worker.
|
|
323
|
+
*/
|
|
324
|
+
interface KillPointWorkerConfig {
|
|
325
|
+
/** Store location the writer script constructs its store over. */
|
|
326
|
+
storePath: string;
|
|
327
|
+
/** The run both processes drive; the referee resumes this id. */
|
|
328
|
+
runId: string;
|
|
329
|
+
/** Lease ttl the writer's store MUST be constructed with. */
|
|
330
|
+
ttlMs: number;
|
|
331
|
+
/** JSONL report file the worker appends its events to. */
|
|
332
|
+
reportPath: string;
|
|
333
|
+
/** Which {@link KILL_POINT_SCENARIOS} entry this worker executes. */
|
|
334
|
+
scenarioId: string;
|
|
335
|
+
}
|
|
336
|
+
/** One JSONL line of a worker's report file. */
|
|
337
|
+
type KillPointEvent = {
|
|
338
|
+
t: "call";
|
|
339
|
+
prompt: string;
|
|
340
|
+
} | {
|
|
341
|
+
t: "tool";
|
|
342
|
+
target: string;
|
|
343
|
+
} | {
|
|
344
|
+
t: "kill";
|
|
345
|
+
point: KillPointName;
|
|
346
|
+
phase: KillPointPhase;
|
|
347
|
+
site: "append" | "putMeta";
|
|
348
|
+
kind?: string;
|
|
349
|
+
status?: string;
|
|
350
|
+
seq?: number;
|
|
351
|
+
} | {
|
|
352
|
+
t: "ran-to-completion";
|
|
353
|
+
status: string;
|
|
354
|
+
} | {
|
|
355
|
+
t: "fatal";
|
|
356
|
+
message: string;
|
|
357
|
+
};
|
|
358
|
+
/** Reads the worker contract a referee serialized into the child env. */
|
|
359
|
+
declare function killPointWorkerConfigFromEnv(env?: Record<string, string | undefined>): KillPointWorkerConfig;
|
|
360
|
+
/** Parses one report file, tolerating a torn trailing line. */
|
|
361
|
+
declare function parseKillPointReport(path: string): KillPointEvent[];
|
|
362
|
+
/** Consumer hooks for {@link runKillPointWorker}. */
|
|
363
|
+
interface KillPointWorkerHooks {
|
|
364
|
+
/**
|
|
365
|
+
* The death itself; default SIGKILLs the current process and never
|
|
366
|
+
* returns. In-process protocol tests inject a throwing hook instead,
|
|
367
|
+
* which surfaces through the engine as a store failure.
|
|
368
|
+
*/
|
|
369
|
+
kill?: () => void;
|
|
370
|
+
}
|
|
371
|
+
/**
|
|
372
|
+
* The worker protocol: run it in a spawned process against the
|
|
373
|
+
* consumer-constructed store pair. Wraps the journal so the configured
|
|
374
|
+
* write kills the process (`before` = ahead of the write, `after` =
|
|
375
|
+
* once it is durable), appends every observation to the report file
|
|
376
|
+
* first (the appends are synchronous, so the report survives the
|
|
377
|
+
* SIGKILL), and reports `ran-to-completion` when the kill point is
|
|
378
|
+
* never reached, which the referee treats as a violation.
|
|
379
|
+
*/
|
|
380
|
+
declare function runKillPointWorker(fixture: FencedTranscriptsFixture, config: KillPointWorkerConfig, hooks?: KillPointWorkerHooks): Promise<void>;
|
|
381
|
+
/** What a green scenario returns (the observed recovery). */
|
|
382
|
+
interface KillPointObservation {
|
|
383
|
+
scenario: KillPointScenario;
|
|
384
|
+
childCalls: number;
|
|
385
|
+
childToolExecutions: number;
|
|
386
|
+
resumeCalls: number;
|
|
387
|
+
resumeToolExecutions: number;
|
|
388
|
+
/** `kind:status` per final journal entry, in seq order. */
|
|
389
|
+
journal: string[];
|
|
390
|
+
metaStatus: string | undefined;
|
|
391
|
+
}
|
|
392
|
+
interface KillPointScenarioOptions {
|
|
393
|
+
/**
|
|
394
|
+
* Absolute path of the consumer's writer script. It must construct
|
|
395
|
+
* the store over `killPointWorkerConfigFromEnv()` and call
|
|
396
|
+
* {@link runKillPointWorker}.
|
|
397
|
+
*/
|
|
398
|
+
writerScript: string;
|
|
399
|
+
/** Scratch directory for the report file. */
|
|
400
|
+
dir: string;
|
|
401
|
+
/** The scenario to execute, by table entry or id. */
|
|
402
|
+
scenario: KillPointScenario | string;
|
|
403
|
+
/** Store location handed to the worker config; default `join(dir, 'kp.db')`. */
|
|
404
|
+
storePath?: string;
|
|
405
|
+
/**
|
|
406
|
+
* The WORKER'S lease ttl; default 300 ms. The referee waits it out
|
|
407
|
+
* after the kill, so keep it short. It binds only the killed owner:
|
|
408
|
+
* the referee's own store (the `openStore` fixture) should keep its
|
|
409
|
+
* GENEROUS default ttl, so a scheduler stall under a loaded test
|
|
410
|
+
* runner cannot expire the resume's lease mid-scenario (a lost lease
|
|
411
|
+
* cancels the run by contract, and the fence then swallows its
|
|
412
|
+
* settle, which reads as a recovery violation when it is really a
|
|
413
|
+
* harness self-inflicted takeover).
|
|
414
|
+
*/
|
|
415
|
+
ttlMs?: number;
|
|
416
|
+
/**
|
|
417
|
+
* Opens the referee's own fixture over the SAME store location for
|
|
418
|
+
* the resume and the final state verification.
|
|
419
|
+
*/
|
|
420
|
+
openStore: () => Promise<FencedTranscriptsFixture> | FencedTranscriptsFixture;
|
|
421
|
+
/** Closes what {@link KillPointScenarioOptions.openStore} opened. */
|
|
422
|
+
closeStore?: (fixture: FencedTranscriptsFixture) => void | Promise<void>;
|
|
423
|
+
/** Extra environment for the worker process. */
|
|
424
|
+
env?: Record<string, string>;
|
|
425
|
+
/** Extra `node` arguments placed before the writer script. */
|
|
426
|
+
execArgv?: string[];
|
|
427
|
+
/** Ceiling on lease-held resume retries; default 15000 ms. */
|
|
428
|
+
resumeDeadlineMs?: number;
|
|
429
|
+
}
|
|
430
|
+
/**
|
|
431
|
+
* Spawns the worker, asserts it died AT the configured write by
|
|
432
|
+
* SIGKILL, waits out the dead owner's lease, resumes the run over the
|
|
433
|
+
* referee's own store instance, and asserts the scenario's pinned
|
|
434
|
+
* recovery semantics. Throws one Error naming every violation.
|
|
435
|
+
*/
|
|
436
|
+
declare function runKillPointScenario(options: KillPointScenarioOptions): Promise<KillPointObservation>;
|
|
437
|
+
/** Per-scenario isolation a consumer's `prepare` hands the suite. */
|
|
438
|
+
interface KillPointTarget {
|
|
439
|
+
/** Store location for this scenario (worker config + referee). */
|
|
440
|
+
storePath?: string;
|
|
441
|
+
/** Extra environment for the worker process. */
|
|
442
|
+
env?: Record<string, string>;
|
|
443
|
+
openStore: KillPointScenarioOptions["openStore"];
|
|
444
|
+
closeStore?: KillPointScenarioOptions["closeStore"];
|
|
445
|
+
/** Runs after the scenario, pass or fail (drop the schema, etc). */
|
|
446
|
+
cleanup?: () => void | Promise<void>;
|
|
447
|
+
}
|
|
448
|
+
interface KillPointConformanceOptions {
|
|
449
|
+
/** Absolute path of the consumer's writer script. */
|
|
450
|
+
writerScript: string;
|
|
451
|
+
/** Scratch directory for report files. */
|
|
452
|
+
dir: string;
|
|
453
|
+
/** Fresh isolation per scenario: store location and referee opener. */
|
|
454
|
+
prepare: (scenario: KillPointScenario) => Promise<KillPointTarget> | KillPointTarget;
|
|
455
|
+
/** The worker's lease ttl (see {@link KillPointScenarioOptions.ttlMs}); default 300 ms. */
|
|
456
|
+
ttlMs?: number;
|
|
457
|
+
/** Extra `node` arguments placed before the writer script. */
|
|
458
|
+
execArgv?: string[];
|
|
459
|
+
/** Ceiling on lease-held resume retries; default 15000 ms. */
|
|
460
|
+
resumeDeadlineMs?: number;
|
|
461
|
+
}
|
|
462
|
+
/**
|
|
463
|
+
* The whole {@link KILL_POINT_SCENARIOS} table as a conformance suite:
|
|
464
|
+
* one check per scenario, each over the fresh isolation `prepare`
|
|
465
|
+
* returns. Register it with a test API whose `it` allows at least
|
|
466
|
+
* thirty seconds per case (spawn, run, die, lease lapse, resume).
|
|
467
|
+
*/
|
|
468
|
+
declare function killPointConformance(options: KillPointConformanceOptions): ConformanceSuite;
|
|
469
|
+
//#endregion
|
|
281
470
|
//#region src/fixtures/golden-fold.d.ts
|
|
282
471
|
/**
|
|
283
472
|
* seq 0 agent spawn (running; abandoned by seq 6)
|
|
@@ -302,4 +491,4 @@ declare function foldStateSha256(entries: readonly JournalEntry[]): string;
|
|
|
302
491
|
/** The reference hash; computed once from the kernel fold and frozen. */
|
|
303
492
|
declare const GOLDEN_FOLD_STATE_SHA256 = "81e6ccff549fb3e6c1de4d34ba65b912162eba6f66403b5d5f23a3e1ec69243c";
|
|
304
493
|
//#endregion
|
|
305
|
-
export { type ConformanceCheck, type ConformanceSuite, DEFAULT_SOAK_QUORUM, type FencedTranscriptsFixture, GOLDEN_FOLD_JOURNAL, GOLDEN_FOLD_STATE_SHA256, type MultiProcessSoakOptions, type MultiProcessSoakResult, type SoakAcceptSurface, type SoakActivity, type SoakEvent, type SoakProbeSurface, type SoakQuorum, type SoakWriterConfig, type SoakWriterHooks, type StoreFactory, type TestRegistrar, countSoakActivity, fencedTranscriptsConformance, fencedWritesConformance, foldStateSha256, journalStoreConformance, leasableStoreConformance, makeSuite, materializeFoldState, parseSoakReport, registerConformance, runMultiProcessSoak, runSoakWriter, soakWriterConfigFromEnv, stableStringify, verifySoakHistory };
|
|
494
|
+
export { type ConformanceCheck, type ConformanceSuite, DEFAULT_SOAK_QUORUM, type FencedTranscriptsFixture, GOLDEN_FOLD_JOURNAL, GOLDEN_FOLD_STATE_SHA256, KILL_POINT_SCENARIOS, type KillPointConformanceOptions, type KillPointEvent, type KillPointExpectation, type KillPointName, type KillPointObservation, type KillPointPhase, type KillPointScenario, type KillPointScenarioOptions, type KillPointTarget, type KillPointWorkerConfig, type KillPointWorkerHooks, type KillPointWorkflowKind, type MultiProcessSoakOptions, type MultiProcessSoakResult, type SoakAcceptSurface, type SoakActivity, type SoakEvent, type SoakProbeSurface, type SoakQuorum, type SoakWriterConfig, type SoakWriterHooks, type StoreFactory, type TestRegistrar, countSoakActivity, fencedTranscriptsConformance, fencedWritesConformance, foldStateSha256, journalStoreConformance, killPointConformance, killPointWorkerConfigFromEnv, leasableStoreConformance, makeSuite, materializeFoldState, parseKillPointReport, parseSoakReport, registerConformance, runKillPointScenario, runKillPointWorker, runMultiProcessSoak, runSoakWriter, soakWriterConfigFromEnv, stableStringify, verifySoakHistory };
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { JournalOrderViolation, LeaseHeldError, Replayer, ResolutionFold, agentScope, buildDeriverRegistry, deriveContentKey, dispositionHook } from "@rulvar/core";
|
|
1
|
+
import { JournalOrderViolation, LeaseHeldError, Replayer, ResolutionFold, agentScope, buildDeriverRegistry, createCanonicalIdMinter, createEngine, defineWorkflow, deriveContentKey, dispositionHook, tool } from "@rulvar/core";
|
|
2
2
|
import { createHash } from "node:crypto";
|
|
3
3
|
import { spawn } from "node:child_process";
|
|
4
4
|
import { appendFileSync, existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
@@ -631,7 +631,7 @@ function leaseEntry(seq) {
|
|
|
631
631
|
startedAt: (/* @__PURE__ */ new Date(17e11 + seq * 1e3)).toISOString()
|
|
632
632
|
};
|
|
633
633
|
}
|
|
634
|
-
function sleep$
|
|
634
|
+
function sleep$2(ms) {
|
|
635
635
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
636
636
|
}
|
|
637
637
|
async function mustReject(operation) {
|
|
@@ -713,13 +713,13 @@ function leasableStoreConformance(mk, options) {
|
|
|
713
713
|
const cadence = Math.floor(ttlMs / 3);
|
|
714
714
|
const deadline = Date.now() + Math.ceil(ttlMs * 1.5);
|
|
715
715
|
while (Date.now() < deadline) {
|
|
716
|
-
await sleep$
|
|
716
|
+
await sleep$2(cadence);
|
|
717
717
|
await store.renew(held);
|
|
718
718
|
ensure(await mustReject(() => store.acquire(RUN$2, "owner-b")) instanceof LeaseHeldError, "lease-ttl-and-renew-cadence", "a lease renewed at ttl/3 must stay held past the original ttl");
|
|
719
719
|
}
|
|
720
720
|
await store.release(held);
|
|
721
721
|
const abandoned = await store.acquire("expiring-run", "owner-a");
|
|
722
|
-
await sleep$
|
|
722
|
+
await sleep$2(Math.ceil(ttlMs * 1.2));
|
|
723
723
|
ensure((await store.acquire("expiring-run", "owner-b")).epoch > abandoned.epoch, "lease-ttl-and-renew-cadence", "reclaiming an expired lease must advance the fencing epoch");
|
|
724
724
|
}
|
|
725
725
|
});
|
|
@@ -1039,7 +1039,7 @@ function soakWriterConfigFromEnv(env = process.env) {
|
|
|
1039
1039
|
if (raw === void 0) throw new Error(`store-conformance multi-process-soak: the writer expects its config in ${SOAK_CONFIG_ENV}`);
|
|
1040
1040
|
return JSON.parse(raw);
|
|
1041
1041
|
}
|
|
1042
|
-
const wallClock = Date.now.bind(globalThis);
|
|
1042
|
+
const wallClock$1 = Date.now.bind(globalThis);
|
|
1043
1043
|
/** Deterministic PRNG (mulberry32): the soak never uses Math.random. */
|
|
1044
1044
|
function prng(seed) {
|
|
1045
1045
|
let a = seed >>> 0;
|
|
@@ -1051,7 +1051,7 @@ function prng(seed) {
|
|
|
1051
1051
|
return ((t ^ t >>> 14) >>> 0) / 4294967296;
|
|
1052
1052
|
};
|
|
1053
1053
|
}
|
|
1054
|
-
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
1054
|
+
const sleep$1 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
1055
1055
|
function soakEntry(seq, nonce) {
|
|
1056
1056
|
return {
|
|
1057
1057
|
hashVersion: 2,
|
|
@@ -1115,7 +1115,7 @@ async function runSoakWriter(fixture, config, hooks = {}) {
|
|
|
1115
1115
|
w: config.writer,
|
|
1116
1116
|
surface
|
|
1117
1117
|
});
|
|
1118
|
-
await sleep(1 + Math.floor(rnd() * 3));
|
|
1118
|
+
await sleep$1(1 + Math.floor(rnd() * 3));
|
|
1119
1119
|
continue;
|
|
1120
1120
|
}
|
|
1121
1121
|
log({
|
|
@@ -1236,7 +1236,7 @@ async function runSoakWriter(fixture, config, hooks = {}) {
|
|
|
1236
1236
|
lease = await journal.acquire(config.runId, owner);
|
|
1237
1237
|
} catch (thrown) {
|
|
1238
1238
|
if (thrown instanceof LeaseHeldError || retryable(thrown)) {
|
|
1239
|
-
await sleep(3 + Math.floor(rnd() * 12));
|
|
1239
|
+
await sleep$1(3 + Math.floor(rnd() * 12));
|
|
1240
1240
|
return;
|
|
1241
1241
|
}
|
|
1242
1242
|
log({
|
|
@@ -1410,7 +1410,7 @@ async function runSoakWriter(fixture, config, hooks = {}) {
|
|
|
1410
1410
|
w: config.writer,
|
|
1411
1411
|
epoch
|
|
1412
1412
|
});
|
|
1413
|
-
await sleep(Math.ceil(config.ttlMs * 1.4));
|
|
1413
|
+
await sleep$1(Math.ceil(config.ttlMs * 1.4));
|
|
1414
1414
|
await staleSweep(lease);
|
|
1415
1415
|
return;
|
|
1416
1416
|
}
|
|
@@ -1629,28 +1629,28 @@ async function runMultiProcessSoak(options) {
|
|
|
1629
1629
|
stderrOf: () => stderr
|
|
1630
1630
|
};
|
|
1631
1631
|
});
|
|
1632
|
-
const startedAt = wallClock();
|
|
1632
|
+
const startedAt = wallClock$1();
|
|
1633
1633
|
let capped = false;
|
|
1634
1634
|
for (;;) {
|
|
1635
|
-
await sleep(100);
|
|
1635
|
+
await sleep$1(100);
|
|
1636
1636
|
if (quorumMet(countSoakActivity(reportPaths.flatMap((path) => parseSoakReport(path))), quorum)) break;
|
|
1637
|
-
if (wallClock() - startedAt > capMs) {
|
|
1637
|
+
if (wallClock$1() - startedAt > capMs) {
|
|
1638
1638
|
capped = true;
|
|
1639
1639
|
break;
|
|
1640
1640
|
}
|
|
1641
1641
|
}
|
|
1642
1642
|
writeFileSync(stopPath, "");
|
|
1643
|
-
const exitDeadline = wallClock() + 3e4;
|
|
1643
|
+
const exitDeadline = wallClock$1() + 3e4;
|
|
1644
1644
|
const codes = await Promise.all(children.map(async ({ child, exit }) => {
|
|
1645
|
-
const timeout = Math.max(1, exitDeadline - wallClock());
|
|
1646
|
-
const code = await Promise.race([exit, sleep(timeout).then(() => "hung")]);
|
|
1645
|
+
const timeout = Math.max(1, exitDeadline - wallClock$1());
|
|
1646
|
+
const code = await Promise.race([exit, sleep$1(timeout).then(() => "hung")]);
|
|
1647
1647
|
if (code === "hung") {
|
|
1648
1648
|
child.kill("SIGKILL");
|
|
1649
1649
|
return "hung";
|
|
1650
1650
|
}
|
|
1651
1651
|
return code;
|
|
1652
1652
|
}));
|
|
1653
|
-
const stormMs = wallClock() - startedAt;
|
|
1653
|
+
const stormMs = wallClock$1() - startedAt;
|
|
1654
1654
|
const events = reportPaths.flatMap((path) => parseSoakReport(path));
|
|
1655
1655
|
const activity = countSoakActivity(events);
|
|
1656
1656
|
const violations = [];
|
|
@@ -1674,4 +1674,619 @@ async function runMultiProcessSoak(options) {
|
|
|
1674
1674
|
};
|
|
1675
1675
|
}
|
|
1676
1676
|
//#endregion
|
|
1677
|
-
|
|
1677
|
+
//#region src/kill-points.ts
|
|
1678
|
+
/**
|
|
1679
|
+
* Engine-level kill-point conformance (the 1.65.0 experiment review,
|
|
1680
|
+
* P1.10): a real OS child process drives a scripted engine run over the
|
|
1681
|
+
* consumer-constructed store and SIGKILLs ITSELF around one durable
|
|
1682
|
+
* write, then the referee resumes the run from a fresh process image
|
|
1683
|
+
* over its own store instance and asserts the engine's documented
|
|
1684
|
+
* recovery semantics against real death, not a simulated throw.
|
|
1685
|
+
*
|
|
1686
|
+
* The five write points and their brackets (before = the write is lost,
|
|
1687
|
+
* after = the write is durable and everything past it is lost):
|
|
1688
|
+
* - `running`: the two-phase dispatch entry appended BEFORE the
|
|
1689
|
+
* provider call (the "request" write). Either bracket costs nothing
|
|
1690
|
+
* extra: the step re-runs once on resume.
|
|
1691
|
+
* - `ok-terminal`: the terminal entry carrying the response and its
|
|
1692
|
+
* usage. The `before` bracket is THE at-least-once window: the
|
|
1693
|
+
* provider was paid and the acknowledgement died, so resume pays the
|
|
1694
|
+
* step a second time. The `after` bracket replays it for free.
|
|
1695
|
+
* - `limit-terminal`: the `maxToolCalls`-expiry terminal. `before`
|
|
1696
|
+
* resumes as a dangling redispatch RESTORED from the last transcript
|
|
1697
|
+
* boundary (one completed tool turn = one checkpoint), so the re-pay
|
|
1698
|
+
* is exactly the turns since that boundary, not the whole agent;
|
|
1699
|
+
* `after` leaves an UNSTAMPED limit terminal in a never-settled run,
|
|
1700
|
+
* which is the documented second chance: resume re-runs the agent
|
|
1701
|
+
* live in full.
|
|
1702
|
+
* - `settle`: the run_settle decision. Both brackets resume as a pure
|
|
1703
|
+
* replay (zero provider calls); a lost settle is re-appended by the
|
|
1704
|
+
* resume segment (the settlement third guard arm), a durable one is
|
|
1705
|
+
* never duplicated.
|
|
1706
|
+
* - `meta`: the RunMeta projection, ordered strictly AFTER the settle.
|
|
1707
|
+
* `before` is the repairable meta-behind residue and the resume heals
|
|
1708
|
+
* it; `after` is a fully consistent run whose resume changes nothing.
|
|
1709
|
+
*
|
|
1710
|
+
* Every scenario additionally asserts: death by SIGKILL (a worker that
|
|
1711
|
+
* runs to completion means the kill point was never reached, which is a
|
|
1712
|
+
* violation, never a pass), exactly ONE ok run_settle in the final
|
|
1713
|
+
* journal, meta `ok`, contiguous journal seqs, and the exact workflow
|
|
1714
|
+
* value after recovery.
|
|
1715
|
+
*
|
|
1716
|
+
* Three pieces, split like the multi-process soak so the child side
|
|
1717
|
+
* stays a few consumer lines:
|
|
1718
|
+
* - {@link runKillPointWorker} is the child protocol: the consumer's
|
|
1719
|
+
* writer script constructs the store over
|
|
1720
|
+
* `killPointWorkerConfigFromEnv()` and hands it over; the protocol
|
|
1721
|
+
* wraps the journal, runs the scripted workflow, and dies at the
|
|
1722
|
+
* configured write.
|
|
1723
|
+
* - {@link runKillPointScenario} is the referee: spawns the writer,
|
|
1724
|
+
* waits out the killed owner's lease ttl, resumes over its own store
|
|
1725
|
+
* instance, and throws one Error naming every violation.
|
|
1726
|
+
* - {@link killPointConformance} registers the whole
|
|
1727
|
+
* {@link KILL_POINT_SCENARIOS} table as a {@link ConformanceSuite}.
|
|
1728
|
+
*/
|
|
1729
|
+
const HAPPY_VALUE = {
|
|
1730
|
+
one: "alpha",
|
|
1731
|
+
two: "beta"
|
|
1732
|
+
};
|
|
1733
|
+
const LIMIT_VALUE = { agentStatus: "limit" };
|
|
1734
|
+
/**
|
|
1735
|
+
* The full table: both brackets of all five write points. The expected
|
|
1736
|
+
* counts ARE the engine's documented recovery semantics; a count moving
|
|
1737
|
+
* here means the durability contract moved and the change must be
|
|
1738
|
+
* deliberate.
|
|
1739
|
+
*/
|
|
1740
|
+
const KILL_POINT_SCENARIOS = [
|
|
1741
|
+
{
|
|
1742
|
+
id: "happy-running-before",
|
|
1743
|
+
workflow: "happy",
|
|
1744
|
+
point: "running",
|
|
1745
|
+
phase: "before",
|
|
1746
|
+
occurrence: 2,
|
|
1747
|
+
expected: {
|
|
1748
|
+
childCalls: 1,
|
|
1749
|
+
childToolExecutions: 0,
|
|
1750
|
+
resumeCalls: 1,
|
|
1751
|
+
resumeToolExecutions: 0,
|
|
1752
|
+
limitTerminals: 0,
|
|
1753
|
+
value: HAPPY_VALUE
|
|
1754
|
+
}
|
|
1755
|
+
},
|
|
1756
|
+
{
|
|
1757
|
+
id: "happy-running-after",
|
|
1758
|
+
workflow: "happy",
|
|
1759
|
+
point: "running",
|
|
1760
|
+
phase: "after",
|
|
1761
|
+
occurrence: 2,
|
|
1762
|
+
expected: {
|
|
1763
|
+
childCalls: 1,
|
|
1764
|
+
childToolExecutions: 0,
|
|
1765
|
+
resumeCalls: 1,
|
|
1766
|
+
resumeToolExecutions: 0,
|
|
1767
|
+
limitTerminals: 0,
|
|
1768
|
+
value: HAPPY_VALUE
|
|
1769
|
+
}
|
|
1770
|
+
},
|
|
1771
|
+
{
|
|
1772
|
+
id: "happy-ok-terminal-before",
|
|
1773
|
+
workflow: "happy",
|
|
1774
|
+
point: "ok-terminal",
|
|
1775
|
+
phase: "before",
|
|
1776
|
+
occurrence: 2,
|
|
1777
|
+
expected: {
|
|
1778
|
+
childCalls: 2,
|
|
1779
|
+
childToolExecutions: 0,
|
|
1780
|
+
resumeCalls: 1,
|
|
1781
|
+
resumeToolExecutions: 0,
|
|
1782
|
+
limitTerminals: 0,
|
|
1783
|
+
value: HAPPY_VALUE
|
|
1784
|
+
}
|
|
1785
|
+
},
|
|
1786
|
+
{
|
|
1787
|
+
id: "happy-ok-terminal-after",
|
|
1788
|
+
workflow: "happy",
|
|
1789
|
+
point: "ok-terminal",
|
|
1790
|
+
phase: "after",
|
|
1791
|
+
occurrence: 2,
|
|
1792
|
+
expected: {
|
|
1793
|
+
childCalls: 2,
|
|
1794
|
+
childToolExecutions: 0,
|
|
1795
|
+
resumeCalls: 0,
|
|
1796
|
+
resumeToolExecutions: 0,
|
|
1797
|
+
limitTerminals: 0,
|
|
1798
|
+
value: HAPPY_VALUE
|
|
1799
|
+
}
|
|
1800
|
+
},
|
|
1801
|
+
{
|
|
1802
|
+
id: "happy-settle-before",
|
|
1803
|
+
workflow: "happy",
|
|
1804
|
+
point: "settle",
|
|
1805
|
+
phase: "before",
|
|
1806
|
+
occurrence: 1,
|
|
1807
|
+
expected: {
|
|
1808
|
+
childCalls: 2,
|
|
1809
|
+
childToolExecutions: 0,
|
|
1810
|
+
resumeCalls: 0,
|
|
1811
|
+
resumeToolExecutions: 0,
|
|
1812
|
+
limitTerminals: 0,
|
|
1813
|
+
value: HAPPY_VALUE
|
|
1814
|
+
}
|
|
1815
|
+
},
|
|
1816
|
+
{
|
|
1817
|
+
id: "happy-settle-after",
|
|
1818
|
+
workflow: "happy",
|
|
1819
|
+
point: "settle",
|
|
1820
|
+
phase: "after",
|
|
1821
|
+
occurrence: 1,
|
|
1822
|
+
expected: {
|
|
1823
|
+
childCalls: 2,
|
|
1824
|
+
childToolExecutions: 0,
|
|
1825
|
+
resumeCalls: 0,
|
|
1826
|
+
resumeToolExecutions: 0,
|
|
1827
|
+
limitTerminals: 0,
|
|
1828
|
+
value: HAPPY_VALUE
|
|
1829
|
+
}
|
|
1830
|
+
},
|
|
1831
|
+
{
|
|
1832
|
+
id: "happy-meta-before",
|
|
1833
|
+
workflow: "happy",
|
|
1834
|
+
point: "meta",
|
|
1835
|
+
phase: "before",
|
|
1836
|
+
occurrence: 1,
|
|
1837
|
+
expected: {
|
|
1838
|
+
childCalls: 2,
|
|
1839
|
+
childToolExecutions: 0,
|
|
1840
|
+
resumeCalls: 0,
|
|
1841
|
+
resumeToolExecutions: 0,
|
|
1842
|
+
limitTerminals: 0,
|
|
1843
|
+
value: HAPPY_VALUE
|
|
1844
|
+
}
|
|
1845
|
+
},
|
|
1846
|
+
{
|
|
1847
|
+
id: "happy-meta-after",
|
|
1848
|
+
workflow: "happy",
|
|
1849
|
+
point: "meta",
|
|
1850
|
+
phase: "after",
|
|
1851
|
+
occurrence: 1,
|
|
1852
|
+
expected: {
|
|
1853
|
+
childCalls: 2,
|
|
1854
|
+
childToolExecutions: 0,
|
|
1855
|
+
resumeCalls: 0,
|
|
1856
|
+
resumeToolExecutions: 0,
|
|
1857
|
+
limitTerminals: 0,
|
|
1858
|
+
value: HAPPY_VALUE
|
|
1859
|
+
}
|
|
1860
|
+
},
|
|
1861
|
+
{
|
|
1862
|
+
id: "limit-limit-terminal-before",
|
|
1863
|
+
workflow: "limit",
|
|
1864
|
+
point: "limit-terminal",
|
|
1865
|
+
phase: "before",
|
|
1866
|
+
occurrence: 1,
|
|
1867
|
+
expected: {
|
|
1868
|
+
childCalls: 3,
|
|
1869
|
+
childToolExecutions: 2,
|
|
1870
|
+
resumeCalls: 1,
|
|
1871
|
+
resumeToolExecutions: 0,
|
|
1872
|
+
limitTerminals: 1,
|
|
1873
|
+
value: LIMIT_VALUE
|
|
1874
|
+
}
|
|
1875
|
+
},
|
|
1876
|
+
{
|
|
1877
|
+
id: "limit-limit-terminal-after",
|
|
1878
|
+
workflow: "limit",
|
|
1879
|
+
point: "limit-terminal",
|
|
1880
|
+
phase: "after",
|
|
1881
|
+
occurrence: 1,
|
|
1882
|
+
expected: {
|
|
1883
|
+
childCalls: 3,
|
|
1884
|
+
childToolExecutions: 2,
|
|
1885
|
+
resumeCalls: 3,
|
|
1886
|
+
resumeToolExecutions: 2,
|
|
1887
|
+
limitTerminals: 2,
|
|
1888
|
+
value: LIMIT_VALUE
|
|
1889
|
+
}
|
|
1890
|
+
}
|
|
1891
|
+
];
|
|
1892
|
+
const KILL_POINT_CONFIG_ENV = "RULVAR_KILL_POINT_CONFIG";
|
|
1893
|
+
/** Reads the worker contract a referee serialized into the child env. */
|
|
1894
|
+
function killPointWorkerConfigFromEnv(env = process.env) {
|
|
1895
|
+
const raw = env[KILL_POINT_CONFIG_ENV];
|
|
1896
|
+
if (raw === void 0) throw new Error(`store-conformance kill-points: the worker expects its config in ${KILL_POINT_CONFIG_ENV}`);
|
|
1897
|
+
return JSON.parse(raw);
|
|
1898
|
+
}
|
|
1899
|
+
/** Parses one report file, tolerating a torn trailing line. */
|
|
1900
|
+
function parseKillPointReport(path) {
|
|
1901
|
+
if (!existsSync(path)) return [];
|
|
1902
|
+
const events = [];
|
|
1903
|
+
for (const line of readFileSync(path, "utf8").split("\n")) {
|
|
1904
|
+
if (line.trim() === "") continue;
|
|
1905
|
+
try {
|
|
1906
|
+
events.push(JSON.parse(line));
|
|
1907
|
+
} catch {}
|
|
1908
|
+
}
|
|
1909
|
+
return events;
|
|
1910
|
+
}
|
|
1911
|
+
const KILL_POINT_CAPS = {
|
|
1912
|
+
structuredOutput: "native",
|
|
1913
|
+
supportsTemperature: false,
|
|
1914
|
+
supportsParallelTools: true,
|
|
1915
|
+
reasoningEfforts: [
|
|
1916
|
+
"low",
|
|
1917
|
+
"medium",
|
|
1918
|
+
"high"
|
|
1919
|
+
],
|
|
1920
|
+
contextWindow: 2e5,
|
|
1921
|
+
maxOutputTokens: 4096
|
|
1922
|
+
};
|
|
1923
|
+
function lastUserText(req) {
|
|
1924
|
+
for (let i = req.messages.length - 1; i >= 0; i -= 1) {
|
|
1925
|
+
const msg = req.messages[i];
|
|
1926
|
+
if (msg?.role !== "user") continue;
|
|
1927
|
+
return msg.parts.filter((part) => part.type === "text").map((part) => part.text).join("\n");
|
|
1928
|
+
}
|
|
1929
|
+
return "";
|
|
1930
|
+
}
|
|
1931
|
+
/**
|
|
1932
|
+
* The scripted model: `step one`/`step two` answer with fixed texts;
|
|
1933
|
+
* every other prompt is the limit script, whose model NEVER stops
|
|
1934
|
+
* asking for the probe tool, so the executed-call cap is what ends it.
|
|
1935
|
+
*/
|
|
1936
|
+
function makeKillPointAdapter(onCall) {
|
|
1937
|
+
const mintId = createCanonicalIdMinter();
|
|
1938
|
+
let limitTurn = 0;
|
|
1939
|
+
return {
|
|
1940
|
+
id: "kp",
|
|
1941
|
+
caps: () => KILL_POINT_CAPS,
|
|
1942
|
+
async *stream(req) {
|
|
1943
|
+
const prompt = lastUserText(req);
|
|
1944
|
+
onCall(prompt);
|
|
1945
|
+
const usage = {
|
|
1946
|
+
inputTokens: Math.max(1, Math.ceil(prompt.length / 4)),
|
|
1947
|
+
outputTokens: 4,
|
|
1948
|
+
cacheReadTokens: 0,
|
|
1949
|
+
cacheWriteTokens: 0
|
|
1950
|
+
};
|
|
1951
|
+
if (prompt.includes("step one")) {
|
|
1952
|
+
yield {
|
|
1953
|
+
type: "text-delta",
|
|
1954
|
+
text: "alpha"
|
|
1955
|
+
};
|
|
1956
|
+
yield {
|
|
1957
|
+
type: "finish",
|
|
1958
|
+
finish: { reason: "stop" },
|
|
1959
|
+
usage
|
|
1960
|
+
};
|
|
1961
|
+
return;
|
|
1962
|
+
}
|
|
1963
|
+
if (prompt.includes("step two")) {
|
|
1964
|
+
yield {
|
|
1965
|
+
type: "text-delta",
|
|
1966
|
+
text: "beta"
|
|
1967
|
+
};
|
|
1968
|
+
yield {
|
|
1969
|
+
type: "finish",
|
|
1970
|
+
finish: { reason: "stop" },
|
|
1971
|
+
usage
|
|
1972
|
+
};
|
|
1973
|
+
return;
|
|
1974
|
+
}
|
|
1975
|
+
limitTurn += 1;
|
|
1976
|
+
const id = mintId();
|
|
1977
|
+
yield {
|
|
1978
|
+
type: "tool-call-start",
|
|
1979
|
+
id,
|
|
1980
|
+
name: "probe"
|
|
1981
|
+
};
|
|
1982
|
+
yield {
|
|
1983
|
+
type: "tool-call-end",
|
|
1984
|
+
id,
|
|
1985
|
+
args: { target: `t${String(limitTurn)}` }
|
|
1986
|
+
};
|
|
1987
|
+
yield {
|
|
1988
|
+
type: "finish",
|
|
1989
|
+
finish: { reason: "tool-calls" },
|
|
1990
|
+
usage
|
|
1991
|
+
};
|
|
1992
|
+
}
|
|
1993
|
+
};
|
|
1994
|
+
}
|
|
1995
|
+
/** Both processes run the SAME workflow; resume replays through it. */
|
|
1996
|
+
function makeKillPointWorkflow(kind, onTool) {
|
|
1997
|
+
if (kind === "limit") {
|
|
1998
|
+
const probe = tool({
|
|
1999
|
+
name: "probe",
|
|
2000
|
+
description: "reads one evidence source",
|
|
2001
|
+
parameters: {
|
|
2002
|
+
type: "object",
|
|
2003
|
+
properties: { target: { type: "string" } },
|
|
2004
|
+
required: ["target"],
|
|
2005
|
+
additionalProperties: false
|
|
2006
|
+
},
|
|
2007
|
+
execute: (input) => {
|
|
2008
|
+
const target = input.target;
|
|
2009
|
+
onTool(target);
|
|
2010
|
+
return Promise.resolve(`evidence ${target}`);
|
|
2011
|
+
}
|
|
2012
|
+
});
|
|
2013
|
+
return defineWorkflow({ name: "kp-limit" }, async (ctx) => {
|
|
2014
|
+
return { agentStatus: (await ctx.agent("probe until the cap", {
|
|
2015
|
+
tools: [probe],
|
|
2016
|
+
limits: {
|
|
2017
|
+
maxTurns: 4,
|
|
2018
|
+
maxToolCalls: 2
|
|
2019
|
+
},
|
|
2020
|
+
result: "full"
|
|
2021
|
+
})).status };
|
|
2022
|
+
});
|
|
2023
|
+
}
|
|
2024
|
+
return defineWorkflow({ name: "kp-happy" }, async (ctx) => {
|
|
2025
|
+
return {
|
|
2026
|
+
one: await ctx.agent("step one"),
|
|
2027
|
+
two: await ctx.agent("step two")
|
|
2028
|
+
};
|
|
2029
|
+
});
|
|
2030
|
+
}
|
|
2031
|
+
function isRunSettle(entry) {
|
|
2032
|
+
return entry.kind === "decision" && entry.value?.decisionType === "run_settle";
|
|
2033
|
+
}
|
|
2034
|
+
/**
|
|
2035
|
+
* The worker protocol: run it in a spawned process against the
|
|
2036
|
+
* consumer-constructed store pair. Wraps the journal so the configured
|
|
2037
|
+
* write kills the process (`before` = ahead of the write, `after` =
|
|
2038
|
+
* once it is durable), appends every observation to the report file
|
|
2039
|
+
* first (the appends are synchronous, so the report survives the
|
|
2040
|
+
* SIGKILL), and reports `ran-to-completion` when the kill point is
|
|
2041
|
+
* never reached, which the referee treats as a violation.
|
|
2042
|
+
*/
|
|
2043
|
+
async function runKillPointWorker(fixture, config, hooks = {}) {
|
|
2044
|
+
const scenario = KILL_POINT_SCENARIOS.find((s) => s.id === config.scenarioId);
|
|
2045
|
+
if (scenario === void 0) throw new Error(`store-conformance kill-points: unknown scenario '${config.scenarioId}'`);
|
|
2046
|
+
const log = (event) => {
|
|
2047
|
+
appendFileSync(config.reportPath, `${JSON.stringify(event)}\n`);
|
|
2048
|
+
};
|
|
2049
|
+
const die = hooks.kill ?? (() => {
|
|
2050
|
+
process.kill(process.pid, "SIGKILL");
|
|
2051
|
+
});
|
|
2052
|
+
const counters = { hits: 0 };
|
|
2053
|
+
const matches = (entry) => {
|
|
2054
|
+
if (scenario.point === "running") return entry.kind === "agent" && entry.status === "running";
|
|
2055
|
+
if (scenario.point === "ok-terminal") return entry.kind === "agent" && entry.ref !== void 0 && entry.status === "ok";
|
|
2056
|
+
if (scenario.point === "limit-terminal") return entry.kind === "agent" && entry.ref !== void 0 && entry.status === "limit";
|
|
2057
|
+
if (scenario.point === "settle") return isRunSettle(entry);
|
|
2058
|
+
return false;
|
|
2059
|
+
};
|
|
2060
|
+
const kill = (site, extra) => {
|
|
2061
|
+
log({
|
|
2062
|
+
t: "kill",
|
|
2063
|
+
point: scenario.point,
|
|
2064
|
+
phase: scenario.phase,
|
|
2065
|
+
site,
|
|
2066
|
+
...extra
|
|
2067
|
+
});
|
|
2068
|
+
die();
|
|
2069
|
+
};
|
|
2070
|
+
const journal = new Proxy(fixture.journal, { get(target, prop, receiver) {
|
|
2071
|
+
if (prop === "append") return async (runId, entry, lease) => {
|
|
2072
|
+
let hit = false;
|
|
2073
|
+
if (matches(entry)) {
|
|
2074
|
+
counters.hits += 1;
|
|
2075
|
+
hit = counters.hits === scenario.occurrence;
|
|
2076
|
+
}
|
|
2077
|
+
const detail = {
|
|
2078
|
+
kind: entry.kind,
|
|
2079
|
+
status: entry.status,
|
|
2080
|
+
seq: entry.seq
|
|
2081
|
+
};
|
|
2082
|
+
if (hit && scenario.phase === "before") kill("append", detail);
|
|
2083
|
+
const out = await target.append(runId, entry, lease);
|
|
2084
|
+
if (hit && scenario.phase === "after") kill("append", detail);
|
|
2085
|
+
return out;
|
|
2086
|
+
};
|
|
2087
|
+
if (prop === "putMeta") return async (meta, lease) => {
|
|
2088
|
+
const hit = scenario.point === "meta" && meta.status !== "running";
|
|
2089
|
+
if (hit && scenario.phase === "before") kill("putMeta", { status: meta.status });
|
|
2090
|
+
const out = await target.putMeta(meta, lease);
|
|
2091
|
+
if (hit && scenario.phase === "after") kill("putMeta", { status: meta.status });
|
|
2092
|
+
return out;
|
|
2093
|
+
};
|
|
2094
|
+
const value = Reflect.get(target, prop, receiver);
|
|
2095
|
+
return typeof value === "function" ? value.bind(target) : value;
|
|
2096
|
+
} });
|
|
2097
|
+
const adapter = makeKillPointAdapter((prompt) => {
|
|
2098
|
+
log({
|
|
2099
|
+
t: "call",
|
|
2100
|
+
prompt
|
|
2101
|
+
});
|
|
2102
|
+
});
|
|
2103
|
+
const workflow = makeKillPointWorkflow(scenario.workflow, (target) => {
|
|
2104
|
+
log({
|
|
2105
|
+
t: "tool",
|
|
2106
|
+
target
|
|
2107
|
+
});
|
|
2108
|
+
});
|
|
2109
|
+
const engine = createEngine({
|
|
2110
|
+
adapters: [adapter],
|
|
2111
|
+
stores: {
|
|
2112
|
+
journal,
|
|
2113
|
+
transcripts: fixture.transcripts
|
|
2114
|
+
},
|
|
2115
|
+
defaults: { routing: { loop: "kp:kp-model" } }
|
|
2116
|
+
});
|
|
2117
|
+
try {
|
|
2118
|
+
log({
|
|
2119
|
+
t: "ran-to-completion",
|
|
2120
|
+
status: (await engine.run(workflow, void 0, { runId: config.runId }).result).status
|
|
2121
|
+
});
|
|
2122
|
+
} catch (thrown) {
|
|
2123
|
+
log({
|
|
2124
|
+
t: "fatal",
|
|
2125
|
+
message: String(thrown)
|
|
2126
|
+
});
|
|
2127
|
+
throw thrown;
|
|
2128
|
+
}
|
|
2129
|
+
}
|
|
2130
|
+
const wallClock = Date.now.bind(globalThis);
|
|
2131
|
+
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
2132
|
+
/**
|
|
2133
|
+
* Spawns the worker, asserts it died AT the configured write by
|
|
2134
|
+
* SIGKILL, waits out the dead owner's lease, resumes the run over the
|
|
2135
|
+
* referee's own store instance, and asserts the scenario's pinned
|
|
2136
|
+
* recovery semantics. Throws one Error naming every violation.
|
|
2137
|
+
*/
|
|
2138
|
+
async function runKillPointScenario(options) {
|
|
2139
|
+
const scenario = typeof options.scenario === "string" ? KILL_POINT_SCENARIOS.find((s) => s.id === options.scenario) : options.scenario;
|
|
2140
|
+
if (scenario === void 0) {
|
|
2141
|
+
const named = typeof options.scenario === "string" ? options.scenario : options.scenario.id;
|
|
2142
|
+
throw new Error(`store-conformance kill-points: unknown scenario '${named}'`);
|
|
2143
|
+
}
|
|
2144
|
+
const ttlMs = options.ttlMs ?? 300;
|
|
2145
|
+
const storePath = options.storePath ?? join(options.dir, "kp.db");
|
|
2146
|
+
const reportPath = join(options.dir, `kill-report-${scenario.id}.jsonl`);
|
|
2147
|
+
const runId = `kp-${scenario.id}`;
|
|
2148
|
+
const config = {
|
|
2149
|
+
storePath,
|
|
2150
|
+
runId,
|
|
2151
|
+
ttlMs,
|
|
2152
|
+
reportPath,
|
|
2153
|
+
scenarioId: scenario.id
|
|
2154
|
+
};
|
|
2155
|
+
const child = spawn(process.execPath, [...options.execArgv ?? [], options.writerScript], {
|
|
2156
|
+
env: {
|
|
2157
|
+
...process.env,
|
|
2158
|
+
...options.env,
|
|
2159
|
+
[KILL_POINT_CONFIG_ENV]: JSON.stringify(config)
|
|
2160
|
+
},
|
|
2161
|
+
stdio: [
|
|
2162
|
+
"ignore",
|
|
2163
|
+
"ignore",
|
|
2164
|
+
"pipe"
|
|
2165
|
+
]
|
|
2166
|
+
});
|
|
2167
|
+
let stderr = "";
|
|
2168
|
+
child.stderr.on("data", (chunk) => {
|
|
2169
|
+
stderr += String(chunk);
|
|
2170
|
+
});
|
|
2171
|
+
const exitPromise = new Promise((resolve) => {
|
|
2172
|
+
child.on("exit", (code, signal) => resolve({
|
|
2173
|
+
code,
|
|
2174
|
+
signal
|
|
2175
|
+
}));
|
|
2176
|
+
});
|
|
2177
|
+
const exit = await Promise.race([exitPromise, sleep(3e4).then(() => "hung")]);
|
|
2178
|
+
if (exit === "hung") {
|
|
2179
|
+
child.kill("SIGKILL");
|
|
2180
|
+
throw new Error(`store-conformance kill-points ${scenario.id}: the worker did not exit within 30 s`);
|
|
2181
|
+
}
|
|
2182
|
+
const violations = [];
|
|
2183
|
+
const events = parseKillPointReport(reportPath);
|
|
2184
|
+
const childCalls = events.filter((e) => e.t === "call").length;
|
|
2185
|
+
const childTools = events.filter((e) => e.t === "tool").length;
|
|
2186
|
+
const kills = events.filter((e) => e.t === "kill");
|
|
2187
|
+
const completed = events.find((e) => e.t === "ran-to-completion");
|
|
2188
|
+
const fatal = events.find((e) => e.t === "fatal");
|
|
2189
|
+
if (exit.signal !== "SIGKILL") violations.push(`the worker must die by SIGKILL at the configured write; exit code ${String(exit.code)}, signal ${String(exit.signal)}` + (completed === void 0 ? "" : " (it ran to completion: the kill point was never reached)") + (fatal === void 0 ? "" : ` (fatal: ${fatal.message})`) + (stderr === "" ? "" : `; stderr: ${stderr.slice(0, 2e3)}`));
|
|
2190
|
+
if (kills.length !== 1) violations.push(`expected exactly one kill event, saw ${String(kills.length)}`);
|
|
2191
|
+
else {
|
|
2192
|
+
const kill = kills[0];
|
|
2193
|
+
if (kill.point !== scenario.point || kill.phase !== scenario.phase) violations.push(`the kill event names ${kill.point}/${kill.phase}, the scenario is ${scenario.point}/${scenario.phase}`);
|
|
2194
|
+
}
|
|
2195
|
+
if (childCalls !== scenario.expected.childCalls) violations.push(`the child paid ${String(childCalls)} provider calls before dying, expected ` + String(scenario.expected.childCalls));
|
|
2196
|
+
if (childTools !== scenario.expected.childToolExecutions) violations.push(`the child executed ${String(childTools)} tools before dying, expected ` + String(scenario.expected.childToolExecutions));
|
|
2197
|
+
await sleep(ttlMs * 2);
|
|
2198
|
+
let resumeCalls = 0;
|
|
2199
|
+
let resumeTools = 0;
|
|
2200
|
+
const fixture = await options.openStore();
|
|
2201
|
+
try {
|
|
2202
|
+
const adapter = makeKillPointAdapter(() => {
|
|
2203
|
+
resumeCalls += 1;
|
|
2204
|
+
});
|
|
2205
|
+
const workflow = makeKillPointWorkflow(scenario.workflow, () => {
|
|
2206
|
+
resumeTools += 1;
|
|
2207
|
+
});
|
|
2208
|
+
const engine = createEngine({
|
|
2209
|
+
adapters: [adapter],
|
|
2210
|
+
stores: {
|
|
2211
|
+
journal: fixture.journal,
|
|
2212
|
+
transcripts: fixture.transcripts
|
|
2213
|
+
},
|
|
2214
|
+
defaults: { routing: { loop: "kp:kp-model" } }
|
|
2215
|
+
});
|
|
2216
|
+
const deadline = wallClock() + (options.resumeDeadlineMs ?? 15e3);
|
|
2217
|
+
let resumed;
|
|
2218
|
+
for (;;) try {
|
|
2219
|
+
resumed = await engine.resume(runId, workflow).result;
|
|
2220
|
+
break;
|
|
2221
|
+
} catch (thrown) {
|
|
2222
|
+
if (thrown instanceof LeaseHeldError && wallClock() < deadline) {
|
|
2223
|
+
await sleep(Math.max(50, Math.ceil(ttlMs / 3)));
|
|
2224
|
+
continue;
|
|
2225
|
+
}
|
|
2226
|
+
violations.push(`the resume rejected: ${String(thrown)}`);
|
|
2227
|
+
break;
|
|
2228
|
+
}
|
|
2229
|
+
if (resumed !== void 0) {
|
|
2230
|
+
if (resumed.status !== "ok") violations.push(`the resume settled '${resumed.status}', expected 'ok'`);
|
|
2231
|
+
if (stableStringify(resumed.value) !== stableStringify(scenario.expected.value)) violations.push(`the recovered value is ${stableStringify(resumed.value)}, expected ` + stableStringify(scenario.expected.value));
|
|
2232
|
+
if (resumeCalls !== scenario.expected.resumeCalls) violations.push(`the resume paid ${String(resumeCalls)} provider calls, the bracket documents ` + String(scenario.expected.resumeCalls));
|
|
2233
|
+
if (resumeTools !== scenario.expected.resumeToolExecutions) violations.push(`the resume executed ${String(resumeTools)} tools, the bracket documents ` + String(scenario.expected.resumeToolExecutions));
|
|
2234
|
+
}
|
|
2235
|
+
const entries = await fixture.journal.load(runId);
|
|
2236
|
+
const firstSeq = entries[0]?.seq ?? 0;
|
|
2237
|
+
if (!entries.every((entry, i) => entry.seq === firstSeq + i)) violations.push(`journal seqs are not contiguous: ${entries.map((e) => String(e.seq)).join(", ")}`);
|
|
2238
|
+
const settles = entries.filter((entry) => isRunSettle(entry));
|
|
2239
|
+
if (settles.length !== 1) violations.push(`expected exactly one run_settle, the journal holds ${String(settles.length)}`);
|
|
2240
|
+
const lastSettle = settles.at(-1)?.value;
|
|
2241
|
+
if (lastSettle?.runStatus !== "ok") violations.push(`the run settle records '${String(lastSettle?.runStatus)}', expected 'ok'`);
|
|
2242
|
+
const limitTerminals = entries.filter((entry) => entry.kind === "agent" && entry.ref !== void 0 && entry.status === "limit").length;
|
|
2243
|
+
if (limitTerminals !== scenario.expected.limitTerminals) violations.push(`the journal holds ${String(limitTerminals)} limit terminals, expected ` + String(scenario.expected.limitTerminals));
|
|
2244
|
+
const meta = await fixture.journal.getMeta?.(runId);
|
|
2245
|
+
if (meta?.status !== "ok") violations.push(`meta records '${String(meta?.status)}', expected 'ok' after recovery`);
|
|
2246
|
+
if (violations.length > 0) throw new Error(`store-conformance kill-points ${scenario.id}: ${String(violations.length)} violation(s)\n - ` + violations.join("\n - "));
|
|
2247
|
+
return {
|
|
2248
|
+
scenario,
|
|
2249
|
+
childCalls,
|
|
2250
|
+
childToolExecutions: childTools,
|
|
2251
|
+
resumeCalls,
|
|
2252
|
+
resumeToolExecutions: resumeTools,
|
|
2253
|
+
journal: entries.map((entry) => `${entry.kind}:${entry.status}`),
|
|
2254
|
+
metaStatus: meta?.status
|
|
2255
|
+
};
|
|
2256
|
+
} finally {
|
|
2257
|
+
await options.closeStore?.(fixture);
|
|
2258
|
+
}
|
|
2259
|
+
}
|
|
2260
|
+
/**
|
|
2261
|
+
* The whole {@link KILL_POINT_SCENARIOS} table as a conformance suite:
|
|
2262
|
+
* one check per scenario, each over the fresh isolation `prepare`
|
|
2263
|
+
* returns. Register it with a test API whose `it` allows at least
|
|
2264
|
+
* thirty seconds per case (spawn, run, die, lease lapse, resume).
|
|
2265
|
+
*/
|
|
2266
|
+
function killPointConformance(options) {
|
|
2267
|
+
return makeSuite("kill-point conformance (engine recovery under real process death)", KILL_POINT_SCENARIOS.map((scenario) => ({
|
|
2268
|
+
id: `kp-${scenario.id}`,
|
|
2269
|
+
title: `a SIGKILL ${scenario.phase} the ${scenario.point} write recovers on resume (${String(scenario.expected.resumeCalls)} re-paid call(s))`,
|
|
2270
|
+
run: async () => {
|
|
2271
|
+
const target = await options.prepare(scenario);
|
|
2272
|
+
try {
|
|
2273
|
+
await runKillPointScenario({
|
|
2274
|
+
writerScript: options.writerScript,
|
|
2275
|
+
dir: options.dir,
|
|
2276
|
+
scenario,
|
|
2277
|
+
openStore: target.openStore,
|
|
2278
|
+
...target.storePath === void 0 ? {} : { storePath: target.storePath },
|
|
2279
|
+
...target.env === void 0 ? {} : { env: target.env },
|
|
2280
|
+
...target.closeStore === void 0 ? {} : { closeStore: target.closeStore },
|
|
2281
|
+
...options.ttlMs === void 0 ? {} : { ttlMs: options.ttlMs },
|
|
2282
|
+
...options.execArgv === void 0 ? {} : { execArgv: options.execArgv },
|
|
2283
|
+
...options.resumeDeadlineMs === void 0 ? {} : { resumeDeadlineMs: options.resumeDeadlineMs }
|
|
2284
|
+
});
|
|
2285
|
+
} finally {
|
|
2286
|
+
await target.cleanup?.();
|
|
2287
|
+
}
|
|
2288
|
+
}
|
|
2289
|
+
})));
|
|
2290
|
+
}
|
|
2291
|
+
//#endregion
|
|
2292
|
+
export { DEFAULT_SOAK_QUORUM, GOLDEN_FOLD_JOURNAL, GOLDEN_FOLD_STATE_SHA256, KILL_POINT_SCENARIOS, countSoakActivity, fencedTranscriptsConformance, fencedWritesConformance, foldStateSha256, journalStoreConformance, killPointConformance, killPointWorkerConfigFromEnv, leasableStoreConformance, makeSuite, materializeFoldState, parseKillPointReport, parseSoakReport, registerConformance, runKillPointScenario, runKillPointWorker, runMultiProcessSoak, runSoakWriter, soakWriterConfigFromEnv, stableStringify, verifySoakHistory };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rulvar/store-conformance",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.70.0",
|
|
4
4
|
"description": "Rulvar executable store conformance kit (DEF-4).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
"access": "public"
|
|
23
23
|
},
|
|
24
24
|
"dependencies": {
|
|
25
|
-
"@rulvar/core": "1.
|
|
25
|
+
"@rulvar/core": "1.70.0"
|
|
26
26
|
},
|
|
27
27
|
"devDependencies": {
|
|
28
28
|
"@types/node": "^22.20.0",
|