@effect-agent/testing 0.1.0-beta.41 → 0.1.0-beta.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/certification.mjs.map +1 -1
- package/dist/chaos.mjs.map +1 -1
- package/dist/code-executor.mjs.map +1 -1
- package/dist/deterministic-layers-CKyYxBhN.mjs.map +1 -1
- package/dist/docs-researcher.mjs +3 -2
- package/dist/docs-researcher.mjs.map +1 -1
- package/dist/scripted-model-C2y0ztuj.mjs.map +1 -1
- package/dist/travel-planner.mjs.map +1 -1
- package/package.json +1 -71
- package/src/certification.ts +78 -0
- package/src/chaos.ts +96 -0
- package/src/code-executor-conformance.ts +19 -0
- package/src/code-executor-substitute.ts +32 -0
- package/src/fixtures/docs-researcher/definition.ts +7 -0
- package/src/fixtures/docs-researcher/harness.ts +19 -0
- package/src/fixtures/docs-researcher/mcp.ts +14 -2
- package/src/fixtures/travel-planner/definition.ts +12 -0
- package/src/fixtures/travel-planner/deterministic-layers.ts +38 -0
- package/src/fixtures/travel-planner/phase3.ts +4 -0
- package/src/fixtures/travel-planner/phase4.ts +10 -0
- package/src/fixtures/travel-planner/phase5.ts +20 -0
- package/src/fixtures/travel-planner/phase6.ts +12 -0
- package/src/fixtures/travel-planner/scenarios.ts +2 -0
- package/src/fixtures/travel-planner/subagents-durable.ts +13 -0
- package/src/fixtures/travel-planner/subagents.ts +21 -0
- package/src/scripted-model.ts +18 -0
package/src/chaos.ts
CHANGED
|
@@ -92,6 +92,7 @@ export const ChaosScenarioKind = Schema.Literals([
|
|
|
92
92
|
"join",
|
|
93
93
|
"delegation",
|
|
94
94
|
]);
|
|
95
|
+
|
|
95
96
|
export type ChaosScenarioKind = typeof ChaosScenarioKind.Type;
|
|
96
97
|
|
|
97
98
|
const LaneIndex = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0), Schema.isLessThanOrEqualTo(7));
|
|
@@ -117,6 +118,7 @@ export const ChaosResolutionKind = Schema.Literals([
|
|
|
117
118
|
/** Unresolvable: route into the abort path (settles aborted, audit retained). */
|
|
118
119
|
"abort-submission",
|
|
119
120
|
]);
|
|
121
|
+
|
|
120
122
|
export type ChaosResolutionKind = typeof ChaosResolutionKind.Type;
|
|
121
123
|
|
|
122
124
|
export const ChaosApprovalDecision = Schema.Literals(["approved", "denied"]);
|
|
@@ -191,7 +193,9 @@ const decodeChaosSeedFromEnvironment = Schema.decodeUnknownOption(ChaosSeedFromE
|
|
|
191
193
|
/** The root seed for this run: `CHAOS_SEED` when set to an integer, the default otherwise. */
|
|
192
194
|
export const chaosSeedFromEnv = (env: Record<string, string | undefined>): number => {
|
|
193
195
|
const raw = env["CHAOS_SEED"];
|
|
196
|
+
|
|
194
197
|
if (raw === undefined || raw === "") return DEFAULT_CHAOS_SEED;
|
|
198
|
+
|
|
195
199
|
return Option.getOrElse(decodeChaosSeedFromEnvironment(raw), () => DEFAULT_CHAOS_SEED);
|
|
196
200
|
};
|
|
197
201
|
|
|
@@ -268,9 +272,12 @@ const planShapeArbitrary = (
|
|
|
268
272
|
ChaosSubmissionSpec.make({ lane: index, kind: lane.kind }),
|
|
269
273
|
),
|
|
270
274
|
);
|
|
275
|
+
|
|
271
276
|
const [first, ...rest] = submissions;
|
|
277
|
+
|
|
272
278
|
// `lanes` >= 1 and every lane has depth >= 1, so `first` always exists.
|
|
273
279
|
if (first === undefined) throw new Error("chaos generator produced an empty plan");
|
|
280
|
+
|
|
274
281
|
return {
|
|
275
282
|
lanes: shape.lanes.length,
|
|
276
283
|
submissions: [first, ...rest] as const,
|
|
@@ -292,6 +299,7 @@ export const generateChaosPlans = (options: ChaosGeneratorOptions): ReadonlyArra
|
|
|
292
299
|
seed: options.seed,
|
|
293
300
|
numRuns: options.count,
|
|
294
301
|
});
|
|
302
|
+
|
|
295
303
|
return sampled.map((shape, index) =>
|
|
296
304
|
ChaosPlan.make({ ...shape, seed: (Math.imul(options.seed, 31) + index) | 0 }),
|
|
297
305
|
);
|
|
@@ -300,10 +308,13 @@ export const generateChaosPlans = (options: ChaosGeneratorOptions): ReadonlyArra
|
|
|
300
308
|
/** Deterministic PRNG for the runner's small ordering choices (lane drive order). */
|
|
301
309
|
const mulberry32 = (seed: number): (() => number) => {
|
|
302
310
|
let state = seed | 0;
|
|
311
|
+
|
|
303
312
|
return () => {
|
|
304
313
|
state = (state + 0x6d2b79f5) | 0;
|
|
305
314
|
let t = Math.imul(state ^ (state >>> 15), 1 | state);
|
|
315
|
+
|
|
306
316
|
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
|
|
317
|
+
|
|
307
318
|
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
|
308
319
|
};
|
|
309
320
|
};
|
|
@@ -382,7 +393,9 @@ const BookUncertain = Tool.make("book", {
|
|
|
382
393
|
parameters: Schema.Struct({ ref: Schema.String }),
|
|
383
394
|
success: Schema.Struct({ confirmation: Schema.String }),
|
|
384
395
|
});
|
|
396
|
+
|
|
385
397
|
const bookTools = Toolkit.make(BookUncertain);
|
|
398
|
+
|
|
386
399
|
const bookDefinition = Agent.make("chaos-book", {
|
|
387
400
|
input: PlainInput,
|
|
388
401
|
output: PlainOutput,
|
|
@@ -396,7 +409,9 @@ const BookApproval = Tool.make("book", {
|
|
|
396
409
|
success: Schema.Struct({ confirmation: Schema.String }),
|
|
397
410
|
needsApproval: true,
|
|
398
411
|
});
|
|
412
|
+
|
|
399
413
|
const approvalTools = Toolkit.make(BookApproval);
|
|
414
|
+
|
|
400
415
|
const approvalDefinition = Agent.make("chaos-approval", {
|
|
401
416
|
input: PlainInput,
|
|
402
417
|
output: PlainOutput,
|
|
@@ -411,7 +426,9 @@ const Itinerary = Tool.make("itinerary", {
|
|
|
411
426
|
failure: DurableStepError,
|
|
412
427
|
dependencies: [DurableStep],
|
|
413
428
|
}).annotate(ToolExecutionClass, "uncertain");
|
|
429
|
+
|
|
414
430
|
const itineraryTools = Toolkit.make(Itinerary);
|
|
431
|
+
|
|
415
432
|
const itineraryDefinition = Agent.make("chaos-itinerary", {
|
|
416
433
|
input: PlainInput,
|
|
417
434
|
output: PlainOutput,
|
|
@@ -467,16 +484,22 @@ const DELEGATE_CALL_ID = "chaos-delegate-1";
|
|
|
467
484
|
|
|
468
485
|
const HEX = "0123456789abcdef";
|
|
469
486
|
const decodeDigest = Schema.decodeSync(Digest);
|
|
487
|
+
|
|
470
488
|
const laneDigests = (lane: number): DefinitionDigests => {
|
|
471
489
|
const digest = decodeDigest(HEX.charAt(lane % 8).repeat(64));
|
|
490
|
+
|
|
472
491
|
return DefinitionDigests.make({ agent: digest, model: digest, tools: digest });
|
|
473
492
|
};
|
|
493
|
+
|
|
474
494
|
const childDigestStrings = (lane: number) => {
|
|
475
495
|
const char = HEX.charAt(8 + (lane % 8));
|
|
496
|
+
|
|
476
497
|
return { agent: char.repeat(64), model: char.repeat(64), tools: char.repeat(64) } as const;
|
|
477
498
|
};
|
|
499
|
+
|
|
478
500
|
const childLaneDigests = (lane: number): DefinitionDigests => {
|
|
479
501
|
const strings = childDigestStrings(lane);
|
|
502
|
+
|
|
480
503
|
return DefinitionDigests.make({
|
|
481
504
|
agent: decodeDigest(strings.agent),
|
|
482
505
|
model: decodeDigest(strings.model),
|
|
@@ -496,10 +519,12 @@ const chaosIdentifiers = Layer.effect(
|
|
|
496
519
|
IdGenerator,
|
|
497
520
|
Effect.gen(function* () {
|
|
498
521
|
const counter = yield* Ref.make(0);
|
|
522
|
+
|
|
499
523
|
const next = <A>(decode: (value: string) => A, prefix: string) =>
|
|
500
524
|
Ref.getAndUpdate(counter, (value) => value + 1).pipe(
|
|
501
525
|
Effect.map((value) => decode(`${prefix}-${value}`)),
|
|
502
526
|
);
|
|
527
|
+
|
|
503
528
|
return {
|
|
504
529
|
nextThreadId: next(decodeThreadId, "chaos-fixture-thread"),
|
|
505
530
|
nextRunId: next(decodeRunId, "chaos-fixture-run"),
|
|
@@ -521,6 +546,7 @@ interface ChaosDesk {
|
|
|
521
546
|
|
|
522
547
|
const makeChaosDesk: Effect.Effect<ChaosDesk> = Effect.gen(function* () {
|
|
523
548
|
const produced = yield* Ref.make<ReadonlySet<string>>(new Set());
|
|
549
|
+
|
|
524
550
|
return {
|
|
525
551
|
produced: Ref.get(produced),
|
|
526
552
|
record: (value: string) => Ref.update(produced, (current) => new Set(current).add(value)),
|
|
@@ -564,6 +590,7 @@ const tolerateTyped = <A, E, R>(
|
|
|
564
590
|
if (Option.isSome(Cause.findErrorOption(exit.cause))) {
|
|
565
591
|
return Effect.succeed(Option.none<A>());
|
|
566
592
|
}
|
|
593
|
+
|
|
567
594
|
return Effect.die(new Error(`chaos step died: ${Cause.pretty(exit.cause)}`));
|
|
568
595
|
}),
|
|
569
596
|
);
|
|
@@ -666,12 +693,14 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
666
693
|
case "plain":
|
|
667
694
|
case "join": {
|
|
668
695
|
const agent = Agent.withModel(plainDefinition, model);
|
|
696
|
+
|
|
669
697
|
return plainLaneFixture(false, runtime.processThread(agent, threadId), (flatIndex) =>
|
|
670
698
|
runtime.submit(agent, { question: `chaos ${flatIndex}` }, submitOptionsFor(flatIndex)),
|
|
671
699
|
);
|
|
672
700
|
}
|
|
673
701
|
case "uncertain-tool": {
|
|
674
702
|
const agent = Agent.withModel(bookDefinition, model);
|
|
703
|
+
|
|
675
704
|
return plainLaneFixture(
|
|
676
705
|
true,
|
|
677
706
|
runtime.processThread(agent, threadId).pipe(Effect.provide(bookToolLayerFor(bookTools))),
|
|
@@ -681,6 +710,7 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
681
710
|
}
|
|
682
711
|
case "approval": {
|
|
683
712
|
const agent = Agent.withModel(approvalDefinition, model);
|
|
713
|
+
|
|
684
714
|
return plainLaneFixture(
|
|
685
715
|
true,
|
|
686
716
|
runtime
|
|
@@ -692,23 +722,28 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
692
722
|
}
|
|
693
723
|
case "durable-steps": {
|
|
694
724
|
const agent = Agent.withModel(itineraryDefinition, model);
|
|
725
|
+
|
|
695
726
|
const toolLayer = itineraryTools.toLayer({
|
|
696
727
|
itinerary: ({ ref: called }) =>
|
|
697
728
|
Effect.gen(function* () {
|
|
698
729
|
const step = yield* DurableStep;
|
|
730
|
+
|
|
699
731
|
const flight = yield* step.do(
|
|
700
732
|
"reserve-flight",
|
|
701
733
|
Schema.String,
|
|
702
734
|
desk.record(flightValue(called)).pipe(Effect.as(flightValue(called))),
|
|
703
735
|
);
|
|
736
|
+
|
|
704
737
|
const lodging = yield* step.do(
|
|
705
738
|
"reserve-lodging",
|
|
706
739
|
Schema.String,
|
|
707
740
|
desk.record(lodgingValue(called)).pipe(Effect.as(lodgingValue(called))),
|
|
708
741
|
);
|
|
742
|
+
|
|
709
743
|
return { state: `${flight}+${lodging}` };
|
|
710
744
|
}),
|
|
711
745
|
});
|
|
746
|
+
|
|
712
747
|
return plainLaneFixture(
|
|
713
748
|
true,
|
|
714
749
|
runtime.processThread(agent, threadId).pipe(Effect.provide(toolLayer)),
|
|
@@ -718,24 +753,31 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
718
753
|
}
|
|
719
754
|
case "delegation": {
|
|
720
755
|
const parentBinding = Agent.withModel(coordinatorDefinition, model);
|
|
756
|
+
|
|
721
757
|
const childModel = promptScriptedModel(`chaos-child-${laneIndex}`, () =>
|
|
722
758
|
finalParts('{"answer":"child"}'),
|
|
723
759
|
);
|
|
760
|
+
|
|
724
761
|
const childBinding = Agent.withModel(childDefinition, childModel);
|
|
762
|
+
|
|
725
763
|
const delegationLayer = SubagentRuntime.layer(chaosDelegation, childBinding, {
|
|
726
764
|
mapChildFailure: (failure) => ChaosDelegationFailed.make({ childErrorTag: failure._tag }),
|
|
727
765
|
durable: { targetDigests: childDigestStrings(laneIndex) },
|
|
728
766
|
}).pipe(Layer.provide(delegationSupport));
|
|
767
|
+
|
|
729
768
|
const parentResolved: ResolvedBinding = yield* DurableWorkerBinding.make(
|
|
730
769
|
parentBinding,
|
|
731
770
|
digests,
|
|
732
771
|
).pipe(Effect.provide(delegationLayer));
|
|
772
|
+
|
|
733
773
|
const childResolved: ResolvedBinding = yield* DurableWorkerBinding.make(
|
|
734
774
|
childBinding,
|
|
735
775
|
childLaneDigests(laneIndex),
|
|
736
776
|
);
|
|
777
|
+
|
|
737
778
|
const bindings = [parentResolved, childResolved];
|
|
738
779
|
const driveResolved = (thread: ThreadId) => runtime.processThreadResolved(thread, bindings);
|
|
780
|
+
|
|
739
781
|
const fixture: LaneFixture = {
|
|
740
782
|
index: laneIndex,
|
|
741
783
|
kind,
|
|
@@ -753,6 +795,7 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
753
795
|
const drives: Array<
|
|
754
796
|
Effect.Effect<ReadonlyArray<Settlement>, DurableWorkerFailure | DurableBindingFailure>
|
|
755
797
|
> = [driveResolved(threadId)];
|
|
798
|
+
|
|
756
799
|
if (firstReceipt !== undefined) {
|
|
757
800
|
drives.push(
|
|
758
801
|
driveResolved(
|
|
@@ -760,11 +803,13 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
760
803
|
),
|
|
761
804
|
);
|
|
762
805
|
}
|
|
806
|
+
|
|
763
807
|
return drives;
|
|
764
808
|
},
|
|
765
809
|
childThreadOf: (firstReceipt) =>
|
|
766
810
|
childThreadIdFor(firstReceipt.submissionId, decodeToolCallId(DELEGATE_CALL_ID)),
|
|
767
811
|
};
|
|
812
|
+
|
|
768
813
|
return fixture;
|
|
769
814
|
}
|
|
770
815
|
}
|
|
@@ -773,7 +818,9 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
773
818
|
/** Stable per-call index into an injection list (identical across resolution passes). */
|
|
774
819
|
const injectionIndex = (submissionFlatIndex: number, callId: string, length: number): number => {
|
|
775
820
|
let hash = submissionFlatIndex + 1;
|
|
821
|
+
|
|
776
822
|
for (const char of callId) hash = (Math.imul(hash, 31) + char.charCodeAt(0)) | 0;
|
|
823
|
+
|
|
777
824
|
return ((hash % length) + length) % length;
|
|
778
825
|
};
|
|
779
826
|
|
|
@@ -803,6 +850,7 @@ const resolutionFor = (
|
|
|
803
850
|
isFailure: false,
|
|
804
851
|
});
|
|
805
852
|
}
|
|
853
|
+
|
|
806
854
|
// The desk never produced a value for this call — resolving "completed" would fabricate.
|
|
807
855
|
return ResolutionNeverHappened.make();
|
|
808
856
|
}
|
|
@@ -821,8 +869,10 @@ const resolutionPass = Effect.fn("Chaos.resolutionPass")(function* (
|
|
|
821
869
|
const ledger = yield* SubmissionLedger;
|
|
822
870
|
const produced = yield* desk.produced;
|
|
823
871
|
const nonterminal = yield* tolerateTyped(Stream.runCollect(ledger.scanNonterminal));
|
|
872
|
+
|
|
824
873
|
if (Option.isNone(nonterminal)) return;
|
|
825
874
|
const byId = new Map<SubmissionId, SubmissionState>();
|
|
875
|
+
|
|
826
876
|
for (const state of states) {
|
|
827
877
|
if (state.receipt !== undefined) byId.set(state.receipt.submissionId, state);
|
|
828
878
|
}
|
|
@@ -830,18 +880,22 @@ const resolutionPass = Effect.fn("Chaos.resolutionPass")(function* (
|
|
|
830
880
|
if (row.state !== "unknown" && row.state !== "suspended") continue;
|
|
831
881
|
const state = byId.get(row.submissionId);
|
|
832
882
|
const explanation = yield* tolerateTyped(runtime.explain(row.submissionId));
|
|
883
|
+
|
|
833
884
|
if (Option.isNone(explanation)) continue;
|
|
834
885
|
const flatIndex = state?.flatIndex ?? 0;
|
|
835
886
|
const ref = state?.lane.ref ?? "ref-child";
|
|
887
|
+
|
|
836
888
|
if (row.state === "unknown") {
|
|
837
889
|
for (const call of explanation.value.evidence.unknownCalls) {
|
|
838
890
|
if (call.resolved) continue;
|
|
891
|
+
|
|
839
892
|
const kind =
|
|
840
893
|
plan.resolutionInjections.length === 0
|
|
841
894
|
? "never-happened"
|
|
842
895
|
: (plan.resolutionInjections.at(
|
|
843
896
|
injectionIndex(flatIndex, call.toolCallId, plan.resolutionInjections.length),
|
|
844
897
|
) ?? "never-happened");
|
|
898
|
+
|
|
845
899
|
yield* tolerateTyped(
|
|
846
900
|
runtime.resolveUnknown(
|
|
847
901
|
UnknownResolutionCommand.make({
|
|
@@ -862,6 +916,7 @@ const resolutionPass = Effect.fn("Chaos.resolutionPass")(function* (
|
|
|
862
916
|
: (plan.approvalDecisions.at(
|
|
863
917
|
injectionIndex(flatIndex, pending.toolCallId, plan.approvalDecisions.length),
|
|
864
918
|
) ?? "approved");
|
|
919
|
+
|
|
865
920
|
yield* tolerateTyped(
|
|
866
921
|
runtime.resolveApproval(
|
|
867
922
|
ApprovalDecisionCommand.make({
|
|
@@ -882,8 +937,10 @@ const submissionIdsNamedBy = (
|
|
|
882
937
|
records: ReadonlyArray<CanonicalRecordEnvelope>,
|
|
883
938
|
): ReadonlySet<SubmissionId> => {
|
|
884
939
|
const named = new Set<SubmissionId>();
|
|
940
|
+
|
|
885
941
|
for (const envelope of records) {
|
|
886
942
|
const payload = envelope.record.payload;
|
|
943
|
+
|
|
887
944
|
if (
|
|
888
945
|
payload._tag === "UserInputRecorded" ||
|
|
889
946
|
payload._tag === "SubmissionSettled" ||
|
|
@@ -892,6 +949,7 @@ const submissionIdsNamedBy = (
|
|
|
892
949
|
if (payload.submissionId !== undefined) named.add(payload.submissionId);
|
|
893
950
|
}
|
|
894
951
|
}
|
|
952
|
+
|
|
895
953
|
return named;
|
|
896
954
|
};
|
|
897
955
|
|
|
@@ -911,18 +969,23 @@ const assertNoFabrication = (
|
|
|
911
969
|
produced: ReadonlySet<string>,
|
|
912
970
|
): Effect.Effect<void, ChaosConvergenceFailure> => {
|
|
913
971
|
const fabricated: Array<string> = [];
|
|
972
|
+
|
|
914
973
|
const requireProduced = (value: string, label: string): void => {
|
|
915
974
|
if (!produced.has(value)) fabricated.push(`${label} "${value}"`);
|
|
916
975
|
};
|
|
976
|
+
|
|
917
977
|
for (const envelope of records) {
|
|
918
978
|
const payload = envelope.record.payload;
|
|
979
|
+
|
|
919
980
|
if (payload._tag === "ToolCallSettled" && !payload.isFailure) {
|
|
920
981
|
if (payload.toolName === "book") {
|
|
921
982
|
const result = decodeBookResult(payload.result);
|
|
983
|
+
|
|
922
984
|
if (Option.isSome(result)) requireProduced(result.value.confirmation, "book result");
|
|
923
985
|
}
|
|
924
986
|
if (payload.toolName === "itinerary") {
|
|
925
987
|
const result = decodeItineraryResult(payload.result);
|
|
988
|
+
|
|
926
989
|
if (Option.isSome(result)) {
|
|
927
990
|
for (const part of result.value.state.split("+")) {
|
|
928
991
|
requireProduced(part, "itinerary step result");
|
|
@@ -932,9 +995,11 @@ const assertNoFabrication = (
|
|
|
932
995
|
}
|
|
933
996
|
if (payload._tag === "ToolStepSettled") {
|
|
934
997
|
const output = decodeStepOutput(payload.output);
|
|
998
|
+
|
|
935
999
|
if (Option.isSome(output)) requireProduced(output.value, "step output");
|
|
936
1000
|
}
|
|
937
1001
|
}
|
|
1002
|
+
|
|
938
1003
|
return fabricated.length === 0
|
|
939
1004
|
? Effect.void
|
|
940
1005
|
: Effect.fail(
|
|
@@ -964,23 +1029,29 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
964
1029
|
// Lane fixtures: the FIRST spec of each lane fixes the lane's agent kind.
|
|
965
1030
|
const laneKinds = new Map<number, ChaosScenarioKind>();
|
|
966
1031
|
const laneSubmissions = new Map<number, Array<number>>();
|
|
1032
|
+
|
|
967
1033
|
plan.submissions.forEach((spec, flatIndex) => {
|
|
968
1034
|
const lane = spec.lane % plan.lanes;
|
|
1035
|
+
|
|
969
1036
|
if (!laneKinds.has(lane)) laneKinds.set(lane, spec.kind);
|
|
970
1037
|
const list = laneSubmissions.get(lane) ?? [];
|
|
1038
|
+
|
|
971
1039
|
list.push(flatIndex);
|
|
972
1040
|
laneSubmissions.set(lane, list);
|
|
973
1041
|
});
|
|
974
1042
|
const lanes: Array<LaneFixture> = [];
|
|
1043
|
+
|
|
975
1044
|
for (const [lane, kind] of laneKinds) {
|
|
976
1045
|
lanes.push(yield* makeLaneFixture(plan, lane, kind, laneSubmissions.get(lane) ?? [], desk));
|
|
977
1046
|
}
|
|
978
1047
|
|
|
979
1048
|
const lanesByIndex = new Map(lanes.map((lane) => [lane.index, lane]));
|
|
980
1049
|
const states: Array<SubmissionState> = [];
|
|
1050
|
+
|
|
981
1051
|
for (const [flatIndex, spec] of plan.submissions.entries()) {
|
|
982
1052
|
const laneIndex = spec.lane % plan.lanes;
|
|
983
1053
|
const lane = lanesByIndex.get(laneIndex);
|
|
1054
|
+
|
|
984
1055
|
if (lane === undefined) {
|
|
985
1056
|
return yield* ChaosConvergenceFailure.make({
|
|
986
1057
|
seed: plan.seed,
|
|
@@ -994,6 +1065,7 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
994
1065
|
type ArmEntry =
|
|
995
1066
|
| { readonly family: "coordinator"; readonly location: DurableRuntimeFailpointLocation }
|
|
996
1067
|
| { readonly family: "adapter"; readonly location: string };
|
|
1068
|
+
|
|
997
1069
|
const armQueue: Array<ArmEntry> = [
|
|
998
1070
|
...plan.failpointArms.map((location): ArmEntry => ({ family: "coordinator", location })),
|
|
999
1071
|
...plan.adapterArms.map((location): ArmEntry => ({ family: "adapter", location })),
|
|
@@ -1002,17 +1074,21 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1002
1074
|
const allSettled = Effect.gen(function* () {
|
|
1003
1075
|
if (states.some((state) => state.receipt === undefined)) return false;
|
|
1004
1076
|
const nonterminal = yield* tolerateTyped(Stream.runCollect(ledger.scanNonterminal));
|
|
1077
|
+
|
|
1005
1078
|
return Option.isSome(nonterminal) && Array.from(nonterminal.value).length === 0;
|
|
1006
1079
|
});
|
|
1007
1080
|
|
|
1008
1081
|
const maxRounds = armQueue.length + states.length * 2 + 12;
|
|
1009
1082
|
let rounds = 0;
|
|
1010
1083
|
let converged = false;
|
|
1084
|
+
|
|
1011
1085
|
for (let round = 0; round < maxRounds; round++) {
|
|
1012
1086
|
rounds = round + 1;
|
|
1013
1087
|
const arm = armQueue[round];
|
|
1088
|
+
|
|
1014
1089
|
if (arm?.family === "coordinator") {
|
|
1015
1090
|
const location = arm.location;
|
|
1091
|
+
|
|
1016
1092
|
yield* failpoints.setHandler((hit) =>
|
|
1017
1093
|
hit === location
|
|
1018
1094
|
? Effect.fail(DurableRuntimeFailpointError.make({ location: hit }))
|
|
@@ -1027,6 +1103,7 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1027
1103
|
for (const state of states) {
|
|
1028
1104
|
if (state.receipt !== undefined) continue;
|
|
1029
1105
|
const receipt = yield* tolerateTyped(state.lane.submitOne(state.flatIndex));
|
|
1106
|
+
|
|
1030
1107
|
if (Option.isSome(receipt)) state.receipt = receipt.value;
|
|
1031
1108
|
}
|
|
1032
1109
|
|
|
@@ -1035,9 +1112,11 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1035
1112
|
.map((lane) => ({ lane, rank: random() }))
|
|
1036
1113
|
.sort((left, right) => left.rank - right.rank || left.lane.index - right.lane.index)
|
|
1037
1114
|
.map(({ lane }) => lane);
|
|
1115
|
+
|
|
1038
1116
|
for (const lane of order) {
|
|
1039
1117
|
const firstFlat = lane.submissionIndexes[0];
|
|
1040
1118
|
const firstReceipt = firstFlat === undefined ? undefined : states[firstFlat]?.receipt;
|
|
1119
|
+
|
|
1041
1120
|
for (const drive of lane.drives(firstReceipt)) {
|
|
1042
1121
|
yield* tolerateTyped(drive);
|
|
1043
1122
|
}
|
|
@@ -1047,8 +1126,10 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1047
1126
|
if (round >= 1) {
|
|
1048
1127
|
for (const rawIndex of plan.abortInjections) {
|
|
1049
1128
|
const index = rawIndex % states.length;
|
|
1129
|
+
|
|
1050
1130
|
if (appliedAborts.has(index)) continue;
|
|
1051
1131
|
const receipt = states[index]?.receipt;
|
|
1132
|
+
|
|
1052
1133
|
if (receipt === undefined) continue;
|
|
1053
1134
|
appliedAborts.add(index);
|
|
1054
1135
|
yield* tolerateTyped(
|
|
@@ -1082,11 +1163,13 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1082
1163
|
|
|
1083
1164
|
if (!converged) {
|
|
1084
1165
|
const nonterminal = yield* tolerateTyped(Stream.runCollect(ledger.scanNonterminal));
|
|
1166
|
+
|
|
1085
1167
|
const detail = Option.isSome(nonterminal)
|
|
1086
1168
|
? Array.from(nonterminal.value)
|
|
1087
1169
|
.map((row: SubmissionSnapshot) => `${row.submissionId}(${row.state})`)
|
|
1088
1170
|
.join(", ")
|
|
1089
1171
|
: "ledger scan failed";
|
|
1172
|
+
|
|
1090
1173
|
return yield* ChaosConvergenceFailure.make({
|
|
1091
1174
|
seed: plan.seed,
|
|
1092
1175
|
message: `plan did not converge within ${maxRounds} rounds; nonterminal: [${detail}]; pending receipts: ${states.filter((state) => state.receipt === undefined).length}`,
|
|
@@ -1097,6 +1180,7 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1097
1180
|
// with the full digest chain (single known producer), plus the desk non-fabrication sweep.
|
|
1098
1181
|
const produced = yield* desk.produced;
|
|
1099
1182
|
const laneReports: Array<ChaosLaneReport> = [];
|
|
1183
|
+
|
|
1100
1184
|
const verifyThread = Effect.fn("Chaos.verifyThread")(function* (
|
|
1101
1185
|
threadId: ThreadId,
|
|
1102
1186
|
kind: ChaosScenarioKind,
|
|
@@ -1110,7 +1194,9 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1110
1194
|
}),
|
|
1111
1195
|
),
|
|
1112
1196
|
);
|
|
1197
|
+
|
|
1113
1198
|
const rows: Array<SubmissionSnapshot> = [];
|
|
1199
|
+
|
|
1114
1200
|
for (const submissionId of submissionIdsNamedBy(exported.records)) {
|
|
1115
1201
|
const found = yield* ledger.lookup(SubmissionLookupById.make({ submissionId })).pipe(
|
|
1116
1202
|
Effect.mapError((error) =>
|
|
@@ -1120,22 +1206,27 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1120
1206
|
}),
|
|
1121
1207
|
),
|
|
1122
1208
|
);
|
|
1209
|
+
|
|
1123
1210
|
if (Option.isSome(found)) rows.push(found.value);
|
|
1124
1211
|
}
|
|
1212
|
+
|
|
1125
1213
|
const batchProducers = new Map<BatchId, ProducerId>(
|
|
1126
1214
|
exported.records.map((envelope) => [envelope.batchId, config.producerId]),
|
|
1127
1215
|
);
|
|
1216
|
+
|
|
1128
1217
|
const report = yield* verifyThreadInvariants({
|
|
1129
1218
|
export: exported,
|
|
1130
1219
|
submissions: rows,
|
|
1131
1220
|
batchProducers,
|
|
1132
1221
|
requireAllSettled: true,
|
|
1133
1222
|
});
|
|
1223
|
+
|
|
1134
1224
|
if (!report.ok) {
|
|
1135
1225
|
const failed = report.checks
|
|
1136
1226
|
.filter((check) => check.status === "failed")
|
|
1137
1227
|
.map((check) => `${check.name}: ${check.detail ?? "failed"}`)
|
|
1138
1228
|
.join("; ");
|
|
1229
|
+
|
|
1139
1230
|
return yield* ChaosConvergenceFailure.make({
|
|
1140
1231
|
seed: plan.seed,
|
|
1141
1232
|
message: `invariants failed for ${threadId} (${kind}): ${failed}`,
|
|
@@ -1159,12 +1250,16 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1159
1250
|
// Delegation lanes: verify every materialized child Thread too.
|
|
1160
1251
|
for (const flatIndex of lane.submissionIndexes) {
|
|
1161
1252
|
const receipt = states[flatIndex]?.receipt;
|
|
1253
|
+
|
|
1162
1254
|
if (receipt === undefined) continue;
|
|
1163
1255
|
const child = lane.childThreadOf(receipt);
|
|
1256
|
+
|
|
1164
1257
|
if (child === undefined) continue;
|
|
1258
|
+
|
|
1165
1259
|
const childExport = yield* Effect.exit(
|
|
1166
1260
|
store.export(ThreadExportRequest.make({ threadId: child })),
|
|
1167
1261
|
);
|
|
1262
|
+
|
|
1168
1263
|
if (Exit.isSuccess(childExport) && childExport.value.records.length > 0) {
|
|
1169
1264
|
yield* verifyThread(child, "plain", false);
|
|
1170
1265
|
}
|
|
@@ -1181,6 +1276,7 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1181
1276
|
}),
|
|
1182
1277
|
),
|
|
1183
1278
|
);
|
|
1279
|
+
|
|
1184
1280
|
if (obligations.entries.length > 0) {
|
|
1185
1281
|
return yield* ChaosConvergenceFailure.make({
|
|
1186
1282
|
seed: plan.seed,
|
|
@@ -88,12 +88,14 @@ const respondingHost = (
|
|
|
88
88
|
respond: (call: CodeHostCall) => CodeHostCallResult,
|
|
89
89
|
): { readonly host: CodeExecutionHost["Service"]; readonly calls: Array<CodeHostCall> } => {
|
|
90
90
|
const calls: Array<CodeHostCall> = [];
|
|
91
|
+
|
|
91
92
|
return {
|
|
92
93
|
calls,
|
|
93
94
|
host: {
|
|
94
95
|
call: (call) =>
|
|
95
96
|
Effect.sync(() => {
|
|
96
97
|
calls.push(call);
|
|
98
|
+
|
|
97
99
|
return respond(call);
|
|
98
100
|
}),
|
|
99
101
|
},
|
|
@@ -106,6 +108,7 @@ const runPass = (
|
|
|
106
108
|
): Effect.Effect<CodeExecutionResult, CodeExecutionError, CodeExecutor> =>
|
|
107
109
|
Effect.gen(function* () {
|
|
108
110
|
const executor = yield* CodeExecutor;
|
|
111
|
+
|
|
109
112
|
return yield* executor
|
|
110
113
|
.execute(request)
|
|
111
114
|
.pipe(Effect.provideService(CodeExecutionHost, CodeExecutionHost.of(host)));
|
|
@@ -134,6 +137,7 @@ const expectSuccess = (
|
|
|
134
137
|
),
|
|
135
138
|
Effect.flatMap((result) => {
|
|
136
139
|
const complaint = check(result);
|
|
140
|
+
|
|
137
141
|
return complaint === undefined ? Effect.void : Effect.fail(violation(caseName, complaint));
|
|
138
142
|
}),
|
|
139
143
|
);
|
|
@@ -157,6 +161,7 @@ const expectFailure = (
|
|
|
157
161
|
);
|
|
158
162
|
}
|
|
159
163
|
const complaint = check?.(error);
|
|
164
|
+
|
|
160
165
|
return complaint === undefined ? Effect.void : Effect.fail(violation(caseName, complaint));
|
|
161
166
|
}),
|
|
162
167
|
);
|
|
@@ -165,6 +170,7 @@ export const codeExecutorConformanceCases = (
|
|
|
165
170
|
options: CodeExecutorConformanceOptions,
|
|
166
171
|
): ReadonlyArray<CodeExecutorConformanceCase> => {
|
|
167
172
|
const posture = options.implementation;
|
|
173
|
+
|
|
168
174
|
return [
|
|
169
175
|
{
|
|
170
176
|
name: "TEST-015 executes bounded JSON computation and returns the program value",
|
|
@@ -184,9 +190,11 @@ export const codeExecutorConformanceCases = (
|
|
|
184
190
|
name: "CAP-015 reports its isolation posture honestly in results and errors",
|
|
185
191
|
run: Effect.gen(function* () {
|
|
186
192
|
const caseName = "CAP-015 reports its isolation posture honestly in results and errors";
|
|
193
|
+
|
|
187
194
|
const result = yield* runPass(makeRequest("async () => 1"), unusedHost).pipe(
|
|
188
195
|
Effect.mapError((error) => violation(caseName, `expected success, got ${error._tag}`)),
|
|
189
196
|
);
|
|
197
|
+
|
|
190
198
|
if (
|
|
191
199
|
result.implementation.isolation !== posture.isolation ||
|
|
192
200
|
result.implementation.identity !== posture.identity
|
|
@@ -196,10 +204,12 @@ export const codeExecutorConformanceCases = (
|
|
|
196
204
|
`result posture ${preview(result.implementation)} does not match the declared ${preview(posture)}`,
|
|
197
205
|
);
|
|
198
206
|
}
|
|
207
|
+
|
|
199
208
|
const error = yield* runPass(makeRequest("async () => {"), unusedHost).pipe(
|
|
200
209
|
Effect.flip,
|
|
201
210
|
Effect.mapError(() => violation(caseName, "expected the invalid-source pass to fail")),
|
|
202
211
|
);
|
|
212
|
+
|
|
203
213
|
// Every expected execution failure carries the posture; an adapter
|
|
204
214
|
// omitting the field must fail this case, not slip past a probe.
|
|
205
215
|
if (
|
|
@@ -219,11 +229,13 @@ export const codeExecutorConformanceCases = (
|
|
|
219
229
|
run: Effect.gen(function* () {
|
|
220
230
|
const caseName =
|
|
221
231
|
"TEST-015 routes host calls through the CodeExecutionHost in program order";
|
|
232
|
+
|
|
222
233
|
const { host, calls } = respondingHost((call) =>
|
|
223
234
|
call.method === "query"
|
|
224
235
|
? CodeHostCallSuccess.make({ value: { rows: [1, 2, 3] } })
|
|
225
236
|
: CodeHostCallSuccess.make({ value: 3 }),
|
|
226
237
|
);
|
|
238
|
+
|
|
227
239
|
const result = yield* runPass(
|
|
228
240
|
makeRequest(
|
|
229
241
|
"async () => { const q = await warehouse.query({ sql: 'select' }); const c = await warehouse.count({ table: 't' }); return { rows: q.rows, count: c }; }",
|
|
@@ -235,10 +247,12 @@ export const codeExecutorConformanceCases = (
|
|
|
235
247
|
violation(caseName, `expected success, got ${error._tag}: ${preview(error)}`),
|
|
236
248
|
),
|
|
237
249
|
);
|
|
250
|
+
|
|
238
251
|
if (JSON.stringify(result.value) !== JSON.stringify({ rows: [1, 2, 3], count: 3 })) {
|
|
239
252
|
return yield* violation(caseName, `unexpected value ${preview(result.value)}`);
|
|
240
253
|
}
|
|
241
254
|
const observed = calls.map((call) => `${call.namespace}.${call.method}`);
|
|
255
|
+
|
|
242
256
|
if (JSON.stringify(observed) !== JSON.stringify(["warehouse.query", "warehouse.count"])) {
|
|
243
257
|
return yield* violation(caseName, `unexpected host call order ${preview(observed)}`);
|
|
244
258
|
}
|
|
@@ -376,6 +390,7 @@ export const codeExecutorConformanceCases = (
|
|
|
376
390
|
run: Effect.gen(function* () {
|
|
377
391
|
const caseName = "TEST-015 fails typed when host calls exceed the executor cap";
|
|
378
392
|
const { host, calls } = respondingHost(() => CodeHostCallSuccess.make({ value: null }));
|
|
393
|
+
|
|
379
394
|
yield* expectFailure(
|
|
380
395
|
caseName,
|
|
381
396
|
makeRequest(
|
|
@@ -462,6 +477,7 @@ export const codeExecutorConformanceCases = (
|
|
|
462
477
|
const caseName = "TEST-015 interruption reaches in-flight host calls and pass teardown";
|
|
463
478
|
const started = yield* Deferred.make<void>();
|
|
464
479
|
const witness = { hostCallInterrupted: false };
|
|
480
|
+
|
|
465
481
|
const host: CodeExecutionHost["Service"] = {
|
|
466
482
|
call: () =>
|
|
467
483
|
Deferred.succeed(started, undefined).pipe(
|
|
@@ -473,16 +489,19 @@ export const codeExecutorConformanceCases = (
|
|
|
473
489
|
),
|
|
474
490
|
),
|
|
475
491
|
};
|
|
492
|
+
|
|
476
493
|
const fiber = yield* runPass(
|
|
477
494
|
makeRequest("async () => warehouse.query({})", { namespaces: [warehouseNamespace] }),
|
|
478
495
|
host,
|
|
479
496
|
).pipe(Effect.forkChild);
|
|
497
|
+
|
|
480
498
|
// Guard against a broken adapter that settles the pass without ever
|
|
481
499
|
// reaching the host: the case must report a violation, not hang.
|
|
482
500
|
const winner = yield* Effect.raceFirst(
|
|
483
501
|
Deferred.await(started).pipe(Effect.as("started" as const)),
|
|
484
502
|
Fiber.join(fiber).pipe(Effect.exit, Effect.as("exited" as const)),
|
|
485
503
|
);
|
|
504
|
+
|
|
486
505
|
if (winner === "exited") {
|
|
487
506
|
return yield* violation(
|
|
488
507
|
caseName,
|