@effect-agent/testing 0.1.0-beta.42 → 0.1.0-beta.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/certification.mjs +8 -8
- package/dist/certification.mjs.map +1 -1
- package/dist/chaos.d.mts +1 -1
- package/dist/chaos.mjs +5 -3
- package/dist/chaos.mjs.map +1 -1
- package/dist/code-executor.mjs.map +1 -1
- package/dist/deterministic-layers-CKyYxBhN.mjs.map +1 -1
- package/dist/docs-researcher.d.mts +1 -1
- package/dist/docs-researcher.mjs +3 -2
- package/dist/docs-researcher.mjs.map +1 -1
- package/dist/scripted-model-C2y0ztuj.mjs.map +1 -1
- package/dist/travel-planner.d.mts +2 -2
- package/dist/travel-planner.mjs +1 -1
- package/dist/travel-planner.mjs.map +1 -1
- package/package.json +1 -71
- package/src/certification.ts +110 -23
- package/src/chaos.ts +111 -3
- package/src/code-executor-conformance.ts +19 -0
- package/src/code-executor-substitute.ts +32 -0
- package/src/fixtures/docs-researcher/definition.ts +7 -0
- package/src/fixtures/docs-researcher/harness.ts +20 -1
- package/src/fixtures/docs-researcher/mcp.ts +14 -2
- package/src/fixtures/travel-planner/definition.ts +12 -0
- package/src/fixtures/travel-planner/deterministic-layers.ts +38 -0
- package/src/fixtures/travel-planner/phase3.ts +4 -0
- package/src/fixtures/travel-planner/phase4.ts +10 -0
- package/src/fixtures/travel-planner/phase5.ts +20 -0
- package/src/fixtures/travel-planner/phase6.ts +12 -0
- package/src/fixtures/travel-planner/scenarios.ts +2 -0
- package/src/fixtures/travel-planner/subagents-durable.ts +15 -2
- package/src/fixtures/travel-planner/subagents.ts +21 -0
- package/src/scripted-model.ts +18 -0
package/src/chaos.ts
CHANGED
|
@@ -14,7 +14,12 @@ import {
|
|
|
14
14
|
TurnId,
|
|
15
15
|
type SubmissionId,
|
|
16
16
|
} from "@effect-agent/core";
|
|
17
|
-
import {
|
|
17
|
+
import {
|
|
18
|
+
DurableStep,
|
|
19
|
+
DurableStepError,
|
|
20
|
+
RunToolAuthorization,
|
|
21
|
+
ToolExecutionClass,
|
|
22
|
+
} from "@effect-agent/engine";
|
|
18
23
|
import {
|
|
19
24
|
AbortCommand,
|
|
20
25
|
ApprovalDecisionCommand,
|
|
@@ -92,6 +97,7 @@ export const ChaosScenarioKind = Schema.Literals([
|
|
|
92
97
|
"join",
|
|
93
98
|
"delegation",
|
|
94
99
|
]);
|
|
100
|
+
|
|
95
101
|
export type ChaosScenarioKind = typeof ChaosScenarioKind.Type;
|
|
96
102
|
|
|
97
103
|
const LaneIndex = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0), Schema.isLessThanOrEqualTo(7));
|
|
@@ -117,6 +123,7 @@ export const ChaosResolutionKind = Schema.Literals([
|
|
|
117
123
|
/** Unresolvable: route into the abort path (settles aborted, audit retained). */
|
|
118
124
|
"abort-submission",
|
|
119
125
|
]);
|
|
126
|
+
|
|
120
127
|
export type ChaosResolutionKind = typeof ChaosResolutionKind.Type;
|
|
121
128
|
|
|
122
129
|
export const ChaosApprovalDecision = Schema.Literals(["approved", "denied"]);
|
|
@@ -191,7 +198,9 @@ const decodeChaosSeedFromEnvironment = Schema.decodeUnknownOption(ChaosSeedFromE
|
|
|
191
198
|
/** The root seed for this run: `CHAOS_SEED` when set to an integer, the default otherwise. */
|
|
192
199
|
export const chaosSeedFromEnv = (env: Record<string, string | undefined>): number => {
|
|
193
200
|
const raw = env["CHAOS_SEED"];
|
|
201
|
+
|
|
194
202
|
if (raw === undefined || raw === "") return DEFAULT_CHAOS_SEED;
|
|
203
|
+
|
|
195
204
|
return Option.getOrElse(decodeChaosSeedFromEnvironment(raw), () => DEFAULT_CHAOS_SEED);
|
|
196
205
|
};
|
|
197
206
|
|
|
@@ -268,9 +277,12 @@ const planShapeArbitrary = (
|
|
|
268
277
|
ChaosSubmissionSpec.make({ lane: index, kind: lane.kind }),
|
|
269
278
|
),
|
|
270
279
|
);
|
|
280
|
+
|
|
271
281
|
const [first, ...rest] = submissions;
|
|
282
|
+
|
|
272
283
|
// `lanes` >= 1 and every lane has depth >= 1, so `first` always exists.
|
|
273
284
|
if (first === undefined) throw new Error("chaos generator produced an empty plan");
|
|
285
|
+
|
|
274
286
|
return {
|
|
275
287
|
lanes: shape.lanes.length,
|
|
276
288
|
submissions: [first, ...rest] as const,
|
|
@@ -292,6 +304,7 @@ export const generateChaosPlans = (options: ChaosGeneratorOptions): ReadonlyArra
|
|
|
292
304
|
seed: options.seed,
|
|
293
305
|
numRuns: options.count,
|
|
294
306
|
});
|
|
307
|
+
|
|
295
308
|
return sampled.map((shape, index) =>
|
|
296
309
|
ChaosPlan.make({ ...shape, seed: (Math.imul(options.seed, 31) + index) | 0 }),
|
|
297
310
|
);
|
|
@@ -300,10 +313,13 @@ export const generateChaosPlans = (options: ChaosGeneratorOptions): ReadonlyArra
|
|
|
300
313
|
/** Deterministic PRNG for the runner's small ordering choices (lane drive order). */
|
|
301
314
|
const mulberry32 = (seed: number): (() => number) => {
|
|
302
315
|
let state = seed | 0;
|
|
316
|
+
|
|
303
317
|
return () => {
|
|
304
318
|
state = (state + 0x6d2b79f5) | 0;
|
|
305
319
|
let t = Math.imul(state ^ (state >>> 15), 1 | state);
|
|
320
|
+
|
|
306
321
|
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
|
|
322
|
+
|
|
307
323
|
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
|
308
324
|
};
|
|
309
325
|
};
|
|
@@ -382,7 +398,9 @@ const BookUncertain = Tool.make("book", {
|
|
|
382
398
|
parameters: Schema.Struct({ ref: Schema.String }),
|
|
383
399
|
success: Schema.Struct({ confirmation: Schema.String }),
|
|
384
400
|
});
|
|
401
|
+
|
|
385
402
|
const bookTools = Toolkit.make(BookUncertain);
|
|
403
|
+
|
|
386
404
|
const bookDefinition = Agent.make("chaos-book", {
|
|
387
405
|
input: PlainInput,
|
|
388
406
|
output: PlainOutput,
|
|
@@ -396,7 +414,9 @@ const BookApproval = Tool.make("book", {
|
|
|
396
414
|
success: Schema.Struct({ confirmation: Schema.String }),
|
|
397
415
|
needsApproval: true,
|
|
398
416
|
});
|
|
417
|
+
|
|
399
418
|
const approvalTools = Toolkit.make(BookApproval);
|
|
419
|
+
|
|
400
420
|
const approvalDefinition = Agent.make("chaos-approval", {
|
|
401
421
|
input: PlainInput,
|
|
402
422
|
output: PlainOutput,
|
|
@@ -411,7 +431,9 @@ const Itinerary = Tool.make("itinerary", {
|
|
|
411
431
|
failure: DurableStepError,
|
|
412
432
|
dependencies: [DurableStep],
|
|
413
433
|
}).annotate(ToolExecutionClass, "uncertain");
|
|
434
|
+
|
|
414
435
|
const itineraryTools = Toolkit.make(Itinerary);
|
|
436
|
+
|
|
415
437
|
const itineraryDefinition = Agent.make("chaos-itinerary", {
|
|
416
438
|
input: PlainInput,
|
|
417
439
|
output: PlainOutput,
|
|
@@ -467,16 +489,22 @@ const DELEGATE_CALL_ID = "chaos-delegate-1";
|
|
|
467
489
|
|
|
468
490
|
const HEX = "0123456789abcdef";
|
|
469
491
|
const decodeDigest = Schema.decodeSync(Digest);
|
|
492
|
+
|
|
470
493
|
const laneDigests = (lane: number): DefinitionDigests => {
|
|
471
494
|
const digest = decodeDigest(HEX.charAt(lane % 8).repeat(64));
|
|
495
|
+
|
|
472
496
|
return DefinitionDigests.make({ agent: digest, model: digest, tools: digest });
|
|
473
497
|
};
|
|
498
|
+
|
|
474
499
|
const childDigestStrings = (lane: number) => {
|
|
475
500
|
const char = HEX.charAt(8 + (lane % 8));
|
|
501
|
+
|
|
476
502
|
return { agent: char.repeat(64), model: char.repeat(64), tools: char.repeat(64) } as const;
|
|
477
503
|
};
|
|
504
|
+
|
|
478
505
|
const childLaneDigests = (lane: number): DefinitionDigests => {
|
|
479
506
|
const strings = childDigestStrings(lane);
|
|
507
|
+
|
|
480
508
|
return DefinitionDigests.make({
|
|
481
509
|
agent: decodeDigest(strings.agent),
|
|
482
510
|
model: decodeDigest(strings.model),
|
|
@@ -496,10 +524,12 @@ const chaosIdentifiers = Layer.effect(
|
|
|
496
524
|
IdGenerator,
|
|
497
525
|
Effect.gen(function* () {
|
|
498
526
|
const counter = yield* Ref.make(0);
|
|
527
|
+
|
|
499
528
|
const next = <A>(decode: (value: string) => A, prefix: string) =>
|
|
500
529
|
Ref.getAndUpdate(counter, (value) => value + 1).pipe(
|
|
501
530
|
Effect.map((value) => decode(`${prefix}-${value}`)),
|
|
502
531
|
);
|
|
532
|
+
|
|
503
533
|
return {
|
|
504
534
|
nextThreadId: next(decodeThreadId, "chaos-fixture-thread"),
|
|
505
535
|
nextRunId: next(decodeRunId, "chaos-fixture-run"),
|
|
@@ -521,6 +551,7 @@ interface ChaosDesk {
|
|
|
521
551
|
|
|
522
552
|
const makeChaosDesk: Effect.Effect<ChaosDesk> = Effect.gen(function* () {
|
|
523
553
|
const produced = yield* Ref.make<ReadonlySet<string>>(new Set());
|
|
554
|
+
|
|
524
555
|
return {
|
|
525
556
|
produced: Ref.get(produced),
|
|
526
557
|
record: (value: string) => Ref.update(produced, (current) => new Set(current).add(value)),
|
|
@@ -564,6 +595,7 @@ const tolerateTyped = <A, E, R>(
|
|
|
564
595
|
if (Option.isSome(Cause.findErrorOption(exit.cause))) {
|
|
565
596
|
return Effect.succeed(Option.none<A>());
|
|
566
597
|
}
|
|
598
|
+
|
|
567
599
|
return Effect.die(new Error(`chaos step died: ${Cause.pretty(exit.cause)}`));
|
|
568
600
|
}),
|
|
569
601
|
);
|
|
@@ -666,12 +698,14 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
666
698
|
case "plain":
|
|
667
699
|
case "join": {
|
|
668
700
|
const agent = Agent.withModel(plainDefinition, model);
|
|
701
|
+
|
|
669
702
|
return plainLaneFixture(false, runtime.processThread(agent, threadId), (flatIndex) =>
|
|
670
703
|
runtime.submit(agent, { question: `chaos ${flatIndex}` }, submitOptionsFor(flatIndex)),
|
|
671
704
|
);
|
|
672
705
|
}
|
|
673
706
|
case "uncertain-tool": {
|
|
674
707
|
const agent = Agent.withModel(bookDefinition, model);
|
|
708
|
+
|
|
675
709
|
return plainLaneFixture(
|
|
676
710
|
true,
|
|
677
711
|
runtime.processThread(agent, threadId).pipe(Effect.provide(bookToolLayerFor(bookTools))),
|
|
@@ -681,6 +715,7 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
681
715
|
}
|
|
682
716
|
case "approval": {
|
|
683
717
|
const agent = Agent.withModel(approvalDefinition, model);
|
|
718
|
+
|
|
684
719
|
return plainLaneFixture(
|
|
685
720
|
true,
|
|
686
721
|
runtime
|
|
@@ -692,23 +727,28 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
692
727
|
}
|
|
693
728
|
case "durable-steps": {
|
|
694
729
|
const agent = Agent.withModel(itineraryDefinition, model);
|
|
730
|
+
|
|
695
731
|
const toolLayer = itineraryTools.toLayer({
|
|
696
732
|
itinerary: ({ ref: called }) =>
|
|
697
733
|
Effect.gen(function* () {
|
|
698
734
|
const step = yield* DurableStep;
|
|
735
|
+
|
|
699
736
|
const flight = yield* step.do(
|
|
700
737
|
"reserve-flight",
|
|
701
738
|
Schema.String,
|
|
702
739
|
desk.record(flightValue(called)).pipe(Effect.as(flightValue(called))),
|
|
703
740
|
);
|
|
741
|
+
|
|
704
742
|
const lodging = yield* step.do(
|
|
705
743
|
"reserve-lodging",
|
|
706
744
|
Schema.String,
|
|
707
745
|
desk.record(lodgingValue(called)).pipe(Effect.as(lodgingValue(called))),
|
|
708
746
|
);
|
|
747
|
+
|
|
709
748
|
return { state: `${flight}+${lodging}` };
|
|
710
749
|
}),
|
|
711
750
|
});
|
|
751
|
+
|
|
712
752
|
return plainLaneFixture(
|
|
713
753
|
true,
|
|
714
754
|
runtime.processThread(agent, threadId).pipe(Effect.provide(toolLayer)),
|
|
@@ -718,24 +758,38 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
718
758
|
}
|
|
719
759
|
case "delegation": {
|
|
720
760
|
const parentBinding = Agent.withModel(coordinatorDefinition, model);
|
|
761
|
+
|
|
721
762
|
const childModel = promptScriptedModel(`chaos-child-${laneIndex}`, () =>
|
|
722
763
|
finalParts('{"answer":"child"}'),
|
|
723
764
|
);
|
|
765
|
+
|
|
724
766
|
const childBinding = Agent.withModel(childDefinition, childModel);
|
|
767
|
+
|
|
725
768
|
const delegationLayer = SubagentRuntime.layer(chaosDelegation, childBinding, {
|
|
726
769
|
mapChildFailure: (failure) => ChaosDelegationFailed.make({ childErrorTag: failure._tag }),
|
|
727
770
|
durable: { targetDigests: childDigestStrings(laneIndex) },
|
|
728
771
|
}).pipe(Layer.provide(delegationSupport));
|
|
772
|
+
|
|
729
773
|
const parentResolved: ResolvedBinding = yield* DurableWorkerBinding.make(
|
|
730
774
|
parentBinding,
|
|
731
775
|
digests,
|
|
732
776
|
).pipe(Effect.provide(delegationLayer));
|
|
777
|
+
|
|
733
778
|
const childResolved: ResolvedBinding = yield* DurableWorkerBinding.make(
|
|
734
779
|
childBinding,
|
|
735
780
|
childLaneDigests(laneIndex),
|
|
736
781
|
);
|
|
737
|
-
|
|
738
|
-
const
|
|
782
|
+
|
|
783
|
+
const registeredRuntime = yield* DurableAgentRuntime.pipe(
|
|
784
|
+
Effect.provide(
|
|
785
|
+
DurableAgentRuntime.layerWithBindings([parentResolved, childResolved]).pipe(
|
|
786
|
+
Layer.provide(RunToolAuthorization.allowAll),
|
|
787
|
+
),
|
|
788
|
+
),
|
|
789
|
+
);
|
|
790
|
+
|
|
791
|
+
const driveResolved = (thread: ThreadId) => registeredRuntime.processThreadResolved(thread);
|
|
792
|
+
|
|
739
793
|
const fixture: LaneFixture = {
|
|
740
794
|
index: laneIndex,
|
|
741
795
|
kind,
|
|
@@ -753,6 +807,7 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
753
807
|
const drives: Array<
|
|
754
808
|
Effect.Effect<ReadonlyArray<Settlement>, DurableWorkerFailure | DurableBindingFailure>
|
|
755
809
|
> = [driveResolved(threadId)];
|
|
810
|
+
|
|
756
811
|
if (firstReceipt !== undefined) {
|
|
757
812
|
drives.push(
|
|
758
813
|
driveResolved(
|
|
@@ -760,11 +815,13 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
760
815
|
),
|
|
761
816
|
);
|
|
762
817
|
}
|
|
818
|
+
|
|
763
819
|
return drives;
|
|
764
820
|
},
|
|
765
821
|
childThreadOf: (firstReceipt) =>
|
|
766
822
|
childThreadIdFor(firstReceipt.submissionId, decodeToolCallId(DELEGATE_CALL_ID)),
|
|
767
823
|
};
|
|
824
|
+
|
|
768
825
|
return fixture;
|
|
769
826
|
}
|
|
770
827
|
}
|
|
@@ -773,7 +830,9 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
773
830
|
/** Stable per-call index into an injection list (identical across resolution passes). */
|
|
774
831
|
const injectionIndex = (submissionFlatIndex: number, callId: string, length: number): number => {
|
|
775
832
|
let hash = submissionFlatIndex + 1;
|
|
833
|
+
|
|
776
834
|
for (const char of callId) hash = (Math.imul(hash, 31) + char.charCodeAt(0)) | 0;
|
|
835
|
+
|
|
777
836
|
return ((hash % length) + length) % length;
|
|
778
837
|
};
|
|
779
838
|
|
|
@@ -803,6 +862,7 @@ const resolutionFor = (
|
|
|
803
862
|
isFailure: false,
|
|
804
863
|
});
|
|
805
864
|
}
|
|
865
|
+
|
|
806
866
|
// The desk never produced a value for this call — resolving "completed" would fabricate.
|
|
807
867
|
return ResolutionNeverHappened.make();
|
|
808
868
|
}
|
|
@@ -821,8 +881,10 @@ const resolutionPass = Effect.fn("Chaos.resolutionPass")(function* (
|
|
|
821
881
|
const ledger = yield* SubmissionLedger;
|
|
822
882
|
const produced = yield* desk.produced;
|
|
823
883
|
const nonterminal = yield* tolerateTyped(Stream.runCollect(ledger.scanNonterminal));
|
|
884
|
+
|
|
824
885
|
if (Option.isNone(nonterminal)) return;
|
|
825
886
|
const byId = new Map<SubmissionId, SubmissionState>();
|
|
887
|
+
|
|
826
888
|
for (const state of states) {
|
|
827
889
|
if (state.receipt !== undefined) byId.set(state.receipt.submissionId, state);
|
|
828
890
|
}
|
|
@@ -830,18 +892,22 @@ const resolutionPass = Effect.fn("Chaos.resolutionPass")(function* (
|
|
|
830
892
|
if (row.state !== "unknown" && row.state !== "suspended") continue;
|
|
831
893
|
const state = byId.get(row.submissionId);
|
|
832
894
|
const explanation = yield* tolerateTyped(runtime.explain(row.submissionId));
|
|
895
|
+
|
|
833
896
|
if (Option.isNone(explanation)) continue;
|
|
834
897
|
const flatIndex = state?.flatIndex ?? 0;
|
|
835
898
|
const ref = state?.lane.ref ?? "ref-child";
|
|
899
|
+
|
|
836
900
|
if (row.state === "unknown") {
|
|
837
901
|
for (const call of explanation.value.evidence.unknownCalls) {
|
|
838
902
|
if (call.resolved) continue;
|
|
903
|
+
|
|
839
904
|
const kind =
|
|
840
905
|
plan.resolutionInjections.length === 0
|
|
841
906
|
? "never-happened"
|
|
842
907
|
: (plan.resolutionInjections.at(
|
|
843
908
|
injectionIndex(flatIndex, call.toolCallId, plan.resolutionInjections.length),
|
|
844
909
|
) ?? "never-happened");
|
|
910
|
+
|
|
845
911
|
yield* tolerateTyped(
|
|
846
912
|
runtime.resolveUnknown(
|
|
847
913
|
UnknownResolutionCommand.make({
|
|
@@ -862,6 +928,7 @@ const resolutionPass = Effect.fn("Chaos.resolutionPass")(function* (
|
|
|
862
928
|
: (plan.approvalDecisions.at(
|
|
863
929
|
injectionIndex(flatIndex, pending.toolCallId, plan.approvalDecisions.length),
|
|
864
930
|
) ?? "approved");
|
|
931
|
+
|
|
865
932
|
yield* tolerateTyped(
|
|
866
933
|
runtime.resolveApproval(
|
|
867
934
|
ApprovalDecisionCommand.make({
|
|
@@ -882,8 +949,10 @@ const submissionIdsNamedBy = (
|
|
|
882
949
|
records: ReadonlyArray<CanonicalRecordEnvelope>,
|
|
883
950
|
): ReadonlySet<SubmissionId> => {
|
|
884
951
|
const named = new Set<SubmissionId>();
|
|
952
|
+
|
|
885
953
|
for (const envelope of records) {
|
|
886
954
|
const payload = envelope.record.payload;
|
|
955
|
+
|
|
887
956
|
if (
|
|
888
957
|
payload._tag === "UserInputRecorded" ||
|
|
889
958
|
payload._tag === "SubmissionSettled" ||
|
|
@@ -892,6 +961,7 @@ const submissionIdsNamedBy = (
|
|
|
892
961
|
if (payload.submissionId !== undefined) named.add(payload.submissionId);
|
|
893
962
|
}
|
|
894
963
|
}
|
|
964
|
+
|
|
895
965
|
return named;
|
|
896
966
|
};
|
|
897
967
|
|
|
@@ -911,18 +981,23 @@ const assertNoFabrication = (
|
|
|
911
981
|
produced: ReadonlySet<string>,
|
|
912
982
|
): Effect.Effect<void, ChaosConvergenceFailure> => {
|
|
913
983
|
const fabricated: Array<string> = [];
|
|
984
|
+
|
|
914
985
|
const requireProduced = (value: string, label: string): void => {
|
|
915
986
|
if (!produced.has(value)) fabricated.push(`${label} "${value}"`);
|
|
916
987
|
};
|
|
988
|
+
|
|
917
989
|
for (const envelope of records) {
|
|
918
990
|
const payload = envelope.record.payload;
|
|
991
|
+
|
|
919
992
|
if (payload._tag === "ToolCallSettled" && !payload.isFailure) {
|
|
920
993
|
if (payload.toolName === "book") {
|
|
921
994
|
const result = decodeBookResult(payload.result);
|
|
995
|
+
|
|
922
996
|
if (Option.isSome(result)) requireProduced(result.value.confirmation, "book result");
|
|
923
997
|
}
|
|
924
998
|
if (payload.toolName === "itinerary") {
|
|
925
999
|
const result = decodeItineraryResult(payload.result);
|
|
1000
|
+
|
|
926
1001
|
if (Option.isSome(result)) {
|
|
927
1002
|
for (const part of result.value.state.split("+")) {
|
|
928
1003
|
requireProduced(part, "itinerary step result");
|
|
@@ -932,9 +1007,11 @@ const assertNoFabrication = (
|
|
|
932
1007
|
}
|
|
933
1008
|
if (payload._tag === "ToolStepSettled") {
|
|
934
1009
|
const output = decodeStepOutput(payload.output);
|
|
1010
|
+
|
|
935
1011
|
if (Option.isSome(output)) requireProduced(output.value, "step output");
|
|
936
1012
|
}
|
|
937
1013
|
}
|
|
1014
|
+
|
|
938
1015
|
return fabricated.length === 0
|
|
939
1016
|
? Effect.void
|
|
940
1017
|
: Effect.fail(
|
|
@@ -964,23 +1041,29 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
964
1041
|
// Lane fixtures: the FIRST spec of each lane fixes the lane's agent kind.
|
|
965
1042
|
const laneKinds = new Map<number, ChaosScenarioKind>();
|
|
966
1043
|
const laneSubmissions = new Map<number, Array<number>>();
|
|
1044
|
+
|
|
967
1045
|
plan.submissions.forEach((spec, flatIndex) => {
|
|
968
1046
|
const lane = spec.lane % plan.lanes;
|
|
1047
|
+
|
|
969
1048
|
if (!laneKinds.has(lane)) laneKinds.set(lane, spec.kind);
|
|
970
1049
|
const list = laneSubmissions.get(lane) ?? [];
|
|
1050
|
+
|
|
971
1051
|
list.push(flatIndex);
|
|
972
1052
|
laneSubmissions.set(lane, list);
|
|
973
1053
|
});
|
|
974
1054
|
const lanes: Array<LaneFixture> = [];
|
|
1055
|
+
|
|
975
1056
|
for (const [lane, kind] of laneKinds) {
|
|
976
1057
|
lanes.push(yield* makeLaneFixture(plan, lane, kind, laneSubmissions.get(lane) ?? [], desk));
|
|
977
1058
|
}
|
|
978
1059
|
|
|
979
1060
|
const lanesByIndex = new Map(lanes.map((lane) => [lane.index, lane]));
|
|
980
1061
|
const states: Array<SubmissionState> = [];
|
|
1062
|
+
|
|
981
1063
|
for (const [flatIndex, spec] of plan.submissions.entries()) {
|
|
982
1064
|
const laneIndex = spec.lane % plan.lanes;
|
|
983
1065
|
const lane = lanesByIndex.get(laneIndex);
|
|
1066
|
+
|
|
984
1067
|
if (lane === undefined) {
|
|
985
1068
|
return yield* ChaosConvergenceFailure.make({
|
|
986
1069
|
seed: plan.seed,
|
|
@@ -994,6 +1077,7 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
994
1077
|
type ArmEntry =
|
|
995
1078
|
| { readonly family: "coordinator"; readonly location: DurableRuntimeFailpointLocation }
|
|
996
1079
|
| { readonly family: "adapter"; readonly location: string };
|
|
1080
|
+
|
|
997
1081
|
const armQueue: Array<ArmEntry> = [
|
|
998
1082
|
...plan.failpointArms.map((location): ArmEntry => ({ family: "coordinator", location })),
|
|
999
1083
|
...plan.adapterArms.map((location): ArmEntry => ({ family: "adapter", location })),
|
|
@@ -1002,17 +1086,21 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1002
1086
|
const allSettled = Effect.gen(function* () {
|
|
1003
1087
|
if (states.some((state) => state.receipt === undefined)) return false;
|
|
1004
1088
|
const nonterminal = yield* tolerateTyped(Stream.runCollect(ledger.scanNonterminal));
|
|
1089
|
+
|
|
1005
1090
|
return Option.isSome(nonterminal) && Array.from(nonterminal.value).length === 0;
|
|
1006
1091
|
});
|
|
1007
1092
|
|
|
1008
1093
|
const maxRounds = armQueue.length + states.length * 2 + 12;
|
|
1009
1094
|
let rounds = 0;
|
|
1010
1095
|
let converged = false;
|
|
1096
|
+
|
|
1011
1097
|
for (let round = 0; round < maxRounds; round++) {
|
|
1012
1098
|
rounds = round + 1;
|
|
1013
1099
|
const arm = armQueue[round];
|
|
1100
|
+
|
|
1014
1101
|
if (arm?.family === "coordinator") {
|
|
1015
1102
|
const location = arm.location;
|
|
1103
|
+
|
|
1016
1104
|
yield* failpoints.setHandler((hit) =>
|
|
1017
1105
|
hit === location
|
|
1018
1106
|
? Effect.fail(DurableRuntimeFailpointError.make({ location: hit }))
|
|
@@ -1027,6 +1115,7 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1027
1115
|
for (const state of states) {
|
|
1028
1116
|
if (state.receipt !== undefined) continue;
|
|
1029
1117
|
const receipt = yield* tolerateTyped(state.lane.submitOne(state.flatIndex));
|
|
1118
|
+
|
|
1030
1119
|
if (Option.isSome(receipt)) state.receipt = receipt.value;
|
|
1031
1120
|
}
|
|
1032
1121
|
|
|
@@ -1035,9 +1124,11 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1035
1124
|
.map((lane) => ({ lane, rank: random() }))
|
|
1036
1125
|
.sort((left, right) => left.rank - right.rank || left.lane.index - right.lane.index)
|
|
1037
1126
|
.map(({ lane }) => lane);
|
|
1127
|
+
|
|
1038
1128
|
for (const lane of order) {
|
|
1039
1129
|
const firstFlat = lane.submissionIndexes[0];
|
|
1040
1130
|
const firstReceipt = firstFlat === undefined ? undefined : states[firstFlat]?.receipt;
|
|
1131
|
+
|
|
1041
1132
|
for (const drive of lane.drives(firstReceipt)) {
|
|
1042
1133
|
yield* tolerateTyped(drive);
|
|
1043
1134
|
}
|
|
@@ -1047,8 +1138,10 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1047
1138
|
if (round >= 1) {
|
|
1048
1139
|
for (const rawIndex of plan.abortInjections) {
|
|
1049
1140
|
const index = rawIndex % states.length;
|
|
1141
|
+
|
|
1050
1142
|
if (appliedAborts.has(index)) continue;
|
|
1051
1143
|
const receipt = states[index]?.receipt;
|
|
1144
|
+
|
|
1052
1145
|
if (receipt === undefined) continue;
|
|
1053
1146
|
appliedAborts.add(index);
|
|
1054
1147
|
yield* tolerateTyped(
|
|
@@ -1082,11 +1175,13 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1082
1175
|
|
|
1083
1176
|
if (!converged) {
|
|
1084
1177
|
const nonterminal = yield* tolerateTyped(Stream.runCollect(ledger.scanNonterminal));
|
|
1178
|
+
|
|
1085
1179
|
const detail = Option.isSome(nonterminal)
|
|
1086
1180
|
? Array.from(nonterminal.value)
|
|
1087
1181
|
.map((row: SubmissionSnapshot) => `${row.submissionId}(${row.state})`)
|
|
1088
1182
|
.join(", ")
|
|
1089
1183
|
: "ledger scan failed";
|
|
1184
|
+
|
|
1090
1185
|
return yield* ChaosConvergenceFailure.make({
|
|
1091
1186
|
seed: plan.seed,
|
|
1092
1187
|
message: `plan did not converge within ${maxRounds} rounds; nonterminal: [${detail}]; pending receipts: ${states.filter((state) => state.receipt === undefined).length}`,
|
|
@@ -1097,6 +1192,7 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1097
1192
|
// with the full digest chain (single known producer), plus the desk non-fabrication sweep.
|
|
1098
1193
|
const produced = yield* desk.produced;
|
|
1099
1194
|
const laneReports: Array<ChaosLaneReport> = [];
|
|
1195
|
+
|
|
1100
1196
|
const verifyThread = Effect.fn("Chaos.verifyThread")(function* (
|
|
1101
1197
|
threadId: ThreadId,
|
|
1102
1198
|
kind: ChaosScenarioKind,
|
|
@@ -1110,7 +1206,9 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1110
1206
|
}),
|
|
1111
1207
|
),
|
|
1112
1208
|
);
|
|
1209
|
+
|
|
1113
1210
|
const rows: Array<SubmissionSnapshot> = [];
|
|
1211
|
+
|
|
1114
1212
|
for (const submissionId of submissionIdsNamedBy(exported.records)) {
|
|
1115
1213
|
const found = yield* ledger.lookup(SubmissionLookupById.make({ submissionId })).pipe(
|
|
1116
1214
|
Effect.mapError((error) =>
|
|
@@ -1120,22 +1218,27 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1120
1218
|
}),
|
|
1121
1219
|
),
|
|
1122
1220
|
);
|
|
1221
|
+
|
|
1123
1222
|
if (Option.isSome(found)) rows.push(found.value);
|
|
1124
1223
|
}
|
|
1224
|
+
|
|
1125
1225
|
const batchProducers = new Map<BatchId, ProducerId>(
|
|
1126
1226
|
exported.records.map((envelope) => [envelope.batchId, config.producerId]),
|
|
1127
1227
|
);
|
|
1228
|
+
|
|
1128
1229
|
const report = yield* verifyThreadInvariants({
|
|
1129
1230
|
export: exported,
|
|
1130
1231
|
submissions: rows,
|
|
1131
1232
|
batchProducers,
|
|
1132
1233
|
requireAllSettled: true,
|
|
1133
1234
|
});
|
|
1235
|
+
|
|
1134
1236
|
if (!report.ok) {
|
|
1135
1237
|
const failed = report.checks
|
|
1136
1238
|
.filter((check) => check.status === "failed")
|
|
1137
1239
|
.map((check) => `${check.name}: ${check.detail ?? "failed"}`)
|
|
1138
1240
|
.join("; ");
|
|
1241
|
+
|
|
1139
1242
|
return yield* ChaosConvergenceFailure.make({
|
|
1140
1243
|
seed: plan.seed,
|
|
1141
1244
|
message: `invariants failed for ${threadId} (${kind}): ${failed}`,
|
|
@@ -1159,12 +1262,16 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1159
1262
|
// Delegation lanes: verify every materialized child Thread too.
|
|
1160
1263
|
for (const flatIndex of lane.submissionIndexes) {
|
|
1161
1264
|
const receipt = states[flatIndex]?.receipt;
|
|
1265
|
+
|
|
1162
1266
|
if (receipt === undefined) continue;
|
|
1163
1267
|
const child = lane.childThreadOf(receipt);
|
|
1268
|
+
|
|
1164
1269
|
if (child === undefined) continue;
|
|
1270
|
+
|
|
1165
1271
|
const childExport = yield* Effect.exit(
|
|
1166
1272
|
store.export(ThreadExportRequest.make({ threadId: child })),
|
|
1167
1273
|
);
|
|
1274
|
+
|
|
1168
1275
|
if (Exit.isSuccess(childExport) && childExport.value.records.length > 0) {
|
|
1169
1276
|
yield* verifyThread(child, "plain", false);
|
|
1170
1277
|
}
|
|
@@ -1181,6 +1288,7 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1181
1288
|
}),
|
|
1182
1289
|
),
|
|
1183
1290
|
);
|
|
1291
|
+
|
|
1184
1292
|
if (obligations.entries.length > 0) {
|
|
1185
1293
|
return yield* ChaosConvergenceFailure.make({
|
|
1186
1294
|
seed: plan.seed,
|
|
@@ -88,12 +88,14 @@ const respondingHost = (
|
|
|
88
88
|
respond: (call: CodeHostCall) => CodeHostCallResult,
|
|
89
89
|
): { readonly host: CodeExecutionHost["Service"]; readonly calls: Array<CodeHostCall> } => {
|
|
90
90
|
const calls: Array<CodeHostCall> = [];
|
|
91
|
+
|
|
91
92
|
return {
|
|
92
93
|
calls,
|
|
93
94
|
host: {
|
|
94
95
|
call: (call) =>
|
|
95
96
|
Effect.sync(() => {
|
|
96
97
|
calls.push(call);
|
|
98
|
+
|
|
97
99
|
return respond(call);
|
|
98
100
|
}),
|
|
99
101
|
},
|
|
@@ -106,6 +108,7 @@ const runPass = (
|
|
|
106
108
|
): Effect.Effect<CodeExecutionResult, CodeExecutionError, CodeExecutor> =>
|
|
107
109
|
Effect.gen(function* () {
|
|
108
110
|
const executor = yield* CodeExecutor;
|
|
111
|
+
|
|
109
112
|
return yield* executor
|
|
110
113
|
.execute(request)
|
|
111
114
|
.pipe(Effect.provideService(CodeExecutionHost, CodeExecutionHost.of(host)));
|
|
@@ -134,6 +137,7 @@ const expectSuccess = (
|
|
|
134
137
|
),
|
|
135
138
|
Effect.flatMap((result) => {
|
|
136
139
|
const complaint = check(result);
|
|
140
|
+
|
|
137
141
|
return complaint === undefined ? Effect.void : Effect.fail(violation(caseName, complaint));
|
|
138
142
|
}),
|
|
139
143
|
);
|
|
@@ -157,6 +161,7 @@ const expectFailure = (
|
|
|
157
161
|
);
|
|
158
162
|
}
|
|
159
163
|
const complaint = check?.(error);
|
|
164
|
+
|
|
160
165
|
return complaint === undefined ? Effect.void : Effect.fail(violation(caseName, complaint));
|
|
161
166
|
}),
|
|
162
167
|
);
|
|
@@ -165,6 +170,7 @@ export const codeExecutorConformanceCases = (
|
|
|
165
170
|
options: CodeExecutorConformanceOptions,
|
|
166
171
|
): ReadonlyArray<CodeExecutorConformanceCase> => {
|
|
167
172
|
const posture = options.implementation;
|
|
173
|
+
|
|
168
174
|
return [
|
|
169
175
|
{
|
|
170
176
|
name: "TEST-015 executes bounded JSON computation and returns the program value",
|
|
@@ -184,9 +190,11 @@ export const codeExecutorConformanceCases = (
|
|
|
184
190
|
name: "CAP-015 reports its isolation posture honestly in results and errors",
|
|
185
191
|
run: Effect.gen(function* () {
|
|
186
192
|
const caseName = "CAP-015 reports its isolation posture honestly in results and errors";
|
|
193
|
+
|
|
187
194
|
const result = yield* runPass(makeRequest("async () => 1"), unusedHost).pipe(
|
|
188
195
|
Effect.mapError((error) => violation(caseName, `expected success, got ${error._tag}`)),
|
|
189
196
|
);
|
|
197
|
+
|
|
190
198
|
if (
|
|
191
199
|
result.implementation.isolation !== posture.isolation ||
|
|
192
200
|
result.implementation.identity !== posture.identity
|
|
@@ -196,10 +204,12 @@ export const codeExecutorConformanceCases = (
|
|
|
196
204
|
`result posture ${preview(result.implementation)} does not match the declared ${preview(posture)}`,
|
|
197
205
|
);
|
|
198
206
|
}
|
|
207
|
+
|
|
199
208
|
const error = yield* runPass(makeRequest("async () => {"), unusedHost).pipe(
|
|
200
209
|
Effect.flip,
|
|
201
210
|
Effect.mapError(() => violation(caseName, "expected the invalid-source pass to fail")),
|
|
202
211
|
);
|
|
212
|
+
|
|
203
213
|
// Every expected execution failure carries the posture; an adapter
|
|
204
214
|
// omitting the field must fail this case, not slip past a probe.
|
|
205
215
|
if (
|
|
@@ -219,11 +229,13 @@ export const codeExecutorConformanceCases = (
|
|
|
219
229
|
run: Effect.gen(function* () {
|
|
220
230
|
const caseName =
|
|
221
231
|
"TEST-015 routes host calls through the CodeExecutionHost in program order";
|
|
232
|
+
|
|
222
233
|
const { host, calls } = respondingHost((call) =>
|
|
223
234
|
call.method === "query"
|
|
224
235
|
? CodeHostCallSuccess.make({ value: { rows: [1, 2, 3] } })
|
|
225
236
|
: CodeHostCallSuccess.make({ value: 3 }),
|
|
226
237
|
);
|
|
238
|
+
|
|
227
239
|
const result = yield* runPass(
|
|
228
240
|
makeRequest(
|
|
229
241
|
"async () => { const q = await warehouse.query({ sql: 'select' }); const c = await warehouse.count({ table: 't' }); return { rows: q.rows, count: c }; }",
|
|
@@ -235,10 +247,12 @@ export const codeExecutorConformanceCases = (
|
|
|
235
247
|
violation(caseName, `expected success, got ${error._tag}: ${preview(error)}`),
|
|
236
248
|
),
|
|
237
249
|
);
|
|
250
|
+
|
|
238
251
|
if (JSON.stringify(result.value) !== JSON.stringify({ rows: [1, 2, 3], count: 3 })) {
|
|
239
252
|
return yield* violation(caseName, `unexpected value ${preview(result.value)}`);
|
|
240
253
|
}
|
|
241
254
|
const observed = calls.map((call) => `${call.namespace}.${call.method}`);
|
|
255
|
+
|
|
242
256
|
if (JSON.stringify(observed) !== JSON.stringify(["warehouse.query", "warehouse.count"])) {
|
|
243
257
|
return yield* violation(caseName, `unexpected host call order ${preview(observed)}`);
|
|
244
258
|
}
|
|
@@ -376,6 +390,7 @@ export const codeExecutorConformanceCases = (
|
|
|
376
390
|
run: Effect.gen(function* () {
|
|
377
391
|
const caseName = "TEST-015 fails typed when host calls exceed the executor cap";
|
|
378
392
|
const { host, calls } = respondingHost(() => CodeHostCallSuccess.make({ value: null }));
|
|
393
|
+
|
|
379
394
|
yield* expectFailure(
|
|
380
395
|
caseName,
|
|
381
396
|
makeRequest(
|
|
@@ -462,6 +477,7 @@ export const codeExecutorConformanceCases = (
|
|
|
462
477
|
const caseName = "TEST-015 interruption reaches in-flight host calls and pass teardown";
|
|
463
478
|
const started = yield* Deferred.make<void>();
|
|
464
479
|
const witness = { hostCallInterrupted: false };
|
|
480
|
+
|
|
465
481
|
const host: CodeExecutionHost["Service"] = {
|
|
466
482
|
call: () =>
|
|
467
483
|
Deferred.succeed(started, undefined).pipe(
|
|
@@ -473,16 +489,19 @@ export const codeExecutorConformanceCases = (
|
|
|
473
489
|
),
|
|
474
490
|
),
|
|
475
491
|
};
|
|
492
|
+
|
|
476
493
|
const fiber = yield* runPass(
|
|
477
494
|
makeRequest("async () => warehouse.query({})", { namespaces: [warehouseNamespace] }),
|
|
478
495
|
host,
|
|
479
496
|
).pipe(Effect.forkChild);
|
|
497
|
+
|
|
480
498
|
// Guard against a broken adapter that settles the pass without ever
|
|
481
499
|
// reaching the host: the case must report a violation, not hang.
|
|
482
500
|
const winner = yield* Effect.raceFirst(
|
|
483
501
|
Deferred.await(started).pipe(Effect.as("started" as const)),
|
|
484
502
|
Fiber.join(fiber).pipe(Effect.exit, Effect.as("exited" as const)),
|
|
485
503
|
);
|
|
504
|
+
|
|
486
505
|
if (winner === "exited") {
|
|
487
506
|
return yield* violation(
|
|
488
507
|
caseName,
|