@effect-agent/testing 0.1.0-beta.42 → 0.1.0-beta.45

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/chaos.ts CHANGED
@@ -14,7 +14,12 @@ import {
14
14
  TurnId,
15
15
  type SubmissionId,
16
16
  } from "@effect-agent/core";
17
- import { DurableStep, DurableStepError, ToolExecutionClass } from "@effect-agent/engine";
17
+ import {
18
+ DurableStep,
19
+ DurableStepError,
20
+ RunToolAuthorization,
21
+ ToolExecutionClass,
22
+ } from "@effect-agent/engine";
18
23
  import {
19
24
  AbortCommand,
20
25
  ApprovalDecisionCommand,
@@ -92,6 +97,7 @@ export const ChaosScenarioKind = Schema.Literals([
92
97
  "join",
93
98
  "delegation",
94
99
  ]);
100
+
95
101
  export type ChaosScenarioKind = typeof ChaosScenarioKind.Type;
96
102
 
97
103
  const LaneIndex = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0), Schema.isLessThanOrEqualTo(7));
@@ -117,6 +123,7 @@ export const ChaosResolutionKind = Schema.Literals([
117
123
  /** Unresolvable: route into the abort path (settles aborted, audit retained). */
118
124
  "abort-submission",
119
125
  ]);
126
+
120
127
  export type ChaosResolutionKind = typeof ChaosResolutionKind.Type;
121
128
 
122
129
  export const ChaosApprovalDecision = Schema.Literals(["approved", "denied"]);
@@ -191,7 +198,9 @@ const decodeChaosSeedFromEnvironment = Schema.decodeUnknownOption(ChaosSeedFromE
191
198
  /** The root seed for this run: `CHAOS_SEED` when set to an integer, the default otherwise. */
192
199
  export const chaosSeedFromEnv = (env: Record<string, string | undefined>): number => {
193
200
  const raw = env["CHAOS_SEED"];
201
+
194
202
  if (raw === undefined || raw === "") return DEFAULT_CHAOS_SEED;
203
+
195
204
  return Option.getOrElse(decodeChaosSeedFromEnvironment(raw), () => DEFAULT_CHAOS_SEED);
196
205
  };
197
206
 
@@ -268,9 +277,12 @@ const planShapeArbitrary = (
268
277
  ChaosSubmissionSpec.make({ lane: index, kind: lane.kind }),
269
278
  ),
270
279
  );
280
+
271
281
  const [first, ...rest] = submissions;
282
+
272
283
  // `lanes` >= 1 and every lane has depth >= 1, so `first` always exists.
273
284
  if (first === undefined) throw new Error("chaos generator produced an empty plan");
285
+
274
286
  return {
275
287
  lanes: shape.lanes.length,
276
288
  submissions: [first, ...rest] as const,
@@ -292,6 +304,7 @@ export const generateChaosPlans = (options: ChaosGeneratorOptions): ReadonlyArra
292
304
  seed: options.seed,
293
305
  numRuns: options.count,
294
306
  });
307
+
295
308
  return sampled.map((shape, index) =>
296
309
  ChaosPlan.make({ ...shape, seed: (Math.imul(options.seed, 31) + index) | 0 }),
297
310
  );
@@ -300,10 +313,13 @@ export const generateChaosPlans = (options: ChaosGeneratorOptions): ReadonlyArra
300
313
  /** Deterministic PRNG for the runner's small ordering choices (lane drive order). */
301
314
  const mulberry32 = (seed: number): (() => number) => {
302
315
  let state = seed | 0;
316
+
303
317
  return () => {
304
318
  state = (state + 0x6d2b79f5) | 0;
305
319
  let t = Math.imul(state ^ (state >>> 15), 1 | state);
320
+
306
321
  t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
322
+
307
323
  return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
308
324
  };
309
325
  };
@@ -382,7 +398,9 @@ const BookUncertain = Tool.make("book", {
382
398
  parameters: Schema.Struct({ ref: Schema.String }),
383
399
  success: Schema.Struct({ confirmation: Schema.String }),
384
400
  });
401
+
385
402
  const bookTools = Toolkit.make(BookUncertain);
403
+
386
404
  const bookDefinition = Agent.make("chaos-book", {
387
405
  input: PlainInput,
388
406
  output: PlainOutput,
@@ -396,7 +414,9 @@ const BookApproval = Tool.make("book", {
396
414
  success: Schema.Struct({ confirmation: Schema.String }),
397
415
  needsApproval: true,
398
416
  });
417
+
399
418
  const approvalTools = Toolkit.make(BookApproval);
419
+
400
420
  const approvalDefinition = Agent.make("chaos-approval", {
401
421
  input: PlainInput,
402
422
  output: PlainOutput,
@@ -411,7 +431,9 @@ const Itinerary = Tool.make("itinerary", {
411
431
  failure: DurableStepError,
412
432
  dependencies: [DurableStep],
413
433
  }).annotate(ToolExecutionClass, "uncertain");
434
+
414
435
  const itineraryTools = Toolkit.make(Itinerary);
436
+
415
437
  const itineraryDefinition = Agent.make("chaos-itinerary", {
416
438
  input: PlainInput,
417
439
  output: PlainOutput,
@@ -467,16 +489,22 @@ const DELEGATE_CALL_ID = "chaos-delegate-1";
467
489
 
468
490
  const HEX = "0123456789abcdef";
469
491
  const decodeDigest = Schema.decodeSync(Digest);
492
+
470
493
  const laneDigests = (lane: number): DefinitionDigests => {
471
494
  const digest = decodeDigest(HEX.charAt(lane % 8).repeat(64));
495
+
472
496
  return DefinitionDigests.make({ agent: digest, model: digest, tools: digest });
473
497
  };
498
+
474
499
  const childDigestStrings = (lane: number) => {
475
500
  const char = HEX.charAt(8 + (lane % 8));
501
+
476
502
  return { agent: char.repeat(64), model: char.repeat(64), tools: char.repeat(64) } as const;
477
503
  };
504
+
478
505
  const childLaneDigests = (lane: number): DefinitionDigests => {
479
506
  const strings = childDigestStrings(lane);
507
+
480
508
  return DefinitionDigests.make({
481
509
  agent: decodeDigest(strings.agent),
482
510
  model: decodeDigest(strings.model),
@@ -496,10 +524,12 @@ const chaosIdentifiers = Layer.effect(
496
524
  IdGenerator,
497
525
  Effect.gen(function* () {
498
526
  const counter = yield* Ref.make(0);
527
+
499
528
  const next = <A>(decode: (value: string) => A, prefix: string) =>
500
529
  Ref.getAndUpdate(counter, (value) => value + 1).pipe(
501
530
  Effect.map((value) => decode(`${prefix}-${value}`)),
502
531
  );
532
+
503
533
  return {
504
534
  nextThreadId: next(decodeThreadId, "chaos-fixture-thread"),
505
535
  nextRunId: next(decodeRunId, "chaos-fixture-run"),
@@ -521,6 +551,7 @@ interface ChaosDesk {
521
551
 
522
552
  const makeChaosDesk: Effect.Effect<ChaosDesk> = Effect.gen(function* () {
523
553
  const produced = yield* Ref.make<ReadonlySet<string>>(new Set());
554
+
524
555
  return {
525
556
  produced: Ref.get(produced),
526
557
  record: (value: string) => Ref.update(produced, (current) => new Set(current).add(value)),
@@ -564,6 +595,7 @@ const tolerateTyped = <A, E, R>(
564
595
  if (Option.isSome(Cause.findErrorOption(exit.cause))) {
565
596
  return Effect.succeed(Option.none<A>());
566
597
  }
598
+
567
599
  return Effect.die(new Error(`chaos step died: ${Cause.pretty(exit.cause)}`));
568
600
  }),
569
601
  );
@@ -666,12 +698,14 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
666
698
  case "plain":
667
699
  case "join": {
668
700
  const agent = Agent.withModel(plainDefinition, model);
701
+
669
702
  return plainLaneFixture(false, runtime.processThread(agent, threadId), (flatIndex) =>
670
703
  runtime.submit(agent, { question: `chaos ${flatIndex}` }, submitOptionsFor(flatIndex)),
671
704
  );
672
705
  }
673
706
  case "uncertain-tool": {
674
707
  const agent = Agent.withModel(bookDefinition, model);
708
+
675
709
  return plainLaneFixture(
676
710
  true,
677
711
  runtime.processThread(agent, threadId).pipe(Effect.provide(bookToolLayerFor(bookTools))),
@@ -681,6 +715,7 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
681
715
  }
682
716
  case "approval": {
683
717
  const agent = Agent.withModel(approvalDefinition, model);
718
+
684
719
  return plainLaneFixture(
685
720
  true,
686
721
  runtime
@@ -692,23 +727,28 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
692
727
  }
693
728
  case "durable-steps": {
694
729
  const agent = Agent.withModel(itineraryDefinition, model);
730
+
695
731
  const toolLayer = itineraryTools.toLayer({
696
732
  itinerary: ({ ref: called }) =>
697
733
  Effect.gen(function* () {
698
734
  const step = yield* DurableStep;
735
+
699
736
  const flight = yield* step.do(
700
737
  "reserve-flight",
701
738
  Schema.String,
702
739
  desk.record(flightValue(called)).pipe(Effect.as(flightValue(called))),
703
740
  );
741
+
704
742
  const lodging = yield* step.do(
705
743
  "reserve-lodging",
706
744
  Schema.String,
707
745
  desk.record(lodgingValue(called)).pipe(Effect.as(lodgingValue(called))),
708
746
  );
747
+
709
748
  return { state: `${flight}+${lodging}` };
710
749
  }),
711
750
  });
751
+
712
752
  return plainLaneFixture(
713
753
  true,
714
754
  runtime.processThread(agent, threadId).pipe(Effect.provide(toolLayer)),
@@ -718,24 +758,38 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
718
758
  }
719
759
  case "delegation": {
720
760
  const parentBinding = Agent.withModel(coordinatorDefinition, model);
761
+
721
762
  const childModel = promptScriptedModel(`chaos-child-${laneIndex}`, () =>
722
763
  finalParts('{"answer":"child"}'),
723
764
  );
765
+
724
766
  const childBinding = Agent.withModel(childDefinition, childModel);
767
+
725
768
  const delegationLayer = SubagentRuntime.layer(chaosDelegation, childBinding, {
726
769
  mapChildFailure: (failure) => ChaosDelegationFailed.make({ childErrorTag: failure._tag }),
727
770
  durable: { targetDigests: childDigestStrings(laneIndex) },
728
771
  }).pipe(Layer.provide(delegationSupport));
772
+
729
773
  const parentResolved: ResolvedBinding = yield* DurableWorkerBinding.make(
730
774
  parentBinding,
731
775
  digests,
732
776
  ).pipe(Effect.provide(delegationLayer));
777
+
733
778
  const childResolved: ResolvedBinding = yield* DurableWorkerBinding.make(
734
779
  childBinding,
735
780
  childLaneDigests(laneIndex),
736
781
  );
737
- const bindings = [parentResolved, childResolved];
738
- const driveResolved = (thread: ThreadId) => runtime.processThreadResolved(thread, bindings);
782
+
783
+ const registeredRuntime = yield* DurableAgentRuntime.pipe(
784
+ Effect.provide(
785
+ DurableAgentRuntime.layerWithBindings([parentResolved, childResolved]).pipe(
786
+ Layer.provide(RunToolAuthorization.allowAll),
787
+ ),
788
+ ),
789
+ );
790
+
791
+ const driveResolved = (thread: ThreadId) => registeredRuntime.processThreadResolved(thread);
792
+
739
793
  const fixture: LaneFixture = {
740
794
  index: laneIndex,
741
795
  kind,
@@ -753,6 +807,7 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
753
807
  const drives: Array<
754
808
  Effect.Effect<ReadonlyArray<Settlement>, DurableWorkerFailure | DurableBindingFailure>
755
809
  > = [driveResolved(threadId)];
810
+
756
811
  if (firstReceipt !== undefined) {
757
812
  drives.push(
758
813
  driveResolved(
@@ -760,11 +815,13 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
760
815
  ),
761
816
  );
762
817
  }
818
+
763
819
  return drives;
764
820
  },
765
821
  childThreadOf: (firstReceipt) =>
766
822
  childThreadIdFor(firstReceipt.submissionId, decodeToolCallId(DELEGATE_CALL_ID)),
767
823
  };
824
+
768
825
  return fixture;
769
826
  }
770
827
  }
@@ -773,7 +830,9 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
773
830
  /** Stable per-call index into an injection list (identical across resolution passes). */
774
831
  const injectionIndex = (submissionFlatIndex: number, callId: string, length: number): number => {
775
832
  let hash = submissionFlatIndex + 1;
833
+
776
834
  for (const char of callId) hash = (Math.imul(hash, 31) + char.charCodeAt(0)) | 0;
835
+
777
836
  return ((hash % length) + length) % length;
778
837
  };
779
838
 
@@ -803,6 +862,7 @@ const resolutionFor = (
803
862
  isFailure: false,
804
863
  });
805
864
  }
865
+
806
866
  // The desk never produced a value for this call — resolving "completed" would fabricate.
807
867
  return ResolutionNeverHappened.make();
808
868
  }
@@ -821,8 +881,10 @@ const resolutionPass = Effect.fn("Chaos.resolutionPass")(function* (
821
881
  const ledger = yield* SubmissionLedger;
822
882
  const produced = yield* desk.produced;
823
883
  const nonterminal = yield* tolerateTyped(Stream.runCollect(ledger.scanNonterminal));
884
+
824
885
  if (Option.isNone(nonterminal)) return;
825
886
  const byId = new Map<SubmissionId, SubmissionState>();
887
+
826
888
  for (const state of states) {
827
889
  if (state.receipt !== undefined) byId.set(state.receipt.submissionId, state);
828
890
  }
@@ -830,18 +892,22 @@ const resolutionPass = Effect.fn("Chaos.resolutionPass")(function* (
830
892
  if (row.state !== "unknown" && row.state !== "suspended") continue;
831
893
  const state = byId.get(row.submissionId);
832
894
  const explanation = yield* tolerateTyped(runtime.explain(row.submissionId));
895
+
833
896
  if (Option.isNone(explanation)) continue;
834
897
  const flatIndex = state?.flatIndex ?? 0;
835
898
  const ref = state?.lane.ref ?? "ref-child";
899
+
836
900
  if (row.state === "unknown") {
837
901
  for (const call of explanation.value.evidence.unknownCalls) {
838
902
  if (call.resolved) continue;
903
+
839
904
  const kind =
840
905
  plan.resolutionInjections.length === 0
841
906
  ? "never-happened"
842
907
  : (plan.resolutionInjections.at(
843
908
  injectionIndex(flatIndex, call.toolCallId, plan.resolutionInjections.length),
844
909
  ) ?? "never-happened");
910
+
845
911
  yield* tolerateTyped(
846
912
  runtime.resolveUnknown(
847
913
  UnknownResolutionCommand.make({
@@ -862,6 +928,7 @@ const resolutionPass = Effect.fn("Chaos.resolutionPass")(function* (
862
928
  : (plan.approvalDecisions.at(
863
929
  injectionIndex(flatIndex, pending.toolCallId, plan.approvalDecisions.length),
864
930
  ) ?? "approved");
931
+
865
932
  yield* tolerateTyped(
866
933
  runtime.resolveApproval(
867
934
  ApprovalDecisionCommand.make({
@@ -882,8 +949,10 @@ const submissionIdsNamedBy = (
882
949
  records: ReadonlyArray<CanonicalRecordEnvelope>,
883
950
  ): ReadonlySet<SubmissionId> => {
884
951
  const named = new Set<SubmissionId>();
952
+
885
953
  for (const envelope of records) {
886
954
  const payload = envelope.record.payload;
955
+
887
956
  if (
888
957
  payload._tag === "UserInputRecorded" ||
889
958
  payload._tag === "SubmissionSettled" ||
@@ -892,6 +961,7 @@ const submissionIdsNamedBy = (
892
961
  if (payload.submissionId !== undefined) named.add(payload.submissionId);
893
962
  }
894
963
  }
964
+
895
965
  return named;
896
966
  };
897
967
 
@@ -911,18 +981,23 @@ const assertNoFabrication = (
911
981
  produced: ReadonlySet<string>,
912
982
  ): Effect.Effect<void, ChaosConvergenceFailure> => {
913
983
  const fabricated: Array<string> = [];
984
+
914
985
  const requireProduced = (value: string, label: string): void => {
915
986
  if (!produced.has(value)) fabricated.push(`${label} "${value}"`);
916
987
  };
988
+
917
989
  for (const envelope of records) {
918
990
  const payload = envelope.record.payload;
991
+
919
992
  if (payload._tag === "ToolCallSettled" && !payload.isFailure) {
920
993
  if (payload.toolName === "book") {
921
994
  const result = decodeBookResult(payload.result);
995
+
922
996
  if (Option.isSome(result)) requireProduced(result.value.confirmation, "book result");
923
997
  }
924
998
  if (payload.toolName === "itinerary") {
925
999
  const result = decodeItineraryResult(payload.result);
1000
+
926
1001
  if (Option.isSome(result)) {
927
1002
  for (const part of result.value.state.split("+")) {
928
1003
  requireProduced(part, "itinerary step result");
@@ -932,9 +1007,11 @@ const assertNoFabrication = (
932
1007
  }
933
1008
  if (payload._tag === "ToolStepSettled") {
934
1009
  const output = decodeStepOutput(payload.output);
1010
+
935
1011
  if (Option.isSome(output)) requireProduced(output.value, "step output");
936
1012
  }
937
1013
  }
1014
+
938
1015
  return fabricated.length === 0
939
1016
  ? Effect.void
940
1017
  : Effect.fail(
@@ -964,23 +1041,29 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
964
1041
  // Lane fixtures: the FIRST spec of each lane fixes the lane's agent kind.
965
1042
  const laneKinds = new Map<number, ChaosScenarioKind>();
966
1043
  const laneSubmissions = new Map<number, Array<number>>();
1044
+
967
1045
  plan.submissions.forEach((spec, flatIndex) => {
968
1046
  const lane = spec.lane % plan.lanes;
1047
+
969
1048
  if (!laneKinds.has(lane)) laneKinds.set(lane, spec.kind);
970
1049
  const list = laneSubmissions.get(lane) ?? [];
1050
+
971
1051
  list.push(flatIndex);
972
1052
  laneSubmissions.set(lane, list);
973
1053
  });
974
1054
  const lanes: Array<LaneFixture> = [];
1055
+
975
1056
  for (const [lane, kind] of laneKinds) {
976
1057
  lanes.push(yield* makeLaneFixture(plan, lane, kind, laneSubmissions.get(lane) ?? [], desk));
977
1058
  }
978
1059
 
979
1060
  const lanesByIndex = new Map(lanes.map((lane) => [lane.index, lane]));
980
1061
  const states: Array<SubmissionState> = [];
1062
+
981
1063
  for (const [flatIndex, spec] of plan.submissions.entries()) {
982
1064
  const laneIndex = spec.lane % plan.lanes;
983
1065
  const lane = lanesByIndex.get(laneIndex);
1066
+
984
1067
  if (lane === undefined) {
985
1068
  return yield* ChaosConvergenceFailure.make({
986
1069
  seed: plan.seed,
@@ -994,6 +1077,7 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
994
1077
  type ArmEntry =
995
1078
  | { readonly family: "coordinator"; readonly location: DurableRuntimeFailpointLocation }
996
1079
  | { readonly family: "adapter"; readonly location: string };
1080
+
997
1081
  const armQueue: Array<ArmEntry> = [
998
1082
  ...plan.failpointArms.map((location): ArmEntry => ({ family: "coordinator", location })),
999
1083
  ...plan.adapterArms.map((location): ArmEntry => ({ family: "adapter", location })),
@@ -1002,17 +1086,21 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
1002
1086
  const allSettled = Effect.gen(function* () {
1003
1087
  if (states.some((state) => state.receipt === undefined)) return false;
1004
1088
  const nonterminal = yield* tolerateTyped(Stream.runCollect(ledger.scanNonterminal));
1089
+
1005
1090
  return Option.isSome(nonterminal) && Array.from(nonterminal.value).length === 0;
1006
1091
  });
1007
1092
 
1008
1093
  const maxRounds = armQueue.length + states.length * 2 + 12;
1009
1094
  let rounds = 0;
1010
1095
  let converged = false;
1096
+
1011
1097
  for (let round = 0; round < maxRounds; round++) {
1012
1098
  rounds = round + 1;
1013
1099
  const arm = armQueue[round];
1100
+
1014
1101
  if (arm?.family === "coordinator") {
1015
1102
  const location = arm.location;
1103
+
1016
1104
  yield* failpoints.setHandler((hit) =>
1017
1105
  hit === location
1018
1106
  ? Effect.fail(DurableRuntimeFailpointError.make({ location: hit }))
@@ -1027,6 +1115,7 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
1027
1115
  for (const state of states) {
1028
1116
  if (state.receipt !== undefined) continue;
1029
1117
  const receipt = yield* tolerateTyped(state.lane.submitOne(state.flatIndex));
1118
+
1030
1119
  if (Option.isSome(receipt)) state.receipt = receipt.value;
1031
1120
  }
1032
1121
 
@@ -1035,9 +1124,11 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
1035
1124
  .map((lane) => ({ lane, rank: random() }))
1036
1125
  .sort((left, right) => left.rank - right.rank || left.lane.index - right.lane.index)
1037
1126
  .map(({ lane }) => lane);
1127
+
1038
1128
  for (const lane of order) {
1039
1129
  const firstFlat = lane.submissionIndexes[0];
1040
1130
  const firstReceipt = firstFlat === undefined ? undefined : states[firstFlat]?.receipt;
1131
+
1041
1132
  for (const drive of lane.drives(firstReceipt)) {
1042
1133
  yield* tolerateTyped(drive);
1043
1134
  }
@@ -1047,8 +1138,10 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
1047
1138
  if (round >= 1) {
1048
1139
  for (const rawIndex of plan.abortInjections) {
1049
1140
  const index = rawIndex % states.length;
1141
+
1050
1142
  if (appliedAborts.has(index)) continue;
1051
1143
  const receipt = states[index]?.receipt;
1144
+
1052
1145
  if (receipt === undefined) continue;
1053
1146
  appliedAborts.add(index);
1054
1147
  yield* tolerateTyped(
@@ -1082,11 +1175,13 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
1082
1175
 
1083
1176
  if (!converged) {
1084
1177
  const nonterminal = yield* tolerateTyped(Stream.runCollect(ledger.scanNonterminal));
1178
+
1085
1179
  const detail = Option.isSome(nonterminal)
1086
1180
  ? Array.from(nonterminal.value)
1087
1181
  .map((row: SubmissionSnapshot) => `${row.submissionId}(${row.state})`)
1088
1182
  .join(", ")
1089
1183
  : "ledger scan failed";
1184
+
1090
1185
  return yield* ChaosConvergenceFailure.make({
1091
1186
  seed: plan.seed,
1092
1187
  message: `plan did not converge within ${maxRounds} rounds; nonterminal: [${detail}]; pending receipts: ${states.filter((state) => state.receipt === undefined).length}`,
@@ -1097,6 +1192,7 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
1097
1192
  // with the full digest chain (single known producer), plus the desk non-fabrication sweep.
1098
1193
  const produced = yield* desk.produced;
1099
1194
  const laneReports: Array<ChaosLaneReport> = [];
1195
+
1100
1196
  const verifyThread = Effect.fn("Chaos.verifyThread")(function* (
1101
1197
  threadId: ThreadId,
1102
1198
  kind: ChaosScenarioKind,
@@ -1110,7 +1206,9 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
1110
1206
  }),
1111
1207
  ),
1112
1208
  );
1209
+
1113
1210
  const rows: Array<SubmissionSnapshot> = [];
1211
+
1114
1212
  for (const submissionId of submissionIdsNamedBy(exported.records)) {
1115
1213
  const found = yield* ledger.lookup(SubmissionLookupById.make({ submissionId })).pipe(
1116
1214
  Effect.mapError((error) =>
@@ -1120,22 +1218,27 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
1120
1218
  }),
1121
1219
  ),
1122
1220
  );
1221
+
1123
1222
  if (Option.isSome(found)) rows.push(found.value);
1124
1223
  }
1224
+
1125
1225
  const batchProducers = new Map<BatchId, ProducerId>(
1126
1226
  exported.records.map((envelope) => [envelope.batchId, config.producerId]),
1127
1227
  );
1228
+
1128
1229
  const report = yield* verifyThreadInvariants({
1129
1230
  export: exported,
1130
1231
  submissions: rows,
1131
1232
  batchProducers,
1132
1233
  requireAllSettled: true,
1133
1234
  });
1235
+
1134
1236
  if (!report.ok) {
1135
1237
  const failed = report.checks
1136
1238
  .filter((check) => check.status === "failed")
1137
1239
  .map((check) => `${check.name}: ${check.detail ?? "failed"}`)
1138
1240
  .join("; ");
1241
+
1139
1242
  return yield* ChaosConvergenceFailure.make({
1140
1243
  seed: plan.seed,
1141
1244
  message: `invariants failed for ${threadId} (${kind}): ${failed}`,
@@ -1159,12 +1262,16 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
1159
1262
  // Delegation lanes: verify every materialized child Thread too.
1160
1263
  for (const flatIndex of lane.submissionIndexes) {
1161
1264
  const receipt = states[flatIndex]?.receipt;
1265
+
1162
1266
  if (receipt === undefined) continue;
1163
1267
  const child = lane.childThreadOf(receipt);
1268
+
1164
1269
  if (child === undefined) continue;
1270
+
1165
1271
  const childExport = yield* Effect.exit(
1166
1272
  store.export(ThreadExportRequest.make({ threadId: child })),
1167
1273
  );
1274
+
1168
1275
  if (Exit.isSuccess(childExport) && childExport.value.records.length > 0) {
1169
1276
  yield* verifyThread(child, "plain", false);
1170
1277
  }
@@ -1181,6 +1288,7 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
1181
1288
  }),
1182
1289
  ),
1183
1290
  );
1291
+
1184
1292
  if (obligations.entries.length > 0) {
1185
1293
  return yield* ChaosConvergenceFailure.make({
1186
1294
  seed: plan.seed,
@@ -88,12 +88,14 @@ const respondingHost = (
88
88
  respond: (call: CodeHostCall) => CodeHostCallResult,
89
89
  ): { readonly host: CodeExecutionHost["Service"]; readonly calls: Array<CodeHostCall> } => {
90
90
  const calls: Array<CodeHostCall> = [];
91
+
91
92
  return {
92
93
  calls,
93
94
  host: {
94
95
  call: (call) =>
95
96
  Effect.sync(() => {
96
97
  calls.push(call);
98
+
97
99
  return respond(call);
98
100
  }),
99
101
  },
@@ -106,6 +108,7 @@ const runPass = (
106
108
  ): Effect.Effect<CodeExecutionResult, CodeExecutionError, CodeExecutor> =>
107
109
  Effect.gen(function* () {
108
110
  const executor = yield* CodeExecutor;
111
+
109
112
  return yield* executor
110
113
  .execute(request)
111
114
  .pipe(Effect.provideService(CodeExecutionHost, CodeExecutionHost.of(host)));
@@ -134,6 +137,7 @@ const expectSuccess = (
134
137
  ),
135
138
  Effect.flatMap((result) => {
136
139
  const complaint = check(result);
140
+
137
141
  return complaint === undefined ? Effect.void : Effect.fail(violation(caseName, complaint));
138
142
  }),
139
143
  );
@@ -157,6 +161,7 @@ const expectFailure = (
157
161
  );
158
162
  }
159
163
  const complaint = check?.(error);
164
+
160
165
  return complaint === undefined ? Effect.void : Effect.fail(violation(caseName, complaint));
161
166
  }),
162
167
  );
@@ -165,6 +170,7 @@ export const codeExecutorConformanceCases = (
165
170
  options: CodeExecutorConformanceOptions,
166
171
  ): ReadonlyArray<CodeExecutorConformanceCase> => {
167
172
  const posture = options.implementation;
173
+
168
174
  return [
169
175
  {
170
176
  name: "TEST-015 executes bounded JSON computation and returns the program value",
@@ -184,9 +190,11 @@ export const codeExecutorConformanceCases = (
184
190
  name: "CAP-015 reports its isolation posture honestly in results and errors",
185
191
  run: Effect.gen(function* () {
186
192
  const caseName = "CAP-015 reports its isolation posture honestly in results and errors";
193
+
187
194
  const result = yield* runPass(makeRequest("async () => 1"), unusedHost).pipe(
188
195
  Effect.mapError((error) => violation(caseName, `expected success, got ${error._tag}`)),
189
196
  );
197
+
190
198
  if (
191
199
  result.implementation.isolation !== posture.isolation ||
192
200
  result.implementation.identity !== posture.identity
@@ -196,10 +204,12 @@ export const codeExecutorConformanceCases = (
196
204
  `result posture ${preview(result.implementation)} does not match the declared ${preview(posture)}`,
197
205
  );
198
206
  }
207
+
199
208
  const error = yield* runPass(makeRequest("async () => {"), unusedHost).pipe(
200
209
  Effect.flip,
201
210
  Effect.mapError(() => violation(caseName, "expected the invalid-source pass to fail")),
202
211
  );
212
+
203
213
  // Every expected execution failure carries the posture; an adapter
204
214
  // omitting the field must fail this case, not slip past a probe.
205
215
  if (
@@ -219,11 +229,13 @@ export const codeExecutorConformanceCases = (
219
229
  run: Effect.gen(function* () {
220
230
  const caseName =
221
231
  "TEST-015 routes host calls through the CodeExecutionHost in program order";
232
+
222
233
  const { host, calls } = respondingHost((call) =>
223
234
  call.method === "query"
224
235
  ? CodeHostCallSuccess.make({ value: { rows: [1, 2, 3] } })
225
236
  : CodeHostCallSuccess.make({ value: 3 }),
226
237
  );
238
+
227
239
  const result = yield* runPass(
228
240
  makeRequest(
229
241
  "async () => { const q = await warehouse.query({ sql: 'select' }); const c = await warehouse.count({ table: 't' }); return { rows: q.rows, count: c }; }",
@@ -235,10 +247,12 @@ export const codeExecutorConformanceCases = (
235
247
  violation(caseName, `expected success, got ${error._tag}: ${preview(error)}`),
236
248
  ),
237
249
  );
250
+
238
251
  if (JSON.stringify(result.value) !== JSON.stringify({ rows: [1, 2, 3], count: 3 })) {
239
252
  return yield* violation(caseName, `unexpected value ${preview(result.value)}`);
240
253
  }
241
254
  const observed = calls.map((call) => `${call.namespace}.${call.method}`);
255
+
242
256
  if (JSON.stringify(observed) !== JSON.stringify(["warehouse.query", "warehouse.count"])) {
243
257
  return yield* violation(caseName, `unexpected host call order ${preview(observed)}`);
244
258
  }
@@ -376,6 +390,7 @@ export const codeExecutorConformanceCases = (
376
390
  run: Effect.gen(function* () {
377
391
  const caseName = "TEST-015 fails typed when host calls exceed the executor cap";
378
392
  const { host, calls } = respondingHost(() => CodeHostCallSuccess.make({ value: null }));
393
+
379
394
  yield* expectFailure(
380
395
  caseName,
381
396
  makeRequest(
@@ -462,6 +477,7 @@ export const codeExecutorConformanceCases = (
462
477
  const caseName = "TEST-015 interruption reaches in-flight host calls and pass teardown";
463
478
  const started = yield* Deferred.make<void>();
464
479
  const witness = { hostCallInterrupted: false };
480
+
465
481
  const host: CodeExecutionHost["Service"] = {
466
482
  call: () =>
467
483
  Deferred.succeed(started, undefined).pipe(
@@ -473,16 +489,19 @@ export const codeExecutorConformanceCases = (
473
489
  ),
474
490
  ),
475
491
  };
492
+
476
493
  const fiber = yield* runPass(
477
494
  makeRequest("async () => warehouse.query({})", { namespaces: [warehouseNamespace] }),
478
495
  host,
479
496
  ).pipe(Effect.forkChild);
497
+
480
498
  // Guard against a broken adapter that settles the pass without ever
481
499
  // reaching the host: the case must report a violation, not hang.
482
500
  const winner = yield* Effect.raceFirst(
483
501
  Deferred.await(started).pipe(Effect.as("started" as const)),
484
502
  Fiber.join(fiber).pipe(Effect.exit, Effect.as("exited" as const)),
485
503
  );
504
+
486
505
  if (winner === "exited") {
487
506
  return yield* violation(
488
507
  caseName,