@effect-agent/testing 0.1.0-beta.8 → 0.1.0-beta.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Certification.d.mts +105 -0
- package/dist/Certification.mjs +592 -0
- package/dist/Certification.mjs.map +1 -0
- package/dist/Chaos.d.mts +126 -0
- package/dist/Chaos.mjs +755 -0
- package/dist/Chaos.mjs.map +1 -0
- package/dist/CodeExecutorConformance.d.mts +33 -0
- package/dist/CodeExecutorConformance.mjs +231 -0
- package/dist/CodeExecutorConformance.mjs.map +1 -0
- package/dist/CodeExecutorSubstitute.d.mts +21 -0
- package/dist/CodeExecutorSubstitute.mjs +368 -0
- package/dist/CodeExecutorSubstitute.mjs.map +1 -0
- package/dist/DocsResearcher.d.mts +270 -0
- package/dist/DocsResearcher.mjs +490 -0
- package/dist/DocsResearcher.mjs.map +1 -0
- package/dist/ScriptedModel-DAvxIiud.d.mts +220 -0
- package/dist/ScriptedModel.d.mts +2 -0
- package/dist/ScriptedModel.mjs +155 -0
- package/dist/ScriptedModel.mjs.map +1 -0
- package/dist/TravelPlanner.d.mts +1665 -0
- package/dist/TravelPlanner.mjs +1963 -0
- package/dist/TravelPlanner.mjs.map +1 -0
- package/dist/deterministic-layers-Eka0fMZq.mjs +358 -0
- package/dist/deterministic-layers-Eka0fMZq.mjs.map +1 -0
- package/dist/index.d.mts +2 -3406
- package/dist/index.mjs +2 -4771
- package/dist/rolldown-runtime-D7D4PA-g.mjs +13 -0
- package/package.json +1 -48
- package/src/{certification.ts → Certification.ts} +251 -117
- package/src/{chaos.ts → Chaos.ts} +246 -122
- package/src/{code-executor-conformance.ts → CodeExecutorConformance.ts} +29 -6
- package/src/{code-executor-substitute.ts → CodeExecutorSubstitute.ts} +112 -45
- package/src/{fixtures/docs-researcher/index.ts → DocsResearcher.ts} +68 -5
- package/src/{scripted-model.ts → ScriptedModel.ts} +22 -25
- package/src/TravelPlanner.ts +232 -0
- package/src/fixtures/docs-researcher/definition.ts +19 -10
- package/src/fixtures/docs-researcher/harness.ts +41 -30
- package/src/fixtures/docs-researcher/mcp.ts +49 -3
- package/src/fixtures/travel-planner/definition.ts +15 -2
- package/src/fixtures/travel-planner/deterministic-layers.ts +47 -4
- package/src/fixtures/travel-planner/phase2.ts +4 -3
- package/src/fixtures/travel-planner/phase3.ts +18 -41
- package/src/fixtures/travel-planner/phase4.ts +23 -36
- package/src/fixtures/travel-planner/phase5.ts +38 -41
- package/src/fixtures/travel-planner/phase6.ts +187 -84
- package/src/fixtures/travel-planner/phase7.ts +4 -102
- package/src/fixtures/travel-planner/scenarios.ts +3 -4
- package/src/fixtures/travel-planner/subagents-durable.ts +33 -57
- package/src/fixtures/travel-planner/subagents.ts +35 -13
- package/src/index.ts +1 -11
- package/dist/index.mjs.map +0 -1
- package/src/code-executor-conformance.d.ts +0 -30
- package/src/fixtures/travel-planner/index.ts +0 -11
- package/src/fixtures/warehouse/index.ts +0 -412
|
@@ -1,36 +1,51 @@
|
|
|
1
|
+
import * as Subagent from "@effect-agent/capabilities/Subagent";
|
|
2
|
+
import { SubagentPolicy, SubagentRuntime } from "@effect-agent/capabilities/Subagent";
|
|
3
|
+
import { SubagentReservationsMemoryLive } from "@effect-agent/capabilities/SubagentReservations";
|
|
4
|
+
import * as Agent from "@effect-agent/core/Agent";
|
|
5
|
+
import { AgentPolicy } from "@effect-agent/core/AgentPolicy";
|
|
1
6
|
import {
|
|
2
|
-
|
|
3
|
-
SubagentPolicy,
|
|
4
|
-
SubagentReservationsMemoryLive,
|
|
5
|
-
SubagentRuntime,
|
|
6
|
-
} from "@effect-agent/capabilities";
|
|
7
|
-
import {
|
|
8
|
-
Agent,
|
|
9
|
-
AgentPolicy,
|
|
10
|
-
ConversationId,
|
|
11
|
-
IdGenerator,
|
|
7
|
+
ThreadId,
|
|
12
8
|
RunId,
|
|
13
|
-
SubmissionId,
|
|
14
9
|
ToolCallId,
|
|
15
10
|
TurnId,
|
|
16
|
-
|
|
17
|
-
|
|
11
|
+
type SubmissionId,
|
|
12
|
+
} from "@effect-agent/core/Identifiers";
|
|
13
|
+
import { IdGenerator } from "@effect-agent/core/IdGenerator";
|
|
14
|
+
import {
|
|
15
|
+
DurableStep,
|
|
16
|
+
DurableStepError,
|
|
17
|
+
ToolExecutionClass,
|
|
18
|
+
} from "@effect-agent/engine/DurableStep";
|
|
19
|
+
import { RunToolAuthorization } from "@effect-agent/engine/RunOptions";
|
|
20
|
+
import { ObligationThresholds } from "@effect-agent/thread/Admin";
|
|
21
|
+
import {
|
|
22
|
+
DurableWorkerBinding,
|
|
23
|
+
type DurableBindingFailure,
|
|
24
|
+
type ResolvedBinding,
|
|
25
|
+
} from "@effect-agent/thread/AgentRegistration";
|
|
18
26
|
import {
|
|
19
|
-
AbortCommand,
|
|
20
|
-
AgentBindingResolver,
|
|
21
|
-
ApprovalDecisionCommand,
|
|
22
|
-
ConversationExportRequest,
|
|
23
|
-
ConversationStore,
|
|
24
|
-
DefinitionDigests,
|
|
25
|
-
Digest,
|
|
26
27
|
DurableAgentRuntime,
|
|
27
28
|
DurableRuntimeConfig,
|
|
29
|
+
type DurableSubmitFailure,
|
|
30
|
+
type DurableWorkerFailure,
|
|
31
|
+
type Receipt,
|
|
32
|
+
} from "@effect-agent/thread/DurableAgentRuntime";
|
|
33
|
+
import {
|
|
28
34
|
DurableRuntimeFailpointError,
|
|
29
35
|
DurableRuntimeFailpointLocation,
|
|
30
|
-
|
|
31
|
-
|
|
36
|
+
} from "@effect-agent/thread/DurableFailpoint";
|
|
37
|
+
import {
|
|
38
|
+
DefinitionDigests,
|
|
39
|
+
Digest,
|
|
40
|
+
type CanonicalRecordEnvelope,
|
|
41
|
+
type BatchId,
|
|
42
|
+
type ProducerId,
|
|
43
|
+
} from "@effect-agent/thread/Records";
|
|
44
|
+
import { childThreadIdFor } from "@effect-agent/thread/RunJournal";
|
|
45
|
+
import {
|
|
46
|
+
AbortCommand,
|
|
47
|
+
ApprovalDecisionCommand,
|
|
32
48
|
IdempotencyKey,
|
|
33
|
-
ObligationThresholds,
|
|
34
49
|
Principal,
|
|
35
50
|
ResolutionAbortSubmission,
|
|
36
51
|
ResolutionCompletedWithResult,
|
|
@@ -38,20 +53,23 @@ import {
|
|
|
38
53
|
SubmissionLedger,
|
|
39
54
|
SubmissionLookupById,
|
|
40
55
|
UnknownResolutionCommand,
|
|
41
|
-
childConversationIdFor,
|
|
42
|
-
verifyConversationInvariants,
|
|
43
|
-
type CanonicalRecordEnvelope,
|
|
44
|
-
type BatchId,
|
|
45
|
-
type ProducerId,
|
|
46
|
-
type Receipt,
|
|
47
|
-
type ResolvedBinding,
|
|
48
56
|
type Settlement,
|
|
49
57
|
type SubmissionSnapshot,
|
|
50
58
|
type UnknownResolution,
|
|
51
|
-
} from "@effect-agent/
|
|
59
|
+
} from "@effect-agent/thread/SubmissionLedger";
|
|
60
|
+
import { DurableRuntimeFailpointTestControl } from "@effect-agent/thread/testing/DurableFailpointTestControl";
|
|
61
|
+
import { verifyThreadInvariants } from "@effect-agent/thread/ThreadInvariants";
|
|
62
|
+
import { ThreadExportRequest, ThreadStore } from "@effect-agent/thread/ThreadStore";
|
|
52
63
|
import { Cause, Effect, Exit, Layer, Option, Ref, Schema, Stream } from "effect";
|
|
53
64
|
import { FastCheck } from "effect/testing";
|
|
54
|
-
import {
|
|
65
|
+
import {
|
|
66
|
+
LanguageModel,
|
|
67
|
+
Model,
|
|
68
|
+
Tool,
|
|
69
|
+
Toolkit,
|
|
70
|
+
type Prompt,
|
|
71
|
+
type Response,
|
|
72
|
+
} from "effect/unstable/ai";
|
|
55
73
|
|
|
56
74
|
/**
|
|
57
75
|
* P7 WP4 chaos machinery (plan §5): a Schema-first `ChaosPlan`, a seeded generator over
|
|
@@ -59,7 +77,7 @@ import { LanguageModel, Model, Prompt, Tool, Toolkit, type Response } from "effe
|
|
|
59
77
|
* deterministic runner that drives the durable coordinator over whatever adapter pair the test
|
|
60
78
|
* provides. Every plan ends in the SAME claims the crash matrices make:
|
|
61
79
|
*
|
|
62
|
-
* 1. `
|
|
80
|
+
* 1. `verifyThreadInvariants` in convergence mode over every touched Thread (the
|
|
63
81
|
* shared WP1 checker — one set of claims for admin verify, certification, chaos, and soak);
|
|
64
82
|
* 2. `scanObligations` returning ZERO entries (everything settled; nothing invisibly stuck);
|
|
65
83
|
* 3. supplier non-fabrication wherever the deterministic desk was in play (durability §10: no
|
|
@@ -83,6 +101,7 @@ export const ChaosScenarioKind = Schema.Literals([
|
|
|
83
101
|
"join",
|
|
84
102
|
"delegation",
|
|
85
103
|
]);
|
|
104
|
+
|
|
86
105
|
export type ChaosScenarioKind = typeof ChaosScenarioKind.Type;
|
|
87
106
|
|
|
88
107
|
const LaneIndex = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0), Schema.isLessThanOrEqualTo(7));
|
|
@@ -108,6 +127,7 @@ export const ChaosResolutionKind = Schema.Literals([
|
|
|
108
127
|
/** Unresolvable: route into the abort path (settles aborted, audit retained). */
|
|
109
128
|
"abort-submission",
|
|
110
129
|
]);
|
|
130
|
+
|
|
111
131
|
export type ChaosResolutionKind = typeof ChaosResolutionKind.Type;
|
|
112
132
|
|
|
113
133
|
export const ChaosApprovalDecision = Schema.Literals(["approved", "denied"]);
|
|
@@ -142,10 +162,10 @@ export class ChaosPlan extends Schema.Class<ChaosPlan>("@effect-agent/testing/Ch
|
|
|
142
162
|
export class ChaosLaneReport extends Schema.Class<ChaosLaneReport>(
|
|
143
163
|
"@effect-agent/testing/ChaosLaneReport",
|
|
144
164
|
)({
|
|
145
|
-
|
|
165
|
+
threadId: ThreadId,
|
|
146
166
|
kind: ChaosScenarioKind,
|
|
147
167
|
submissionCount: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
148
|
-
/** Verdict of `
|
|
168
|
+
/** Verdict of `verifyThreadInvariants` in convergence mode. */
|
|
149
169
|
verified: Schema.Boolean,
|
|
150
170
|
}) {}
|
|
151
171
|
|
|
@@ -176,12 +196,16 @@ export class ChaosConvergenceFailure extends Schema.TaggedError<ChaosConvergence
|
|
|
176
196
|
/** Default root seed for chaos suites; override with the `CHAOS_SEED` environment variable. */
|
|
177
197
|
export const DEFAULT_CHAOS_SEED = 20260813;
|
|
178
198
|
|
|
199
|
+
const ChaosSeedFromEnvironment = Schema.FiniteFromString.check(Schema.isInt());
|
|
200
|
+
const decodeChaosSeedFromEnvironment = Schema.decodeUnknownOption(ChaosSeedFromEnvironment);
|
|
201
|
+
|
|
179
202
|
/** The root seed for this run: `CHAOS_SEED` when set to an integer, the default otherwise. */
|
|
180
203
|
export const chaosSeedFromEnv = (env: Record<string, string | undefined>): number => {
|
|
181
204
|
const raw = env["CHAOS_SEED"];
|
|
205
|
+
|
|
182
206
|
if (raw === undefined || raw === "") return DEFAULT_CHAOS_SEED;
|
|
183
|
-
|
|
184
|
-
return
|
|
207
|
+
|
|
208
|
+
return Option.getOrElse(decodeChaosSeedFromEnvironment(raw), () => DEFAULT_CHAOS_SEED);
|
|
185
209
|
};
|
|
186
210
|
|
|
187
211
|
export interface ChaosGeneratorOptions {
|
|
@@ -205,13 +229,12 @@ const laneArbitrary: FastCheck.Arbitrary<GeneratedLane> = FastCheck.constantFrom
|
|
|
205
229
|
"approval",
|
|
206
230
|
"join",
|
|
207
231
|
"delegation",
|
|
208
|
-
).chain(
|
|
209
|
-
|
|
210
|
-
kind
|
|
211
|
-
|
|
212
|
-
: kind
|
|
213
|
-
|
|
214
|
-
: FastCheck.constant<GeneratedLane>({ kind, depth: 1 }),
|
|
232
|
+
).chain((kind): FastCheck.Arbitrary<GeneratedLane> =>
|
|
233
|
+
kind === "join"
|
|
234
|
+
? FastCheck.integer({ min: 2, max: 3 }).map((depth): GeneratedLane => ({ kind, depth }))
|
|
235
|
+
: kind === "plain"
|
|
236
|
+
? FastCheck.integer({ min: 1, max: 2 }).map((depth): GeneratedLane => ({ kind, depth }))
|
|
237
|
+
: FastCheck.constant<GeneratedLane>({ kind, depth: 1 }),
|
|
215
238
|
);
|
|
216
239
|
|
|
217
240
|
interface ChaosPlanShape {
|
|
@@ -258,9 +281,12 @@ const planShapeArbitrary = (
|
|
|
258
281
|
ChaosSubmissionSpec.make({ lane: index, kind: lane.kind }),
|
|
259
282
|
),
|
|
260
283
|
);
|
|
284
|
+
|
|
261
285
|
const [first, ...rest] = submissions;
|
|
286
|
+
|
|
262
287
|
// `lanes` >= 1 and every lane has depth >= 1, so `first` always exists.
|
|
263
288
|
if (first === undefined) throw new Error("chaos generator produced an empty plan");
|
|
289
|
+
|
|
264
290
|
return {
|
|
265
291
|
lanes: shape.lanes.length,
|
|
266
292
|
submissions: [first, ...rest] as const,
|
|
@@ -282,6 +308,7 @@ export const generateChaosPlans = (options: ChaosGeneratorOptions): ReadonlyArra
|
|
|
282
308
|
seed: options.seed,
|
|
283
309
|
numRuns: options.count,
|
|
284
310
|
});
|
|
311
|
+
|
|
285
312
|
return sampled.map((shape, index) =>
|
|
286
313
|
ChaosPlan.make({ ...shape, seed: (Math.imul(options.seed, 31) + index) | 0 }),
|
|
287
314
|
);
|
|
@@ -290,10 +317,13 @@ export const generateChaosPlans = (options: ChaosGeneratorOptions): ReadonlyArra
|
|
|
290
317
|
/** Deterministic PRNG for the runner's small ordering choices (lane drive order). */
|
|
291
318
|
const mulberry32 = (seed: number): (() => number) => {
|
|
292
319
|
let state = seed | 0;
|
|
320
|
+
|
|
293
321
|
return () => {
|
|
294
322
|
state = (state + 0x6d2b79f5) | 0;
|
|
295
323
|
let t = Math.imul(state ^ (state >>> 15), 1 | state);
|
|
324
|
+
|
|
296
325
|
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
|
|
326
|
+
|
|
297
327
|
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
|
298
328
|
};
|
|
299
329
|
};
|
|
@@ -359,7 +389,7 @@ const policy = AgentPolicy.make({
|
|
|
359
389
|
const PlainInput = Schema.Struct({ question: Schema.String });
|
|
360
390
|
const PlainOutput = Schema.Struct({ answer: Schema.String });
|
|
361
391
|
|
|
362
|
-
const plainDefinition = Agent.
|
|
392
|
+
const plainDefinition = Agent.make("chaos-plain", {
|
|
363
393
|
input: PlainInput,
|
|
364
394
|
output: PlainOutput,
|
|
365
395
|
instructions: "Answer as JSON.",
|
|
@@ -372,8 +402,10 @@ const BookUncertain = Tool.make("book", {
|
|
|
372
402
|
parameters: Schema.Struct({ ref: Schema.String }),
|
|
373
403
|
success: Schema.Struct({ confirmation: Schema.String }),
|
|
374
404
|
});
|
|
405
|
+
|
|
375
406
|
const bookTools = Toolkit.make(BookUncertain);
|
|
376
|
-
|
|
407
|
+
|
|
408
|
+
const bookDefinition = Agent.make("chaos-book", {
|
|
377
409
|
input: PlainInput,
|
|
378
410
|
output: PlainOutput,
|
|
379
411
|
instructions: "Book it.",
|
|
@@ -386,8 +418,10 @@ const BookApproval = Tool.make("book", {
|
|
|
386
418
|
success: Schema.Struct({ confirmation: Schema.String }),
|
|
387
419
|
needsApproval: true,
|
|
388
420
|
});
|
|
421
|
+
|
|
389
422
|
const approvalTools = Toolkit.make(BookApproval);
|
|
390
|
-
|
|
423
|
+
|
|
424
|
+
const approvalDefinition = Agent.make("chaos-approval", {
|
|
391
425
|
input: PlainInput,
|
|
392
426
|
output: PlainOutput,
|
|
393
427
|
instructions: "Book after approval.",
|
|
@@ -401,8 +435,10 @@ const Itinerary = Tool.make("itinerary", {
|
|
|
401
435
|
failure: DurableStepError,
|
|
402
436
|
dependencies: [DurableStep],
|
|
403
437
|
}).annotate(ToolExecutionClass, "uncertain");
|
|
438
|
+
|
|
404
439
|
const itineraryTools = Toolkit.make(Itinerary);
|
|
405
|
-
|
|
440
|
+
|
|
441
|
+
const itineraryDefinition = Agent.make("chaos-itinerary", {
|
|
406
442
|
input: PlainInput,
|
|
407
443
|
output: PlainOutput,
|
|
408
444
|
instructions: "Reserve the itinerary.",
|
|
@@ -410,7 +446,7 @@ const itineraryDefinition = Agent.define("chaos-itinerary", {
|
|
|
410
446
|
policy,
|
|
411
447
|
});
|
|
412
448
|
|
|
413
|
-
const childDefinition = Agent.
|
|
449
|
+
const childDefinition = Agent.make("chaos-child", {
|
|
414
450
|
input: PlainInput,
|
|
415
451
|
output: PlainOutput,
|
|
416
452
|
instructions: "Answer as JSON.",
|
|
@@ -445,7 +481,7 @@ const chaosDelegation = Subagent.define("delegate_chaos", {
|
|
|
445
481
|
}),
|
|
446
482
|
});
|
|
447
483
|
|
|
448
|
-
const coordinatorDefinition = Agent.
|
|
484
|
+
const coordinatorDefinition = Agent.make("chaos-coordinator", {
|
|
449
485
|
input: Schema.Struct({ mission: Schema.String }),
|
|
450
486
|
output: Schema.Struct({ report: Schema.String }),
|
|
451
487
|
instructions: "Delegate, then report as JSON.",
|
|
@@ -457,16 +493,22 @@ const DELEGATE_CALL_ID = "chaos-delegate-1";
|
|
|
457
493
|
|
|
458
494
|
const HEX = "0123456789abcdef";
|
|
459
495
|
const decodeDigest = Schema.decodeSync(Digest);
|
|
496
|
+
|
|
460
497
|
const laneDigests = (lane: number): DefinitionDigests => {
|
|
461
|
-
const digest = decodeDigest(HEX
|
|
498
|
+
const digest = decodeDigest(HEX.charAt(lane % 8).repeat(64));
|
|
499
|
+
|
|
462
500
|
return DefinitionDigests.make({ agent: digest, model: digest, tools: digest });
|
|
463
501
|
};
|
|
502
|
+
|
|
464
503
|
const childDigestStrings = (lane: number) => {
|
|
465
|
-
const char = HEX
|
|
504
|
+
const char = HEX.charAt(8 + (lane % 8));
|
|
505
|
+
|
|
466
506
|
return { agent: char.repeat(64), model: char.repeat(64), tools: char.repeat(64) } as const;
|
|
467
507
|
};
|
|
508
|
+
|
|
468
509
|
const childLaneDigests = (lane: number): DefinitionDigests => {
|
|
469
510
|
const strings = childDigestStrings(lane);
|
|
511
|
+
|
|
470
512
|
return DefinitionDigests.make({
|
|
471
513
|
agent: decodeDigest(strings.agent),
|
|
472
514
|
model: decodeDigest(strings.model),
|
|
@@ -475,7 +517,7 @@ const childLaneDigests = (lane: number): DefinitionDigests => {
|
|
|
475
517
|
};
|
|
476
518
|
|
|
477
519
|
const CHAOS_PRINCIPAL = Schema.decodeSync(Principal)("principal-chaos");
|
|
478
|
-
const
|
|
520
|
+
const decodeThreadId = Schema.decodeSync(ThreadId);
|
|
479
521
|
const decodeIdempotencyKey = Schema.decodeSync(IdempotencyKey);
|
|
480
522
|
const decodeToolCallId = Schema.decodeSync(ToolCallId);
|
|
481
523
|
const decodeRunId = Schema.decodeSync(RunId);
|
|
@@ -486,12 +528,14 @@ const chaosIdentifiers = Layer.effect(
|
|
|
486
528
|
IdGenerator,
|
|
487
529
|
Effect.gen(function* () {
|
|
488
530
|
const counter = yield* Ref.make(0);
|
|
531
|
+
|
|
489
532
|
const next = <A>(decode: (value: string) => A, prefix: string) =>
|
|
490
533
|
Ref.getAndUpdate(counter, (value) => value + 1).pipe(
|
|
491
534
|
Effect.map((value) => decode(`${prefix}-${value}`)),
|
|
492
535
|
);
|
|
536
|
+
|
|
493
537
|
return {
|
|
494
|
-
|
|
538
|
+
nextThreadId: next(decodeThreadId, "chaos-fixture-thread"),
|
|
495
539
|
nextRunId: next(decodeRunId, "chaos-fixture-run"),
|
|
496
540
|
nextTurnId: next(decodeTurnId, "chaos-fixture-turn"),
|
|
497
541
|
};
|
|
@@ -511,6 +555,7 @@ interface ChaosDesk {
|
|
|
511
555
|
|
|
512
556
|
const makeChaosDesk: Effect.Effect<ChaosDesk> = Effect.gen(function* () {
|
|
513
557
|
const produced = yield* Ref.make<ReadonlySet<string>>(new Set());
|
|
558
|
+
|
|
514
559
|
return {
|
|
515
560
|
produced: Ref.get(produced),
|
|
516
561
|
record: (value: string) => Ref.update(produced, (current) => new Set(current).add(value)),
|
|
@@ -543,7 +588,7 @@ export interface ChaosRunOptions {
|
|
|
543
588
|
readonly betweenRounds?: Effect.Effect<void> | undefined;
|
|
544
589
|
}
|
|
545
590
|
|
|
546
|
-
/**
|
|
591
|
+
/** Tolerate typed failures while preserving every defect and interruption reason. */
|
|
547
592
|
const tolerateTyped = <A, E, R>(
|
|
548
593
|
effect: Effect.Effect<A, E, R>,
|
|
549
594
|
): Effect.Effect<Option.Option<A>, never, R> =>
|
|
@@ -551,25 +596,28 @@ const tolerateTyped = <A, E, R>(
|
|
|
551
596
|
Effect.exit,
|
|
552
597
|
Effect.flatMap((exit) => {
|
|
553
598
|
if (Exit.isSuccess(exit)) return Effect.succeed(Option.some(exit.value));
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
599
|
+
const unexpected = exit.cause.reasons.filter((reason) => reason._tag !== "Fail");
|
|
600
|
+
|
|
601
|
+
return unexpected.length === 0
|
|
602
|
+
? Effect.succeed(Option.none<A>())
|
|
603
|
+
: Effect.failCause(Cause.fromReasons<never>(unexpected));
|
|
558
604
|
}),
|
|
559
605
|
);
|
|
560
606
|
|
|
561
607
|
interface LaneFixture {
|
|
562
608
|
readonly index: number;
|
|
563
609
|
readonly kind: ChaosScenarioKind;
|
|
564
|
-
readonly
|
|
610
|
+
readonly threadId: ThreadId;
|
|
565
611
|
readonly ref: string;
|
|
566
612
|
readonly deskInPlay: boolean;
|
|
567
613
|
readonly submissionIndexes: ReadonlyArray<number>;
|
|
568
|
-
readonly submitOne: (flatIndex: number) => Effect.Effect<Receipt,
|
|
614
|
+
readonly submitOne: (flatIndex: number) => Effect.Effect<Receipt, DurableSubmitFailure>;
|
|
569
615
|
readonly drives: (
|
|
570
616
|
firstReceipt: Receipt | undefined,
|
|
571
|
-
) => ReadonlyArray<
|
|
572
|
-
|
|
617
|
+
) => ReadonlyArray<
|
|
618
|
+
Effect.Effect<ReadonlyArray<Settlement>, DurableWorkerFailure | DurableBindingFailure>
|
|
619
|
+
>;
|
|
620
|
+
readonly childThreadOf: (firstReceipt: Receipt) => ThreadId | undefined;
|
|
573
621
|
}
|
|
574
622
|
|
|
575
623
|
interface SubmissionState {
|
|
@@ -613,14 +661,14 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
613
661
|
desk: ChaosDesk,
|
|
614
662
|
) {
|
|
615
663
|
const runtime = yield* DurableAgentRuntime;
|
|
616
|
-
const
|
|
664
|
+
const threadId = decodeThreadId(`chaos-${plan.seed}-lane-${laneIndex}`);
|
|
617
665
|
const ref = `ref-l${laneIndex}`;
|
|
618
666
|
const script = scriptFor(kind, ref);
|
|
619
667
|
const model = promptScriptedModel(`chaos-${kind}-${laneIndex}`, script);
|
|
620
668
|
const digests = laneDigests(laneIndex);
|
|
621
669
|
|
|
622
670
|
const submitOptionsFor = (flatIndex: number) => ({
|
|
623
|
-
|
|
671
|
+
threadId,
|
|
624
672
|
principal: CHAOS_PRINCIPAL,
|
|
625
673
|
idempotencyKey: decodeIdempotencyKey(`chaos-${plan.seed}-s${flatIndex}`),
|
|
626
674
|
definitions: digests,
|
|
@@ -636,48 +684,46 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
636
684
|
|
|
637
685
|
const plainLaneFixture = (
|
|
638
686
|
deskInPlay: boolean,
|
|
639
|
-
drive: Effect.Effect<ReadonlyArray<Settlement>,
|
|
640
|
-
submitOne: (flatIndex: number) => Effect.Effect<Receipt,
|
|
687
|
+
drive: Effect.Effect<ReadonlyArray<Settlement>, DurableWorkerFailure | DurableBindingFailure>,
|
|
688
|
+
submitOne: (flatIndex: number) => Effect.Effect<Receipt, DurableSubmitFailure>,
|
|
641
689
|
): LaneFixture => ({
|
|
642
690
|
index: laneIndex,
|
|
643
691
|
kind,
|
|
644
|
-
|
|
692
|
+
threadId,
|
|
645
693
|
ref,
|
|
646
694
|
deskInPlay,
|
|
647
695
|
submissionIndexes,
|
|
648
696
|
submitOne,
|
|
649
697
|
drives: () => [drive],
|
|
650
|
-
|
|
698
|
+
childThreadOf: () => undefined,
|
|
651
699
|
});
|
|
652
700
|
|
|
653
701
|
switch (kind) {
|
|
654
702
|
case "plain":
|
|
655
703
|
case "join": {
|
|
656
704
|
const agent = Agent.withModel(plainDefinition, model);
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
runtime.
|
|
660
|
-
(flatIndex) =>
|
|
661
|
-
runtime.submit(agent, { question: `chaos ${flatIndex}` }, submitOptionsFor(flatIndex)),
|
|
705
|
+
|
|
706
|
+
return plainLaneFixture(false, runtime.processThread(agent, threadId), (flatIndex) =>
|
|
707
|
+
runtime.submit(agent, { question: `chaos ${flatIndex}` }, submitOptionsFor(flatIndex)),
|
|
662
708
|
);
|
|
663
709
|
}
|
|
664
710
|
case "uncertain-tool": {
|
|
665
711
|
const agent = Agent.withModel(bookDefinition, model);
|
|
712
|
+
|
|
666
713
|
return plainLaneFixture(
|
|
667
714
|
true,
|
|
668
|
-
runtime
|
|
669
|
-
.processConversation(agent, conversationId)
|
|
670
|
-
.pipe(Effect.provide(bookToolLayerFor(bookTools))),
|
|
715
|
+
runtime.processThread(agent, threadId).pipe(Effect.provide(bookToolLayerFor(bookTools))),
|
|
671
716
|
(flatIndex) =>
|
|
672
717
|
runtime.submit(agent, { question: `chaos ${flatIndex}` }, submitOptionsFor(flatIndex)),
|
|
673
718
|
);
|
|
674
719
|
}
|
|
675
720
|
case "approval": {
|
|
676
721
|
const agent = Agent.withModel(approvalDefinition, model);
|
|
722
|
+
|
|
677
723
|
return plainLaneFixture(
|
|
678
724
|
true,
|
|
679
725
|
runtime
|
|
680
|
-
.
|
|
726
|
+
.processThread(agent, threadId)
|
|
681
727
|
.pipe(Effect.provide(bookToolLayerFor(approvalTools))),
|
|
682
728
|
(flatIndex) =>
|
|
683
729
|
runtime.submit(agent, { question: `chaos ${flatIndex}` }, submitOptionsFor(flatIndex)),
|
|
@@ -685,57 +731,73 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
685
731
|
}
|
|
686
732
|
case "durable-steps": {
|
|
687
733
|
const agent = Agent.withModel(itineraryDefinition, model);
|
|
734
|
+
|
|
688
735
|
const toolLayer = itineraryTools.toLayer({
|
|
689
736
|
itinerary: ({ ref: called }) =>
|
|
690
737
|
Effect.gen(function* () {
|
|
691
738
|
const step = yield* DurableStep;
|
|
739
|
+
|
|
692
740
|
const flight = yield* step.do(
|
|
693
741
|
"reserve-flight",
|
|
694
742
|
Schema.String,
|
|
695
743
|
desk.record(flightValue(called)).pipe(Effect.as(flightValue(called))),
|
|
696
744
|
);
|
|
745
|
+
|
|
697
746
|
const lodging = yield* step.do(
|
|
698
747
|
"reserve-lodging",
|
|
699
748
|
Schema.String,
|
|
700
749
|
desk.record(lodgingValue(called)).pipe(Effect.as(lodgingValue(called))),
|
|
701
750
|
);
|
|
751
|
+
|
|
702
752
|
return { state: `${flight}+${lodging}` };
|
|
703
753
|
}),
|
|
704
754
|
});
|
|
755
|
+
|
|
705
756
|
return plainLaneFixture(
|
|
706
757
|
true,
|
|
707
|
-
runtime.
|
|
758
|
+
runtime.processThread(agent, threadId).pipe(Effect.provide(toolLayer)),
|
|
708
759
|
(flatIndex) =>
|
|
709
760
|
runtime.submit(agent, { question: `chaos ${flatIndex}` }, submitOptionsFor(flatIndex)),
|
|
710
761
|
);
|
|
711
762
|
}
|
|
712
763
|
case "delegation": {
|
|
713
764
|
const parentBinding = Agent.withModel(coordinatorDefinition, model);
|
|
765
|
+
|
|
714
766
|
const childModel = promptScriptedModel(`chaos-child-${laneIndex}`, () =>
|
|
715
767
|
finalParts('{"answer":"child"}'),
|
|
716
768
|
);
|
|
769
|
+
|
|
717
770
|
const childBinding = Agent.withModel(childDefinition, childModel);
|
|
771
|
+
|
|
718
772
|
const delegationLayer = SubagentRuntime.layer(chaosDelegation, childBinding, {
|
|
719
773
|
mapChildFailure: (failure) => ChaosDelegationFailed.make({ childErrorTag: failure._tag }),
|
|
720
774
|
durable: { targetDigests: childDigestStrings(laneIndex) },
|
|
721
775
|
}).pipe(Layer.provide(delegationSupport));
|
|
776
|
+
|
|
722
777
|
const parentResolved: ResolvedBinding = yield* DurableWorkerBinding.make(
|
|
723
778
|
parentBinding,
|
|
724
779
|
digests,
|
|
725
780
|
).pipe(Effect.provide(delegationLayer));
|
|
781
|
+
|
|
726
782
|
const childResolved: ResolvedBinding = yield* DurableWorkerBinding.make(
|
|
727
783
|
childBinding,
|
|
728
784
|
childLaneDigests(laneIndex),
|
|
729
785
|
);
|
|
730
|
-
|
|
731
|
-
const
|
|
732
|
-
|
|
733
|
-
.
|
|
734
|
-
|
|
786
|
+
|
|
787
|
+
const registeredRuntime = yield* DurableAgentRuntime.pipe(
|
|
788
|
+
Effect.provide(
|
|
789
|
+
DurableAgentRuntime.layerWithBindings([parentResolved, childResolved]).pipe(
|
|
790
|
+
Layer.provide(RunToolAuthorization.allowAll),
|
|
791
|
+
),
|
|
792
|
+
),
|
|
793
|
+
);
|
|
794
|
+
|
|
795
|
+
const driveResolved = (thread: ThreadId) => registeredRuntime.processThreadResolved(thread);
|
|
796
|
+
|
|
735
797
|
const fixture: LaneFixture = {
|
|
736
798
|
index: laneIndex,
|
|
737
799
|
kind,
|
|
738
|
-
|
|
800
|
+
threadId,
|
|
739
801
|
ref,
|
|
740
802
|
deskInPlay: false,
|
|
741
803
|
submissionIndexes,
|
|
@@ -746,24 +808,24 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
746
808
|
submitOptionsFor(flatIndex),
|
|
747
809
|
),
|
|
748
810
|
drives: (firstReceipt) => {
|
|
749
|
-
const drives: Array<
|
|
750
|
-
|
|
751
|
-
];
|
|
811
|
+
const drives: Array<
|
|
812
|
+
Effect.Effect<ReadonlyArray<Settlement>, DurableWorkerFailure | DurableBindingFailure>
|
|
813
|
+
> = [driveResolved(threadId)];
|
|
814
|
+
|
|
752
815
|
if (firstReceipt !== undefined) {
|
|
753
816
|
drives.push(
|
|
754
817
|
driveResolved(
|
|
755
|
-
|
|
756
|
-
firstReceipt.submissionId,
|
|
757
|
-
decodeToolCallId(DELEGATE_CALL_ID),
|
|
758
|
-
),
|
|
818
|
+
childThreadIdFor(firstReceipt.submissionId, decodeToolCallId(DELEGATE_CALL_ID)),
|
|
759
819
|
),
|
|
760
820
|
);
|
|
761
821
|
}
|
|
822
|
+
|
|
762
823
|
return drives;
|
|
763
824
|
},
|
|
764
|
-
|
|
765
|
-
|
|
825
|
+
childThreadOf: (firstReceipt) =>
|
|
826
|
+
childThreadIdFor(firstReceipt.submissionId, decodeToolCallId(DELEGATE_CALL_ID)),
|
|
766
827
|
};
|
|
828
|
+
|
|
767
829
|
return fixture;
|
|
768
830
|
}
|
|
769
831
|
}
|
|
@@ -772,7 +834,9 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
772
834
|
/** Stable per-call index into an injection list (identical across resolution passes). */
|
|
773
835
|
const injectionIndex = (submissionFlatIndex: number, callId: string, length: number): number => {
|
|
774
836
|
let hash = submissionFlatIndex + 1;
|
|
837
|
+
|
|
775
838
|
for (const char of callId) hash = (Math.imul(hash, 31) + char.charCodeAt(0)) | 0;
|
|
839
|
+
|
|
776
840
|
return ((hash % length) + length) % length;
|
|
777
841
|
};
|
|
778
842
|
|
|
@@ -802,6 +866,7 @@ const resolutionFor = (
|
|
|
802
866
|
isFailure: false,
|
|
803
867
|
});
|
|
804
868
|
}
|
|
869
|
+
|
|
805
870
|
// The desk never produced a value for this call — resolving "completed" would fabricate.
|
|
806
871
|
return ResolutionNeverHappened.make();
|
|
807
872
|
}
|
|
@@ -820,8 +885,10 @@ const resolutionPass = Effect.fn("Chaos.resolutionPass")(function* (
|
|
|
820
885
|
const ledger = yield* SubmissionLedger;
|
|
821
886
|
const produced = yield* desk.produced;
|
|
822
887
|
const nonterminal = yield* tolerateTyped(Stream.runCollect(ledger.scanNonterminal));
|
|
888
|
+
|
|
823
889
|
if (Option.isNone(nonterminal)) return;
|
|
824
890
|
const byId = new Map<SubmissionId, SubmissionState>();
|
|
891
|
+
|
|
825
892
|
for (const state of states) {
|
|
826
893
|
if (state.receipt !== undefined) byId.set(state.receipt.submissionId, state);
|
|
827
894
|
}
|
|
@@ -829,18 +896,22 @@ const resolutionPass = Effect.fn("Chaos.resolutionPass")(function* (
|
|
|
829
896
|
if (row.state !== "unknown" && row.state !== "suspended") continue;
|
|
830
897
|
const state = byId.get(row.submissionId);
|
|
831
898
|
const explanation = yield* tolerateTyped(runtime.explain(row.submissionId));
|
|
899
|
+
|
|
832
900
|
if (Option.isNone(explanation)) continue;
|
|
833
901
|
const flatIndex = state?.flatIndex ?? 0;
|
|
834
902
|
const ref = state?.lane.ref ?? "ref-child";
|
|
903
|
+
|
|
835
904
|
if (row.state === "unknown") {
|
|
836
905
|
for (const call of explanation.value.evidence.unknownCalls) {
|
|
837
906
|
if (call.resolved) continue;
|
|
907
|
+
|
|
838
908
|
const kind =
|
|
839
909
|
plan.resolutionInjections.length === 0
|
|
840
910
|
? "never-happened"
|
|
841
|
-
: plan.resolutionInjections
|
|
842
|
-
injectionIndex(flatIndex, call.toolCallId, plan.resolutionInjections.length)
|
|
843
|
-
|
|
911
|
+
: (plan.resolutionInjections.at(
|
|
912
|
+
injectionIndex(flatIndex, call.toolCallId, plan.resolutionInjections.length),
|
|
913
|
+
) ?? "never-happened");
|
|
914
|
+
|
|
844
915
|
yield* tolerateTyped(
|
|
845
916
|
runtime.resolveUnknown(
|
|
846
917
|
UnknownResolutionCommand.make({
|
|
@@ -858,9 +929,10 @@ const resolutionPass = Effect.fn("Chaos.resolutionPass")(function* (
|
|
|
858
929
|
const decision =
|
|
859
930
|
plan.approvalDecisions.length === 0
|
|
860
931
|
? "approved"
|
|
861
|
-
: plan.approvalDecisions
|
|
862
|
-
injectionIndex(flatIndex, pending.toolCallId, plan.approvalDecisions.length)
|
|
863
|
-
|
|
932
|
+
: (plan.approvalDecisions.at(
|
|
933
|
+
injectionIndex(flatIndex, pending.toolCallId, plan.approvalDecisions.length),
|
|
934
|
+
) ?? "approved");
|
|
935
|
+
|
|
864
936
|
yield* tolerateTyped(
|
|
865
937
|
runtime.resolveApproval(
|
|
866
938
|
ApprovalDecisionCommand.make({
|
|
@@ -881,16 +953,19 @@ const submissionIdsNamedBy = (
|
|
|
881
953
|
records: ReadonlyArray<CanonicalRecordEnvelope>,
|
|
882
954
|
): ReadonlySet<SubmissionId> => {
|
|
883
955
|
const named = new Set<SubmissionId>();
|
|
956
|
+
|
|
884
957
|
for (const envelope of records) {
|
|
885
958
|
const payload = envelope.record.payload;
|
|
959
|
+
|
|
886
960
|
if (
|
|
887
961
|
payload._tag === "UserInputRecorded" ||
|
|
888
962
|
payload._tag === "SubmissionSettled" ||
|
|
889
963
|
payload._tag === "AbortRequested"
|
|
890
964
|
) {
|
|
891
|
-
named.add(payload.submissionId);
|
|
965
|
+
if (payload.submissionId !== undefined) named.add(payload.submissionId);
|
|
892
966
|
}
|
|
893
967
|
}
|
|
968
|
+
|
|
894
969
|
return named;
|
|
895
970
|
};
|
|
896
971
|
|
|
@@ -910,18 +985,23 @@ const assertNoFabrication = (
|
|
|
910
985
|
produced: ReadonlySet<string>,
|
|
911
986
|
): Effect.Effect<void, ChaosConvergenceFailure> => {
|
|
912
987
|
const fabricated: Array<string> = [];
|
|
988
|
+
|
|
913
989
|
const requireProduced = (value: string, label: string): void => {
|
|
914
990
|
if (!produced.has(value)) fabricated.push(`${label} "${value}"`);
|
|
915
991
|
};
|
|
992
|
+
|
|
916
993
|
for (const envelope of records) {
|
|
917
994
|
const payload = envelope.record.payload;
|
|
995
|
+
|
|
918
996
|
if (payload._tag === "ToolCallSettled" && !payload.isFailure) {
|
|
919
997
|
if (payload.toolName === "book") {
|
|
920
998
|
const result = decodeBookResult(payload.result);
|
|
999
|
+
|
|
921
1000
|
if (Option.isSome(result)) requireProduced(result.value.confirmation, "book result");
|
|
922
1001
|
}
|
|
923
1002
|
if (payload.toolName === "itinerary") {
|
|
924
1003
|
const result = decodeItineraryResult(payload.result);
|
|
1004
|
+
|
|
925
1005
|
if (Option.isSome(result)) {
|
|
926
1006
|
for (const part of result.value.state.split("+")) {
|
|
927
1007
|
requireProduced(part, "itinerary step result");
|
|
@@ -931,9 +1011,11 @@ const assertNoFabrication = (
|
|
|
931
1011
|
}
|
|
932
1012
|
if (payload._tag === "ToolStepSettled") {
|
|
933
1013
|
const output = decodeStepOutput(payload.output);
|
|
1014
|
+
|
|
934
1015
|
if (Option.isSome(output)) requireProduced(output.value, "step output");
|
|
935
1016
|
}
|
|
936
1017
|
}
|
|
1018
|
+
|
|
937
1019
|
return fabricated.length === 0
|
|
938
1020
|
? Effect.void
|
|
939
1021
|
: Effect.fail(
|
|
@@ -954,7 +1036,7 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
954
1036
|
) {
|
|
955
1037
|
const runtime = yield* DurableAgentRuntime;
|
|
956
1038
|
const ledger = yield* SubmissionLedger;
|
|
957
|
-
const store = yield*
|
|
1039
|
+
const store = yield* ThreadStore;
|
|
958
1040
|
const config = yield* DurableRuntimeConfig;
|
|
959
1041
|
const failpoints = yield* DurableRuntimeFailpointTestControl;
|
|
960
1042
|
const random = mulberry32(plan.seed);
|
|
@@ -963,28 +1045,43 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
963
1045
|
// Lane fixtures: the FIRST spec of each lane fixes the lane's agent kind.
|
|
964
1046
|
const laneKinds = new Map<number, ChaosScenarioKind>();
|
|
965
1047
|
const laneSubmissions = new Map<number, Array<number>>();
|
|
1048
|
+
|
|
966
1049
|
plan.submissions.forEach((spec, flatIndex) => {
|
|
967
1050
|
const lane = spec.lane % plan.lanes;
|
|
1051
|
+
|
|
968
1052
|
if (!laneKinds.has(lane)) laneKinds.set(lane, spec.kind);
|
|
969
1053
|
const list = laneSubmissions.get(lane) ?? [];
|
|
1054
|
+
|
|
970
1055
|
list.push(flatIndex);
|
|
971
1056
|
laneSubmissions.set(lane, list);
|
|
972
1057
|
});
|
|
973
1058
|
const lanes: Array<LaneFixture> = [];
|
|
1059
|
+
|
|
974
1060
|
for (const [lane, kind] of laneKinds) {
|
|
975
1061
|
lanes.push(yield* makeLaneFixture(plan, lane, kind, laneSubmissions.get(lane) ?? [], desk));
|
|
976
1062
|
}
|
|
977
1063
|
|
|
978
|
-
const
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
1064
|
+
const lanesByIndex = new Map(lanes.map((lane) => [lane.index, lane]));
|
|
1065
|
+
const states: Array<SubmissionState> = [];
|
|
1066
|
+
|
|
1067
|
+
for (const [flatIndex, spec] of plan.submissions.entries()) {
|
|
1068
|
+
const laneIndex = spec.lane % plan.lanes;
|
|
1069
|
+
const lane = lanesByIndex.get(laneIndex);
|
|
1070
|
+
|
|
1071
|
+
if (lane === undefined) {
|
|
1072
|
+
return yield* ChaosConvergenceFailure.make({
|
|
1073
|
+
seed: plan.seed,
|
|
1074
|
+
message: `submission ${flatIndex} addresses missing lane ${laneIndex}`,
|
|
1075
|
+
});
|
|
1076
|
+
}
|
|
1077
|
+
states.push({ flatIndex, lane, receipt: undefined });
|
|
1078
|
+
}
|
|
983
1079
|
const appliedAborts = new Set<number>();
|
|
984
1080
|
|
|
985
1081
|
type ArmEntry =
|
|
986
1082
|
| { readonly family: "coordinator"; readonly location: DurableRuntimeFailpointLocation }
|
|
987
1083
|
| { readonly family: "adapter"; readonly location: string };
|
|
1084
|
+
|
|
988
1085
|
const armQueue: Array<ArmEntry> = [
|
|
989
1086
|
...plan.failpointArms.map((location): ArmEntry => ({ family: "coordinator", location })),
|
|
990
1087
|
...plan.adapterArms.map((location): ArmEntry => ({ family: "adapter", location })),
|
|
@@ -993,17 +1090,21 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
993
1090
|
const allSettled = Effect.gen(function* () {
|
|
994
1091
|
if (states.some((state) => state.receipt === undefined)) return false;
|
|
995
1092
|
const nonterminal = yield* tolerateTyped(Stream.runCollect(ledger.scanNonterminal));
|
|
1093
|
+
|
|
996
1094
|
return Option.isSome(nonterminal) && Array.from(nonterminal.value).length === 0;
|
|
997
1095
|
});
|
|
998
1096
|
|
|
999
1097
|
const maxRounds = armQueue.length + states.length * 2 + 12;
|
|
1000
1098
|
let rounds = 0;
|
|
1001
1099
|
let converged = false;
|
|
1100
|
+
|
|
1002
1101
|
for (let round = 0; round < maxRounds; round++) {
|
|
1003
1102
|
rounds = round + 1;
|
|
1004
1103
|
const arm = armQueue[round];
|
|
1104
|
+
|
|
1005
1105
|
if (arm?.family === "coordinator") {
|
|
1006
1106
|
const location = arm.location;
|
|
1107
|
+
|
|
1007
1108
|
yield* failpoints.setHandler((hit) =>
|
|
1008
1109
|
hit === location
|
|
1009
1110
|
? Effect.fail(DurableRuntimeFailpointError.make({ location: hit }))
|
|
@@ -1014,18 +1115,24 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1014
1115
|
}
|
|
1015
1116
|
|
|
1016
1117
|
// Admission chaos: pending submissions retry under the active arm until a Receipt lands;
|
|
1017
|
-
// the identical (
|
|
1118
|
+
// the identical (thread, principal, key) triple reattaches, never duplicates.
|
|
1018
1119
|
for (const state of states) {
|
|
1019
1120
|
if (state.receipt !== undefined) continue;
|
|
1020
1121
|
const receipt = yield* tolerateTyped(state.lane.submitOne(state.flatIndex));
|
|
1122
|
+
|
|
1021
1123
|
if (Option.isSome(receipt)) state.receipt = receipt.value;
|
|
1022
1124
|
}
|
|
1023
1125
|
|
|
1024
1126
|
// Drive every lane (and discovered child lanes) in seeded order under the active arm.
|
|
1025
|
-
const order =
|
|
1127
|
+
const order = lanes
|
|
1128
|
+
.map((lane) => ({ lane, rank: random() }))
|
|
1129
|
+
.sort((left, right) => left.rank - right.rank || left.lane.index - right.lane.index)
|
|
1130
|
+
.map(({ lane }) => lane);
|
|
1131
|
+
|
|
1026
1132
|
for (const lane of order) {
|
|
1027
1133
|
const firstFlat = lane.submissionIndexes[0];
|
|
1028
1134
|
const firstReceipt = firstFlat === undefined ? undefined : states[firstFlat]?.receipt;
|
|
1135
|
+
|
|
1029
1136
|
for (const drive of lane.drives(firstReceipt)) {
|
|
1030
1137
|
yield* tolerateTyped(drive);
|
|
1031
1138
|
}
|
|
@@ -1035,8 +1142,10 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1035
1142
|
if (round >= 1) {
|
|
1036
1143
|
for (const rawIndex of plan.abortInjections) {
|
|
1037
1144
|
const index = rawIndex % states.length;
|
|
1145
|
+
|
|
1038
1146
|
if (appliedAborts.has(index)) continue;
|
|
1039
1147
|
const receipt = states[index]?.receipt;
|
|
1148
|
+
|
|
1040
1149
|
if (receipt === undefined) continue;
|
|
1041
1150
|
appliedAborts.add(index);
|
|
1042
1151
|
yield* tolerateTyped(
|
|
@@ -1070,35 +1179,40 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1070
1179
|
|
|
1071
1180
|
if (!converged) {
|
|
1072
1181
|
const nonterminal = yield* tolerateTyped(Stream.runCollect(ledger.scanNonterminal));
|
|
1182
|
+
|
|
1073
1183
|
const detail = Option.isSome(nonterminal)
|
|
1074
1184
|
? Array.from(nonterminal.value)
|
|
1075
1185
|
.map((row: SubmissionSnapshot) => `${row.submissionId}(${row.state})`)
|
|
1076
1186
|
.join(", ")
|
|
1077
1187
|
: "ledger scan failed";
|
|
1188
|
+
|
|
1078
1189
|
return yield* ChaosConvergenceFailure.make({
|
|
1079
1190
|
seed: plan.seed,
|
|
1080
1191
|
message: `plan did not converge within ${maxRounds} rounds; nonterminal: [${detail}]; pending receipts: ${states.filter((state) => state.receipt === undefined).length}`,
|
|
1081
1192
|
});
|
|
1082
1193
|
}
|
|
1083
1194
|
|
|
1084
|
-
// Final claims: the shared invariant checker per touched
|
|
1195
|
+
// Final claims: the shared invariant checker per touched Thread, in convergence mode,
|
|
1085
1196
|
// with the full digest chain (single known producer), plus the desk non-fabrication sweep.
|
|
1086
1197
|
const produced = yield* desk.produced;
|
|
1087
1198
|
const laneReports: Array<ChaosLaneReport> = [];
|
|
1088
|
-
|
|
1089
|
-
|
|
1199
|
+
|
|
1200
|
+
const verifyThread = Effect.fn("Chaos.verifyThread")(function* (
|
|
1201
|
+
threadId: ThreadId,
|
|
1090
1202
|
kind: ChaosScenarioKind,
|
|
1091
1203
|
deskInPlay: boolean,
|
|
1092
1204
|
) {
|
|
1093
|
-
const exported = yield* store.export(
|
|
1205
|
+
const exported = yield* store.export(ThreadExportRequest.make({ threadId })).pipe(
|
|
1094
1206
|
Effect.mapError((error) =>
|
|
1095
1207
|
ChaosConvergenceFailure.make({
|
|
1096
1208
|
seed: plan.seed,
|
|
1097
|
-
message: `export of ${
|
|
1209
|
+
message: `export of ${threadId} failed: ${String(error)}`,
|
|
1098
1210
|
}),
|
|
1099
1211
|
),
|
|
1100
1212
|
);
|
|
1213
|
+
|
|
1101
1214
|
const rows: Array<SubmissionSnapshot> = [];
|
|
1215
|
+
|
|
1102
1216
|
for (const submissionId of submissionIdsNamedBy(exported.records)) {
|
|
1103
1217
|
const found = yield* ledger.lookup(SubmissionLookupById.make({ submissionId })).pipe(
|
|
1104
1218
|
Effect.mapError((error) =>
|
|
@@ -1108,25 +1222,30 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1108
1222
|
}),
|
|
1109
1223
|
),
|
|
1110
1224
|
);
|
|
1225
|
+
|
|
1111
1226
|
if (Option.isSome(found)) rows.push(found.value);
|
|
1112
1227
|
}
|
|
1228
|
+
|
|
1113
1229
|
const batchProducers = new Map<BatchId, ProducerId>(
|
|
1114
1230
|
exported.records.map((envelope) => [envelope.batchId, config.producerId]),
|
|
1115
1231
|
);
|
|
1116
|
-
|
|
1232
|
+
|
|
1233
|
+
const report = yield* verifyThreadInvariants({
|
|
1117
1234
|
export: exported,
|
|
1118
1235
|
submissions: rows,
|
|
1119
1236
|
batchProducers,
|
|
1120
1237
|
requireAllSettled: true,
|
|
1121
1238
|
});
|
|
1239
|
+
|
|
1122
1240
|
if (!report.ok) {
|
|
1123
1241
|
const failed = report.checks
|
|
1124
1242
|
.filter((check) => check.status === "failed")
|
|
1125
1243
|
.map((check) => `${check.name}: ${check.detail ?? "failed"}`)
|
|
1126
1244
|
.join("; ");
|
|
1245
|
+
|
|
1127
1246
|
return yield* ChaosConvergenceFailure.make({
|
|
1128
1247
|
seed: plan.seed,
|
|
1129
|
-
message: `invariants failed for ${
|
|
1248
|
+
message: `invariants failed for ${threadId} (${kind}): ${failed}`,
|
|
1130
1249
|
});
|
|
1131
1250
|
}
|
|
1132
1251
|
if (deskInPlay) {
|
|
@@ -1134,7 +1253,7 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1134
1253
|
}
|
|
1135
1254
|
laneReports.push(
|
|
1136
1255
|
ChaosLaneReport.make({
|
|
1137
|
-
|
|
1256
|
+
threadId,
|
|
1138
1257
|
kind,
|
|
1139
1258
|
submissionCount: rows.length,
|
|
1140
1259
|
verified: report.ok,
|
|
@@ -1143,18 +1262,22 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1143
1262
|
});
|
|
1144
1263
|
|
|
1145
1264
|
for (const lane of lanes) {
|
|
1146
|
-
yield*
|
|
1147
|
-
// Delegation lanes: verify every materialized child
|
|
1265
|
+
yield* verifyThread(lane.threadId, lane.kind, lane.deskInPlay);
|
|
1266
|
+
// Delegation lanes: verify every materialized child Thread too.
|
|
1148
1267
|
for (const flatIndex of lane.submissionIndexes) {
|
|
1149
1268
|
const receipt = states[flatIndex]?.receipt;
|
|
1269
|
+
|
|
1150
1270
|
if (receipt === undefined) continue;
|
|
1151
|
-
const child = lane.
|
|
1271
|
+
const child = lane.childThreadOf(receipt);
|
|
1272
|
+
|
|
1152
1273
|
if (child === undefined) continue;
|
|
1274
|
+
|
|
1153
1275
|
const childExport = yield* Effect.exit(
|
|
1154
|
-
store.export(
|
|
1276
|
+
store.export(ThreadExportRequest.make({ threadId: child })),
|
|
1155
1277
|
);
|
|
1278
|
+
|
|
1156
1279
|
if (Exit.isSuccess(childExport) && childExport.value.records.length > 0) {
|
|
1157
|
-
yield*
|
|
1280
|
+
yield* verifyThread(child, "plain", false);
|
|
1158
1281
|
}
|
|
1159
1282
|
}
|
|
1160
1283
|
}
|
|
@@ -1169,6 +1292,7 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1169
1292
|
}),
|
|
1170
1293
|
),
|
|
1171
1294
|
);
|
|
1295
|
+
|
|
1172
1296
|
if (obligations.entries.length > 0) {
|
|
1173
1297
|
return yield* ChaosConvergenceFailure.make({
|
|
1174
1298
|
seed: plan.seed,
|