@effect-agent/testing 0.1.0-beta.9 → 0.1.0-beta.91
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Certification.d.mts +106 -0
- package/dist/Certification.mjs +633 -0
- package/dist/Certification.mjs.map +1 -0
- package/dist/Chaos.d.mts +126 -0
- package/dist/Chaos.mjs +755 -0
- package/dist/Chaos.mjs.map +1 -0
- package/dist/CodeExecutorConformance.d.mts +33 -0
- package/dist/CodeExecutorConformance.mjs +231 -0
- package/dist/CodeExecutorConformance.mjs.map +1 -0
- package/dist/CodeExecutorSubstitute.d.mts +21 -0
- package/dist/CodeExecutorSubstitute.mjs +368 -0
- package/dist/CodeExecutorSubstitute.mjs.map +1 -0
- package/dist/DocsResearcher.d.mts +276 -0
- package/dist/DocsResearcher.mjs +490 -0
- package/dist/DocsResearcher.mjs.map +1 -0
- package/dist/ScriptedModel-DAvxIiud.d.mts +220 -0
- package/dist/ScriptedModel.d.mts +2 -0
- package/dist/ScriptedModel.mjs +155 -0
- package/dist/ScriptedModel.mjs.map +1 -0
- package/dist/TravelPlanner.d.mts +1680 -0
- package/dist/TravelPlanner.mjs +1963 -0
- package/dist/TravelPlanner.mjs.map +1 -0
- package/dist/deterministic-layers-D5owIoke.mjs +358 -0
- package/dist/deterministic-layers-D5owIoke.mjs.map +1 -0
- package/dist/index.d.mts +2 -3408
- package/dist/index.mjs +2 -4771
- package/dist/rolldown-runtime-D7D4PA-g.mjs +13 -0
- package/package.json +1 -48
- package/src/{certification.ts → Certification.ts} +366 -171
- package/src/{chaos.ts → Chaos.ts} +241 -127
- package/src/{code-executor-conformance.ts → CodeExecutorConformance.ts} +30 -7
- package/src/{code-executor-substitute.ts → CodeExecutorSubstitute.ts} +113 -46
- package/src/{fixtures/docs-researcher/index.ts → DocsResearcher.ts} +68 -5
- package/src/{scripted-model.ts → ScriptedModel.ts} +25 -29
- package/src/TravelPlanner.ts +232 -0
- package/src/fixtures/docs-researcher/definition.ts +20 -11
- package/src/fixtures/docs-researcher/harness.ts +41 -34
- package/src/fixtures/docs-researcher/mcp.ts +49 -3
- package/src/fixtures/travel-planner/definition.ts +15 -2
- package/src/fixtures/travel-planner/deterministic-layers.ts +47 -4
- package/src/fixtures/travel-planner/phase2.ts +4 -3
- package/src/fixtures/travel-planner/phase3.ts +19 -42
- package/src/fixtures/travel-planner/phase4.ts +24 -37
- package/src/fixtures/travel-planner/phase5.ts +35 -42
- package/src/fixtures/travel-planner/phase6.ts +191 -88
- package/src/fixtures/travel-planner/phase7.ts +4 -102
- package/src/fixtures/travel-planner/scenarios.ts +3 -4
- package/src/fixtures/travel-planner/subagents-durable.ts +35 -61
- package/src/fixtures/travel-planner/subagents.ts +36 -14
- package/src/index.ts +1 -11
- package/src/internal/certification-report.ts +25 -0
- package/dist/index.mjs.map +0 -1
- package/src/code-executor-conformance.d.ts +0 -30
- package/src/fixtures/travel-planner/index.ts +0 -11
- package/src/fixtures/warehouse/index.ts +0 -412
|
@@ -1,36 +1,42 @@
|
|
|
1
|
+
import { Cause, Effect, Exit, Layer, Option, Ref, Schema, Stream } from "effect";
|
|
2
|
+
import { ObligationThresholds } from "effect-agent/admin";
|
|
3
|
+
import * as Agent from "effect-agent/agent";
|
|
4
|
+
import { AgentPolicy } from "effect-agent/agent-policy";
|
|
1
5
|
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
} from "@effect-agent/capabilities";
|
|
7
|
-
import {
|
|
8
|
-
Agent,
|
|
9
|
-
AgentPolicy,
|
|
10
|
-
ConversationId,
|
|
11
|
-
IdGenerator,
|
|
12
|
-
RunId,
|
|
13
|
-
SubmissionId,
|
|
14
|
-
ToolCallId,
|
|
15
|
-
TurnId,
|
|
16
|
-
} from "@effect-agent/core";
|
|
17
|
-
import { DurableStep, DurableStepError, ToolExecutionClass } from "@effect-agent/engine";
|
|
6
|
+
DurableWorkerBinding,
|
|
7
|
+
type DurableBindingFailure,
|
|
8
|
+
type ResolvedBinding,
|
|
9
|
+
} from "effect-agent/agent-registration";
|
|
18
10
|
import {
|
|
19
|
-
AbortCommand,
|
|
20
|
-
AgentBindingResolver,
|
|
21
|
-
ApprovalDecisionCommand,
|
|
22
|
-
ConversationExportRequest,
|
|
23
|
-
ConversationStore,
|
|
24
|
-
DefinitionDigests,
|
|
25
|
-
Digest,
|
|
26
11
|
DurableAgentRuntime,
|
|
27
12
|
DurableRuntimeConfig,
|
|
13
|
+
type DurableSubmitFailure,
|
|
14
|
+
type DurableWorkerFailure,
|
|
15
|
+
type Receipt,
|
|
16
|
+
} from "effect-agent/durable-agent-runtime";
|
|
17
|
+
import {
|
|
28
18
|
DurableRuntimeFailpointError,
|
|
29
19
|
DurableRuntimeFailpointLocation,
|
|
30
|
-
|
|
31
|
-
|
|
20
|
+
} from "effect-agent/durable-failpoint";
|
|
21
|
+
import { DurableStep, DurableStepError, ToolExecutionClass } from "effect-agent/durable-step";
|
|
22
|
+
import { IdGenerator } from "effect-agent/id-generator";
|
|
23
|
+
import { ThreadId, RunId, ToolCallId, TurnId, type SubmissionId } from "effect-agent/identifiers";
|
|
24
|
+
import {
|
|
25
|
+
DefinitionDigests,
|
|
26
|
+
Digest,
|
|
27
|
+
type CanonicalRecordEnvelope,
|
|
28
|
+
type BatchId,
|
|
29
|
+
type ProducerId,
|
|
30
|
+
} from "effect-agent/records";
|
|
31
|
+
import { childThreadIdFor } from "effect-agent/run-journal";
|
|
32
|
+
import { RunToolAuthorization } from "effect-agent/run-options";
|
|
33
|
+
import * as Subagent from "effect-agent/subagent";
|
|
34
|
+
import { SubagentPolicy } from "effect-agent/subagent";
|
|
35
|
+
import { SubagentReservationsMemoryLive } from "effect-agent/subagent-reservations";
|
|
36
|
+
import {
|
|
37
|
+
AbortCommand,
|
|
38
|
+
ApprovalDecisionCommand,
|
|
32
39
|
IdempotencyKey,
|
|
33
|
-
ObligationThresholds,
|
|
34
40
|
Principal,
|
|
35
41
|
ResolutionAbortSubmission,
|
|
36
42
|
ResolutionCompletedWithResult,
|
|
@@ -38,20 +44,22 @@ import {
|
|
|
38
44
|
SubmissionLedger,
|
|
39
45
|
SubmissionLookupById,
|
|
40
46
|
UnknownResolutionCommand,
|
|
41
|
-
childConversationIdFor,
|
|
42
|
-
verifyConversationInvariants,
|
|
43
|
-
type CanonicalRecordEnvelope,
|
|
44
|
-
type BatchId,
|
|
45
|
-
type ProducerId,
|
|
46
|
-
type Receipt,
|
|
47
|
-
type ResolvedBinding,
|
|
48
47
|
type Settlement,
|
|
49
48
|
type SubmissionSnapshot,
|
|
50
49
|
type UnknownResolution,
|
|
51
|
-
} from "
|
|
52
|
-
import {
|
|
50
|
+
} from "effect-agent/submission-ledger";
|
|
51
|
+
import { DurableRuntimeFailpointTestControl } from "effect-agent/testing/durable-failpoint-test-control";
|
|
52
|
+
import { verifyThreadInvariants } from "effect-agent/thread-invariants";
|
|
53
|
+
import { ThreadExportRequest, ThreadStore } from "effect-agent/thread-store";
|
|
53
54
|
import { FastCheck } from "effect/testing";
|
|
54
|
-
import {
|
|
55
|
+
import {
|
|
56
|
+
LanguageModel,
|
|
57
|
+
Model,
|
|
58
|
+
Tool,
|
|
59
|
+
Toolkit,
|
|
60
|
+
type Prompt,
|
|
61
|
+
type Response,
|
|
62
|
+
} from "effect/unstable/ai";
|
|
55
63
|
|
|
56
64
|
/**
|
|
57
65
|
* P7 WP4 chaos machinery (plan §5): a Schema-first `ChaosPlan`, a seeded generator over
|
|
@@ -59,7 +67,7 @@ import { LanguageModel, Model, Prompt, Tool, Toolkit, type Response } from "effe
|
|
|
59
67
|
* deterministic runner that drives the durable coordinator over whatever adapter pair the test
|
|
60
68
|
* provides. Every plan ends in the SAME claims the crash matrices make:
|
|
61
69
|
*
|
|
62
|
-
* 1. `
|
|
70
|
+
* 1. `verifyThreadInvariants` in convergence mode over every touched Thread (the
|
|
63
71
|
* shared WP1 checker — one set of claims for admin verify, certification, chaos, and soak);
|
|
64
72
|
* 2. `scanObligations` returning ZERO entries (everything settled; nothing invisibly stuck);
|
|
65
73
|
* 3. supplier non-fabrication wherever the deterministic desk was in play (durability §10: no
|
|
@@ -83,6 +91,7 @@ export const ChaosScenarioKind = Schema.Literals([
|
|
|
83
91
|
"join",
|
|
84
92
|
"delegation",
|
|
85
93
|
]);
|
|
94
|
+
|
|
86
95
|
export type ChaosScenarioKind = typeof ChaosScenarioKind.Type;
|
|
87
96
|
|
|
88
97
|
const LaneIndex = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0), Schema.isLessThanOrEqualTo(7));
|
|
@@ -108,6 +117,7 @@ export const ChaosResolutionKind = Schema.Literals([
|
|
|
108
117
|
/** Unresolvable: route into the abort path (settles aborted, audit retained). */
|
|
109
118
|
"abort-submission",
|
|
110
119
|
]);
|
|
120
|
+
|
|
111
121
|
export type ChaosResolutionKind = typeof ChaosResolutionKind.Type;
|
|
112
122
|
|
|
113
123
|
export const ChaosApprovalDecision = Schema.Literals(["approved", "denied"]);
|
|
@@ -142,10 +152,10 @@ export class ChaosPlan extends Schema.Class<ChaosPlan>("@effect-agent/testing/Ch
|
|
|
142
152
|
export class ChaosLaneReport extends Schema.Class<ChaosLaneReport>(
|
|
143
153
|
"@effect-agent/testing/ChaosLaneReport",
|
|
144
154
|
)({
|
|
145
|
-
|
|
155
|
+
threadId: ThreadId,
|
|
146
156
|
kind: ChaosScenarioKind,
|
|
147
157
|
submissionCount: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
148
|
-
/** Verdict of `
|
|
158
|
+
/** Verdict of `verifyThreadInvariants` in convergence mode. */
|
|
149
159
|
verified: Schema.Boolean,
|
|
150
160
|
}) {}
|
|
151
161
|
|
|
@@ -176,12 +186,16 @@ export class ChaosConvergenceFailure extends Schema.TaggedError<ChaosConvergence
|
|
|
176
186
|
/** Default root seed for chaos suites; override with the `CHAOS_SEED` environment variable. */
|
|
177
187
|
export const DEFAULT_CHAOS_SEED = 20260813;
|
|
178
188
|
|
|
189
|
+
const ChaosSeedFromEnvironment = Schema.FiniteFromString.check(Schema.isInt());
|
|
190
|
+
const decodeChaosSeedFromEnvironment = Schema.decodeUnknownOption(ChaosSeedFromEnvironment);
|
|
191
|
+
|
|
179
192
|
/** The root seed for this run: `CHAOS_SEED` when set to an integer, the default otherwise. */
|
|
180
193
|
export const chaosSeedFromEnv = (env: Record<string, string | undefined>): number => {
|
|
181
194
|
const raw = env["CHAOS_SEED"];
|
|
195
|
+
|
|
182
196
|
if (raw === undefined || raw === "") return DEFAULT_CHAOS_SEED;
|
|
183
|
-
|
|
184
|
-
return
|
|
197
|
+
|
|
198
|
+
return Option.getOrElse(decodeChaosSeedFromEnvironment(raw), () => DEFAULT_CHAOS_SEED);
|
|
185
199
|
};
|
|
186
200
|
|
|
187
201
|
export interface ChaosGeneratorOptions {
|
|
@@ -205,13 +219,12 @@ const laneArbitrary: FastCheck.Arbitrary<GeneratedLane> = FastCheck.constantFrom
|
|
|
205
219
|
"approval",
|
|
206
220
|
"join",
|
|
207
221
|
"delegation",
|
|
208
|
-
).chain(
|
|
209
|
-
|
|
210
|
-
kind
|
|
211
|
-
|
|
212
|
-
: kind
|
|
213
|
-
|
|
214
|
-
: FastCheck.constant<GeneratedLane>({ kind, depth: 1 }),
|
|
222
|
+
).chain((kind): FastCheck.Arbitrary<GeneratedLane> =>
|
|
223
|
+
kind === "join"
|
|
224
|
+
? FastCheck.integer({ min: 2, max: 3 }).map((depth): GeneratedLane => ({ kind, depth }))
|
|
225
|
+
: kind === "plain"
|
|
226
|
+
? FastCheck.integer({ min: 1, max: 2 }).map((depth): GeneratedLane => ({ kind, depth }))
|
|
227
|
+
: FastCheck.constant<GeneratedLane>({ kind, depth: 1 }),
|
|
215
228
|
);
|
|
216
229
|
|
|
217
230
|
interface ChaosPlanShape {
|
|
@@ -258,9 +271,12 @@ const planShapeArbitrary = (
|
|
|
258
271
|
ChaosSubmissionSpec.make({ lane: index, kind: lane.kind }),
|
|
259
272
|
),
|
|
260
273
|
);
|
|
274
|
+
|
|
261
275
|
const [first, ...rest] = submissions;
|
|
276
|
+
|
|
262
277
|
// `lanes` >= 1 and every lane has depth >= 1, so `first` always exists.
|
|
263
278
|
if (first === undefined) throw new Error("chaos generator produced an empty plan");
|
|
279
|
+
|
|
264
280
|
return {
|
|
265
281
|
lanes: shape.lanes.length,
|
|
266
282
|
submissions: [first, ...rest] as const,
|
|
@@ -282,6 +298,7 @@ export const generateChaosPlans = (options: ChaosGeneratorOptions): ReadonlyArra
|
|
|
282
298
|
seed: options.seed,
|
|
283
299
|
numRuns: options.count,
|
|
284
300
|
});
|
|
301
|
+
|
|
285
302
|
return sampled.map((shape, index) =>
|
|
286
303
|
ChaosPlan.make({ ...shape, seed: (Math.imul(options.seed, 31) + index) | 0 }),
|
|
287
304
|
);
|
|
@@ -290,10 +307,13 @@ export const generateChaosPlans = (options: ChaosGeneratorOptions): ReadonlyArra
|
|
|
290
307
|
/** Deterministic PRNG for the runner's small ordering choices (lane drive order). */
|
|
291
308
|
const mulberry32 = (seed: number): (() => number) => {
|
|
292
309
|
let state = seed | 0;
|
|
310
|
+
|
|
293
311
|
return () => {
|
|
294
312
|
state = (state + 0x6d2b79f5) | 0;
|
|
295
313
|
let t = Math.imul(state ^ (state >>> 15), 1 | state);
|
|
314
|
+
|
|
296
315
|
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
|
|
316
|
+
|
|
297
317
|
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
|
298
318
|
};
|
|
299
319
|
};
|
|
@@ -359,7 +379,7 @@ const policy = AgentPolicy.make({
|
|
|
359
379
|
const PlainInput = Schema.Struct({ question: Schema.String });
|
|
360
380
|
const PlainOutput = Schema.Struct({ answer: Schema.String });
|
|
361
381
|
|
|
362
|
-
const plainDefinition = Agent.
|
|
382
|
+
const plainDefinition = Agent.make("chaos-plain", {
|
|
363
383
|
input: PlainInput,
|
|
364
384
|
output: PlainOutput,
|
|
365
385
|
instructions: "Answer as JSON.",
|
|
@@ -372,8 +392,10 @@ const BookUncertain = Tool.make("book", {
|
|
|
372
392
|
parameters: Schema.Struct({ ref: Schema.String }),
|
|
373
393
|
success: Schema.Struct({ confirmation: Schema.String }),
|
|
374
394
|
});
|
|
395
|
+
|
|
375
396
|
const bookTools = Toolkit.make(BookUncertain);
|
|
376
|
-
|
|
397
|
+
|
|
398
|
+
const bookDefinition = Agent.make("chaos-book", {
|
|
377
399
|
input: PlainInput,
|
|
378
400
|
output: PlainOutput,
|
|
379
401
|
instructions: "Book it.",
|
|
@@ -386,8 +408,10 @@ const BookApproval = Tool.make("book", {
|
|
|
386
408
|
success: Schema.Struct({ confirmation: Schema.String }),
|
|
387
409
|
needsApproval: true,
|
|
388
410
|
});
|
|
411
|
+
|
|
389
412
|
const approvalTools = Toolkit.make(BookApproval);
|
|
390
|
-
|
|
413
|
+
|
|
414
|
+
const approvalDefinition = Agent.make("chaos-approval", {
|
|
391
415
|
input: PlainInput,
|
|
392
416
|
output: PlainOutput,
|
|
393
417
|
instructions: "Book after approval.",
|
|
@@ -401,8 +425,10 @@ const Itinerary = Tool.make("itinerary", {
|
|
|
401
425
|
failure: DurableStepError,
|
|
402
426
|
dependencies: [DurableStep],
|
|
403
427
|
}).annotate(ToolExecutionClass, "uncertain");
|
|
428
|
+
|
|
404
429
|
const itineraryTools = Toolkit.make(Itinerary);
|
|
405
|
-
|
|
430
|
+
|
|
431
|
+
const itineraryDefinition = Agent.make("chaos-itinerary", {
|
|
406
432
|
input: PlainInput,
|
|
407
433
|
output: PlainOutput,
|
|
408
434
|
instructions: "Reserve the itinerary.",
|
|
@@ -410,7 +436,7 @@ const itineraryDefinition = Agent.define("chaos-itinerary", {
|
|
|
410
436
|
policy,
|
|
411
437
|
});
|
|
412
438
|
|
|
413
|
-
const childDefinition = Agent.
|
|
439
|
+
const childDefinition = Agent.make("chaos-child", {
|
|
414
440
|
input: PlainInput,
|
|
415
441
|
output: PlainOutput,
|
|
416
442
|
instructions: "Answer as JSON.",
|
|
@@ -445,7 +471,7 @@ const chaosDelegation = Subagent.define("delegate_chaos", {
|
|
|
445
471
|
}),
|
|
446
472
|
});
|
|
447
473
|
|
|
448
|
-
const coordinatorDefinition = Agent.
|
|
474
|
+
const coordinatorDefinition = Agent.make("chaos-coordinator", {
|
|
449
475
|
input: Schema.Struct({ mission: Schema.String }),
|
|
450
476
|
output: Schema.Struct({ report: Schema.String }),
|
|
451
477
|
instructions: "Delegate, then report as JSON.",
|
|
@@ -457,16 +483,22 @@ const DELEGATE_CALL_ID = "chaos-delegate-1";
|
|
|
457
483
|
|
|
458
484
|
const HEX = "0123456789abcdef";
|
|
459
485
|
const decodeDigest = Schema.decodeSync(Digest);
|
|
486
|
+
|
|
460
487
|
const laneDigests = (lane: number): DefinitionDigests => {
|
|
461
|
-
const digest = decodeDigest(HEX
|
|
488
|
+
const digest = decodeDigest(HEX.charAt(lane % 8).repeat(64));
|
|
489
|
+
|
|
462
490
|
return DefinitionDigests.make({ agent: digest, model: digest, tools: digest });
|
|
463
491
|
};
|
|
492
|
+
|
|
464
493
|
const childDigestStrings = (lane: number) => {
|
|
465
|
-
const char = HEX
|
|
494
|
+
const char = HEX.charAt(8 + (lane % 8));
|
|
495
|
+
|
|
466
496
|
return { agent: char.repeat(64), model: char.repeat(64), tools: char.repeat(64) } as const;
|
|
467
497
|
};
|
|
498
|
+
|
|
468
499
|
const childLaneDigests = (lane: number): DefinitionDigests => {
|
|
469
500
|
const strings = childDigestStrings(lane);
|
|
501
|
+
|
|
470
502
|
return DefinitionDigests.make({
|
|
471
503
|
agent: decodeDigest(strings.agent),
|
|
472
504
|
model: decodeDigest(strings.model),
|
|
@@ -475,7 +507,7 @@ const childLaneDigests = (lane: number): DefinitionDigests => {
|
|
|
475
507
|
};
|
|
476
508
|
|
|
477
509
|
const CHAOS_PRINCIPAL = Schema.decodeSync(Principal)("principal-chaos");
|
|
478
|
-
const
|
|
510
|
+
const decodeThreadId = Schema.decodeSync(ThreadId);
|
|
479
511
|
const decodeIdempotencyKey = Schema.decodeSync(IdempotencyKey);
|
|
480
512
|
const decodeToolCallId = Schema.decodeSync(ToolCallId);
|
|
481
513
|
const decodeRunId = Schema.decodeSync(RunId);
|
|
@@ -486,12 +518,14 @@ const chaosIdentifiers = Layer.effect(
|
|
|
486
518
|
IdGenerator,
|
|
487
519
|
Effect.gen(function* () {
|
|
488
520
|
const counter = yield* Ref.make(0);
|
|
521
|
+
|
|
489
522
|
const next = <A>(decode: (value: string) => A, prefix: string) =>
|
|
490
523
|
Ref.getAndUpdate(counter, (value) => value + 1).pipe(
|
|
491
524
|
Effect.map((value) => decode(`${prefix}-${value}`)),
|
|
492
525
|
);
|
|
526
|
+
|
|
493
527
|
return {
|
|
494
|
-
|
|
528
|
+
nextThreadId: next(decodeThreadId, "chaos-fixture-thread"),
|
|
495
529
|
nextRunId: next(decodeRunId, "chaos-fixture-run"),
|
|
496
530
|
nextTurnId: next(decodeTurnId, "chaos-fixture-turn"),
|
|
497
531
|
};
|
|
@@ -511,6 +545,7 @@ interface ChaosDesk {
|
|
|
511
545
|
|
|
512
546
|
const makeChaosDesk: Effect.Effect<ChaosDesk> = Effect.gen(function* () {
|
|
513
547
|
const produced = yield* Ref.make<ReadonlySet<string>>(new Set());
|
|
548
|
+
|
|
514
549
|
return {
|
|
515
550
|
produced: Ref.get(produced),
|
|
516
551
|
record: (value: string) => Ref.update(produced, (current) => new Set(current).add(value)),
|
|
@@ -543,7 +578,7 @@ export interface ChaosRunOptions {
|
|
|
543
578
|
readonly betweenRounds?: Effect.Effect<void> | undefined;
|
|
544
579
|
}
|
|
545
580
|
|
|
546
|
-
/**
|
|
581
|
+
/** Tolerate typed failures while preserving every defect and interruption reason. */
|
|
547
582
|
const tolerateTyped = <A, E, R>(
|
|
548
583
|
effect: Effect.Effect<A, E, R>,
|
|
549
584
|
): Effect.Effect<Option.Option<A>, never, R> =>
|
|
@@ -551,25 +586,28 @@ const tolerateTyped = <A, E, R>(
|
|
|
551
586
|
Effect.exit,
|
|
552
587
|
Effect.flatMap((exit) => {
|
|
553
588
|
if (Exit.isSuccess(exit)) return Effect.succeed(Option.some(exit.value));
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
589
|
+
const unexpected = exit.cause.reasons.filter((reason) => reason._tag !== "Fail");
|
|
590
|
+
|
|
591
|
+
return unexpected.length === 0
|
|
592
|
+
? Effect.succeed(Option.none<A>())
|
|
593
|
+
: Effect.failCause(Cause.fromReasons<never>(unexpected));
|
|
558
594
|
}),
|
|
559
595
|
);
|
|
560
596
|
|
|
561
597
|
interface LaneFixture {
|
|
562
598
|
readonly index: number;
|
|
563
599
|
readonly kind: ChaosScenarioKind;
|
|
564
|
-
readonly
|
|
600
|
+
readonly threadId: ThreadId;
|
|
565
601
|
readonly ref: string;
|
|
566
602
|
readonly deskInPlay: boolean;
|
|
567
603
|
readonly submissionIndexes: ReadonlyArray<number>;
|
|
568
|
-
readonly submitOne: (flatIndex: number) => Effect.Effect<Receipt,
|
|
604
|
+
readonly submitOne: (flatIndex: number) => Effect.Effect<Receipt, DurableSubmitFailure>;
|
|
569
605
|
readonly drives: (
|
|
570
606
|
firstReceipt: Receipt | undefined,
|
|
571
|
-
) => ReadonlyArray<
|
|
572
|
-
|
|
607
|
+
) => ReadonlyArray<
|
|
608
|
+
Effect.Effect<ReadonlyArray<Settlement>, DurableWorkerFailure | DurableBindingFailure>
|
|
609
|
+
>;
|
|
610
|
+
readonly childThreadOf: (firstReceipt: Receipt) => ThreadId | undefined;
|
|
573
611
|
}
|
|
574
612
|
|
|
575
613
|
interface SubmissionState {
|
|
@@ -613,14 +651,14 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
613
651
|
desk: ChaosDesk,
|
|
614
652
|
) {
|
|
615
653
|
const runtime = yield* DurableAgentRuntime;
|
|
616
|
-
const
|
|
654
|
+
const threadId = decodeThreadId(`chaos-${plan.seed}-lane-${laneIndex}`);
|
|
617
655
|
const ref = `ref-l${laneIndex}`;
|
|
618
656
|
const script = scriptFor(kind, ref);
|
|
619
657
|
const model = promptScriptedModel(`chaos-${kind}-${laneIndex}`, script);
|
|
620
658
|
const digests = laneDigests(laneIndex);
|
|
621
659
|
|
|
622
660
|
const submitOptionsFor = (flatIndex: number) => ({
|
|
623
|
-
|
|
661
|
+
threadId,
|
|
624
662
|
principal: CHAOS_PRINCIPAL,
|
|
625
663
|
idempotencyKey: decodeIdempotencyKey(`chaos-${plan.seed}-s${flatIndex}`),
|
|
626
664
|
definitions: digests,
|
|
@@ -636,48 +674,46 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
636
674
|
|
|
637
675
|
const plainLaneFixture = (
|
|
638
676
|
deskInPlay: boolean,
|
|
639
|
-
drive: Effect.Effect<ReadonlyArray<Settlement>,
|
|
640
|
-
submitOne: (flatIndex: number) => Effect.Effect<Receipt,
|
|
677
|
+
drive: Effect.Effect<ReadonlyArray<Settlement>, DurableWorkerFailure | DurableBindingFailure>,
|
|
678
|
+
submitOne: (flatIndex: number) => Effect.Effect<Receipt, DurableSubmitFailure>,
|
|
641
679
|
): LaneFixture => ({
|
|
642
680
|
index: laneIndex,
|
|
643
681
|
kind,
|
|
644
|
-
|
|
682
|
+
threadId,
|
|
645
683
|
ref,
|
|
646
684
|
deskInPlay,
|
|
647
685
|
submissionIndexes,
|
|
648
686
|
submitOne,
|
|
649
687
|
drives: () => [drive],
|
|
650
|
-
|
|
688
|
+
childThreadOf: () => undefined,
|
|
651
689
|
});
|
|
652
690
|
|
|
653
691
|
switch (kind) {
|
|
654
692
|
case "plain":
|
|
655
693
|
case "join": {
|
|
656
694
|
const agent = Agent.withModel(plainDefinition, model);
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
runtime.
|
|
660
|
-
(flatIndex) =>
|
|
661
|
-
runtime.submit(agent, { question: `chaos ${flatIndex}` }, submitOptionsFor(flatIndex)),
|
|
695
|
+
|
|
696
|
+
return plainLaneFixture(false, runtime.processThread(agent, threadId), (flatIndex) =>
|
|
697
|
+
runtime.submit(agent, { question: `chaos ${flatIndex}` }, submitOptionsFor(flatIndex)),
|
|
662
698
|
);
|
|
663
699
|
}
|
|
664
700
|
case "uncertain-tool": {
|
|
665
701
|
const agent = Agent.withModel(bookDefinition, model);
|
|
702
|
+
|
|
666
703
|
return plainLaneFixture(
|
|
667
704
|
true,
|
|
668
|
-
runtime
|
|
669
|
-
.processConversation(agent, conversationId)
|
|
670
|
-
.pipe(Effect.provide(bookToolLayerFor(bookTools))),
|
|
705
|
+
runtime.processThread(agent, threadId).pipe(Effect.provide(bookToolLayerFor(bookTools))),
|
|
671
706
|
(flatIndex) =>
|
|
672
707
|
runtime.submit(agent, { question: `chaos ${flatIndex}` }, submitOptionsFor(flatIndex)),
|
|
673
708
|
);
|
|
674
709
|
}
|
|
675
710
|
case "approval": {
|
|
676
711
|
const agent = Agent.withModel(approvalDefinition, model);
|
|
712
|
+
|
|
677
713
|
return plainLaneFixture(
|
|
678
714
|
true,
|
|
679
715
|
runtime
|
|
680
|
-
.
|
|
716
|
+
.processThread(agent, threadId)
|
|
681
717
|
.pipe(Effect.provide(bookToolLayerFor(approvalTools))),
|
|
682
718
|
(flatIndex) =>
|
|
683
719
|
runtime.submit(agent, { question: `chaos ${flatIndex}` }, submitOptionsFor(flatIndex)),
|
|
@@ -685,57 +721,73 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
685
721
|
}
|
|
686
722
|
case "durable-steps": {
|
|
687
723
|
const agent = Agent.withModel(itineraryDefinition, model);
|
|
724
|
+
|
|
688
725
|
const toolLayer = itineraryTools.toLayer({
|
|
689
726
|
itinerary: ({ ref: called }) =>
|
|
690
727
|
Effect.gen(function* () {
|
|
691
728
|
const step = yield* DurableStep;
|
|
729
|
+
|
|
692
730
|
const flight = yield* step.do(
|
|
693
731
|
"reserve-flight",
|
|
694
732
|
Schema.String,
|
|
695
733
|
desk.record(flightValue(called)).pipe(Effect.as(flightValue(called))),
|
|
696
734
|
);
|
|
735
|
+
|
|
697
736
|
const lodging = yield* step.do(
|
|
698
737
|
"reserve-lodging",
|
|
699
738
|
Schema.String,
|
|
700
739
|
desk.record(lodgingValue(called)).pipe(Effect.as(lodgingValue(called))),
|
|
701
740
|
);
|
|
741
|
+
|
|
702
742
|
return { state: `${flight}+${lodging}` };
|
|
703
743
|
}),
|
|
704
744
|
});
|
|
745
|
+
|
|
705
746
|
return plainLaneFixture(
|
|
706
747
|
true,
|
|
707
|
-
runtime.
|
|
748
|
+
runtime.processThread(agent, threadId).pipe(Effect.provide(toolLayer)),
|
|
708
749
|
(flatIndex) =>
|
|
709
750
|
runtime.submit(agent, { question: `chaos ${flatIndex}` }, submitOptionsFor(flatIndex)),
|
|
710
751
|
);
|
|
711
752
|
}
|
|
712
753
|
case "delegation": {
|
|
713
754
|
const parentBinding = Agent.withModel(coordinatorDefinition, model);
|
|
755
|
+
|
|
714
756
|
const childModel = promptScriptedModel(`chaos-child-${laneIndex}`, () =>
|
|
715
757
|
finalParts('{"answer":"child"}'),
|
|
716
758
|
);
|
|
759
|
+
|
|
717
760
|
const childBinding = Agent.withModel(childDefinition, childModel);
|
|
718
|
-
|
|
761
|
+
|
|
762
|
+
const delegationLayer = Subagent.layer(chaosDelegation, childBinding, {
|
|
719
763
|
mapChildFailure: (failure) => ChaosDelegationFailed.make({ childErrorTag: failure._tag }),
|
|
720
764
|
durable: { targetDigests: childDigestStrings(laneIndex) },
|
|
721
765
|
}).pipe(Layer.provide(delegationSupport));
|
|
766
|
+
|
|
722
767
|
const parentResolved: ResolvedBinding = yield* DurableWorkerBinding.make(
|
|
723
768
|
parentBinding,
|
|
724
769
|
digests,
|
|
725
770
|
).pipe(Effect.provide(delegationLayer));
|
|
771
|
+
|
|
726
772
|
const childResolved: ResolvedBinding = yield* DurableWorkerBinding.make(
|
|
727
773
|
childBinding,
|
|
728
774
|
childLaneDigests(laneIndex),
|
|
729
775
|
);
|
|
730
|
-
|
|
731
|
-
const
|
|
732
|
-
|
|
733
|
-
.
|
|
734
|
-
|
|
776
|
+
|
|
777
|
+
const registeredRuntime = yield* DurableAgentRuntime.pipe(
|
|
778
|
+
Effect.provide(
|
|
779
|
+
DurableAgentRuntime.layerWithBindings([parentResolved, childResolved]).pipe(
|
|
780
|
+
Layer.provide(RunToolAuthorization.allowAll),
|
|
781
|
+
),
|
|
782
|
+
),
|
|
783
|
+
);
|
|
784
|
+
|
|
785
|
+
const driveResolved = (thread: ThreadId) => registeredRuntime.processThreadResolved(thread);
|
|
786
|
+
|
|
735
787
|
const fixture: LaneFixture = {
|
|
736
788
|
index: laneIndex,
|
|
737
789
|
kind,
|
|
738
|
-
|
|
790
|
+
threadId,
|
|
739
791
|
ref,
|
|
740
792
|
deskInPlay: false,
|
|
741
793
|
submissionIndexes,
|
|
@@ -746,24 +798,24 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
746
798
|
submitOptionsFor(flatIndex),
|
|
747
799
|
),
|
|
748
800
|
drives: (firstReceipt) => {
|
|
749
|
-
const drives: Array<
|
|
750
|
-
|
|
751
|
-
];
|
|
801
|
+
const drives: Array<
|
|
802
|
+
Effect.Effect<ReadonlyArray<Settlement>, DurableWorkerFailure | DurableBindingFailure>
|
|
803
|
+
> = [driveResolved(threadId)];
|
|
804
|
+
|
|
752
805
|
if (firstReceipt !== undefined) {
|
|
753
806
|
drives.push(
|
|
754
807
|
driveResolved(
|
|
755
|
-
|
|
756
|
-
firstReceipt.submissionId,
|
|
757
|
-
decodeToolCallId(DELEGATE_CALL_ID),
|
|
758
|
-
),
|
|
808
|
+
childThreadIdFor(firstReceipt.submissionId, decodeToolCallId(DELEGATE_CALL_ID)),
|
|
759
809
|
),
|
|
760
810
|
);
|
|
761
811
|
}
|
|
812
|
+
|
|
762
813
|
return drives;
|
|
763
814
|
},
|
|
764
|
-
|
|
765
|
-
|
|
815
|
+
childThreadOf: (firstReceipt) =>
|
|
816
|
+
childThreadIdFor(firstReceipt.submissionId, decodeToolCallId(DELEGATE_CALL_ID)),
|
|
766
817
|
};
|
|
818
|
+
|
|
767
819
|
return fixture;
|
|
768
820
|
}
|
|
769
821
|
}
|
|
@@ -772,7 +824,9 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
772
824
|
/** Stable per-call index into an injection list (identical across resolution passes). */
|
|
773
825
|
const injectionIndex = (submissionFlatIndex: number, callId: string, length: number): number => {
|
|
774
826
|
let hash = submissionFlatIndex + 1;
|
|
827
|
+
|
|
775
828
|
for (const char of callId) hash = (Math.imul(hash, 31) + char.charCodeAt(0)) | 0;
|
|
829
|
+
|
|
776
830
|
return ((hash % length) + length) % length;
|
|
777
831
|
};
|
|
778
832
|
|
|
@@ -802,6 +856,7 @@ const resolutionFor = (
|
|
|
802
856
|
isFailure: false,
|
|
803
857
|
});
|
|
804
858
|
}
|
|
859
|
+
|
|
805
860
|
// The desk never produced a value for this call — resolving "completed" would fabricate.
|
|
806
861
|
return ResolutionNeverHappened.make();
|
|
807
862
|
}
|
|
@@ -820,8 +875,10 @@ const resolutionPass = Effect.fn("Chaos.resolutionPass")(function* (
|
|
|
820
875
|
const ledger = yield* SubmissionLedger;
|
|
821
876
|
const produced = yield* desk.produced;
|
|
822
877
|
const nonterminal = yield* tolerateTyped(Stream.runCollect(ledger.scanNonterminal));
|
|
878
|
+
|
|
823
879
|
if (Option.isNone(nonterminal)) return;
|
|
824
880
|
const byId = new Map<SubmissionId, SubmissionState>();
|
|
881
|
+
|
|
825
882
|
for (const state of states) {
|
|
826
883
|
if (state.receipt !== undefined) byId.set(state.receipt.submissionId, state);
|
|
827
884
|
}
|
|
@@ -829,18 +886,22 @@ const resolutionPass = Effect.fn("Chaos.resolutionPass")(function* (
|
|
|
829
886
|
if (row.state !== "unknown" && row.state !== "suspended") continue;
|
|
830
887
|
const state = byId.get(row.submissionId);
|
|
831
888
|
const explanation = yield* tolerateTyped(runtime.explain(row.submissionId));
|
|
889
|
+
|
|
832
890
|
if (Option.isNone(explanation)) continue;
|
|
833
891
|
const flatIndex = state?.flatIndex ?? 0;
|
|
834
892
|
const ref = state?.lane.ref ?? "ref-child";
|
|
893
|
+
|
|
835
894
|
if (row.state === "unknown") {
|
|
836
895
|
for (const call of explanation.value.evidence.unknownCalls) {
|
|
837
896
|
if (call.resolved) continue;
|
|
897
|
+
|
|
838
898
|
const kind =
|
|
839
899
|
plan.resolutionInjections.length === 0
|
|
840
900
|
? "never-happened"
|
|
841
|
-
: plan.resolutionInjections
|
|
842
|
-
injectionIndex(flatIndex, call.toolCallId, plan.resolutionInjections.length)
|
|
843
|
-
|
|
901
|
+
: (plan.resolutionInjections.at(
|
|
902
|
+
injectionIndex(flatIndex, call.toolCallId, plan.resolutionInjections.length),
|
|
903
|
+
) ?? "never-happened");
|
|
904
|
+
|
|
844
905
|
yield* tolerateTyped(
|
|
845
906
|
runtime.resolveUnknown(
|
|
846
907
|
UnknownResolutionCommand.make({
|
|
@@ -858,9 +919,10 @@ const resolutionPass = Effect.fn("Chaos.resolutionPass")(function* (
|
|
|
858
919
|
const decision =
|
|
859
920
|
plan.approvalDecisions.length === 0
|
|
860
921
|
? "approved"
|
|
861
|
-
: plan.approvalDecisions
|
|
862
|
-
injectionIndex(flatIndex, pending.toolCallId, plan.approvalDecisions.length)
|
|
863
|
-
|
|
922
|
+
: (plan.approvalDecisions.at(
|
|
923
|
+
injectionIndex(flatIndex, pending.toolCallId, plan.approvalDecisions.length),
|
|
924
|
+
) ?? "approved");
|
|
925
|
+
|
|
864
926
|
yield* tolerateTyped(
|
|
865
927
|
runtime.resolveApproval(
|
|
866
928
|
ApprovalDecisionCommand.make({
|
|
@@ -881,16 +943,19 @@ const submissionIdsNamedBy = (
|
|
|
881
943
|
records: ReadonlyArray<CanonicalRecordEnvelope>,
|
|
882
944
|
): ReadonlySet<SubmissionId> => {
|
|
883
945
|
const named = new Set<SubmissionId>();
|
|
946
|
+
|
|
884
947
|
for (const envelope of records) {
|
|
885
948
|
const payload = envelope.record.payload;
|
|
949
|
+
|
|
886
950
|
if (
|
|
887
951
|
payload._tag === "UserInputRecorded" ||
|
|
888
952
|
payload._tag === "SubmissionSettled" ||
|
|
889
953
|
payload._tag === "AbortRequested"
|
|
890
954
|
) {
|
|
891
|
-
named.add(payload.submissionId);
|
|
955
|
+
if (payload.submissionId !== undefined) named.add(payload.submissionId);
|
|
892
956
|
}
|
|
893
957
|
}
|
|
958
|
+
|
|
894
959
|
return named;
|
|
895
960
|
};
|
|
896
961
|
|
|
@@ -910,18 +975,23 @@ const assertNoFabrication = (
|
|
|
910
975
|
produced: ReadonlySet<string>,
|
|
911
976
|
): Effect.Effect<void, ChaosConvergenceFailure> => {
|
|
912
977
|
const fabricated: Array<string> = [];
|
|
978
|
+
|
|
913
979
|
const requireProduced = (value: string, label: string): void => {
|
|
914
980
|
if (!produced.has(value)) fabricated.push(`${label} "${value}"`);
|
|
915
981
|
};
|
|
982
|
+
|
|
916
983
|
for (const envelope of records) {
|
|
917
984
|
const payload = envelope.record.payload;
|
|
985
|
+
|
|
918
986
|
if (payload._tag === "ToolCallSettled" && !payload.isFailure) {
|
|
919
987
|
if (payload.toolName === "book") {
|
|
920
988
|
const result = decodeBookResult(payload.result);
|
|
989
|
+
|
|
921
990
|
if (Option.isSome(result)) requireProduced(result.value.confirmation, "book result");
|
|
922
991
|
}
|
|
923
992
|
if (payload.toolName === "itinerary") {
|
|
924
993
|
const result = decodeItineraryResult(payload.result);
|
|
994
|
+
|
|
925
995
|
if (Option.isSome(result)) {
|
|
926
996
|
for (const part of result.value.state.split("+")) {
|
|
927
997
|
requireProduced(part, "itinerary step result");
|
|
@@ -931,9 +1001,11 @@ const assertNoFabrication = (
|
|
|
931
1001
|
}
|
|
932
1002
|
if (payload._tag === "ToolStepSettled") {
|
|
933
1003
|
const output = decodeStepOutput(payload.output);
|
|
1004
|
+
|
|
934
1005
|
if (Option.isSome(output)) requireProduced(output.value, "step output");
|
|
935
1006
|
}
|
|
936
1007
|
}
|
|
1008
|
+
|
|
937
1009
|
return fabricated.length === 0
|
|
938
1010
|
? Effect.void
|
|
939
1011
|
: Effect.fail(
|
|
@@ -954,7 +1026,7 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
954
1026
|
) {
|
|
955
1027
|
const runtime = yield* DurableAgentRuntime;
|
|
956
1028
|
const ledger = yield* SubmissionLedger;
|
|
957
|
-
const store = yield*
|
|
1029
|
+
const store = yield* ThreadStore;
|
|
958
1030
|
const config = yield* DurableRuntimeConfig;
|
|
959
1031
|
const failpoints = yield* DurableRuntimeFailpointTestControl;
|
|
960
1032
|
const random = mulberry32(plan.seed);
|
|
@@ -963,28 +1035,43 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
963
1035
|
// Lane fixtures: the FIRST spec of each lane fixes the lane's agent kind.
|
|
964
1036
|
const laneKinds = new Map<number, ChaosScenarioKind>();
|
|
965
1037
|
const laneSubmissions = new Map<number, Array<number>>();
|
|
1038
|
+
|
|
966
1039
|
plan.submissions.forEach((spec, flatIndex) => {
|
|
967
1040
|
const lane = spec.lane % plan.lanes;
|
|
1041
|
+
|
|
968
1042
|
if (!laneKinds.has(lane)) laneKinds.set(lane, spec.kind);
|
|
969
1043
|
const list = laneSubmissions.get(lane) ?? [];
|
|
1044
|
+
|
|
970
1045
|
list.push(flatIndex);
|
|
971
1046
|
laneSubmissions.set(lane, list);
|
|
972
1047
|
});
|
|
973
1048
|
const lanes: Array<LaneFixture> = [];
|
|
1049
|
+
|
|
974
1050
|
for (const [lane, kind] of laneKinds) {
|
|
975
1051
|
lanes.push(yield* makeLaneFixture(plan, lane, kind, laneSubmissions.get(lane) ?? [], desk));
|
|
976
1052
|
}
|
|
977
1053
|
|
|
978
|
-
const
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
1054
|
+
const lanesByIndex = new Map(lanes.map((lane) => [lane.index, lane]));
|
|
1055
|
+
const states: Array<SubmissionState> = [];
|
|
1056
|
+
|
|
1057
|
+
for (const [flatIndex, spec] of plan.submissions.entries()) {
|
|
1058
|
+
const laneIndex = spec.lane % plan.lanes;
|
|
1059
|
+
const lane = lanesByIndex.get(laneIndex);
|
|
1060
|
+
|
|
1061
|
+
if (lane === undefined) {
|
|
1062
|
+
return yield* ChaosConvergenceFailure.make({
|
|
1063
|
+
seed: plan.seed,
|
|
1064
|
+
message: `submission ${flatIndex} addresses missing lane ${laneIndex}`,
|
|
1065
|
+
});
|
|
1066
|
+
}
|
|
1067
|
+
states.push({ flatIndex, lane, receipt: undefined });
|
|
1068
|
+
}
|
|
983
1069
|
const appliedAborts = new Set<number>();
|
|
984
1070
|
|
|
985
1071
|
type ArmEntry =
|
|
986
1072
|
| { readonly family: "coordinator"; readonly location: DurableRuntimeFailpointLocation }
|
|
987
1073
|
| { readonly family: "adapter"; readonly location: string };
|
|
1074
|
+
|
|
988
1075
|
const armQueue: Array<ArmEntry> = [
|
|
989
1076
|
...plan.failpointArms.map((location): ArmEntry => ({ family: "coordinator", location })),
|
|
990
1077
|
...plan.adapterArms.map((location): ArmEntry => ({ family: "adapter", location })),
|
|
@@ -993,17 +1080,21 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
993
1080
|
const allSettled = Effect.gen(function* () {
|
|
994
1081
|
if (states.some((state) => state.receipt === undefined)) return false;
|
|
995
1082
|
const nonterminal = yield* tolerateTyped(Stream.runCollect(ledger.scanNonterminal));
|
|
1083
|
+
|
|
996
1084
|
return Option.isSome(nonterminal) && Array.from(nonterminal.value).length === 0;
|
|
997
1085
|
});
|
|
998
1086
|
|
|
999
1087
|
const maxRounds = armQueue.length + states.length * 2 + 12;
|
|
1000
1088
|
let rounds = 0;
|
|
1001
1089
|
let converged = false;
|
|
1090
|
+
|
|
1002
1091
|
for (let round = 0; round < maxRounds; round++) {
|
|
1003
1092
|
rounds = round + 1;
|
|
1004
1093
|
const arm = armQueue[round];
|
|
1094
|
+
|
|
1005
1095
|
if (arm?.family === "coordinator") {
|
|
1006
1096
|
const location = arm.location;
|
|
1097
|
+
|
|
1007
1098
|
yield* failpoints.setHandler((hit) =>
|
|
1008
1099
|
hit === location
|
|
1009
1100
|
? Effect.fail(DurableRuntimeFailpointError.make({ location: hit }))
|
|
@@ -1014,18 +1105,24 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1014
1105
|
}
|
|
1015
1106
|
|
|
1016
1107
|
// Admission chaos: pending submissions retry under the active arm until a Receipt lands;
|
|
1017
|
-
// the identical (
|
|
1108
|
+
// the identical (thread, principal, key) triple reattaches, never duplicates.
|
|
1018
1109
|
for (const state of states) {
|
|
1019
1110
|
if (state.receipt !== undefined) continue;
|
|
1020
1111
|
const receipt = yield* tolerateTyped(state.lane.submitOne(state.flatIndex));
|
|
1112
|
+
|
|
1021
1113
|
if (Option.isSome(receipt)) state.receipt = receipt.value;
|
|
1022
1114
|
}
|
|
1023
1115
|
|
|
1024
1116
|
// Drive every lane (and discovered child lanes) in seeded order under the active arm.
|
|
1025
|
-
const order =
|
|
1117
|
+
const order = lanes
|
|
1118
|
+
.map((lane) => ({ lane, rank: random() }))
|
|
1119
|
+
.sort((left, right) => left.rank - right.rank || left.lane.index - right.lane.index)
|
|
1120
|
+
.map(({ lane }) => lane);
|
|
1121
|
+
|
|
1026
1122
|
for (const lane of order) {
|
|
1027
1123
|
const firstFlat = lane.submissionIndexes[0];
|
|
1028
1124
|
const firstReceipt = firstFlat === undefined ? undefined : states[firstFlat]?.receipt;
|
|
1125
|
+
|
|
1029
1126
|
for (const drive of lane.drives(firstReceipt)) {
|
|
1030
1127
|
yield* tolerateTyped(drive);
|
|
1031
1128
|
}
|
|
@@ -1035,8 +1132,10 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1035
1132
|
if (round >= 1) {
|
|
1036
1133
|
for (const rawIndex of plan.abortInjections) {
|
|
1037
1134
|
const index = rawIndex % states.length;
|
|
1135
|
+
|
|
1038
1136
|
if (appliedAborts.has(index)) continue;
|
|
1039
1137
|
const receipt = states[index]?.receipt;
|
|
1138
|
+
|
|
1040
1139
|
if (receipt === undefined) continue;
|
|
1041
1140
|
appliedAborts.add(index);
|
|
1042
1141
|
yield* tolerateTyped(
|
|
@@ -1070,35 +1169,40 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1070
1169
|
|
|
1071
1170
|
if (!converged) {
|
|
1072
1171
|
const nonterminal = yield* tolerateTyped(Stream.runCollect(ledger.scanNonterminal));
|
|
1172
|
+
|
|
1073
1173
|
const detail = Option.isSome(nonterminal)
|
|
1074
1174
|
? Array.from(nonterminal.value)
|
|
1075
1175
|
.map((row: SubmissionSnapshot) => `${row.submissionId}(${row.state})`)
|
|
1076
1176
|
.join(", ")
|
|
1077
1177
|
: "ledger scan failed";
|
|
1178
|
+
|
|
1078
1179
|
return yield* ChaosConvergenceFailure.make({
|
|
1079
1180
|
seed: plan.seed,
|
|
1080
1181
|
message: `plan did not converge within ${maxRounds} rounds; nonterminal: [${detail}]; pending receipts: ${states.filter((state) => state.receipt === undefined).length}`,
|
|
1081
1182
|
});
|
|
1082
1183
|
}
|
|
1083
1184
|
|
|
1084
|
-
// Final claims: the shared invariant checker per touched
|
|
1185
|
+
// Final claims: the shared invariant checker per touched Thread, in convergence mode,
|
|
1085
1186
|
// with the full digest chain (single known producer), plus the desk non-fabrication sweep.
|
|
1086
1187
|
const produced = yield* desk.produced;
|
|
1087
1188
|
const laneReports: Array<ChaosLaneReport> = [];
|
|
1088
|
-
|
|
1089
|
-
|
|
1189
|
+
|
|
1190
|
+
const verifyThread = Effect.fn("Chaos.verifyThread")(function* (
|
|
1191
|
+
threadId: ThreadId,
|
|
1090
1192
|
kind: ChaosScenarioKind,
|
|
1091
1193
|
deskInPlay: boolean,
|
|
1092
1194
|
) {
|
|
1093
|
-
const exported = yield* store.export(
|
|
1195
|
+
const exported = yield* store.export(ThreadExportRequest.make({ threadId })).pipe(
|
|
1094
1196
|
Effect.mapError((error) =>
|
|
1095
1197
|
ChaosConvergenceFailure.make({
|
|
1096
1198
|
seed: plan.seed,
|
|
1097
|
-
message: `export of ${
|
|
1199
|
+
message: `export of ${threadId} failed: ${String(error)}`,
|
|
1098
1200
|
}),
|
|
1099
1201
|
),
|
|
1100
1202
|
);
|
|
1203
|
+
|
|
1101
1204
|
const rows: Array<SubmissionSnapshot> = [];
|
|
1205
|
+
|
|
1102
1206
|
for (const submissionId of submissionIdsNamedBy(exported.records)) {
|
|
1103
1207
|
const found = yield* ledger.lookup(SubmissionLookupById.make({ submissionId })).pipe(
|
|
1104
1208
|
Effect.mapError((error) =>
|
|
@@ -1108,25 +1212,30 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1108
1212
|
}),
|
|
1109
1213
|
),
|
|
1110
1214
|
);
|
|
1215
|
+
|
|
1111
1216
|
if (Option.isSome(found)) rows.push(found.value);
|
|
1112
1217
|
}
|
|
1218
|
+
|
|
1113
1219
|
const batchProducers = new Map<BatchId, ProducerId>(
|
|
1114
1220
|
exported.records.map((envelope) => [envelope.batchId, config.producerId]),
|
|
1115
1221
|
);
|
|
1116
|
-
|
|
1222
|
+
|
|
1223
|
+
const report = yield* verifyThreadInvariants({
|
|
1117
1224
|
export: exported,
|
|
1118
1225
|
submissions: rows,
|
|
1119
1226
|
batchProducers,
|
|
1120
1227
|
requireAllSettled: true,
|
|
1121
1228
|
});
|
|
1229
|
+
|
|
1122
1230
|
if (!report.ok) {
|
|
1123
1231
|
const failed = report.checks
|
|
1124
1232
|
.filter((check) => check.status === "failed")
|
|
1125
1233
|
.map((check) => `${check.name}: ${check.detail ?? "failed"}`)
|
|
1126
1234
|
.join("; ");
|
|
1235
|
+
|
|
1127
1236
|
return yield* ChaosConvergenceFailure.make({
|
|
1128
1237
|
seed: plan.seed,
|
|
1129
|
-
message: `invariants failed for ${
|
|
1238
|
+
message: `invariants failed for ${threadId} (${kind}): ${failed}`,
|
|
1130
1239
|
});
|
|
1131
1240
|
}
|
|
1132
1241
|
if (deskInPlay) {
|
|
@@ -1134,7 +1243,7 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1134
1243
|
}
|
|
1135
1244
|
laneReports.push(
|
|
1136
1245
|
ChaosLaneReport.make({
|
|
1137
|
-
|
|
1246
|
+
threadId,
|
|
1138
1247
|
kind,
|
|
1139
1248
|
submissionCount: rows.length,
|
|
1140
1249
|
verified: report.ok,
|
|
@@ -1143,18 +1252,22 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1143
1252
|
});
|
|
1144
1253
|
|
|
1145
1254
|
for (const lane of lanes) {
|
|
1146
|
-
yield*
|
|
1147
|
-
// Delegation lanes: verify every materialized child
|
|
1255
|
+
yield* verifyThread(lane.threadId, lane.kind, lane.deskInPlay);
|
|
1256
|
+
// Delegation lanes: verify every materialized child Thread too.
|
|
1148
1257
|
for (const flatIndex of lane.submissionIndexes) {
|
|
1149
1258
|
const receipt = states[flatIndex]?.receipt;
|
|
1259
|
+
|
|
1150
1260
|
if (receipt === undefined) continue;
|
|
1151
|
-
const child = lane.
|
|
1261
|
+
const child = lane.childThreadOf(receipt);
|
|
1262
|
+
|
|
1152
1263
|
if (child === undefined) continue;
|
|
1264
|
+
|
|
1153
1265
|
const childExport = yield* Effect.exit(
|
|
1154
|
-
store.export(
|
|
1266
|
+
store.export(ThreadExportRequest.make({ threadId: child })),
|
|
1155
1267
|
);
|
|
1268
|
+
|
|
1156
1269
|
if (Exit.isSuccess(childExport) && childExport.value.records.length > 0) {
|
|
1157
|
-
yield*
|
|
1270
|
+
yield* verifyThread(child, "plain", false);
|
|
1158
1271
|
}
|
|
1159
1272
|
}
|
|
1160
1273
|
}
|
|
@@ -1169,6 +1282,7 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1169
1282
|
}),
|
|
1170
1283
|
),
|
|
1171
1284
|
);
|
|
1285
|
+
|
|
1172
1286
|
if (obligations.entries.length > 0) {
|
|
1173
1287
|
return yield* ChaosConvergenceFailure.make({
|
|
1174
1288
|
seed: plan.seed,
|