@effect-agent/testing 0.1.0-beta.11 → 0.1.0-beta.110
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Certification.d.mts +105 -0
- package/dist/Certification.mjs +637 -0
- package/dist/Certification.mjs.map +1 -0
- package/dist/Chaos.d.mts +127 -0
- package/dist/Chaos.mjs +756 -0
- package/dist/Chaos.mjs.map +1 -0
- package/{src/code-executor-conformance.d.ts → dist/CodeExecutorConformance.d.mts} +12 -10
- package/dist/CodeExecutorConformance.mjs +231 -0
- package/dist/CodeExecutorConformance.mjs.map +1 -0
- package/dist/CodeExecutorSubstitute.d.mts +20 -0
- package/dist/CodeExecutorSubstitute.mjs +368 -0
- package/dist/CodeExecutorSubstitute.mjs.map +1 -0
- package/dist/DocsResearcher.d.mts +276 -0
- package/dist/DocsResearcher.mjs +490 -0
- package/dist/DocsResearcher.mjs.map +1 -0
- package/dist/ScriptedModel-HIvFn-5K.d.mts +220 -0
- package/dist/ScriptedModel.d.mts +2 -0
- package/dist/ScriptedModel.mjs +155 -0
- package/dist/ScriptedModel.mjs.map +1 -0
- package/dist/TravelPlanner.d.mts +1680 -0
- package/dist/TravelPlanner.mjs +1985 -0
- package/dist/TravelPlanner.mjs.map +1 -0
- package/dist/deterministic-layers-D5owIoke.mjs +358 -0
- package/dist/deterministic-layers-D5owIoke.mjs.map +1 -0
- package/dist/index.d.mts +2 -3408
- package/dist/index.mjs +2 -4771
- package/dist/rolldown-runtime-D7D4PA-g.mjs +13 -0
- package/package.json +1 -48
- package/src/{certification.ts → Certification.ts} +375 -171
- package/src/{chaos.ts → Chaos.ts} +290 -191
- package/src/{code-executor-conformance.ts → CodeExecutorConformance.ts} +30 -7
- package/src/{code-executor-substitute.ts → CodeExecutorSubstitute.ts} +113 -46
- package/src/{fixtures/docs-researcher/index.ts → DocsResearcher.ts} +68 -5
- package/src/{scripted-model.ts → ScriptedModel.ts} +25 -29
- package/src/TravelPlanner.ts +232 -0
- package/src/fixtures/docs-researcher/definition.ts +20 -11
- package/src/fixtures/docs-researcher/harness.ts +41 -34
- package/src/fixtures/docs-researcher/mcp.ts +49 -3
- package/src/fixtures/travel-planner/definition.ts +15 -2
- package/src/fixtures/travel-planner/deterministic-layers.ts +47 -4
- package/src/fixtures/travel-planner/phase2.ts +4 -3
- package/src/fixtures/travel-planner/phase3.ts +19 -42
- package/src/fixtures/travel-planner/phase4.ts +24 -37
- package/src/fixtures/travel-planner/phase5.ts +35 -42
- package/src/fixtures/travel-planner/phase6.ts +224 -96
- package/src/fixtures/travel-planner/phase7.ts +4 -102
- package/src/fixtures/travel-planner/scenarios.ts +3 -4
- package/src/fixtures/travel-planner/subagents-durable.ts +35 -61
- package/src/fixtures/travel-planner/subagents.ts +36 -14
- package/src/index.ts +1 -11
- package/src/internal/certification-report.ts +25 -0
- package/dist/index.mjs.map +0 -1
- package/src/fixtures/travel-planner/index.ts +0 -11
- package/src/fixtures/warehouse/index.ts +0 -412
|
@@ -1,36 +1,42 @@
|
|
|
1
|
+
import { Cause, Effect, Exit, Layer, Option, Ref, Schema, Stream } from "effect";
|
|
2
|
+
import { ObligationThresholds } from "effect-agent/admin";
|
|
3
|
+
import * as Agent from "effect-agent/agent";
|
|
4
|
+
import { AgentPolicy } from "effect-agent/agent-policy";
|
|
1
5
|
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
} from "@effect-agent/capabilities";
|
|
7
|
-
import {
|
|
8
|
-
Agent,
|
|
9
|
-
AgentPolicy,
|
|
10
|
-
ConversationId,
|
|
11
|
-
IdGenerator,
|
|
12
|
-
RunId,
|
|
13
|
-
SubmissionId,
|
|
14
|
-
ToolCallId,
|
|
15
|
-
TurnId,
|
|
16
|
-
} from "@effect-agent/core";
|
|
17
|
-
import { DurableStep, DurableStepError, ToolExecutionClass } from "@effect-agent/engine";
|
|
6
|
+
DurableWorkerBinding,
|
|
7
|
+
type DurableBindingFailure,
|
|
8
|
+
type ResolvedBinding,
|
|
9
|
+
} from "effect-agent/agent-registration";
|
|
18
10
|
import {
|
|
19
|
-
AbortCommand,
|
|
20
|
-
AgentBindingResolver,
|
|
21
|
-
ApprovalDecisionCommand,
|
|
22
|
-
ConversationExportRequest,
|
|
23
|
-
ConversationStore,
|
|
24
|
-
DefinitionDigests,
|
|
25
|
-
Digest,
|
|
26
11
|
DurableAgentRuntime,
|
|
27
12
|
DurableRuntimeConfig,
|
|
13
|
+
type DurableSubmitFailure,
|
|
14
|
+
type DurableWorkerFailure,
|
|
15
|
+
type Receipt,
|
|
16
|
+
} from "effect-agent/durable-agent-runtime";
|
|
17
|
+
import {
|
|
28
18
|
DurableRuntimeFailpointError,
|
|
29
19
|
DurableRuntimeFailpointLocation,
|
|
30
|
-
|
|
31
|
-
|
|
20
|
+
} from "effect-agent/durable-failpoint";
|
|
21
|
+
import { DurableStep, DurableStepError, ToolExecutionClass } from "effect-agent/durable-step";
|
|
22
|
+
import { IdGenerator } from "effect-agent/id-generator";
|
|
23
|
+
import { ThreadId, RunId, ToolCallId, TurnId, type SubmissionId } from "effect-agent/identifiers";
|
|
24
|
+
import {
|
|
25
|
+
DefinitionDigests,
|
|
26
|
+
Digest,
|
|
27
|
+
type CanonicalRecordEnvelope,
|
|
28
|
+
type BatchId,
|
|
29
|
+
type ProducerId,
|
|
30
|
+
} from "effect-agent/records";
|
|
31
|
+
import { childThreadIdFor } from "effect-agent/run-journal";
|
|
32
|
+
import { RunToolAuthorization } from "effect-agent/run-options";
|
|
33
|
+
import * as Subagent from "effect-agent/subagent";
|
|
34
|
+
import { SubagentPolicy } from "effect-agent/subagent";
|
|
35
|
+
import { SubagentReservationsMemoryLive } from "effect-agent/subagent-reservations";
|
|
36
|
+
import {
|
|
37
|
+
AbortCommand,
|
|
38
|
+
ApprovalDecisionCommand,
|
|
32
39
|
IdempotencyKey,
|
|
33
|
-
ObligationThresholds,
|
|
34
40
|
Principal,
|
|
35
41
|
ResolutionAbortSubmission,
|
|
36
42
|
ResolutionCompletedWithResult,
|
|
@@ -38,28 +44,30 @@ import {
|
|
|
38
44
|
SubmissionLedger,
|
|
39
45
|
SubmissionLookupById,
|
|
40
46
|
UnknownResolutionCommand,
|
|
41
|
-
childConversationIdFor,
|
|
42
|
-
verifyConversationInvariants,
|
|
43
|
-
type CanonicalRecordEnvelope,
|
|
44
|
-
type BatchId,
|
|
45
|
-
type ProducerId,
|
|
46
|
-
type Receipt,
|
|
47
|
-
type ResolvedBinding,
|
|
48
47
|
type Settlement,
|
|
49
48
|
type SubmissionSnapshot,
|
|
50
49
|
type UnknownResolution,
|
|
51
|
-
} from "
|
|
52
|
-
import {
|
|
53
|
-
import {
|
|
54
|
-
import {
|
|
50
|
+
} from "effect-agent/submission-ledger";
|
|
51
|
+
import { DurableRuntimeFailpointTestControl } from "effect-agent/testing/durable-failpoint-test-control";
|
|
52
|
+
import { verifyThreadInvariants } from "effect-agent/thread-invariants";
|
|
53
|
+
import { ThreadExportRequest, ThreadStore } from "effect-agent/thread-store";
|
|
54
|
+
import {
|
|
55
|
+
LanguageModel,
|
|
56
|
+
Model,
|
|
57
|
+
Tool,
|
|
58
|
+
Toolkit,
|
|
59
|
+
type Prompt,
|
|
60
|
+
type Response,
|
|
61
|
+
} from "effect/unstable/ai";
|
|
62
|
+
import { Arbitrary } from "effect/unstable/arbitrary";
|
|
55
63
|
|
|
56
64
|
/**
|
|
57
65
|
* P7 WP4 chaos machinery (plan §5): a Schema-first `ChaosPlan`, a seeded generator over
|
|
58
|
-
* `effect/
|
|
66
|
+
* `effect/unstable/arbitrary`, and a
|
|
59
67
|
* deterministic runner that drives the durable coordinator over whatever adapter pair the test
|
|
60
68
|
* provides. Every plan ends in the SAME claims the crash matrices make:
|
|
61
69
|
*
|
|
62
|
-
* 1. `
|
|
70
|
+
* 1. `verifyThreadInvariants` in convergence mode over every touched Thread (the
|
|
63
71
|
* shared WP1 checker — one set of claims for admin verify, certification, chaos, and soak);
|
|
64
72
|
* 2. `scanObligations` returning ZERO entries (everything settled; nothing invisibly stuck);
|
|
65
73
|
* 3. supplier non-fabrication wherever the deterministic desk was in play (durability §10: no
|
|
@@ -83,6 +91,7 @@ export const ChaosScenarioKind = Schema.Literals([
|
|
|
83
91
|
"join",
|
|
84
92
|
"delegation",
|
|
85
93
|
]);
|
|
94
|
+
|
|
86
95
|
export type ChaosScenarioKind = typeof ChaosScenarioKind.Type;
|
|
87
96
|
|
|
88
97
|
const LaneIndex = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0), Schema.isLessThanOrEqualTo(7));
|
|
@@ -108,6 +117,7 @@ export const ChaosResolutionKind = Schema.Literals([
|
|
|
108
117
|
/** Unresolvable: route into the abort path (settles aborted, audit retained). */
|
|
109
118
|
"abort-submission",
|
|
110
119
|
]);
|
|
120
|
+
|
|
111
121
|
export type ChaosResolutionKind = typeof ChaosResolutionKind.Type;
|
|
112
122
|
|
|
113
123
|
export const ChaosApprovalDecision = Schema.Literals(["approved", "denied"]);
|
|
@@ -142,10 +152,10 @@ export class ChaosPlan extends Schema.Class<ChaosPlan>("@effect-agent/testing/Ch
|
|
|
142
152
|
export class ChaosLaneReport extends Schema.Class<ChaosLaneReport>(
|
|
143
153
|
"@effect-agent/testing/ChaosLaneReport",
|
|
144
154
|
)({
|
|
145
|
-
|
|
155
|
+
threadId: ThreadId,
|
|
146
156
|
kind: ChaosScenarioKind,
|
|
147
157
|
submissionCount: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
|
|
148
|
-
/** Verdict of `
|
|
158
|
+
/** Verdict of `verifyThreadInvariants` in convergence mode. */
|
|
149
159
|
verified: Schema.Boolean,
|
|
150
160
|
}) {}
|
|
151
161
|
|
|
@@ -176,12 +186,16 @@ export class ChaosConvergenceFailure extends Schema.TaggedError<ChaosConvergence
|
|
|
176
186
|
/** Default root seed for chaos suites; override with the `CHAOS_SEED` environment variable. */
|
|
177
187
|
export const DEFAULT_CHAOS_SEED = 20260813;
|
|
178
188
|
|
|
189
|
+
const ChaosSeedFromEnvironment = Schema.FiniteFromString.check(Schema.isInt());
|
|
190
|
+
const decodeChaosSeedFromEnvironment = Schema.decodeUnknownOption(ChaosSeedFromEnvironment);
|
|
191
|
+
|
|
179
192
|
/** The root seed for this run: `CHAOS_SEED` when set to an integer, the default otherwise. */
|
|
180
193
|
export const chaosSeedFromEnv = (env: Record<string, string | undefined>): number => {
|
|
181
194
|
const raw = env["CHAOS_SEED"];
|
|
195
|
+
|
|
182
196
|
if (raw === undefined || raw === "") return DEFAULT_CHAOS_SEED;
|
|
183
|
-
|
|
184
|
-
return
|
|
197
|
+
|
|
198
|
+
return Option.getOrElse(decodeChaosSeedFromEnvironment(raw), () => DEFAULT_CHAOS_SEED);
|
|
185
199
|
};
|
|
186
200
|
|
|
187
201
|
export interface ChaosGeneratorOptions {
|
|
@@ -193,26 +207,14 @@ export interface ChaosGeneratorOptions {
|
|
|
193
207
|
readonly adapterArms?: ReadonlyArray<string> | undefined;
|
|
194
208
|
}
|
|
195
209
|
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
"durable-steps",
|
|
205
|
-
"approval",
|
|
206
|
-
"join",
|
|
207
|
-
"delegation",
|
|
208
|
-
).chain(
|
|
209
|
-
(kind): FastCheck.Arbitrary<GeneratedLane> =>
|
|
210
|
-
kind === "join"
|
|
211
|
-
? FastCheck.integer({ min: 2, max: 3 }).map((depth): GeneratedLane => ({ kind, depth }))
|
|
212
|
-
: kind === "plain"
|
|
213
|
-
? FastCheck.integer({ min: 1, max: 2 }).map((depth): GeneratedLane => ({ kind, depth }))
|
|
214
|
-
: FastCheck.constant<GeneratedLane>({ kind, depth: 1 }),
|
|
215
|
-
);
|
|
210
|
+
const GeneratedLane = Schema.Union([
|
|
211
|
+
Schema.Struct({ kind: Schema.Literal("join"), depth: Schema.Literals([2, 3]) }),
|
|
212
|
+
Schema.Struct({ kind: Schema.Literal("plain"), depth: Schema.Literals([1, 2]) }),
|
|
213
|
+
Schema.Struct({
|
|
214
|
+
kind: Schema.Literals(["uncertain-tool", "durable-steps", "approval", "delegation"]),
|
|
215
|
+
depth: Schema.Literal(1),
|
|
216
|
+
}),
|
|
217
|
+
]);
|
|
216
218
|
|
|
217
219
|
interface ChaosPlanShape {
|
|
218
220
|
readonly lanes: number;
|
|
@@ -226,74 +228,77 @@ interface ChaosPlanShape {
|
|
|
226
228
|
|
|
227
229
|
const planShapeArbitrary = (
|
|
228
230
|
adapterArms: ReadonlyArray<string>,
|
|
229
|
-
):
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
adapterArms:
|
|
237
|
-
adapterArms.length === 0
|
|
238
|
-
? FastCheck.constant<Array<string>>([])
|
|
239
|
-
: FastCheck.uniqueArray(FastCheck.constantFrom(...adapterArms), { maxLength: 2 }),
|
|
240
|
-
abortInjections: FastCheck.uniqueArray(FastCheck.integer({ min: 0, max: 15 }), {
|
|
241
|
-
maxLength: 2,
|
|
242
|
-
}),
|
|
243
|
-
resolutionInjections: FastCheck.array(
|
|
244
|
-
FastCheck.constantFrom<ChaosResolutionKind>(
|
|
245
|
-
"never-happened",
|
|
246
|
-
"completed-from-supplier",
|
|
247
|
-
"abort-submission",
|
|
231
|
+
): Arbitrary.Arbitrary<ChaosPlanShape> =>
|
|
232
|
+
Arbitrary.schema(
|
|
233
|
+
Schema.Struct({
|
|
234
|
+
lanes: Schema.Array(GeneratedLane).check(Schema.isMinLength(1), Schema.isMaxLength(3)),
|
|
235
|
+
failpointArms: Schema.Array(DurableRuntimeFailpointLocation).check(
|
|
236
|
+
Schema.isUnique(),
|
|
237
|
+
Schema.isMaxLength(3),
|
|
248
238
|
),
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
239
|
+
adapterArms: Schema.Array(
|
|
240
|
+
adapterArms.length === 0 ? Schema.String : Schema.Literals(adapterArms),
|
|
241
|
+
).check(Schema.isUnique(), Schema.isMaxLength(adapterArms.length === 0 ? 0 : 2)),
|
|
242
|
+
abortInjections: Schema.Array(
|
|
243
|
+
Schema.Int.check(Schema.isBetween({ minimum: 0, maximum: 15 })),
|
|
244
|
+
).check(Schema.isUnique(), Schema.isMaxLength(2)),
|
|
245
|
+
resolutionInjections: Schema.Array(ChaosResolutionKind).check(Schema.isMaxLength(4)),
|
|
246
|
+
approvalDecisions: Schema.Array(ChaosApprovalDecision).check(Schema.isMaxLength(2)),
|
|
247
|
+
}),
|
|
248
|
+
).pipe(
|
|
249
|
+
Arbitrary.map((shape) => {
|
|
250
|
+
const submissions = shape.lanes.flatMap((lane, index) =>
|
|
251
|
+
Array.from({ length: lane.depth }, () =>
|
|
252
|
+
ChaosSubmissionSpec.make({ lane: index, kind: lane.kind }),
|
|
253
|
+
),
|
|
254
|
+
);
|
|
255
|
+
|
|
256
|
+
const [first, ...rest] = submissions;
|
|
257
|
+
|
|
258
|
+
// `lanes` >= 1 and every lane has depth >= 1, so `first` always exists.
|
|
259
|
+
if (first === undefined) throw new Error("chaos generator produced an empty plan");
|
|
260
|
+
|
|
261
|
+
return {
|
|
262
|
+
lanes: shape.lanes.length,
|
|
263
|
+
submissions: [first, ...rest] as const,
|
|
264
|
+
failpointArms: shape.failpointArms,
|
|
265
|
+
adapterArms: shape.adapterArms,
|
|
266
|
+
abortInjections: shape.abortInjections,
|
|
267
|
+
resolutionInjections: shape.resolutionInjections,
|
|
268
|
+
approvalDecisions: shape.approvalDecisions,
|
|
269
|
+
};
|
|
270
|
+
}),
|
|
271
|
+
);
|
|
274
272
|
|
|
275
273
|
/**
|
|
276
274
|
* Derive `count` chaos plans deterministically from one root seed. The same
|
|
277
275
|
* `{seed, count, adapterArms}` triple always yields byte-identical plans, so a failure line
|
|
278
|
-
* `CHAOS_SEED=<seed>` replays the exact schedule.
|
|
276
|
+
* `CHAOS_SEED=<seed>` replays the exact schedule with the same pinned generator version.
|
|
277
|
+
* Sampling is interruptible and reports bounded generation exhaustion as Arbitrary.SampleError.
|
|
279
278
|
*/
|
|
280
|
-
export const generateChaosPlans = (
|
|
281
|
-
|
|
279
|
+
export const generateChaosPlans = Effect.fnUntraced(function* (
|
|
280
|
+
options: ChaosGeneratorOptions,
|
|
281
|
+
): Effect.fn.Return<ReadonlyArray<ChaosPlan>, Arbitrary.SampleError> {
|
|
282
|
+
const sampled = yield* Arbitrary.sampleEffect(planShapeArbitrary(options.adapterArms ?? []), {
|
|
282
283
|
seed: options.seed,
|
|
283
|
-
|
|
284
|
+
count: options.count,
|
|
284
285
|
});
|
|
286
|
+
|
|
285
287
|
return sampled.map((shape, index) =>
|
|
286
288
|
ChaosPlan.make({ ...shape, seed: (Math.imul(options.seed, 31) + index) | 0 }),
|
|
287
289
|
);
|
|
288
|
-
};
|
|
290
|
+
});
|
|
289
291
|
|
|
290
292
|
/** Deterministic PRNG for the runner's small ordering choices (lane drive order). */
|
|
291
293
|
const mulberry32 = (seed: number): (() => number) => {
|
|
292
294
|
let state = seed | 0;
|
|
295
|
+
|
|
293
296
|
return () => {
|
|
294
297
|
state = (state + 0x6d2b79f5) | 0;
|
|
295
298
|
let t = Math.imul(state ^ (state >>> 15), 1 | state);
|
|
299
|
+
|
|
296
300
|
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
|
|
301
|
+
|
|
297
302
|
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
|
298
303
|
};
|
|
299
304
|
};
|
|
@@ -359,7 +364,7 @@ const policy = AgentPolicy.make({
|
|
|
359
364
|
const PlainInput = Schema.Struct({ question: Schema.String });
|
|
360
365
|
const PlainOutput = Schema.Struct({ answer: Schema.String });
|
|
361
366
|
|
|
362
|
-
const plainDefinition = Agent.
|
|
367
|
+
const plainDefinition = Agent.make("chaos-plain", {
|
|
363
368
|
input: PlainInput,
|
|
364
369
|
output: PlainOutput,
|
|
365
370
|
instructions: "Answer as JSON.",
|
|
@@ -372,8 +377,10 @@ const BookUncertain = Tool.make("book", {
|
|
|
372
377
|
parameters: Schema.Struct({ ref: Schema.String }),
|
|
373
378
|
success: Schema.Struct({ confirmation: Schema.String }),
|
|
374
379
|
});
|
|
380
|
+
|
|
375
381
|
const bookTools = Toolkit.make(BookUncertain);
|
|
376
|
-
|
|
382
|
+
|
|
383
|
+
const bookDefinition = Agent.make("chaos-book", {
|
|
377
384
|
input: PlainInput,
|
|
378
385
|
output: PlainOutput,
|
|
379
386
|
instructions: "Book it.",
|
|
@@ -386,8 +393,10 @@ const BookApproval = Tool.make("book", {
|
|
|
386
393
|
success: Schema.Struct({ confirmation: Schema.String }),
|
|
387
394
|
needsApproval: true,
|
|
388
395
|
});
|
|
396
|
+
|
|
389
397
|
const approvalTools = Toolkit.make(BookApproval);
|
|
390
|
-
|
|
398
|
+
|
|
399
|
+
const approvalDefinition = Agent.make("chaos-approval", {
|
|
391
400
|
input: PlainInput,
|
|
392
401
|
output: PlainOutput,
|
|
393
402
|
instructions: "Book after approval.",
|
|
@@ -401,8 +410,10 @@ const Itinerary = Tool.make("itinerary", {
|
|
|
401
410
|
failure: DurableStepError,
|
|
402
411
|
dependencies: [DurableStep],
|
|
403
412
|
}).annotate(ToolExecutionClass, "uncertain");
|
|
413
|
+
|
|
404
414
|
const itineraryTools = Toolkit.make(Itinerary);
|
|
405
|
-
|
|
415
|
+
|
|
416
|
+
const itineraryDefinition = Agent.make("chaos-itinerary", {
|
|
406
417
|
input: PlainInput,
|
|
407
418
|
output: PlainOutput,
|
|
408
419
|
instructions: "Reserve the itinerary.",
|
|
@@ -410,7 +421,7 @@ const itineraryDefinition = Agent.define("chaos-itinerary", {
|
|
|
410
421
|
policy,
|
|
411
422
|
});
|
|
412
423
|
|
|
413
|
-
const childDefinition = Agent.
|
|
424
|
+
const childDefinition = Agent.make("chaos-child", {
|
|
414
425
|
input: PlainInput,
|
|
415
426
|
output: PlainOutput,
|
|
416
427
|
instructions: "Answer as JSON.",
|
|
@@ -445,7 +456,7 @@ const chaosDelegation = Subagent.define("delegate_chaos", {
|
|
|
445
456
|
}),
|
|
446
457
|
});
|
|
447
458
|
|
|
448
|
-
const coordinatorDefinition = Agent.
|
|
459
|
+
const coordinatorDefinition = Agent.make("chaos-coordinator", {
|
|
449
460
|
input: Schema.Struct({ mission: Schema.String }),
|
|
450
461
|
output: Schema.Struct({ report: Schema.String }),
|
|
451
462
|
instructions: "Delegate, then report as JSON.",
|
|
@@ -457,16 +468,22 @@ const DELEGATE_CALL_ID = "chaos-delegate-1";
|
|
|
457
468
|
|
|
458
469
|
const HEX = "0123456789abcdef";
|
|
459
470
|
const decodeDigest = Schema.decodeSync(Digest);
|
|
471
|
+
|
|
460
472
|
const laneDigests = (lane: number): DefinitionDigests => {
|
|
461
|
-
const digest = decodeDigest(HEX
|
|
473
|
+
const digest = decodeDigest(HEX.charAt(lane % 8).repeat(64));
|
|
474
|
+
|
|
462
475
|
return DefinitionDigests.make({ agent: digest, model: digest, tools: digest });
|
|
463
476
|
};
|
|
477
|
+
|
|
464
478
|
const childDigestStrings = (lane: number) => {
|
|
465
|
-
const char = HEX
|
|
479
|
+
const char = HEX.charAt(8 + (lane % 8));
|
|
480
|
+
|
|
466
481
|
return { agent: char.repeat(64), model: char.repeat(64), tools: char.repeat(64) } as const;
|
|
467
482
|
};
|
|
483
|
+
|
|
468
484
|
const childLaneDigests = (lane: number): DefinitionDigests => {
|
|
469
485
|
const strings = childDigestStrings(lane);
|
|
486
|
+
|
|
470
487
|
return DefinitionDigests.make({
|
|
471
488
|
agent: decodeDigest(strings.agent),
|
|
472
489
|
model: decodeDigest(strings.model),
|
|
@@ -475,7 +492,7 @@ const childLaneDigests = (lane: number): DefinitionDigests => {
|
|
|
475
492
|
};
|
|
476
493
|
|
|
477
494
|
const CHAOS_PRINCIPAL = Schema.decodeSync(Principal)("principal-chaos");
|
|
478
|
-
const
|
|
495
|
+
const decodeThreadId = Schema.decodeSync(ThreadId);
|
|
479
496
|
const decodeIdempotencyKey = Schema.decodeSync(IdempotencyKey);
|
|
480
497
|
const decodeToolCallId = Schema.decodeSync(ToolCallId);
|
|
481
498
|
const decodeRunId = Schema.decodeSync(RunId);
|
|
@@ -486,12 +503,14 @@ const chaosIdentifiers = Layer.effect(
|
|
|
486
503
|
IdGenerator,
|
|
487
504
|
Effect.gen(function* () {
|
|
488
505
|
const counter = yield* Ref.make(0);
|
|
506
|
+
|
|
489
507
|
const next = <A>(decode: (value: string) => A, prefix: string) =>
|
|
490
508
|
Ref.getAndUpdate(counter, (value) => value + 1).pipe(
|
|
491
509
|
Effect.map((value) => decode(`${prefix}-${value}`)),
|
|
492
510
|
);
|
|
511
|
+
|
|
493
512
|
return {
|
|
494
|
-
|
|
513
|
+
nextThreadId: next(decodeThreadId, "chaos-fixture-thread"),
|
|
495
514
|
nextRunId: next(decodeRunId, "chaos-fixture-run"),
|
|
496
515
|
nextTurnId: next(decodeTurnId, "chaos-fixture-turn"),
|
|
497
516
|
};
|
|
@@ -511,6 +530,7 @@ interface ChaosDesk {
|
|
|
511
530
|
|
|
512
531
|
const makeChaosDesk: Effect.Effect<ChaosDesk> = Effect.gen(function* () {
|
|
513
532
|
const produced = yield* Ref.make<ReadonlySet<string>>(new Set());
|
|
533
|
+
|
|
514
534
|
return {
|
|
515
535
|
produced: Ref.get(produced),
|
|
516
536
|
record: (value: string) => Ref.update(produced, (current) => new Set(current).add(value)),
|
|
@@ -543,7 +563,7 @@ export interface ChaosRunOptions {
|
|
|
543
563
|
readonly betweenRounds?: Effect.Effect<void> | undefined;
|
|
544
564
|
}
|
|
545
565
|
|
|
546
|
-
/**
|
|
566
|
+
/** Tolerate typed failures while preserving every defect and interruption reason. */
|
|
547
567
|
const tolerateTyped = <A, E, R>(
|
|
548
568
|
effect: Effect.Effect<A, E, R>,
|
|
549
569
|
): Effect.Effect<Option.Option<A>, never, R> =>
|
|
@@ -551,25 +571,28 @@ const tolerateTyped = <A, E, R>(
|
|
|
551
571
|
Effect.exit,
|
|
552
572
|
Effect.flatMap((exit) => {
|
|
553
573
|
if (Exit.isSuccess(exit)) return Effect.succeed(Option.some(exit.value));
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
574
|
+
const unexpected = exit.cause.reasons.filter((reason) => reason._tag !== "Fail");
|
|
575
|
+
|
|
576
|
+
return unexpected.length === 0
|
|
577
|
+
? Effect.succeed(Option.none<A>())
|
|
578
|
+
: Effect.failCause(Cause.fromReasons<never>(unexpected));
|
|
558
579
|
}),
|
|
559
580
|
);
|
|
560
581
|
|
|
561
582
|
interface LaneFixture {
|
|
562
583
|
readonly index: number;
|
|
563
584
|
readonly kind: ChaosScenarioKind;
|
|
564
|
-
readonly
|
|
585
|
+
readonly threadId: ThreadId;
|
|
565
586
|
readonly ref: string;
|
|
566
587
|
readonly deskInPlay: boolean;
|
|
567
588
|
readonly submissionIndexes: ReadonlyArray<number>;
|
|
568
|
-
readonly submitOne: (flatIndex: number) => Effect.Effect<Receipt,
|
|
589
|
+
readonly submitOne: (flatIndex: number) => Effect.Effect<Receipt, DurableSubmitFailure>;
|
|
569
590
|
readonly drives: (
|
|
570
591
|
firstReceipt: Receipt | undefined,
|
|
571
|
-
) => ReadonlyArray<
|
|
572
|
-
|
|
592
|
+
) => ReadonlyArray<
|
|
593
|
+
Effect.Effect<ReadonlyArray<Settlement>, DurableWorkerFailure | DurableBindingFailure>
|
|
594
|
+
>;
|
|
595
|
+
readonly childThreadOf: (firstReceipt: Receipt) => ThreadId | undefined;
|
|
573
596
|
}
|
|
574
597
|
|
|
575
598
|
interface SubmissionState {
|
|
@@ -613,14 +636,14 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
613
636
|
desk: ChaosDesk,
|
|
614
637
|
) {
|
|
615
638
|
const runtime = yield* DurableAgentRuntime;
|
|
616
|
-
const
|
|
639
|
+
const threadId = decodeThreadId(`chaos-${plan.seed}-lane-${laneIndex}`);
|
|
617
640
|
const ref = `ref-l${laneIndex}`;
|
|
618
641
|
const script = scriptFor(kind, ref);
|
|
619
642
|
const model = promptScriptedModel(`chaos-${kind}-${laneIndex}`, script);
|
|
620
643
|
const digests = laneDigests(laneIndex);
|
|
621
644
|
|
|
622
645
|
const submitOptionsFor = (flatIndex: number) => ({
|
|
623
|
-
|
|
646
|
+
threadId,
|
|
624
647
|
principal: CHAOS_PRINCIPAL,
|
|
625
648
|
idempotencyKey: decodeIdempotencyKey(`chaos-${plan.seed}-s${flatIndex}`),
|
|
626
649
|
definitions: digests,
|
|
@@ -636,48 +659,46 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
636
659
|
|
|
637
660
|
const plainLaneFixture = (
|
|
638
661
|
deskInPlay: boolean,
|
|
639
|
-
drive: Effect.Effect<ReadonlyArray<Settlement>,
|
|
640
|
-
submitOne: (flatIndex: number) => Effect.Effect<Receipt,
|
|
662
|
+
drive: Effect.Effect<ReadonlyArray<Settlement>, DurableWorkerFailure | DurableBindingFailure>,
|
|
663
|
+
submitOne: (flatIndex: number) => Effect.Effect<Receipt, DurableSubmitFailure>,
|
|
641
664
|
): LaneFixture => ({
|
|
642
665
|
index: laneIndex,
|
|
643
666
|
kind,
|
|
644
|
-
|
|
667
|
+
threadId,
|
|
645
668
|
ref,
|
|
646
669
|
deskInPlay,
|
|
647
670
|
submissionIndexes,
|
|
648
671
|
submitOne,
|
|
649
672
|
drives: () => [drive],
|
|
650
|
-
|
|
673
|
+
childThreadOf: () => undefined,
|
|
651
674
|
});
|
|
652
675
|
|
|
653
676
|
switch (kind) {
|
|
654
677
|
case "plain":
|
|
655
678
|
case "join": {
|
|
656
679
|
const agent = Agent.withModel(plainDefinition, model);
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
runtime.
|
|
660
|
-
(flatIndex) =>
|
|
661
|
-
runtime.submit(agent, { question: `chaos ${flatIndex}` }, submitOptionsFor(flatIndex)),
|
|
680
|
+
|
|
681
|
+
return plainLaneFixture(false, runtime.processThread(agent, threadId), (flatIndex) =>
|
|
682
|
+
runtime.submit(agent, { question: `chaos ${flatIndex}` }, submitOptionsFor(flatIndex)),
|
|
662
683
|
);
|
|
663
684
|
}
|
|
664
685
|
case "uncertain-tool": {
|
|
665
686
|
const agent = Agent.withModel(bookDefinition, model);
|
|
687
|
+
|
|
666
688
|
return plainLaneFixture(
|
|
667
689
|
true,
|
|
668
|
-
runtime
|
|
669
|
-
.processConversation(agent, conversationId)
|
|
670
|
-
.pipe(Effect.provide(bookToolLayerFor(bookTools))),
|
|
690
|
+
runtime.processThread(agent, threadId).pipe(Effect.provide(bookToolLayerFor(bookTools))),
|
|
671
691
|
(flatIndex) =>
|
|
672
692
|
runtime.submit(agent, { question: `chaos ${flatIndex}` }, submitOptionsFor(flatIndex)),
|
|
673
693
|
);
|
|
674
694
|
}
|
|
675
695
|
case "approval": {
|
|
676
696
|
const agent = Agent.withModel(approvalDefinition, model);
|
|
697
|
+
|
|
677
698
|
return plainLaneFixture(
|
|
678
699
|
true,
|
|
679
700
|
runtime
|
|
680
|
-
.
|
|
701
|
+
.processThread(agent, threadId)
|
|
681
702
|
.pipe(Effect.provide(bookToolLayerFor(approvalTools))),
|
|
682
703
|
(flatIndex) =>
|
|
683
704
|
runtime.submit(agent, { question: `chaos ${flatIndex}` }, submitOptionsFor(flatIndex)),
|
|
@@ -685,57 +706,73 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
685
706
|
}
|
|
686
707
|
case "durable-steps": {
|
|
687
708
|
const agent = Agent.withModel(itineraryDefinition, model);
|
|
709
|
+
|
|
688
710
|
const toolLayer = itineraryTools.toLayer({
|
|
689
711
|
itinerary: ({ ref: called }) =>
|
|
690
712
|
Effect.gen(function* () {
|
|
691
713
|
const step = yield* DurableStep;
|
|
714
|
+
|
|
692
715
|
const flight = yield* step.do(
|
|
693
716
|
"reserve-flight",
|
|
694
717
|
Schema.String,
|
|
695
718
|
desk.record(flightValue(called)).pipe(Effect.as(flightValue(called))),
|
|
696
719
|
);
|
|
720
|
+
|
|
697
721
|
const lodging = yield* step.do(
|
|
698
722
|
"reserve-lodging",
|
|
699
723
|
Schema.String,
|
|
700
724
|
desk.record(lodgingValue(called)).pipe(Effect.as(lodgingValue(called))),
|
|
701
725
|
);
|
|
726
|
+
|
|
702
727
|
return { state: `${flight}+${lodging}` };
|
|
703
728
|
}),
|
|
704
729
|
});
|
|
730
|
+
|
|
705
731
|
return plainLaneFixture(
|
|
706
732
|
true,
|
|
707
|
-
runtime.
|
|
733
|
+
runtime.processThread(agent, threadId).pipe(Effect.provide(toolLayer)),
|
|
708
734
|
(flatIndex) =>
|
|
709
735
|
runtime.submit(agent, { question: `chaos ${flatIndex}` }, submitOptionsFor(flatIndex)),
|
|
710
736
|
);
|
|
711
737
|
}
|
|
712
738
|
case "delegation": {
|
|
713
739
|
const parentBinding = Agent.withModel(coordinatorDefinition, model);
|
|
740
|
+
|
|
714
741
|
const childModel = promptScriptedModel(`chaos-child-${laneIndex}`, () =>
|
|
715
742
|
finalParts('{"answer":"child"}'),
|
|
716
743
|
);
|
|
744
|
+
|
|
717
745
|
const childBinding = Agent.withModel(childDefinition, childModel);
|
|
718
|
-
|
|
746
|
+
|
|
747
|
+
const delegationLayer = Subagent.layer(chaosDelegation, childBinding, {
|
|
719
748
|
mapChildFailure: (failure) => ChaosDelegationFailed.make({ childErrorTag: failure._tag }),
|
|
720
749
|
durable: { targetDigests: childDigestStrings(laneIndex) },
|
|
721
750
|
}).pipe(Layer.provide(delegationSupport));
|
|
751
|
+
|
|
722
752
|
const parentResolved: ResolvedBinding = yield* DurableWorkerBinding.make(
|
|
723
753
|
parentBinding,
|
|
724
754
|
digests,
|
|
725
755
|
).pipe(Effect.provide(delegationLayer));
|
|
756
|
+
|
|
726
757
|
const childResolved: ResolvedBinding = yield* DurableWorkerBinding.make(
|
|
727
758
|
childBinding,
|
|
728
759
|
childLaneDigests(laneIndex),
|
|
729
760
|
);
|
|
730
|
-
|
|
731
|
-
const
|
|
732
|
-
|
|
733
|
-
.
|
|
734
|
-
|
|
761
|
+
|
|
762
|
+
const registeredRuntime = yield* DurableAgentRuntime.pipe(
|
|
763
|
+
Effect.provide(
|
|
764
|
+
DurableAgentRuntime.layerWithBindings([parentResolved, childResolved]).pipe(
|
|
765
|
+
Layer.provide(RunToolAuthorization.allowAll),
|
|
766
|
+
),
|
|
767
|
+
),
|
|
768
|
+
);
|
|
769
|
+
|
|
770
|
+
const driveResolved = (thread: ThreadId) => registeredRuntime.processThreadResolved(thread);
|
|
771
|
+
|
|
735
772
|
const fixture: LaneFixture = {
|
|
736
773
|
index: laneIndex,
|
|
737
774
|
kind,
|
|
738
|
-
|
|
775
|
+
threadId,
|
|
739
776
|
ref,
|
|
740
777
|
deskInPlay: false,
|
|
741
778
|
submissionIndexes,
|
|
@@ -746,24 +783,24 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
746
783
|
submitOptionsFor(flatIndex),
|
|
747
784
|
),
|
|
748
785
|
drives: (firstReceipt) => {
|
|
749
|
-
const drives: Array<
|
|
750
|
-
|
|
751
|
-
];
|
|
786
|
+
const drives: Array<
|
|
787
|
+
Effect.Effect<ReadonlyArray<Settlement>, DurableWorkerFailure | DurableBindingFailure>
|
|
788
|
+
> = [driveResolved(threadId)];
|
|
789
|
+
|
|
752
790
|
if (firstReceipt !== undefined) {
|
|
753
791
|
drives.push(
|
|
754
792
|
driveResolved(
|
|
755
|
-
|
|
756
|
-
firstReceipt.submissionId,
|
|
757
|
-
decodeToolCallId(DELEGATE_CALL_ID),
|
|
758
|
-
),
|
|
793
|
+
childThreadIdFor(firstReceipt.submissionId, decodeToolCallId(DELEGATE_CALL_ID)),
|
|
759
794
|
),
|
|
760
795
|
);
|
|
761
796
|
}
|
|
797
|
+
|
|
762
798
|
return drives;
|
|
763
799
|
},
|
|
764
|
-
|
|
765
|
-
|
|
800
|
+
childThreadOf: (firstReceipt) =>
|
|
801
|
+
childThreadIdFor(firstReceipt.submissionId, decodeToolCallId(DELEGATE_CALL_ID)),
|
|
766
802
|
};
|
|
803
|
+
|
|
767
804
|
return fixture;
|
|
768
805
|
}
|
|
769
806
|
}
|
|
@@ -772,7 +809,9 @@ const makeLaneFixture = Effect.fn("Chaos.makeLaneFixture")(function* (
|
|
|
772
809
|
/** Stable per-call index into an injection list (identical across resolution passes). */
|
|
773
810
|
const injectionIndex = (submissionFlatIndex: number, callId: string, length: number): number => {
|
|
774
811
|
let hash = submissionFlatIndex + 1;
|
|
812
|
+
|
|
775
813
|
for (const char of callId) hash = (Math.imul(hash, 31) + char.charCodeAt(0)) | 0;
|
|
814
|
+
|
|
776
815
|
return ((hash % length) + length) % length;
|
|
777
816
|
};
|
|
778
817
|
|
|
@@ -802,6 +841,7 @@ const resolutionFor = (
|
|
|
802
841
|
isFailure: false,
|
|
803
842
|
});
|
|
804
843
|
}
|
|
844
|
+
|
|
805
845
|
// The desk never produced a value for this call — resolving "completed" would fabricate.
|
|
806
846
|
return ResolutionNeverHappened.make();
|
|
807
847
|
}
|
|
@@ -820,8 +860,10 @@ const resolutionPass = Effect.fn("Chaos.resolutionPass")(function* (
|
|
|
820
860
|
const ledger = yield* SubmissionLedger;
|
|
821
861
|
const produced = yield* desk.produced;
|
|
822
862
|
const nonterminal = yield* tolerateTyped(Stream.runCollect(ledger.scanNonterminal));
|
|
863
|
+
|
|
823
864
|
if (Option.isNone(nonterminal)) return;
|
|
824
865
|
const byId = new Map<SubmissionId, SubmissionState>();
|
|
866
|
+
|
|
825
867
|
for (const state of states) {
|
|
826
868
|
if (state.receipt !== undefined) byId.set(state.receipt.submissionId, state);
|
|
827
869
|
}
|
|
@@ -829,18 +871,22 @@ const resolutionPass = Effect.fn("Chaos.resolutionPass")(function* (
|
|
|
829
871
|
if (row.state !== "unknown" && row.state !== "suspended") continue;
|
|
830
872
|
const state = byId.get(row.submissionId);
|
|
831
873
|
const explanation = yield* tolerateTyped(runtime.explain(row.submissionId));
|
|
874
|
+
|
|
832
875
|
if (Option.isNone(explanation)) continue;
|
|
833
876
|
const flatIndex = state?.flatIndex ?? 0;
|
|
834
877
|
const ref = state?.lane.ref ?? "ref-child";
|
|
878
|
+
|
|
835
879
|
if (row.state === "unknown") {
|
|
836
880
|
for (const call of explanation.value.evidence.unknownCalls) {
|
|
837
881
|
if (call.resolved) continue;
|
|
882
|
+
|
|
838
883
|
const kind =
|
|
839
884
|
plan.resolutionInjections.length === 0
|
|
840
885
|
? "never-happened"
|
|
841
|
-
: plan.resolutionInjections
|
|
842
|
-
injectionIndex(flatIndex, call.toolCallId, plan.resolutionInjections.length)
|
|
843
|
-
|
|
886
|
+
: (plan.resolutionInjections.at(
|
|
887
|
+
injectionIndex(flatIndex, call.toolCallId, plan.resolutionInjections.length),
|
|
888
|
+
) ?? "never-happened");
|
|
889
|
+
|
|
844
890
|
yield* tolerateTyped(
|
|
845
891
|
runtime.resolveUnknown(
|
|
846
892
|
UnknownResolutionCommand.make({
|
|
@@ -858,9 +904,10 @@ const resolutionPass = Effect.fn("Chaos.resolutionPass")(function* (
|
|
|
858
904
|
const decision =
|
|
859
905
|
plan.approvalDecisions.length === 0
|
|
860
906
|
? "approved"
|
|
861
|
-
: plan.approvalDecisions
|
|
862
|
-
injectionIndex(flatIndex, pending.toolCallId, plan.approvalDecisions.length)
|
|
863
|
-
|
|
907
|
+
: (plan.approvalDecisions.at(
|
|
908
|
+
injectionIndex(flatIndex, pending.toolCallId, plan.approvalDecisions.length),
|
|
909
|
+
) ?? "approved");
|
|
910
|
+
|
|
864
911
|
yield* tolerateTyped(
|
|
865
912
|
runtime.resolveApproval(
|
|
866
913
|
ApprovalDecisionCommand.make({
|
|
@@ -881,16 +928,19 @@ const submissionIdsNamedBy = (
|
|
|
881
928
|
records: ReadonlyArray<CanonicalRecordEnvelope>,
|
|
882
929
|
): ReadonlySet<SubmissionId> => {
|
|
883
930
|
const named = new Set<SubmissionId>();
|
|
931
|
+
|
|
884
932
|
for (const envelope of records) {
|
|
885
933
|
const payload = envelope.record.payload;
|
|
934
|
+
|
|
886
935
|
if (
|
|
887
936
|
payload._tag === "UserInputRecorded" ||
|
|
888
937
|
payload._tag === "SubmissionSettled" ||
|
|
889
938
|
payload._tag === "AbortRequested"
|
|
890
939
|
) {
|
|
891
|
-
named.add(payload.submissionId);
|
|
940
|
+
if (payload.submissionId !== undefined) named.add(payload.submissionId);
|
|
892
941
|
}
|
|
893
942
|
}
|
|
943
|
+
|
|
894
944
|
return named;
|
|
895
945
|
};
|
|
896
946
|
|
|
@@ -910,18 +960,23 @@ const assertNoFabrication = (
|
|
|
910
960
|
produced: ReadonlySet<string>,
|
|
911
961
|
): Effect.Effect<void, ChaosConvergenceFailure> => {
|
|
912
962
|
const fabricated: Array<string> = [];
|
|
963
|
+
|
|
913
964
|
const requireProduced = (value: string, label: string): void => {
|
|
914
965
|
if (!produced.has(value)) fabricated.push(`${label} "${value}"`);
|
|
915
966
|
};
|
|
967
|
+
|
|
916
968
|
for (const envelope of records) {
|
|
917
969
|
const payload = envelope.record.payload;
|
|
970
|
+
|
|
918
971
|
if (payload._tag === "ToolCallSettled" && !payload.isFailure) {
|
|
919
972
|
if (payload.toolName === "book") {
|
|
920
973
|
const result = decodeBookResult(payload.result);
|
|
974
|
+
|
|
921
975
|
if (Option.isSome(result)) requireProduced(result.value.confirmation, "book result");
|
|
922
976
|
}
|
|
923
977
|
if (payload.toolName === "itinerary") {
|
|
924
978
|
const result = decodeItineraryResult(payload.result);
|
|
979
|
+
|
|
925
980
|
if (Option.isSome(result)) {
|
|
926
981
|
for (const part of result.value.state.split("+")) {
|
|
927
982
|
requireProduced(part, "itinerary step result");
|
|
@@ -931,9 +986,11 @@ const assertNoFabrication = (
|
|
|
931
986
|
}
|
|
932
987
|
if (payload._tag === "ToolStepSettled") {
|
|
933
988
|
const output = decodeStepOutput(payload.output);
|
|
989
|
+
|
|
934
990
|
if (Option.isSome(output)) requireProduced(output.value, "step output");
|
|
935
991
|
}
|
|
936
992
|
}
|
|
993
|
+
|
|
937
994
|
return fabricated.length === 0
|
|
938
995
|
? Effect.void
|
|
939
996
|
: Effect.fail(
|
|
@@ -954,7 +1011,7 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
954
1011
|
) {
|
|
955
1012
|
const runtime = yield* DurableAgentRuntime;
|
|
956
1013
|
const ledger = yield* SubmissionLedger;
|
|
957
|
-
const store = yield*
|
|
1014
|
+
const store = yield* ThreadStore;
|
|
958
1015
|
const config = yield* DurableRuntimeConfig;
|
|
959
1016
|
const failpoints = yield* DurableRuntimeFailpointTestControl;
|
|
960
1017
|
const random = mulberry32(plan.seed);
|
|
@@ -963,28 +1020,43 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
963
1020
|
// Lane fixtures: the FIRST spec of each lane fixes the lane's agent kind.
|
|
964
1021
|
const laneKinds = new Map<number, ChaosScenarioKind>();
|
|
965
1022
|
const laneSubmissions = new Map<number, Array<number>>();
|
|
1023
|
+
|
|
966
1024
|
plan.submissions.forEach((spec, flatIndex) => {
|
|
967
1025
|
const lane = spec.lane % plan.lanes;
|
|
1026
|
+
|
|
968
1027
|
if (!laneKinds.has(lane)) laneKinds.set(lane, spec.kind);
|
|
969
1028
|
const list = laneSubmissions.get(lane) ?? [];
|
|
1029
|
+
|
|
970
1030
|
list.push(flatIndex);
|
|
971
1031
|
laneSubmissions.set(lane, list);
|
|
972
1032
|
});
|
|
973
1033
|
const lanes: Array<LaneFixture> = [];
|
|
1034
|
+
|
|
974
1035
|
for (const [lane, kind] of laneKinds) {
|
|
975
1036
|
lanes.push(yield* makeLaneFixture(plan, lane, kind, laneSubmissions.get(lane) ?? [], desk));
|
|
976
1037
|
}
|
|
977
1038
|
|
|
978
|
-
const
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
1039
|
+
const lanesByIndex = new Map(lanes.map((lane) => [lane.index, lane]));
|
|
1040
|
+
const states: Array<SubmissionState> = [];
|
|
1041
|
+
|
|
1042
|
+
for (const [flatIndex, spec] of plan.submissions.entries()) {
|
|
1043
|
+
const laneIndex = spec.lane % plan.lanes;
|
|
1044
|
+
const lane = lanesByIndex.get(laneIndex);
|
|
1045
|
+
|
|
1046
|
+
if (lane === undefined) {
|
|
1047
|
+
return yield* ChaosConvergenceFailure.make({
|
|
1048
|
+
seed: plan.seed,
|
|
1049
|
+
message: `submission ${flatIndex} addresses missing lane ${laneIndex}`,
|
|
1050
|
+
});
|
|
1051
|
+
}
|
|
1052
|
+
states.push({ flatIndex, lane, receipt: undefined });
|
|
1053
|
+
}
|
|
983
1054
|
const appliedAborts = new Set<number>();
|
|
984
1055
|
|
|
985
1056
|
type ArmEntry =
|
|
986
1057
|
| { readonly family: "coordinator"; readonly location: DurableRuntimeFailpointLocation }
|
|
987
1058
|
| { readonly family: "adapter"; readonly location: string };
|
|
1059
|
+
|
|
988
1060
|
const armQueue: Array<ArmEntry> = [
|
|
989
1061
|
...plan.failpointArms.map((location): ArmEntry => ({ family: "coordinator", location })),
|
|
990
1062
|
...plan.adapterArms.map((location): ArmEntry => ({ family: "adapter", location })),
|
|
@@ -993,17 +1065,21 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
993
1065
|
const allSettled = Effect.gen(function* () {
|
|
994
1066
|
if (states.some((state) => state.receipt === undefined)) return false;
|
|
995
1067
|
const nonterminal = yield* tolerateTyped(Stream.runCollect(ledger.scanNonterminal));
|
|
1068
|
+
|
|
996
1069
|
return Option.isSome(nonterminal) && Array.from(nonterminal.value).length === 0;
|
|
997
1070
|
});
|
|
998
1071
|
|
|
999
1072
|
const maxRounds = armQueue.length + states.length * 2 + 12;
|
|
1000
1073
|
let rounds = 0;
|
|
1001
1074
|
let converged = false;
|
|
1075
|
+
|
|
1002
1076
|
for (let round = 0; round < maxRounds; round++) {
|
|
1003
1077
|
rounds = round + 1;
|
|
1004
1078
|
const arm = armQueue[round];
|
|
1079
|
+
|
|
1005
1080
|
if (arm?.family === "coordinator") {
|
|
1006
1081
|
const location = arm.location;
|
|
1082
|
+
|
|
1007
1083
|
yield* failpoints.setHandler((hit) =>
|
|
1008
1084
|
hit === location
|
|
1009
1085
|
? Effect.fail(DurableRuntimeFailpointError.make({ location: hit }))
|
|
@@ -1014,18 +1090,24 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1014
1090
|
}
|
|
1015
1091
|
|
|
1016
1092
|
// Admission chaos: pending submissions retry under the active arm until a Receipt lands;
|
|
1017
|
-
// the identical (
|
|
1093
|
+
// the identical (thread, principal, key) triple reattaches, never duplicates.
|
|
1018
1094
|
for (const state of states) {
|
|
1019
1095
|
if (state.receipt !== undefined) continue;
|
|
1020
1096
|
const receipt = yield* tolerateTyped(state.lane.submitOne(state.flatIndex));
|
|
1097
|
+
|
|
1021
1098
|
if (Option.isSome(receipt)) state.receipt = receipt.value;
|
|
1022
1099
|
}
|
|
1023
1100
|
|
|
1024
1101
|
// Drive every lane (and discovered child lanes) in seeded order under the active arm.
|
|
1025
|
-
const order =
|
|
1102
|
+
const order = lanes
|
|
1103
|
+
.map((lane) => ({ lane, rank: random() }))
|
|
1104
|
+
.sort((left, right) => left.rank - right.rank || left.lane.index - right.lane.index)
|
|
1105
|
+
.map(({ lane }) => lane);
|
|
1106
|
+
|
|
1026
1107
|
for (const lane of order) {
|
|
1027
1108
|
const firstFlat = lane.submissionIndexes[0];
|
|
1028
1109
|
const firstReceipt = firstFlat === undefined ? undefined : states[firstFlat]?.receipt;
|
|
1110
|
+
|
|
1029
1111
|
for (const drive of lane.drives(firstReceipt)) {
|
|
1030
1112
|
yield* tolerateTyped(drive);
|
|
1031
1113
|
}
|
|
@@ -1035,8 +1117,10 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1035
1117
|
if (round >= 1) {
|
|
1036
1118
|
for (const rawIndex of plan.abortInjections) {
|
|
1037
1119
|
const index = rawIndex % states.length;
|
|
1120
|
+
|
|
1038
1121
|
if (appliedAborts.has(index)) continue;
|
|
1039
1122
|
const receipt = states[index]?.receipt;
|
|
1123
|
+
|
|
1040
1124
|
if (receipt === undefined) continue;
|
|
1041
1125
|
appliedAborts.add(index);
|
|
1042
1126
|
yield* tolerateTyped(
|
|
@@ -1070,35 +1154,40 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1070
1154
|
|
|
1071
1155
|
if (!converged) {
|
|
1072
1156
|
const nonterminal = yield* tolerateTyped(Stream.runCollect(ledger.scanNonterminal));
|
|
1157
|
+
|
|
1073
1158
|
const detail = Option.isSome(nonterminal)
|
|
1074
1159
|
? Array.from(nonterminal.value)
|
|
1075
1160
|
.map((row: SubmissionSnapshot) => `${row.submissionId}(${row.state})`)
|
|
1076
1161
|
.join(", ")
|
|
1077
1162
|
: "ledger scan failed";
|
|
1163
|
+
|
|
1078
1164
|
return yield* ChaosConvergenceFailure.make({
|
|
1079
1165
|
seed: plan.seed,
|
|
1080
1166
|
message: `plan did not converge within ${maxRounds} rounds; nonterminal: [${detail}]; pending receipts: ${states.filter((state) => state.receipt === undefined).length}`,
|
|
1081
1167
|
});
|
|
1082
1168
|
}
|
|
1083
1169
|
|
|
1084
|
-
// Final claims: the shared invariant checker per touched
|
|
1170
|
+
// Final claims: the shared invariant checker per touched Thread, in convergence mode,
|
|
1085
1171
|
// with the full digest chain (single known producer), plus the desk non-fabrication sweep.
|
|
1086
1172
|
const produced = yield* desk.produced;
|
|
1087
1173
|
const laneReports: Array<ChaosLaneReport> = [];
|
|
1088
|
-
|
|
1089
|
-
|
|
1174
|
+
|
|
1175
|
+
const verifyThread = Effect.fn("Chaos.verifyThread")(function* (
|
|
1176
|
+
threadId: ThreadId,
|
|
1090
1177
|
kind: ChaosScenarioKind,
|
|
1091
1178
|
deskInPlay: boolean,
|
|
1092
1179
|
) {
|
|
1093
|
-
const exported = yield* store.export(
|
|
1180
|
+
const exported = yield* store.export(ThreadExportRequest.make({ threadId })).pipe(
|
|
1094
1181
|
Effect.mapError((error) =>
|
|
1095
1182
|
ChaosConvergenceFailure.make({
|
|
1096
1183
|
seed: plan.seed,
|
|
1097
|
-
message: `export of ${
|
|
1184
|
+
message: `export of ${threadId} failed: ${String(error)}`,
|
|
1098
1185
|
}),
|
|
1099
1186
|
),
|
|
1100
1187
|
);
|
|
1188
|
+
|
|
1101
1189
|
const rows: Array<SubmissionSnapshot> = [];
|
|
1190
|
+
|
|
1102
1191
|
for (const submissionId of submissionIdsNamedBy(exported.records)) {
|
|
1103
1192
|
const found = yield* ledger.lookup(SubmissionLookupById.make({ submissionId })).pipe(
|
|
1104
1193
|
Effect.mapError((error) =>
|
|
@@ -1108,25 +1197,30 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1108
1197
|
}),
|
|
1109
1198
|
),
|
|
1110
1199
|
);
|
|
1200
|
+
|
|
1111
1201
|
if (Option.isSome(found)) rows.push(found.value);
|
|
1112
1202
|
}
|
|
1203
|
+
|
|
1113
1204
|
const batchProducers = new Map<BatchId, ProducerId>(
|
|
1114
1205
|
exported.records.map((envelope) => [envelope.batchId, config.producerId]),
|
|
1115
1206
|
);
|
|
1116
|
-
|
|
1207
|
+
|
|
1208
|
+
const report = yield* verifyThreadInvariants({
|
|
1117
1209
|
export: exported,
|
|
1118
1210
|
submissions: rows,
|
|
1119
1211
|
batchProducers,
|
|
1120
1212
|
requireAllSettled: true,
|
|
1121
1213
|
});
|
|
1214
|
+
|
|
1122
1215
|
if (!report.ok) {
|
|
1123
1216
|
const failed = report.checks
|
|
1124
1217
|
.filter((check) => check.status === "failed")
|
|
1125
1218
|
.map((check) => `${check.name}: ${check.detail ?? "failed"}`)
|
|
1126
1219
|
.join("; ");
|
|
1220
|
+
|
|
1127
1221
|
return yield* ChaosConvergenceFailure.make({
|
|
1128
1222
|
seed: plan.seed,
|
|
1129
|
-
message: `invariants failed for ${
|
|
1223
|
+
message: `invariants failed for ${threadId} (${kind}): ${failed}`,
|
|
1130
1224
|
});
|
|
1131
1225
|
}
|
|
1132
1226
|
if (deskInPlay) {
|
|
@@ -1134,7 +1228,7 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1134
1228
|
}
|
|
1135
1229
|
laneReports.push(
|
|
1136
1230
|
ChaosLaneReport.make({
|
|
1137
|
-
|
|
1231
|
+
threadId,
|
|
1138
1232
|
kind,
|
|
1139
1233
|
submissionCount: rows.length,
|
|
1140
1234
|
verified: report.ok,
|
|
@@ -1143,18 +1237,22 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1143
1237
|
});
|
|
1144
1238
|
|
|
1145
1239
|
for (const lane of lanes) {
|
|
1146
|
-
yield*
|
|
1147
|
-
// Delegation lanes: verify every materialized child
|
|
1240
|
+
yield* verifyThread(lane.threadId, lane.kind, lane.deskInPlay);
|
|
1241
|
+
// Delegation lanes: verify every materialized child Thread too.
|
|
1148
1242
|
for (const flatIndex of lane.submissionIndexes) {
|
|
1149
1243
|
const receipt = states[flatIndex]?.receipt;
|
|
1244
|
+
|
|
1150
1245
|
if (receipt === undefined) continue;
|
|
1151
|
-
const child = lane.
|
|
1246
|
+
const child = lane.childThreadOf(receipt);
|
|
1247
|
+
|
|
1152
1248
|
if (child === undefined) continue;
|
|
1249
|
+
|
|
1153
1250
|
const childExport = yield* Effect.exit(
|
|
1154
|
-
store.export(
|
|
1251
|
+
store.export(ThreadExportRequest.make({ threadId: child })),
|
|
1155
1252
|
);
|
|
1253
|
+
|
|
1156
1254
|
if (Exit.isSuccess(childExport) && childExport.value.records.length > 0) {
|
|
1157
|
-
yield*
|
|
1255
|
+
yield* verifyThread(child, "plain", false);
|
|
1158
1256
|
}
|
|
1159
1257
|
}
|
|
1160
1258
|
}
|
|
@@ -1169,6 +1267,7 @@ export const runChaosPlan = Effect.fn("Chaos.runChaosPlan")(function* (
|
|
|
1169
1267
|
}),
|
|
1170
1268
|
),
|
|
1171
1269
|
);
|
|
1270
|
+
|
|
1172
1271
|
if (obligations.entries.length > 0) {
|
|
1173
1272
|
return yield* ChaosConvergenceFailure.make({
|
|
1174
1273
|
seed: plan.seed,
|