@effect-agent/testing 0.1.0-beta.9 → 0.1.0-beta.91
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Certification.d.mts +106 -0
- package/dist/Certification.mjs +633 -0
- package/dist/Certification.mjs.map +1 -0
- package/dist/Chaos.d.mts +126 -0
- package/dist/Chaos.mjs +755 -0
- package/dist/Chaos.mjs.map +1 -0
- package/dist/CodeExecutorConformance.d.mts +33 -0
- package/dist/CodeExecutorConformance.mjs +231 -0
- package/dist/CodeExecutorConformance.mjs.map +1 -0
- package/dist/CodeExecutorSubstitute.d.mts +21 -0
- package/dist/CodeExecutorSubstitute.mjs +368 -0
- package/dist/CodeExecutorSubstitute.mjs.map +1 -0
- package/dist/DocsResearcher.d.mts +276 -0
- package/dist/DocsResearcher.mjs +490 -0
- package/dist/DocsResearcher.mjs.map +1 -0
- package/dist/ScriptedModel-DAvxIiud.d.mts +220 -0
- package/dist/ScriptedModel.d.mts +2 -0
- package/dist/ScriptedModel.mjs +155 -0
- package/dist/ScriptedModel.mjs.map +1 -0
- package/dist/TravelPlanner.d.mts +1680 -0
- package/dist/TravelPlanner.mjs +1963 -0
- package/dist/TravelPlanner.mjs.map +1 -0
- package/dist/deterministic-layers-D5owIoke.mjs +358 -0
- package/dist/deterministic-layers-D5owIoke.mjs.map +1 -0
- package/dist/index.d.mts +2 -3408
- package/dist/index.mjs +2 -4771
- package/dist/rolldown-runtime-D7D4PA-g.mjs +13 -0
- package/package.json +1 -48
- package/src/{certification.ts → Certification.ts} +366 -171
- package/src/{chaos.ts → Chaos.ts} +241 -127
- package/src/{code-executor-conformance.ts → CodeExecutorConformance.ts} +30 -7
- package/src/{code-executor-substitute.ts → CodeExecutorSubstitute.ts} +113 -46
- package/src/{fixtures/docs-researcher/index.ts → DocsResearcher.ts} +68 -5
- package/src/{scripted-model.ts → ScriptedModel.ts} +25 -29
- package/src/TravelPlanner.ts +232 -0
- package/src/fixtures/docs-researcher/definition.ts +20 -11
- package/src/fixtures/docs-researcher/harness.ts +41 -34
- package/src/fixtures/docs-researcher/mcp.ts +49 -3
- package/src/fixtures/travel-planner/definition.ts +15 -2
- package/src/fixtures/travel-planner/deterministic-layers.ts +47 -4
- package/src/fixtures/travel-planner/phase2.ts +4 -3
- package/src/fixtures/travel-planner/phase3.ts +19 -42
- package/src/fixtures/travel-planner/phase4.ts +24 -37
- package/src/fixtures/travel-planner/phase5.ts +35 -42
- package/src/fixtures/travel-planner/phase6.ts +191 -88
- package/src/fixtures/travel-planner/phase7.ts +4 -102
- package/src/fixtures/travel-planner/scenarios.ts +3 -4
- package/src/fixtures/travel-planner/subagents-durable.ts +35 -61
- package/src/fixtures/travel-planner/subagents.ts +36 -14
- package/src/index.ts +1 -11
- package/src/internal/certification-report.ts +25 -0
- package/dist/index.mjs.map +0 -1
- package/src/code-executor-conformance.d.ts +0 -30
- package/src/fixtures/travel-planner/index.ts +0 -11
- package/src/fixtures/warehouse/index.ts +0 -412
|
@@ -1,80 +1,83 @@
|
|
|
1
|
+
import type { Crypto } from "effect";
|
|
1
2
|
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
Cause,
|
|
4
|
+
Clock,
|
|
5
|
+
DateTime,
|
|
6
|
+
Duration,
|
|
7
|
+
Effect,
|
|
8
|
+
Exit,
|
|
9
|
+
Layer,
|
|
10
|
+
Option,
|
|
11
|
+
Ref,
|
|
12
|
+
Schema,
|
|
13
|
+
Stream,
|
|
14
|
+
} from "effect";
|
|
15
|
+
import * as Agent from "effect-agent/agent";
|
|
16
|
+
import { AgentPolicy } from "effect-agent/agent-policy";
|
|
17
|
+
import { DurableWorkerBinding, type ResolvedBinding } from "effect-agent/agent-registration";
|
|
7
18
|
import {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
19
|
+
DurableAgentRuntime,
|
|
20
|
+
DurableRuntimeConfig,
|
|
21
|
+
type DurableSubmitFailure,
|
|
22
|
+
type DurableSubmitOptions,
|
|
23
|
+
type Receipt,
|
|
24
|
+
} from "effect-agent/durable-agent-runtime";
|
|
25
|
+
import {
|
|
26
|
+
DurableRuntimeFailpointError,
|
|
27
|
+
DurableRuntimeFailpointLocation,
|
|
28
|
+
} from "effect-agent/durable-failpoint";
|
|
29
|
+
import { DurableStep, DurableStepError } from "effect-agent/durable-step";
|
|
30
|
+
import { IdGenerator } from "effect-agent/id-generator";
|
|
31
|
+
import {
|
|
32
|
+
ThreadId,
|
|
12
33
|
RunId,
|
|
13
34
|
ToolCallId,
|
|
14
35
|
TurnId,
|
|
15
36
|
type AgentId,
|
|
16
37
|
type SubmissionId,
|
|
17
|
-
} from "
|
|
18
|
-
import { DurableStep, DurableStepError } from "@effect-agent/engine";
|
|
38
|
+
} from "effect-agent/identifiers";
|
|
19
39
|
import {
|
|
20
|
-
AgentBindingResolver,
|
|
21
|
-
ApprovalDecisionCommand,
|
|
22
|
-
CertificationCaseResult,
|
|
23
|
-
CertificationReport,
|
|
24
|
-
CertificationSweepResult,
|
|
25
|
-
CertificationTierThreeReport,
|
|
26
|
-
CertifiedAdapterIdentity,
|
|
27
|
-
ConversationExportRequest,
|
|
28
|
-
ConversationStore,
|
|
29
40
|
DefinitionDigests,
|
|
30
41
|
DeploymentId,
|
|
31
42
|
Digest,
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
43
|
+
ProducerId,
|
|
44
|
+
type BatchId,
|
|
45
|
+
} from "effect-agent/records";
|
|
46
|
+
import { childThreadIdFor } from "effect-agent/run-journal";
|
|
47
|
+
import { RunToolAuthorization } from "effect-agent/run-options";
|
|
48
|
+
import * as Subagent from "effect-agent/subagent";
|
|
49
|
+
import { SubagentPolicy } from "effect-agent/subagent";
|
|
50
|
+
import { SubagentReservationsMemoryLive } from "effect-agent/subagent-reservations";
|
|
51
|
+
import {
|
|
52
|
+
ApprovalDecisionCommand,
|
|
39
53
|
IdempotencyKey,
|
|
40
|
-
LoadCheckpointRequest,
|
|
41
54
|
Principal,
|
|
42
|
-
ProducerId,
|
|
43
55
|
ResolutionSafeToRetry,
|
|
44
56
|
SubmissionLedger,
|
|
45
57
|
SubmissionLookupById,
|
|
46
|
-
ToolReconciler,
|
|
47
58
|
UnknownResolutionCommand,
|
|
48
|
-
WakeScheduler,
|
|
49
59
|
DEFAULT_OWNERSHIP_LEASE_DURATION,
|
|
50
|
-
certifyPorts,
|
|
51
|
-
childConversationIdFor,
|
|
52
|
-
verifyConversationInvariants,
|
|
53
|
-
type BatchId,
|
|
54
|
-
type CertificationScenario,
|
|
55
|
-
type DurableSubmitFailure,
|
|
56
|
-
type DurableSubmitOptions,
|
|
57
|
-
type Receipt,
|
|
58
|
-
type ResolvedBinding,
|
|
59
60
|
type SubmissionSnapshot,
|
|
60
|
-
} from "
|
|
61
|
+
} from "effect-agent/submission-ledger";
|
|
61
62
|
import {
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
} from "effect";
|
|
63
|
+
type CertificationCaseResult,
|
|
64
|
+
type CertificationReport,
|
|
65
|
+
CertificationSweepResult,
|
|
66
|
+
CertificationTierThreeReport,
|
|
67
|
+
CertifiedAdapterIdentity,
|
|
68
|
+
certifyPorts,
|
|
69
|
+
type CertificationScenario,
|
|
70
|
+
} from "effect-agent/testing/certification";
|
|
71
|
+
import { DurableRuntimeFailpointTestControl } from "effect-agent/testing/durable-failpoint-test-control";
|
|
72
|
+
import { verifyThreadInvariants } from "effect-agent/thread-invariants";
|
|
73
|
+
import { ThreadExportRequest, ThreadStore, LoadCheckpointRequest } from "effect-agent/thread-store";
|
|
74
|
+
import { ToolReconciler } from "effect-agent/tool-reconciler";
|
|
75
|
+
import { WakeScheduler } from "effect-agent/wake-scheduler";
|
|
75
76
|
import { TestClock } from "effect/testing";
|
|
76
77
|
import { LanguageModel, Model, Tool, Toolkit, type Response } from "effect/unstable/ai";
|
|
77
78
|
|
|
79
|
+
import { makeCertificationReport } from "./internal/certification-report.ts";
|
|
80
|
+
|
|
78
81
|
/**
|
|
79
82
|
* P7 WP2 — `certifyDurableAdapters` (plan §1): the one certification entry point a durable
|
|
80
83
|
* adapter pair runs to earn a Schema-encoded certificate.
|
|
@@ -83,12 +86,13 @@ import { LanguageModel, Model, Tool, Toolkit, type Response } from "effect/unsta
|
|
|
83
86
|
* `certifyPorts` (TEST-004/STORE-010).
|
|
84
87
|
* - **Tier 2 — coordinator protocol + failpoint convergence**: the durable coordinator is
|
|
85
88
|
* assembled over the CANDIDATE Layer pair with a scripted deterministic model; every
|
|
86
|
-
* `DurableRuntimeFailpointLocation` is armed one-shot across the six scenario shapes
|
|
87
|
-
* (plain / uncertain-tool / durable-steps / approval / join / delegation).
|
|
88
|
-
*
|
|
89
|
-
*
|
|
90
|
-
*
|
|
91
|
-
*
|
|
89
|
+
* reached `DurableRuntimeFailpointLocation` is armed one-shot across the six scenario shapes
|
|
90
|
+
* (plain / uncertain-tool / durable-steps / approval / join / delegation). Unreached locations
|
|
91
|
+
* share a verified clean run per shape; new unaccounted-for locations get a full sweep.
|
|
92
|
+
* After the injected fault the runner asserts the state stays CLASSIFIABLE (recovery +
|
|
93
|
+
* the public unblocking operations `resolveUnknown`/`resolveApproval` are the only levers
|
|
94
|
+
* used) and that the re-drive CONVERGES to `verifyThreadInvariants` with `requireAllSettled`,
|
|
95
|
+
* including a fully discharged digest-chain check, because the runner captures per-batch producer
|
|
92
96
|
* identity at append time.
|
|
93
97
|
* - **Tier 3 — real loss lever**: recorded honestly. A durable adapter either supplies a
|
|
94
98
|
* `CertificationCrashLever` executed in this run, cites its committed real-loss evidence
|
|
@@ -126,7 +130,7 @@ export interface CertifyDurableAdaptersOptions<LedgerE = never, StoreE = never>
|
|
|
126
130
|
* does); the certification's own environment supplies nothing else.
|
|
127
131
|
*/
|
|
128
132
|
readonly submissionLedger: Layer.Layer<SubmissionLedger, LedgerE, Crypto.Crypto>;
|
|
129
|
-
readonly
|
|
133
|
+
readonly threadStore: Layer.Layer<ThreadStore, StoreE, Crypto.Crypto>;
|
|
130
134
|
/** Defaults to `WakeScheduler.layerNoop`; the runner re-drives lanes explicitly. */
|
|
131
135
|
readonly wakeScheduler?: Layer.Layer<WakeScheduler> | undefined;
|
|
132
136
|
/** Executes Tier 3 in this run; takes precedence over `tierThreeEvidence`. */
|
|
@@ -156,23 +160,58 @@ export const CERTIFICATION_SCENARIOS: ReadonlyArray<CertificationScenario> = [
|
|
|
156
160
|
|
|
157
161
|
/**
|
|
158
162
|
* Coordinator failpoint locations that none of the six scenario shapes can reach, recorded
|
|
159
|
-
* honestly instead of silently claimed
|
|
160
|
-
* not take. They are pinned in-process
|
|
161
|
-
* (`packages/testing/test/durable-tools.test.ts` "resolveUnknown is idempotent across the
|
|
163
|
+
* honestly instead of silently claimed. These require operator, compaction, reservation,
|
|
164
|
+
* background-worker, or Agent-update paths the shapes do not take. They are pinned in-process
|
|
165
|
+
* by the P5/S2 suites (`packages/testing/test/durable-tools.test.ts` "resolveUnknown is idempotent across the
|
|
162
166
|
* intent failpoint", `durable-runtime.test.ts` abort rows,
|
|
163
167
|
* `durable-subagents.test.ts` abort propagation) and by the process-kill/eviction crash
|
|
164
168
|
* matrices. Runner tests assert the observed never-fired set equals EXACTLY this list, so a
|
|
165
169
|
* protocol change that silently stops exercising a location fails the certification.
|
|
166
170
|
*/
|
|
167
171
|
export const TIER2_UNREACHED_LOCATIONS: ReadonlyArray<DurableRuntimeFailpointLocation> = [
|
|
172
|
+
// Tier 2 has no native compaction; dedicated checkpoint process-loss tests cover these.
|
|
173
|
+
"checkpoint:before-save",
|
|
174
|
+
"checkpoint:after-save",
|
|
168
175
|
"abort:after-intent",
|
|
169
176
|
// Compaction requires a `contextTokenLimit` policy plus prior-Run history
|
|
170
177
|
// none of the six scenario shapes carries; pinned in-process by the
|
|
171
178
|
// RUN-026 rows in `packages/testing/test/durable-runtime.test.ts`
|
|
172
179
|
// (compaction failpoint idempotence across re-drive).
|
|
180
|
+
"compaction:before-canonical-append",
|
|
173
181
|
"compaction:after-canonical-append",
|
|
182
|
+
// These shapes use neither programmatic Tools nor grace finalization. Their reservations
|
|
183
|
+
// are exercised before/after append in the public durable-runtime regression suite.
|
|
184
|
+
"policy:before-reservation-append",
|
|
185
|
+
"policy:after-reservation-append",
|
|
174
186
|
"resolve:after-intent",
|
|
175
187
|
"subagent:after-child-abort-intent",
|
|
188
|
+
// Background workers use retained delivery, source capacity, and child-origin paths absent
|
|
189
|
+
// from these six attached/ordinary scenarios. The before/after creation and completion
|
|
190
|
+
// boundaries are exercised by packages/effect-agent/test/durable/worker-host.test.ts.
|
|
191
|
+
"worker:before-source-append",
|
|
192
|
+
"worker:after-source-append",
|
|
193
|
+
"worker:before-origin-append",
|
|
194
|
+
"worker:after-origin-append",
|
|
195
|
+
"worker:before-completion-append",
|
|
196
|
+
"worker:after-completion-append",
|
|
197
|
+
// Root attached declarations keep their independent pool; nested shared subtree mutations
|
|
198
|
+
// are covered by the same focused worker-host failpoint suite.
|
|
199
|
+
"worker:before-subtree-append",
|
|
200
|
+
"worker:after-subtree-append",
|
|
201
|
+
// Automatic report decisions and retained delivery insertion have dedicated worker crash tests.
|
|
202
|
+
"worker:before-report-append",
|
|
203
|
+
"worker:after-report-append",
|
|
204
|
+
"worker:before-report-delivery",
|
|
205
|
+
"worker:after-report-delivery",
|
|
206
|
+
// None of the six shapes emits Agent updates. All four boundaries are exercised separately
|
|
207
|
+
// by packages/effect-agent/test/durable/worker-host.test.ts ("repairs an accepted parent update after ...").
|
|
208
|
+
// Node restart/lost-ack coverage is in packages/platform-node/test/worker-updates.test.ts;
|
|
209
|
+
// Cloudflare eviction/alarm recovery is in packages/platform-cloudflare/test/background-workers.test.ts.
|
|
210
|
+
// These suites are not executed by this certification runner; its update rows remain not-triggered.
|
|
211
|
+
"update:before-canonical-append",
|
|
212
|
+
"update:after-canonical-append",
|
|
213
|
+
"update:before-delivery-insert",
|
|
214
|
+
"update:after-delivery-insert",
|
|
176
215
|
];
|
|
177
216
|
|
|
178
217
|
/** Locations of `tier2` rows whose armed fault never fired in ANY scenario, sorted. */
|
|
@@ -180,9 +219,11 @@ export const tier2NeverFiredLocations = (
|
|
|
180
219
|
tier2: ReadonlyArray<CertificationSweepResult>,
|
|
181
220
|
): ReadonlyArray<DurableRuntimeFailpointLocation> => {
|
|
182
221
|
const fired = new Set<DurableRuntimeFailpointLocation>();
|
|
222
|
+
|
|
183
223
|
for (const row of tier2) {
|
|
184
224
|
if (row.failpointFired) fired.add(row.location);
|
|
185
225
|
}
|
|
226
|
+
|
|
186
227
|
return DurableRuntimeFailpointLocation.literals.filter((location) => !fired.has(location)).sort();
|
|
187
228
|
};
|
|
188
229
|
|
|
@@ -192,18 +233,21 @@ export const tier2NeverFiredLocations = (
|
|
|
192
233
|
|
|
193
234
|
const SHA_A = Schema.decodeSync(Digest)("a".repeat(64));
|
|
194
235
|
const DIGESTS = DefinitionDigests.make({ agent: SHA_A, model: SHA_A, tools: SHA_A });
|
|
236
|
+
|
|
195
237
|
const CHILD_DIGEST_STRINGS = {
|
|
196
238
|
agent: "b".repeat(64),
|
|
197
239
|
model: "c".repeat(64),
|
|
198
240
|
tools: "d".repeat(64),
|
|
199
241
|
} as const;
|
|
242
|
+
|
|
200
243
|
const CHILD_DIGESTS = DefinitionDigests.make({
|
|
201
244
|
agent: Schema.decodeSync(Digest)(CHILD_DIGEST_STRINGS.agent),
|
|
202
245
|
model: Schema.decodeSync(Digest)(CHILD_DIGEST_STRINGS.model),
|
|
203
246
|
tools: Schema.decodeSync(Digest)(CHILD_DIGEST_STRINGS.tools),
|
|
204
247
|
});
|
|
248
|
+
|
|
205
249
|
const PRINCIPAL = Schema.decodeSync(Principal)("principal-certification");
|
|
206
|
-
const
|
|
250
|
+
const decodeThreadId = Schema.decodeSync(ThreadId);
|
|
207
251
|
const decodeIdempotencyKey = Schema.decodeSync(IdempotencyKey);
|
|
208
252
|
const decodeToolCallId = Schema.decodeSync(ToolCallId);
|
|
209
253
|
|
|
@@ -253,18 +297,26 @@ const promptShapeModel = (
|
|
|
253
297
|
generateText: () => Effect.succeed([]),
|
|
254
298
|
streamText: (request) => {
|
|
255
299
|
const hasToolResult = request.prompt.content.some((message) => message.role === "tool");
|
|
300
|
+
|
|
256
301
|
const parts =
|
|
257
302
|
toolParts === undefined || hasToolResult ? finalParts(finalText) : toolParts;
|
|
303
|
+
|
|
258
304
|
return Stream.fromIterable(parts);
|
|
259
305
|
},
|
|
260
306
|
}),
|
|
261
307
|
),
|
|
262
308
|
);
|
|
263
309
|
|
|
310
|
+
// Tier-2 intentionally advances virtual time past a 30-second ownership lease on every
|
|
311
|
+
// re-drive round. Keep its business-policy duration above the full eight-round sweep so this
|
|
312
|
+
// fixture certifies crash convergence rather than accidentally relying on a fresh duration
|
|
313
|
+
// allowance per replacement Attempt. RUN-030 owns the dedicated duration-expiry coverage.
|
|
314
|
+
const CERTIFICATION_MAX_DURATION = "10 minutes" as const;
|
|
315
|
+
|
|
264
316
|
const policy = AgentPolicy.make({
|
|
265
317
|
maxTurns: 4,
|
|
266
318
|
maxToolCalls: 4,
|
|
267
|
-
maxDuration:
|
|
319
|
+
maxDuration: CERTIFICATION_MAX_DURATION,
|
|
268
320
|
toolConcurrency: 2,
|
|
269
321
|
});
|
|
270
322
|
|
|
@@ -272,7 +324,7 @@ const QuestionInput = Schema.Struct({ question: Schema.String });
|
|
|
272
324
|
const AnswerOutput = Schema.Struct({ answer: Schema.String });
|
|
273
325
|
|
|
274
326
|
/** plain / join: no tools — the pure Turn/submission/join seams. */
|
|
275
|
-
const plainDefinition = Agent.
|
|
327
|
+
const plainDefinition = Agent.make("certify-plain", {
|
|
276
328
|
input: QuestionInput,
|
|
277
329
|
output: AnswerOutput,
|
|
278
330
|
instructions: "Answer as JSON.",
|
|
@@ -285,8 +337,10 @@ const Book = Tool.make("book", {
|
|
|
285
337
|
parameters: Schema.Struct({ ref: Schema.String }),
|
|
286
338
|
success: Schema.Struct({ confirmation: Schema.String }),
|
|
287
339
|
});
|
|
340
|
+
|
|
288
341
|
const bookToolkit = Toolkit.make(Book);
|
|
289
|
-
|
|
342
|
+
|
|
343
|
+
const uncertainDefinition = Agent.make("certify-uncertain", {
|
|
290
344
|
input: QuestionInput,
|
|
291
345
|
output: AnswerOutput,
|
|
292
346
|
instructions: "Book it.",
|
|
@@ -301,8 +355,10 @@ const Itinerary = Tool.make("itinerary", {
|
|
|
301
355
|
failure: DurableStepError,
|
|
302
356
|
dependencies: [DurableStep],
|
|
303
357
|
});
|
|
358
|
+
|
|
304
359
|
const itineraryToolkit = Toolkit.make(Itinerary);
|
|
305
|
-
|
|
360
|
+
|
|
361
|
+
const stepsDefinition = Agent.make("certify-steps", {
|
|
306
362
|
input: QuestionInput,
|
|
307
363
|
output: AnswerOutput,
|
|
308
364
|
instructions: "Reserve the itinerary.",
|
|
@@ -316,8 +372,10 @@ const BookApproval = Tool.make("book", {
|
|
|
316
372
|
success: Schema.Struct({ confirmation: Schema.String }),
|
|
317
373
|
needsApproval: true,
|
|
318
374
|
});
|
|
375
|
+
|
|
319
376
|
const approvalToolkit = Toolkit.make(BookApproval);
|
|
320
|
-
|
|
377
|
+
|
|
378
|
+
const approvalDefinition = Agent.make("certify-approval", {
|
|
321
379
|
input: QuestionInput,
|
|
322
380
|
output: AnswerOutput,
|
|
323
381
|
instructions: "Book after approval.",
|
|
@@ -326,7 +384,7 @@ const approvalDefinition = Agent.define("certify-approval", {
|
|
|
326
384
|
});
|
|
327
385
|
|
|
328
386
|
/** delegation: durable attached child plus an ordinary uncertain sibling in ONE batch. */
|
|
329
|
-
const childDefinition = Agent.
|
|
387
|
+
const childDefinition = Agent.make("certify-child", {
|
|
330
388
|
input: QuestionInput,
|
|
331
389
|
output: AnswerOutput,
|
|
332
390
|
instructions: "Answer as JSON.",
|
|
@@ -334,7 +392,7 @@ const childDefinition = Agent.define("certify-child", {
|
|
|
334
392
|
policy: AgentPolicy.make({
|
|
335
393
|
maxTurns: 2,
|
|
336
394
|
maxToolCalls: 1,
|
|
337
|
-
maxDuration:
|
|
395
|
+
maxDuration: CERTIFICATION_MAX_DURATION,
|
|
338
396
|
toolConcurrency: 1,
|
|
339
397
|
}),
|
|
340
398
|
});
|
|
@@ -357,7 +415,7 @@ const researchDelegation = Subagent.define("delegate_research", {
|
|
|
357
415
|
maxConcurrency: 2,
|
|
358
416
|
maxTurns: 4,
|
|
359
417
|
maxToolCalls: 4,
|
|
360
|
-
maxDuration:
|
|
418
|
+
maxDuration: CERTIFICATION_MAX_DURATION,
|
|
361
419
|
}),
|
|
362
420
|
});
|
|
363
421
|
|
|
@@ -366,7 +424,7 @@ const Lookup = Tool.make("lookup", {
|
|
|
366
424
|
success: Schema.Struct({ value: Schema.String }),
|
|
367
425
|
});
|
|
368
426
|
|
|
369
|
-
const coordinatorDefinition = Agent.
|
|
427
|
+
const coordinatorDefinition = Agent.make("certify-coordinator", {
|
|
370
428
|
input: Schema.Struct({ mission: Schema.String }),
|
|
371
429
|
output: Schema.Struct({ report: Schema.String }),
|
|
372
430
|
instructions: "Delegate and look up, then answer as JSON.",
|
|
@@ -374,7 +432,7 @@ const coordinatorDefinition = Agent.define("certify-coordinator", {
|
|
|
374
432
|
policy: AgentPolicy.make({
|
|
375
433
|
maxTurns: 4,
|
|
376
434
|
maxToolCalls: 3,
|
|
377
|
-
maxDuration:
|
|
435
|
+
maxDuration: CERTIFICATION_MAX_DURATION,
|
|
378
436
|
toolConcurrency: 2,
|
|
379
437
|
}),
|
|
380
438
|
});
|
|
@@ -389,12 +447,14 @@ const identifiers = Layer.effect(
|
|
|
389
447
|
IdGenerator,
|
|
390
448
|
Effect.gen(function* () {
|
|
391
449
|
const counter = yield* Ref.make(0);
|
|
450
|
+
|
|
392
451
|
const next = <A>(decode: (value: string) => A, prefix: string) =>
|
|
393
452
|
Ref.getAndUpdate(counter, (value) => value + 1).pipe(
|
|
394
453
|
Effect.map((value) => decode(`${prefix}-${value}`)),
|
|
395
454
|
);
|
|
455
|
+
|
|
396
456
|
return {
|
|
397
|
-
|
|
457
|
+
nextThreadId: next(decodeThreadId, "certify-fixture-thread"),
|
|
398
458
|
nextRunId: next(Schema.decodeSync(RunId), "certify-fixture-run"),
|
|
399
459
|
nextTurnId: next(Schema.decodeSync(TurnId), "certify-fixture-turn"),
|
|
400
460
|
};
|
|
@@ -408,15 +468,15 @@ const delegationSupport = Layer.mergeAll(SubagentReservationsMemoryLive, identif
|
|
|
408
468
|
// ---------------------------------------------------------------------------
|
|
409
469
|
|
|
410
470
|
interface CertificationCell {
|
|
411
|
-
readonly
|
|
471
|
+
readonly bindings: ReadonlyArray<ResolvedBinding>;
|
|
412
472
|
/** Idempotent submission batch — safe to replay verbatim after a submit-boundary fault. */
|
|
413
473
|
readonly submit: Effect.Effect<ReadonlyArray<Receipt>, DurableSubmitFailure, DurableAgentRuntime>;
|
|
414
474
|
/** All lanes of the cell in drive order, computed from the (possibly replayed) receipts. */
|
|
415
|
-
readonly lanes: (receipts: ReadonlyArray<Receipt>) => ReadonlyArray<
|
|
475
|
+
readonly lanes: (receipts: ReadonlyArray<Receipt>) => ReadonlyArray<ThreadId>;
|
|
416
476
|
}
|
|
417
477
|
|
|
418
|
-
const submitOptionsFor = (slug: string,
|
|
419
|
-
|
|
478
|
+
const submitOptionsFor = (slug: string, threadId: ThreadId): DurableSubmitOptions => ({
|
|
479
|
+
threadId,
|
|
420
480
|
principal: PRINCIPAL,
|
|
421
481
|
idempotencyKey: decodeIdempotencyKey(`certify-key-${slug}`),
|
|
422
482
|
definitions: DIGESTS,
|
|
@@ -428,20 +488,24 @@ const makeSingleAgentCell = (
|
|
|
428
488
|
resolved: ResolvedBinding,
|
|
429
489
|
slug: string,
|
|
430
490
|
): CertificationCell => {
|
|
431
|
-
const
|
|
491
|
+
const threadId = decodeThreadId(`certify-${slug}`);
|
|
492
|
+
|
|
432
493
|
const submit = Effect.gen(function* () {
|
|
433
494
|
const runtime = yield* DurableAgentRuntime;
|
|
495
|
+
|
|
434
496
|
const receipt = yield* runtime.submit(
|
|
435
497
|
{ definition: { id: definition.id, input: definition.input } },
|
|
436
498
|
{ question: `certify ${slug}` },
|
|
437
|
-
submitOptionsFor(slug,
|
|
499
|
+
submitOptionsFor(slug, threadId),
|
|
438
500
|
);
|
|
501
|
+
|
|
439
502
|
return [receipt];
|
|
440
503
|
});
|
|
504
|
+
|
|
441
505
|
return {
|
|
442
|
-
|
|
506
|
+
bindings: [resolved],
|
|
443
507
|
submit,
|
|
444
|
-
lanes: () => [
|
|
508
|
+
lanes: () => [threadId],
|
|
445
509
|
};
|
|
446
510
|
};
|
|
447
511
|
|
|
@@ -455,7 +519,9 @@ const makeCell = Effect.fn("Certification.makeCell")(function* (
|
|
|
455
519
|
plainDefinition,
|
|
456
520
|
promptShapeModel("certify-plain", '{"answer":"done"}'),
|
|
457
521
|
);
|
|
522
|
+
|
|
458
523
|
const resolved = yield* DurableWorkerBinding.make(binding, DIGESTS);
|
|
524
|
+
|
|
459
525
|
return makeSingleAgentCell(plainDefinition, resolved, slug);
|
|
460
526
|
}
|
|
461
527
|
case "uncertain-tool": {
|
|
@@ -467,12 +533,15 @@ const makeCell = Effect.fn("Certification.makeCell")(function* (
|
|
|
467
533
|
toolTurn(toolCallPart("book-1", "book", { ref: `r-${slug}` })),
|
|
468
534
|
),
|
|
469
535
|
);
|
|
536
|
+
|
|
470
537
|
const toolLayer = bookToolkit.toLayer({
|
|
471
538
|
book: ({ ref }) => Effect.succeed({ confirmation: `confirmed-${ref}` }),
|
|
472
539
|
});
|
|
540
|
+
|
|
473
541
|
const resolved = yield* DurableWorkerBinding.make(binding, DIGESTS).pipe(
|
|
474
542
|
Effect.provide(toolLayer),
|
|
475
543
|
);
|
|
544
|
+
|
|
476
545
|
return makeSingleAgentCell(uncertainDefinition, resolved, slug);
|
|
477
546
|
}
|
|
478
547
|
case "durable-steps": {
|
|
@@ -484,26 +553,32 @@ const makeCell = Effect.fn("Certification.makeCell")(function* (
|
|
|
484
553
|
toolTurn(toolCallPart("itinerary-1", "itinerary", { ref: `trip-${slug}` })),
|
|
485
554
|
),
|
|
486
555
|
);
|
|
556
|
+
|
|
487
557
|
const toolLayer = itineraryToolkit.toLayer({
|
|
488
558
|
itinerary: ({ ref }) =>
|
|
489
559
|
Effect.gen(function* () {
|
|
490
560
|
const step = yield* DurableStep;
|
|
561
|
+
|
|
491
562
|
const flight = yield* step.do(
|
|
492
563
|
"reserve-flight",
|
|
493
564
|
Schema.String,
|
|
494
565
|
Effect.succeed(`flight-${ref}`),
|
|
495
566
|
);
|
|
567
|
+
|
|
496
568
|
const lodging = yield* step.do(
|
|
497
569
|
"reserve-lodging",
|
|
498
570
|
Schema.String,
|
|
499
571
|
Effect.succeed(`lodging-${ref}`),
|
|
500
572
|
);
|
|
573
|
+
|
|
501
574
|
return { state: `${flight}+${lodging}` };
|
|
502
575
|
}),
|
|
503
576
|
});
|
|
577
|
+
|
|
504
578
|
const resolved = yield* DurableWorkerBinding.make(binding, DIGESTS).pipe(
|
|
505
579
|
Effect.provide(toolLayer),
|
|
506
580
|
);
|
|
581
|
+
|
|
507
582
|
return makeSingleAgentCell(stepsDefinition, resolved, slug);
|
|
508
583
|
}
|
|
509
584
|
case "approval": {
|
|
@@ -515,12 +590,15 @@ const makeCell = Effect.fn("Certification.makeCell")(function* (
|
|
|
515
590
|
toolTurn(toolCallPart("book-1", "book", { ref: `r-${slug}` })),
|
|
516
591
|
),
|
|
517
592
|
);
|
|
593
|
+
|
|
518
594
|
const toolLayer = approvalToolkit.toLayer({
|
|
519
595
|
book: ({ ref }) => Effect.succeed({ confirmation: `confirmed-${ref}` }),
|
|
520
596
|
});
|
|
597
|
+
|
|
521
598
|
const resolved = yield* DurableWorkerBinding.make(binding, DIGESTS).pipe(
|
|
522
599
|
Effect.provide(toolLayer),
|
|
523
600
|
);
|
|
601
|
+
|
|
524
602
|
return makeSingleAgentCell(approvalDefinition, resolved, slug);
|
|
525
603
|
}
|
|
526
604
|
case "join": {
|
|
@@ -528,31 +606,39 @@ const makeCell = Effect.fn("Certification.makeCell")(function* (
|
|
|
528
606
|
plainDefinition,
|
|
529
607
|
promptShapeModel("certify-join", '{"answer":"host answer"}'),
|
|
530
608
|
);
|
|
609
|
+
|
|
531
610
|
const resolved = yield* DurableWorkerBinding.make(binding, DIGESTS);
|
|
532
|
-
const
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
611
|
+
const threadId = decodeThreadId(`certify-${slug}`);
|
|
612
|
+
|
|
613
|
+
const submitOne = Effect.fn("Certification.submitOne")(function* (
|
|
614
|
+
key: string,
|
|
615
|
+
question: string,
|
|
616
|
+
) {
|
|
617
|
+
const runtime = yield* DurableAgentRuntime;
|
|
618
|
+
|
|
619
|
+
return yield* runtime.submit(
|
|
620
|
+
{ definition: { id: plainDefinition.id, input: plainDefinition.input } },
|
|
621
|
+
{ question },
|
|
622
|
+
{
|
|
623
|
+
threadId,
|
|
624
|
+
principal: PRINCIPAL,
|
|
625
|
+
idempotencyKey: decodeIdempotencyKey(key),
|
|
626
|
+
definitions: DIGESTS,
|
|
627
|
+
},
|
|
628
|
+
);
|
|
629
|
+
});
|
|
630
|
+
|
|
547
631
|
const cell: CertificationCell = {
|
|
548
|
-
|
|
632
|
+
bindings: [resolved],
|
|
549
633
|
submit: Effect.gen(function* () {
|
|
550
634
|
const host = yield* submitOne(`certify-key-${slug}-host`, "host question");
|
|
551
635
|
const queued = yield* submitOne(`certify-key-${slug}-queued`, "queued question");
|
|
636
|
+
|
|
552
637
|
return [host, queued];
|
|
553
638
|
}),
|
|
554
|
-
lanes: () => [
|
|
639
|
+
lanes: () => [threadId],
|
|
555
640
|
};
|
|
641
|
+
|
|
556
642
|
return cell;
|
|
557
643
|
}
|
|
558
644
|
case "delegation": {
|
|
@@ -560,6 +646,7 @@ const makeCell = Effect.fn("Certification.makeCell")(function* (
|
|
|
560
646
|
childDefinition,
|
|
561
647
|
promptShapeModel("certify-child", '{"answer":"child-answer"}'),
|
|
562
648
|
);
|
|
649
|
+
|
|
563
650
|
const parentBinding = Agent.withModel(
|
|
564
651
|
coordinatorDefinition,
|
|
565
652
|
promptShapeModel(
|
|
@@ -571,36 +658,45 @@ const makeCell = Effect.fn("Certification.makeCell")(function* (
|
|
|
571
658
|
),
|
|
572
659
|
),
|
|
573
660
|
);
|
|
574
|
-
|
|
661
|
+
|
|
662
|
+
const delegationLayer = Subagent.layer(researchDelegation, childBinding, {
|
|
575
663
|
mapChildFailure,
|
|
576
664
|
durable: { targetDigests: CHILD_DIGEST_STRINGS },
|
|
577
665
|
}).pipe(Layer.provide(delegationSupport));
|
|
666
|
+
|
|
578
667
|
const lookupLayer = Toolkit.make(Lookup).toLayer({
|
|
579
668
|
lookup: ({ key }) => Effect.succeed({ value: `found-${key}` }),
|
|
580
669
|
});
|
|
670
|
+
|
|
581
671
|
const parentResolved = yield* DurableWorkerBinding.make(parentBinding, DIGESTS).pipe(
|
|
582
672
|
Effect.provide(Layer.mergeAll(delegationLayer, lookupLayer)),
|
|
583
673
|
);
|
|
674
|
+
|
|
584
675
|
const childResolved = yield* DurableWorkerBinding.make(childBinding, CHILD_DIGESTS);
|
|
585
|
-
const
|
|
676
|
+
const threadId = decodeThreadId(`certify-${slug}`);
|
|
677
|
+
|
|
586
678
|
const cell: CertificationCell = {
|
|
587
|
-
|
|
679
|
+
bindings: [parentResolved, childResolved],
|
|
588
680
|
submit: Effect.gen(function* () {
|
|
589
681
|
const runtime = yield* DurableAgentRuntime;
|
|
682
|
+
|
|
590
683
|
const receipt = yield* runtime.submit(
|
|
591
684
|
{ definition: { id: coordinatorDefinition.id, input: coordinatorDefinition.input } },
|
|
592
685
|
{ mission: "plan" },
|
|
593
|
-
submitOptionsFor(slug,
|
|
686
|
+
submitOptionsFor(slug, threadId),
|
|
594
687
|
);
|
|
688
|
+
|
|
595
689
|
return [receipt];
|
|
596
690
|
}),
|
|
597
691
|
lanes: (receipts) => {
|
|
598
692
|
const parent = receipts.at(0);
|
|
693
|
+
|
|
599
694
|
return parent === undefined
|
|
600
|
-
? [
|
|
601
|
-
: [
|
|
695
|
+
? [threadId]
|
|
696
|
+
: [threadId, childThreadIdFor(parent.submissionId, DELEGATE_CALL)];
|
|
602
697
|
},
|
|
603
698
|
};
|
|
699
|
+
|
|
604
700
|
return cell;
|
|
605
701
|
}
|
|
606
702
|
}
|
|
@@ -616,107 +712,139 @@ const MAX_REDRIVE_ROUNDS = 8;
|
|
|
616
712
|
* the digest chain is fully recomputed instead of skipped.
|
|
617
713
|
*/
|
|
618
714
|
const verifyLane = Effect.fn("Certification.verifyLane")(function* (
|
|
619
|
-
lane:
|
|
715
|
+
lane: ThreadId,
|
|
620
716
|
batchProducers: ReadonlyMap<BatchId, ProducerId>,
|
|
621
717
|
) {
|
|
622
|
-
const store = yield*
|
|
718
|
+
const store = yield* ThreadStore;
|
|
623
719
|
const ledger = yield* SubmissionLedger;
|
|
624
|
-
const exported = yield* store.export(
|
|
720
|
+
const exported = yield* store.export(ThreadExportRequest.make({ threadId: lane }));
|
|
625
721
|
const rows = new Map<SubmissionId, SubmissionSnapshot>();
|
|
626
722
|
const nonterminal = yield* Stream.runCollect(ledger.scanNonterminal);
|
|
723
|
+
|
|
627
724
|
for (const submission of nonterminal) {
|
|
628
|
-
if (submission.
|
|
725
|
+
if (submission.threadId === lane) rows.set(submission.submissionId, submission);
|
|
629
726
|
}
|
|
630
727
|
const named = new Set<SubmissionId>();
|
|
728
|
+
|
|
631
729
|
for (const envelope of exported.records) {
|
|
632
730
|
const payload = envelope.record.payload;
|
|
731
|
+
|
|
633
732
|
if (
|
|
634
733
|
payload._tag === "UserInputRecorded" ||
|
|
635
734
|
payload._tag === "SubmissionSettled" ||
|
|
636
735
|
payload._tag === "AbortRequested"
|
|
637
736
|
) {
|
|
638
|
-
named.add(payload.submissionId);
|
|
737
|
+
if (payload.submissionId !== undefined) named.add(payload.submissionId);
|
|
639
738
|
}
|
|
640
739
|
}
|
|
641
740
|
for (const submissionId of named) {
|
|
642
741
|
if (rows.has(submissionId)) continue;
|
|
643
742
|
const found = yield* ledger.lookup(SubmissionLookupById.make({ submissionId }));
|
|
644
|
-
|
|
743
|
+
|
|
744
|
+
if (Option.isSome(found) && found.value.threadId === lane) {
|
|
645
745
|
rows.set(submissionId, found.value);
|
|
646
746
|
}
|
|
647
747
|
}
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
748
|
+
|
|
749
|
+
const checkpoint =
|
|
750
|
+
store.checkpoints === undefined
|
|
751
|
+
? Option.none()
|
|
752
|
+
: yield* store.checkpoints.load(LoadCheckpointRequest.make({ threadId: lane }));
|
|
753
|
+
|
|
754
|
+
return yield* verifyThreadInvariants({
|
|
652
755
|
export: exported,
|
|
653
756
|
submissions: [...rows.values()],
|
|
654
757
|
batchProducers,
|
|
655
758
|
checkpoint: Option.getOrUndefined(checkpoint),
|
|
759
|
+
checkpointsSupported: store.checkpoints !== undefined,
|
|
656
760
|
requireAllSettled: true,
|
|
657
761
|
});
|
|
658
762
|
});
|
|
659
763
|
|
|
660
764
|
const failureTagOf = <E>(cause: Cause.Cause<E>): string => {
|
|
661
765
|
const failure = Cause.findErrorOption(cause);
|
|
766
|
+
|
|
662
767
|
if (Option.isSome(failure)) {
|
|
663
768
|
const error: unknown = failure.value;
|
|
769
|
+
|
|
664
770
|
if (typeof error === "object" && error !== null && "_tag" in error) {
|
|
665
|
-
return String(
|
|
771
|
+
return String(error._tag);
|
|
666
772
|
}
|
|
773
|
+
|
|
667
774
|
return String(error).slice(0, 256);
|
|
668
775
|
}
|
|
776
|
+
|
|
669
777
|
return "defect";
|
|
670
778
|
};
|
|
671
779
|
|
|
672
|
-
|
|
780
|
+
type SweepOutcome = Pick<
|
|
781
|
+
CertificationSweepResult,
|
|
782
|
+
"failpointFired" | "status" | "digestChainVerified" | "detail"
|
|
783
|
+
>;
|
|
784
|
+
|
|
785
|
+
/** Discover a clean path, or arm one location; both drives converge and verify real storage. */
|
|
673
786
|
const runSweepCell = Effect.fn("Certification.runSweepCell")(function* (
|
|
674
787
|
scenario: CertificationScenario,
|
|
675
|
-
location: DurableRuntimeFailpointLocation,
|
|
788
|
+
location: DurableRuntimeFailpointLocation | undefined,
|
|
676
789
|
batchProducers: ReadonlyMap<BatchId, ProducerId>,
|
|
677
790
|
leaseAdvance: Duration.Duration,
|
|
791
|
+
reached: Set<DurableRuntimeFailpointLocation>,
|
|
678
792
|
) {
|
|
679
|
-
const runtime = yield* DurableAgentRuntime;
|
|
680
793
|
const ledger = yield* SubmissionLedger;
|
|
681
794
|
const control = yield* DurableRuntimeFailpointTestControl;
|
|
682
|
-
const slug = `${scenario}-${location
|
|
683
|
-
|
|
684
|
-
const failed = (detail: string, fired: boolean):
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
digestChainVerified: false,
|
|
691
|
-
detail: detail.slice(0, 4_096),
|
|
692
|
-
});
|
|
795
|
+
const slug = `${scenario}-${location?.replaceAll(":", "-") ?? "discovery"}`;
|
|
796
|
+
|
|
797
|
+
const failed = (detail: string, fired: boolean): SweepOutcome => ({
|
|
798
|
+
failpointFired: fired,
|
|
799
|
+
status: "failed",
|
|
800
|
+
digestChainVerified: false,
|
|
801
|
+
detail: detail.slice(0, 4_096),
|
|
802
|
+
});
|
|
693
803
|
|
|
694
804
|
const cell = yield* makeCell(scenario, slug);
|
|
695
805
|
|
|
806
|
+
const runtime = yield* DurableAgentRuntime.pipe(
|
|
807
|
+
Effect.provide(
|
|
808
|
+
DurableAgentRuntime.layerWithBindings(cell.bindings).pipe(
|
|
809
|
+
Layer.provide(RunToolAuthorization.allowAll),
|
|
810
|
+
),
|
|
811
|
+
),
|
|
812
|
+
);
|
|
813
|
+
|
|
814
|
+
const submit = cell.submit.pipe(Effect.provideService(DurableAgentRuntime, runtime));
|
|
815
|
+
|
|
696
816
|
// One-shot arm: the fault fires at most once anywhere in the cell (initial drive OR a
|
|
697
817
|
// re-drive round's public unblocking operation), modelling one crash at this boundary.
|
|
698
818
|
const fired = yield* Ref.make(false);
|
|
819
|
+
|
|
699
820
|
yield* control.setHandler((hit) =>
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
821
|
+
Effect.suspend(() => {
|
|
822
|
+
reached.add(hit);
|
|
823
|
+
|
|
824
|
+
return hit !== location
|
|
825
|
+
? Effect.void
|
|
826
|
+
: Ref.getAndSet(fired, true).pipe(
|
|
827
|
+
Effect.flatMap((already) =>
|
|
828
|
+
already
|
|
829
|
+
? Effect.void
|
|
830
|
+
: Effect.fail(DurableRuntimeFailpointError.make({ location: hit })),
|
|
831
|
+
),
|
|
832
|
+
);
|
|
833
|
+
}),
|
|
709
834
|
);
|
|
710
835
|
|
|
711
836
|
// Submissions are idempotent (DUR-001): one replay recovers a submit-boundary fault.
|
|
712
837
|
let receipts: ReadonlyArray<Receipt>;
|
|
713
|
-
const firstSubmit = yield* Effect.exit(
|
|
838
|
+
const firstSubmit = yield* Effect.exit(submit);
|
|
839
|
+
|
|
714
840
|
if (Exit.isSuccess(firstSubmit)) {
|
|
715
841
|
receipts = firstSubmit.value;
|
|
716
842
|
} else {
|
|
717
|
-
const secondSubmit = yield* Effect.exit(
|
|
843
|
+
const secondSubmit = yield* Effect.exit(submit);
|
|
844
|
+
|
|
718
845
|
if (Exit.isFailure(secondSubmit)) {
|
|
719
846
|
yield* control.clear;
|
|
847
|
+
|
|
720
848
|
return failed(
|
|
721
849
|
`submission replay did not recover: ${failureTagOf(secondSubmit.cause)}`,
|
|
722
850
|
yield* Ref.get(fired),
|
|
@@ -726,24 +854,24 @@ const runSweepCell = Effect.fn("Certification.runSweepCell")(function* (
|
|
|
726
854
|
}
|
|
727
855
|
const lanes = cell.lanes(receipts);
|
|
728
856
|
|
|
729
|
-
const driveLane = (lane:
|
|
730
|
-
runtime
|
|
731
|
-
.processConversationResolved(lane)
|
|
732
|
-
.pipe(Effect.provideService(AgentBindingResolver, cell.resolver));
|
|
857
|
+
const driveLane = (lane: ThreadId) => runtime.processThreadResolved(lane);
|
|
733
858
|
|
|
734
859
|
const allSettled = Effect.gen(function* () {
|
|
735
860
|
for (const receipt of receipts) {
|
|
736
861
|
const snapshot = yield* ledger.lookup(
|
|
737
862
|
SubmissionLookupById.make({ submissionId: receipt.submissionId }),
|
|
738
863
|
);
|
|
864
|
+
|
|
739
865
|
if (Option.isNone(snapshot) || snapshot.value.state !== "settled") return false;
|
|
740
866
|
}
|
|
867
|
+
|
|
741
868
|
return true;
|
|
742
869
|
});
|
|
743
870
|
|
|
744
871
|
// Re-drive to convergence using ONLY public operations: worker drives, recovery passes,
|
|
745
|
-
// and the authorized DUR-017/approval unblocking paths chosen from `
|
|
872
|
+
// and the authorized DUR-017/approval unblocking paths chosen from `explainThread`.
|
|
746
873
|
let converged = false;
|
|
874
|
+
|
|
747
875
|
for (let round = 0; round < MAX_REDRIVE_ROUNDS && !converged; round++) {
|
|
748
876
|
// Expire any lease a faulted Attempt left behind (D5): virtual time is the
|
|
749
877
|
// adapter-neutral reclaim lever — a live lease may block every new claim.
|
|
@@ -753,7 +881,8 @@ const runSweepCell = Effect.fn("Certification.runSweepCell")(function* (
|
|
|
753
881
|
yield* Effect.exit(driveLane(lane));
|
|
754
882
|
}
|
|
755
883
|
for (const lane of lanes) {
|
|
756
|
-
const explains = yield* Effect.exit(runtime.
|
|
884
|
+
const explains = yield* Effect.exit(runtime.explainThread(lane));
|
|
885
|
+
|
|
757
886
|
if (Exit.isFailure(explains)) continue;
|
|
758
887
|
for (const explanation of explains.value) {
|
|
759
888
|
for (const unknown of explanation.evidence.unknownCalls) {
|
|
@@ -774,6 +903,7 @@ const runSweepCell = Effect.fn("Certification.runSweepCell")(function* (
|
|
|
774
903
|
const decided = explanation.evidence.approvalDecisions.some(
|
|
775
904
|
(decision) => decision.toolCallId === pending.toolCallId,
|
|
776
905
|
);
|
|
906
|
+
|
|
777
907
|
if (decided) continue;
|
|
778
908
|
yield* Effect.exit(
|
|
779
909
|
runtime.resolveApproval(
|
|
@@ -790,6 +920,7 @@ const runSweepCell = Effect.fn("Certification.runSweepCell")(function* (
|
|
|
790
920
|
}
|
|
791
921
|
}
|
|
792
922
|
const settled = yield* Effect.exit(allSettled);
|
|
923
|
+
|
|
793
924
|
converged = Exit.isSuccess(settled) && settled.value;
|
|
794
925
|
}
|
|
795
926
|
yield* control.clear;
|
|
@@ -802,8 +933,10 @@ const runSweepCell = Effect.fn("Certification.runSweepCell")(function* (
|
|
|
802
933
|
// Every lane of the cell must verify in convergence mode with a recomputed digest chain.
|
|
803
934
|
let digestChainVerified = true;
|
|
804
935
|
const failedChecks: Array<string> = [];
|
|
936
|
+
|
|
805
937
|
for (const lane of lanes) {
|
|
806
938
|
const verdict = yield* Effect.exit(verifyLane(lane, batchProducers));
|
|
939
|
+
|
|
807
940
|
if (Exit.isFailure(verdict)) {
|
|
808
941
|
return failed(`lane ${lane} could not be verified: ${failureTagOf(verdict.cause)}`, wasFired);
|
|
809
942
|
}
|
|
@@ -827,13 +960,11 @@ const runSweepCell = Effect.fn("Certification.runSweepCell")(function* (
|
|
|
827
960
|
);
|
|
828
961
|
}
|
|
829
962
|
|
|
830
|
-
return
|
|
831
|
-
scenario,
|
|
832
|
-
location,
|
|
963
|
+
return {
|
|
833
964
|
failpointFired: wasFired,
|
|
834
965
|
status: wasFired ? "converged" : "not-triggered",
|
|
835
966
|
digestChainVerified,
|
|
836
|
-
}
|
|
967
|
+
} satisfies SweepOutcome;
|
|
837
968
|
});
|
|
838
969
|
|
|
839
970
|
// ---------------------------------------------------------------------------
|
|
@@ -864,10 +995,14 @@ export const resolveTierThree = Effect.fn("Certification.resolveTierThree")(func
|
|
|
864
995
|
}
|
|
865
996
|
if (options.crashLever !== undefined) {
|
|
866
997
|
const cases = yield* options.crashLever;
|
|
998
|
+
|
|
867
999
|
return CertificationTierThreeReport.make({
|
|
868
|
-
status: "exercised",
|
|
1000
|
+
status: cases.length === 0 ? "not-exercised" : "exercised",
|
|
869
1001
|
evidence: options.tierThreeEvidence ?? [],
|
|
870
1002
|
cases,
|
|
1003
|
+
...(cases.length === 0
|
|
1004
|
+
? { detail: "the supplied crash lever executed no real-loss cases" }
|
|
1005
|
+
: {}),
|
|
871
1006
|
});
|
|
872
1007
|
}
|
|
873
1008
|
if (options.tierThreeEvidence !== undefined && options.tierThreeEvidence.length > 0) {
|
|
@@ -877,6 +1012,7 @@ export const resolveTierThree = Effect.fn("Certification.resolveTierThree")(func
|
|
|
877
1012
|
cases: [],
|
|
878
1013
|
});
|
|
879
1014
|
}
|
|
1015
|
+
|
|
880
1016
|
return CertificationTierThreeReport.make({
|
|
881
1017
|
status: "not-exercised",
|
|
882
1018
|
evidence: [],
|
|
@@ -905,13 +1041,15 @@ export const certifyDurableAdapters = <LedgerE = never, StoreE = never>(
|
|
|
905
1041
|
options: CertifyDurableAdaptersOptions<LedgerE, StoreE>,
|
|
906
1042
|
): Effect.Effect<CertificationReport, LedgerE | StoreE, Crypto.Crypto> => {
|
|
907
1043
|
const batchProducers = new Map<BatchId, ProducerId>();
|
|
1044
|
+
|
|
908
1045
|
// Interpose the candidate store with an append-time capture of each batch's producer
|
|
909
|
-
// identity — the one value the
|
|
1046
|
+
// identity — the one value the ThreadStore port deliberately does not export — so
|
|
910
1047
|
// Tier 2's invariant verification recomputes the FULL digest chain instead of skipping it.
|
|
911
|
-
const capturingStore = Layer.effect(
|
|
1048
|
+
const capturingStore = Layer.effect(ThreadStore)(
|
|
912
1049
|
Effect.gen(function* () {
|
|
913
|
-
const inner = yield*
|
|
914
|
-
|
|
1050
|
+
const inner = yield* ThreadStore;
|
|
1051
|
+
|
|
1052
|
+
return ThreadStore.of({
|
|
915
1053
|
...inner,
|
|
916
1054
|
append: (request) =>
|
|
917
1055
|
Effect.sync(() => {
|
|
@@ -919,13 +1057,15 @@ export const certifyDurableAdapters = <LedgerE = never, StoreE = never>(
|
|
|
919
1057
|
}).pipe(Effect.andThen(inner.append(request))),
|
|
920
1058
|
});
|
|
921
1059
|
}),
|
|
922
|
-
).pipe(Layer.provide(options.
|
|
1060
|
+
).pipe(Layer.provide(options.threadStore));
|
|
923
1061
|
|
|
924
|
-
|
|
1062
|
+
// RUN-036: certification uses the default-none Tool failure observer. Trusted application
|
|
1063
|
+
// reporting adds no durable transition and is verified separately from adapter certification.
|
|
1064
|
+
const environment = Layer.mergeAll(
|
|
925
1065
|
options.submissionLedger,
|
|
926
1066
|
capturingStore,
|
|
927
1067
|
options.wakeScheduler ?? WakeScheduler.layerNoop,
|
|
928
|
-
|
|
1068
|
+
DurableRuntimeFailpointTestControl.layer,
|
|
929
1069
|
ToolReconciler.uncertain,
|
|
930
1070
|
DurableRuntimeConfig.layer({
|
|
931
1071
|
deploymentId: Schema.decodeSync(DeploymentId)("deployment-certification"),
|
|
@@ -935,7 +1075,6 @@ export const certifyDurableAdapters = <LedgerE = never, StoreE = never>(
|
|
|
935
1075
|
abortPollInterval: Duration.millis(50),
|
|
936
1076
|
}),
|
|
937
1077
|
);
|
|
938
|
-
const environment = DurableAgentRuntime.layer.pipe(Layer.provideMerge(support));
|
|
939
1078
|
|
|
940
1079
|
const leaseAdvance = Duration.millis(
|
|
941
1080
|
Duration.toMillis(options.ownershipLeaseDuration ?? DEFAULT_OWNERSHIP_LEASE_DURATION) + 1_000,
|
|
@@ -944,11 +1083,73 @@ export const certifyDurableAdapters = <LedgerE = never, StoreE = never>(
|
|
|
944
1083
|
const program = Effect.gen(function* () {
|
|
945
1084
|
const ledger = yield* SubmissionLedger;
|
|
946
1085
|
|
|
947
|
-
// Tier 2 —
|
|
1086
|
+
// Tier 2 — discover each deterministic shape without a fault. Unreached locations
|
|
1087
|
+
// would all repeat this same clean drive, including its full digest verification.
|
|
948
1088
|
const tier2: Array<CertificationSweepResult> = [];
|
|
1089
|
+
|
|
1090
|
+
const discoveries: Array<{
|
|
1091
|
+
readonly scenario: CertificationScenario;
|
|
1092
|
+
readonly outcome: SweepOutcome;
|
|
1093
|
+
readonly reached: Set<DurableRuntimeFailpointLocation>;
|
|
1094
|
+
}> = [];
|
|
1095
|
+
|
|
949
1096
|
for (const scenario of CERTIFICATION_SCENARIOS) {
|
|
1097
|
+
const reached = new Set<DurableRuntimeFailpointLocation>();
|
|
1098
|
+
|
|
1099
|
+
const outcome = yield* runSweepCell(
|
|
1100
|
+
scenario,
|
|
1101
|
+
undefined,
|
|
1102
|
+
batchProducers,
|
|
1103
|
+
leaseAdvance,
|
|
1104
|
+
reached,
|
|
1105
|
+
);
|
|
1106
|
+
|
|
1107
|
+
discoveries.push({ scenario, outcome, reached });
|
|
1108
|
+
}
|
|
1109
|
+
const observed = new Set(discoveries.flatMap(({ reached }) => [...reached]));
|
|
1110
|
+
|
|
1111
|
+
// Never silently omit a new location: if neither discovery nor the documented
|
|
1112
|
+
// never-fired set accounts for it, arm it in EVERY shape. Tests pin the fired paths
|
|
1113
|
+
// per shape and the never-fired set, so new or lost routes require explicit review.
|
|
1114
|
+
const unaccounted = new Set(
|
|
1115
|
+
DurableRuntimeFailpointLocation.literals.filter(
|
|
1116
|
+
(location) => !observed.has(location) && !TIER2_UNREACHED_LOCATIONS.includes(location),
|
|
1117
|
+
),
|
|
1118
|
+
);
|
|
1119
|
+
|
|
1120
|
+
for (const discovery of discoveries) {
|
|
1121
|
+
const { scenario, reached } = discovery;
|
|
1122
|
+
const outcomes = new Map<DurableRuntimeFailpointLocation, SweepOutcome>();
|
|
1123
|
+
|
|
1124
|
+
if (discovery.outcome.status !== "failed") {
|
|
1125
|
+
while (true) {
|
|
1126
|
+
// Recovery may expose another route. Discover those too, without re-running a
|
|
1127
|
+
// cell already checked. Schema order bounds this to one drive per location.
|
|
1128
|
+
const location = DurableRuntimeFailpointLocation.literals.find(
|
|
1129
|
+
(candidate) =>
|
|
1130
|
+
!outcomes.has(candidate) && (reached.has(candidate) || unaccounted.has(candidate)),
|
|
1131
|
+
);
|
|
1132
|
+
|
|
1133
|
+
if (location === undefined) break;
|
|
1134
|
+
outcomes.set(
|
|
1135
|
+
location,
|
|
1136
|
+
yield* runSweepCell(scenario, location, batchProducers, leaseAdvance, reached),
|
|
1137
|
+
);
|
|
1138
|
+
}
|
|
1139
|
+
}
|
|
950
1140
|
for (const location of DurableRuntimeFailpointLocation.literals) {
|
|
951
|
-
|
|
1141
|
+
const outcome = outcomes.get(location) ?? discovery.outcome;
|
|
1142
|
+
|
|
1143
|
+
tier2.push(
|
|
1144
|
+
CertificationSweepResult.make({
|
|
1145
|
+
scenario,
|
|
1146
|
+
location,
|
|
1147
|
+
...outcome,
|
|
1148
|
+
...(outcomes.has(location) || outcome.status === "failed"
|
|
1149
|
+
? {}
|
|
1150
|
+
: { detail: "verified clean scenario did not reach this location" }),
|
|
1151
|
+
}),
|
|
1152
|
+
);
|
|
952
1153
|
}
|
|
953
1154
|
}
|
|
954
1155
|
|
|
@@ -960,13 +1161,8 @@ export const certifyDurableAdapters = <LedgerE = never, StoreE = never>(
|
|
|
960
1161
|
const tier3 = yield* resolveTierThree(capabilities.durability, options);
|
|
961
1162
|
|
|
962
1163
|
const generatedAt = yield* nowUtc;
|
|
963
|
-
const ok =
|
|
964
|
-
tier1.every((result) => result.status === "passed") &&
|
|
965
|
-
tier2.every((result) => result.status !== "failed") &&
|
|
966
|
-
tier3.cases.every((result) => result.status === "passed");
|
|
967
1164
|
|
|
968
|
-
return
|
|
969
|
-
format: "effect-agent/certification@1",
|
|
1165
|
+
return makeCertificationReport({
|
|
970
1166
|
adapter: CertifiedAdapterIdentity.make({
|
|
971
1167
|
name: options.adapter.name,
|
|
972
1168
|
...(options.adapter.version === undefined ? {} : { version: options.adapter.version }),
|
|
@@ -976,7 +1172,6 @@ export const certifyDurableAdapters = <LedgerE = never, StoreE = never>(
|
|
|
976
1172
|
tier1,
|
|
977
1173
|
tier2,
|
|
978
1174
|
tier3,
|
|
979
|
-
ok,
|
|
980
1175
|
});
|
|
981
1176
|
});
|
|
982
1177
|
|