@effect-agent/testing 0.0.1-beta.6 → 0.1.0-beta.100
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Certification.d.mts +105 -0
- package/dist/Certification.mjs +637 -0
- package/dist/Certification.mjs.map +1 -0
- package/dist/Chaos.d.mts +127 -0
- package/dist/Chaos.mjs +756 -0
- package/dist/Chaos.mjs.map +1 -0
- package/{src/code-executor-conformance.d.ts → dist/CodeExecutorConformance.d.mts} +12 -10
- package/dist/CodeExecutorConformance.mjs +231 -0
- package/dist/CodeExecutorConformance.mjs.map +1 -0
- package/dist/CodeExecutorSubstitute.d.mts +20 -0
- package/dist/CodeExecutorSubstitute.mjs +368 -0
- package/dist/CodeExecutorSubstitute.mjs.map +1 -0
- package/dist/DocsResearcher.d.mts +276 -0
- package/dist/DocsResearcher.mjs +490 -0
- package/dist/DocsResearcher.mjs.map +1 -0
- package/dist/ScriptedModel-HIvFn-5K.d.mts +220 -0
- package/dist/ScriptedModel.d.mts +2 -0
- package/dist/ScriptedModel.mjs +155 -0
- package/dist/ScriptedModel.mjs.map +1 -0
- package/dist/TravelPlanner.d.mts +1680 -0
- package/dist/TravelPlanner.mjs +1985 -0
- package/dist/TravelPlanner.mjs.map +1 -0
- package/dist/deterministic-layers-D5owIoke.mjs +358 -0
- package/dist/deterministic-layers-D5owIoke.mjs.map +1 -0
- package/dist/index.d.mts +2 -3376
- package/dist/index.mjs +2 -4766
- package/dist/rolldown-runtime-D7D4PA-g.mjs +13 -0
- package/package.json +1 -48
- package/src/{certification.ts → Certification.ts} +380 -171
- package/src/{chaos.ts → Chaos.ts} +290 -191
- package/src/{code-executor-conformance.ts → CodeExecutorConformance.ts} +30 -7
- package/src/{code-executor-substitute.ts → CodeExecutorSubstitute.ts} +113 -46
- package/src/{fixtures/docs-researcher/index.ts → DocsResearcher.ts} +68 -5
- package/src/{scripted-model.ts → ScriptedModel.ts} +25 -29
- package/src/TravelPlanner.ts +232 -0
- package/src/fixtures/docs-researcher/definition.ts +20 -11
- package/src/fixtures/docs-researcher/harness.ts +41 -34
- package/src/fixtures/docs-researcher/mcp.ts +49 -3
- package/src/fixtures/travel-planner/definition.ts +15 -2
- package/src/fixtures/travel-planner/deterministic-layers.ts +47 -4
- package/src/fixtures/travel-planner/phase2.ts +4 -3
- package/src/fixtures/travel-planner/phase3.ts +19 -42
- package/src/fixtures/travel-planner/phase4.ts +24 -37
- package/src/fixtures/travel-planner/phase5.ts +35 -42
- package/src/fixtures/travel-planner/phase6.ts +228 -96
- package/src/fixtures/travel-planner/phase7.ts +4 -102
- package/src/fixtures/travel-planner/scenarios.ts +3 -4
- package/src/fixtures/travel-planner/subagents-durable.ts +35 -61
- package/src/fixtures/travel-planner/subagents.ts +36 -14
- package/src/index.ts +1 -11
- package/src/internal/certification-report.ts +25 -0
- package/dist/index.mjs.map +0 -1
- package/src/fixtures/travel-planner/index.ts +0 -11
- package/src/fixtures/warehouse/index.ts +0 -412
|
@@ -1,80 +1,83 @@
|
|
|
1
|
+
import type { Crypto } from "effect";
|
|
1
2
|
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
Cause,
|
|
4
|
+
Clock,
|
|
5
|
+
DateTime,
|
|
6
|
+
Duration,
|
|
7
|
+
Effect,
|
|
8
|
+
Exit,
|
|
9
|
+
Layer,
|
|
10
|
+
Option,
|
|
11
|
+
Ref,
|
|
12
|
+
Schema,
|
|
13
|
+
Stream,
|
|
14
|
+
} from "effect";
|
|
15
|
+
import * as Agent from "effect-agent/agent";
|
|
16
|
+
import { AgentPolicy } from "effect-agent/agent-policy";
|
|
17
|
+
import { DurableWorkerBinding, type ResolvedBinding } from "effect-agent/agent-registration";
|
|
7
18
|
import {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
19
|
+
DurableAgentRuntime,
|
|
20
|
+
DurableRuntimeConfig,
|
|
21
|
+
type DurableSubmitFailure,
|
|
22
|
+
type DurableSubmitOptions,
|
|
23
|
+
type Receipt,
|
|
24
|
+
} from "effect-agent/durable-agent-runtime";
|
|
25
|
+
import {
|
|
26
|
+
DurableRuntimeFailpointError,
|
|
27
|
+
DurableRuntimeFailpointLocation,
|
|
28
|
+
} from "effect-agent/durable-failpoint";
|
|
29
|
+
import { DurableStep, DurableStepError } from "effect-agent/durable-step";
|
|
30
|
+
import { IdGenerator } from "effect-agent/id-generator";
|
|
31
|
+
import {
|
|
32
|
+
ThreadId,
|
|
12
33
|
RunId,
|
|
13
34
|
ToolCallId,
|
|
14
35
|
TurnId,
|
|
15
36
|
type AgentId,
|
|
16
37
|
type SubmissionId,
|
|
17
|
-
} from "
|
|
18
|
-
import { DurableStep, DurableStepError } from "@effect-agent/engine";
|
|
38
|
+
} from "effect-agent/identifiers";
|
|
19
39
|
import {
|
|
20
|
-
AgentBindingResolver,
|
|
21
|
-
ApprovalDecisionCommand,
|
|
22
|
-
CertificationCaseResult,
|
|
23
|
-
CertificationReport,
|
|
24
|
-
CertificationSweepResult,
|
|
25
|
-
CertificationTierThreeReport,
|
|
26
|
-
CertifiedAdapterIdentity,
|
|
27
|
-
ConversationExportRequest,
|
|
28
|
-
ConversationStore,
|
|
29
40
|
DefinitionDigests,
|
|
30
41
|
DeploymentId,
|
|
31
42
|
Digest,
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
43
|
+
ProducerId,
|
|
44
|
+
type BatchId,
|
|
45
|
+
} from "effect-agent/records";
|
|
46
|
+
import { childThreadIdFor } from "effect-agent/run-journal";
|
|
47
|
+
import { RunToolAuthorization } from "effect-agent/run-options";
|
|
48
|
+
import * as Subagent from "effect-agent/subagent";
|
|
49
|
+
import { SubagentPolicy } from "effect-agent/subagent";
|
|
50
|
+
import { SubagentReservationsMemoryLive } from "effect-agent/subagent-reservations";
|
|
51
|
+
import {
|
|
52
|
+
ApprovalDecisionCommand,
|
|
39
53
|
IdempotencyKey,
|
|
40
|
-
LoadCheckpointRequest,
|
|
41
54
|
Principal,
|
|
42
|
-
ProducerId,
|
|
43
55
|
ResolutionSafeToRetry,
|
|
44
56
|
SubmissionLedger,
|
|
45
57
|
SubmissionLookupById,
|
|
46
|
-
ToolReconciler,
|
|
47
58
|
UnknownResolutionCommand,
|
|
48
|
-
WakeScheduler,
|
|
49
59
|
DEFAULT_OWNERSHIP_LEASE_DURATION,
|
|
50
|
-
certifyPorts,
|
|
51
|
-
childConversationIdFor,
|
|
52
|
-
verifyConversationInvariants,
|
|
53
|
-
type BatchId,
|
|
54
|
-
type CertificationScenario,
|
|
55
|
-
type DurableSubmitFailure,
|
|
56
|
-
type DurableSubmitOptions,
|
|
57
|
-
type Receipt,
|
|
58
|
-
type ResolvedBinding,
|
|
59
60
|
type SubmissionSnapshot,
|
|
60
|
-
} from "
|
|
61
|
+
} from "effect-agent/submission-ledger";
|
|
61
62
|
import {
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
} from "effect";
|
|
63
|
+
type CertificationCaseResult,
|
|
64
|
+
type CertificationReport,
|
|
65
|
+
CertificationSweepResult,
|
|
66
|
+
CertificationTierThreeReport,
|
|
67
|
+
CertifiedAdapterIdentity,
|
|
68
|
+
certifyPorts,
|
|
69
|
+
type CertificationScenario,
|
|
70
|
+
} from "effect-agent/testing/certification";
|
|
71
|
+
import { DurableRuntimeFailpointTestControl } from "effect-agent/testing/durable-failpoint-test-control";
|
|
72
|
+
import { verifyThreadInvariants } from "effect-agent/thread-invariants";
|
|
73
|
+
import { ThreadExportRequest, ThreadStore, LoadCheckpointRequest } from "effect-agent/thread-store";
|
|
74
|
+
import { ToolReconciler } from "effect-agent/tool-reconciler";
|
|
75
|
+
import { WakeScheduler } from "effect-agent/wake-scheduler";
|
|
75
76
|
import { TestClock } from "effect/testing";
|
|
76
77
|
import { LanguageModel, Model, Tool, Toolkit, type Response } from "effect/unstable/ai";
|
|
77
78
|
|
|
79
|
+
import { makeCertificationReport } from "./internal/certification-report.ts";
|
|
80
|
+
|
|
78
81
|
/**
|
|
79
82
|
* P7 WP2 — `certifyDurableAdapters` (plan §1): the one certification entry point a durable
|
|
80
83
|
* adapter pair runs to earn a Schema-encoded certificate.
|
|
@@ -83,12 +86,13 @@ import { LanguageModel, Model, Tool, Toolkit, type Response } from "effect/unsta
|
|
|
83
86
|
* `certifyPorts` (TEST-004/STORE-010).
|
|
84
87
|
* - **Tier 2 — coordinator protocol + failpoint convergence**: the durable coordinator is
|
|
85
88
|
* assembled over the CANDIDATE Layer pair with a scripted deterministic model; every
|
|
86
|
-
* `DurableRuntimeFailpointLocation` is armed one-shot across the six scenario shapes
|
|
87
|
-
* (plain / uncertain-tool / durable-steps / approval / join / delegation).
|
|
88
|
-
*
|
|
89
|
-
*
|
|
90
|
-
*
|
|
91
|
-
*
|
|
89
|
+
* reached `DurableRuntimeFailpointLocation` is armed one-shot across the six scenario shapes
|
|
90
|
+
* (plain / uncertain-tool / durable-steps / approval / join / delegation). Unreached locations
|
|
91
|
+
* share a verified clean run per shape; new unaccounted-for locations get a full sweep.
|
|
92
|
+
* After the injected fault the runner asserts the state stays CLASSIFIABLE (recovery +
|
|
93
|
+
* the public unblocking operations `resolveUnknown`/`resolveApproval` are the only levers
|
|
94
|
+
* used) and that the re-drive CONVERGES to `verifyThreadInvariants` with `requireAllSettled`,
|
|
95
|
+
* including a fully discharged digest-chain check, because the runner captures per-batch producer
|
|
92
96
|
* identity at append time.
|
|
93
97
|
* - **Tier 3 — real loss lever**: recorded honestly. A durable adapter either supplies a
|
|
94
98
|
* `CertificationCrashLever` executed in this run, cites its committed real-loss evidence
|
|
@@ -126,7 +130,7 @@ export interface CertifyDurableAdaptersOptions<LedgerE = never, StoreE = never>
|
|
|
126
130
|
* does); the certification's own environment supplies nothing else.
|
|
127
131
|
*/
|
|
128
132
|
readonly submissionLedger: Layer.Layer<SubmissionLedger, LedgerE, Crypto.Crypto>;
|
|
129
|
-
readonly
|
|
133
|
+
readonly threadStore: Layer.Layer<ThreadStore, StoreE, Crypto.Crypto>;
|
|
130
134
|
/** Defaults to `WakeScheduler.layerNoop`; the runner re-drives lanes explicitly. */
|
|
131
135
|
readonly wakeScheduler?: Layer.Layer<WakeScheduler> | undefined;
|
|
132
136
|
/** Executes Tier 3 in this run; takes precedence over `tierThreeEvidence`. */
|
|
@@ -156,18 +160,67 @@ export const CERTIFICATION_SCENARIOS: ReadonlyArray<CertificationScenario> = [
|
|
|
156
160
|
|
|
157
161
|
/**
|
|
158
162
|
* Coordinator failpoint locations that none of the six scenario shapes can reach, recorded
|
|
159
|
-
* honestly instead of silently claimed
|
|
160
|
-
* not take. They are pinned in-process
|
|
161
|
-
* (`packages/testing/test/durable-tools.test.ts` "resolveUnknown is idempotent across the
|
|
163
|
+
* honestly instead of silently claimed. These require operator, compaction, reservation,
|
|
164
|
+
* background-worker, or Agent-update paths the shapes do not take. They are pinned in-process
|
|
165
|
+
* by the P5/S2 suites (`packages/testing/test/durable-tools.test.ts` "resolveUnknown is idempotent across the
|
|
162
166
|
* intent failpoint", `durable-runtime.test.ts` abort rows,
|
|
163
167
|
* `durable-subagents.test.ts` abort propagation) and by the process-kill/eviction crash
|
|
164
168
|
* matrices. Runner tests assert the observed never-fired set equals EXACTLY this list, so a
|
|
165
169
|
* protocol change that silently stops exercising a location fails the certification.
|
|
166
170
|
*/
|
|
167
171
|
export const TIER2_UNREACHED_LOCATIONS: ReadonlyArray<DurableRuntimeFailpointLocation> = [
|
|
172
|
+
// Active timeout recording is covered before/after append by durable-runtime.test.ts.
|
|
173
|
+
"run:before-duration-append",
|
|
174
|
+
"run:after-duration-append",
|
|
175
|
+
// Tier 2 has no native compaction; dedicated checkpoint process-loss tests cover these.
|
|
176
|
+
"checkpoint:before-save",
|
|
177
|
+
"checkpoint:after-save",
|
|
168
178
|
"abort:after-intent",
|
|
179
|
+
// Compaction requires a `contextTokenLimit` policy plus prior-Run history
|
|
180
|
+
// none of the six scenario shapes carries; pinned in-process by the
|
|
181
|
+
// RUN-026 rows in `packages/testing/test/durable-runtime.test.ts`
|
|
182
|
+
// (compaction failpoint idempotence across re-drive).
|
|
183
|
+
"compaction:before-canonical-append",
|
|
184
|
+
"compaction:after-canonical-append",
|
|
185
|
+
// These shapes use neither programmatic Tools nor grace finalization. Their reservations
|
|
186
|
+
// are exercised before/after append in the public durable-runtime regression suite.
|
|
187
|
+
"policy:before-reservation-append",
|
|
188
|
+
"policy:after-reservation-append",
|
|
169
189
|
"resolve:after-intent",
|
|
190
|
+
// All six shapes retain their original Tool contracts, so none retires an unavailable
|
|
191
|
+
// operation. The deployment continuity matrix in durable-runtime.test.ts exercises the
|
|
192
|
+
// actual before/after-unavailable append boundaries, asserting absent/present canonical
|
|
193
|
+
// ToolUnavailable results at the crash and one original-call settlement after re-drive.
|
|
194
|
+
"tools:before-unavailable-append",
|
|
195
|
+
"tools:after-unavailable-append",
|
|
170
196
|
"subagent:after-child-abort-intent",
|
|
197
|
+
// Background workers use retained delivery, source capacity, and child-origin paths absent
|
|
198
|
+
// from these six attached/ordinary scenarios. The before/after creation and completion
|
|
199
|
+
// boundaries are exercised by packages/effect-agent/test/durable/worker-host.test.ts.
|
|
200
|
+
"worker:before-source-append",
|
|
201
|
+
"worker:after-source-append",
|
|
202
|
+
"worker:before-origin-append",
|
|
203
|
+
"worker:after-origin-append",
|
|
204
|
+
"worker:before-completion-append",
|
|
205
|
+
"worker:after-completion-append",
|
|
206
|
+
// Root attached declarations keep their independent pool; nested shared subtree mutations
|
|
207
|
+
// are covered by the same focused worker-host failpoint suite.
|
|
208
|
+
"worker:before-subtree-append",
|
|
209
|
+
"worker:after-subtree-append",
|
|
210
|
+
// Automatic report decisions and retained delivery insertion have dedicated worker crash tests.
|
|
211
|
+
"worker:before-report-append",
|
|
212
|
+
"worker:after-report-append",
|
|
213
|
+
"worker:before-report-delivery",
|
|
214
|
+
"worker:after-report-delivery",
|
|
215
|
+
// None of the six shapes emits Agent updates. All four boundaries are exercised separately
|
|
216
|
+
// by packages/effect-agent/test/durable/worker-host.test.ts ("repairs an accepted parent update after ...").
|
|
217
|
+
// Node restart/lost-ack coverage is in packages/platform-node/test/worker-updates.test.ts;
|
|
218
|
+
// Cloudflare eviction/alarm recovery is in packages/platform-cloudflare/test/background-workers.test.ts.
|
|
219
|
+
// These suites are not executed by this certification runner; its update rows remain not-triggered.
|
|
220
|
+
"update:before-canonical-append",
|
|
221
|
+
"update:after-canonical-append",
|
|
222
|
+
"update:before-delivery-insert",
|
|
223
|
+
"update:after-delivery-insert",
|
|
171
224
|
];
|
|
172
225
|
|
|
173
226
|
/** Locations of `tier2` rows whose armed fault never fired in ANY scenario, sorted. */
|
|
@@ -175,9 +228,11 @@ export const tier2NeverFiredLocations = (
|
|
|
175
228
|
tier2: ReadonlyArray<CertificationSweepResult>,
|
|
176
229
|
): ReadonlyArray<DurableRuntimeFailpointLocation> => {
|
|
177
230
|
const fired = new Set<DurableRuntimeFailpointLocation>();
|
|
231
|
+
|
|
178
232
|
for (const row of tier2) {
|
|
179
233
|
if (row.failpointFired) fired.add(row.location);
|
|
180
234
|
}
|
|
235
|
+
|
|
181
236
|
return DurableRuntimeFailpointLocation.literals.filter((location) => !fired.has(location)).sort();
|
|
182
237
|
};
|
|
183
238
|
|
|
@@ -187,18 +242,21 @@ export const tier2NeverFiredLocations = (
|
|
|
187
242
|
|
|
188
243
|
const SHA_A = Schema.decodeSync(Digest)("a".repeat(64));
|
|
189
244
|
const DIGESTS = DefinitionDigests.make({ agent: SHA_A, model: SHA_A, tools: SHA_A });
|
|
245
|
+
|
|
190
246
|
const CHILD_DIGEST_STRINGS = {
|
|
191
247
|
agent: "b".repeat(64),
|
|
192
248
|
model: "c".repeat(64),
|
|
193
249
|
tools: "d".repeat(64),
|
|
194
250
|
} as const;
|
|
251
|
+
|
|
195
252
|
const CHILD_DIGESTS = DefinitionDigests.make({
|
|
196
253
|
agent: Schema.decodeSync(Digest)(CHILD_DIGEST_STRINGS.agent),
|
|
197
254
|
model: Schema.decodeSync(Digest)(CHILD_DIGEST_STRINGS.model),
|
|
198
255
|
tools: Schema.decodeSync(Digest)(CHILD_DIGEST_STRINGS.tools),
|
|
199
256
|
});
|
|
257
|
+
|
|
200
258
|
const PRINCIPAL = Schema.decodeSync(Principal)("principal-certification");
|
|
201
|
-
const
|
|
259
|
+
const decodeThreadId = Schema.decodeSync(ThreadId);
|
|
202
260
|
const decodeIdempotencyKey = Schema.decodeSync(IdempotencyKey);
|
|
203
261
|
const decodeToolCallId = Schema.decodeSync(ToolCallId);
|
|
204
262
|
|
|
@@ -248,18 +306,26 @@ const promptShapeModel = (
|
|
|
248
306
|
generateText: () => Effect.succeed([]),
|
|
249
307
|
streamText: (request) => {
|
|
250
308
|
const hasToolResult = request.prompt.content.some((message) => message.role === "tool");
|
|
309
|
+
|
|
251
310
|
const parts =
|
|
252
311
|
toolParts === undefined || hasToolResult ? finalParts(finalText) : toolParts;
|
|
312
|
+
|
|
253
313
|
return Stream.fromIterable(parts);
|
|
254
314
|
},
|
|
255
315
|
}),
|
|
256
316
|
),
|
|
257
317
|
);
|
|
258
318
|
|
|
319
|
+
// Tier-2 intentionally advances virtual time past a 30-second ownership lease on every
|
|
320
|
+
// re-drive round. Keep its business-policy duration above the full eight-round sweep so this
|
|
321
|
+
// fixture certifies crash convergence rather than accidentally relying on a fresh duration
|
|
322
|
+
// allowance per replacement Attempt. RUN-030 owns the dedicated duration-expiry coverage.
|
|
323
|
+
const CERTIFICATION_MAX_DURATION = "10 minutes" as const;
|
|
324
|
+
|
|
259
325
|
const policy = AgentPolicy.make({
|
|
260
326
|
maxTurns: 4,
|
|
261
327
|
maxToolCalls: 4,
|
|
262
|
-
maxDuration:
|
|
328
|
+
maxDuration: CERTIFICATION_MAX_DURATION,
|
|
263
329
|
toolConcurrency: 2,
|
|
264
330
|
});
|
|
265
331
|
|
|
@@ -267,7 +333,7 @@ const QuestionInput = Schema.Struct({ question: Schema.String });
|
|
|
267
333
|
const AnswerOutput = Schema.Struct({ answer: Schema.String });
|
|
268
334
|
|
|
269
335
|
/** plain / join: no tools — the pure Turn/submission/join seams. */
|
|
270
|
-
const plainDefinition = Agent.
|
|
336
|
+
const plainDefinition = Agent.make("certify-plain", {
|
|
271
337
|
input: QuestionInput,
|
|
272
338
|
output: AnswerOutput,
|
|
273
339
|
instructions: "Answer as JSON.",
|
|
@@ -280,8 +346,10 @@ const Book = Tool.make("book", {
|
|
|
280
346
|
parameters: Schema.Struct({ ref: Schema.String }),
|
|
281
347
|
success: Schema.Struct({ confirmation: Schema.String }),
|
|
282
348
|
});
|
|
349
|
+
|
|
283
350
|
const bookToolkit = Toolkit.make(Book);
|
|
284
|
-
|
|
351
|
+
|
|
352
|
+
const uncertainDefinition = Agent.make("certify-uncertain", {
|
|
285
353
|
input: QuestionInput,
|
|
286
354
|
output: AnswerOutput,
|
|
287
355
|
instructions: "Book it.",
|
|
@@ -296,8 +364,10 @@ const Itinerary = Tool.make("itinerary", {
|
|
|
296
364
|
failure: DurableStepError,
|
|
297
365
|
dependencies: [DurableStep],
|
|
298
366
|
});
|
|
367
|
+
|
|
299
368
|
const itineraryToolkit = Toolkit.make(Itinerary);
|
|
300
|
-
|
|
369
|
+
|
|
370
|
+
const stepsDefinition = Agent.make("certify-steps", {
|
|
301
371
|
input: QuestionInput,
|
|
302
372
|
output: AnswerOutput,
|
|
303
373
|
instructions: "Reserve the itinerary.",
|
|
@@ -311,8 +381,10 @@ const BookApproval = Tool.make("book", {
|
|
|
311
381
|
success: Schema.Struct({ confirmation: Schema.String }),
|
|
312
382
|
needsApproval: true,
|
|
313
383
|
});
|
|
384
|
+
|
|
314
385
|
const approvalToolkit = Toolkit.make(BookApproval);
|
|
315
|
-
|
|
386
|
+
|
|
387
|
+
const approvalDefinition = Agent.make("certify-approval", {
|
|
316
388
|
input: QuestionInput,
|
|
317
389
|
output: AnswerOutput,
|
|
318
390
|
instructions: "Book after approval.",
|
|
@@ -321,7 +393,7 @@ const approvalDefinition = Agent.define("certify-approval", {
|
|
|
321
393
|
});
|
|
322
394
|
|
|
323
395
|
/** delegation: durable attached child plus an ordinary uncertain sibling in ONE batch. */
|
|
324
|
-
const childDefinition = Agent.
|
|
396
|
+
const childDefinition = Agent.make("certify-child", {
|
|
325
397
|
input: QuestionInput,
|
|
326
398
|
output: AnswerOutput,
|
|
327
399
|
instructions: "Answer as JSON.",
|
|
@@ -329,7 +401,7 @@ const childDefinition = Agent.define("certify-child", {
|
|
|
329
401
|
policy: AgentPolicy.make({
|
|
330
402
|
maxTurns: 2,
|
|
331
403
|
maxToolCalls: 1,
|
|
332
|
-
maxDuration:
|
|
404
|
+
maxDuration: CERTIFICATION_MAX_DURATION,
|
|
333
405
|
toolConcurrency: 1,
|
|
334
406
|
}),
|
|
335
407
|
});
|
|
@@ -352,7 +424,7 @@ const researchDelegation = Subagent.define("delegate_research", {
|
|
|
352
424
|
maxConcurrency: 2,
|
|
353
425
|
maxTurns: 4,
|
|
354
426
|
maxToolCalls: 4,
|
|
355
|
-
maxDuration:
|
|
427
|
+
maxDuration: CERTIFICATION_MAX_DURATION,
|
|
356
428
|
}),
|
|
357
429
|
});
|
|
358
430
|
|
|
@@ -361,7 +433,7 @@ const Lookup = Tool.make("lookup", {
|
|
|
361
433
|
success: Schema.Struct({ value: Schema.String }),
|
|
362
434
|
});
|
|
363
435
|
|
|
364
|
-
const coordinatorDefinition = Agent.
|
|
436
|
+
const coordinatorDefinition = Agent.make("certify-coordinator", {
|
|
365
437
|
input: Schema.Struct({ mission: Schema.String }),
|
|
366
438
|
output: Schema.Struct({ report: Schema.String }),
|
|
367
439
|
instructions: "Delegate and look up, then answer as JSON.",
|
|
@@ -369,7 +441,7 @@ const coordinatorDefinition = Agent.define("certify-coordinator", {
|
|
|
369
441
|
policy: AgentPolicy.make({
|
|
370
442
|
maxTurns: 4,
|
|
371
443
|
maxToolCalls: 3,
|
|
372
|
-
maxDuration:
|
|
444
|
+
maxDuration: CERTIFICATION_MAX_DURATION,
|
|
373
445
|
toolConcurrency: 2,
|
|
374
446
|
}),
|
|
375
447
|
});
|
|
@@ -384,12 +456,14 @@ const identifiers = Layer.effect(
|
|
|
384
456
|
IdGenerator,
|
|
385
457
|
Effect.gen(function* () {
|
|
386
458
|
const counter = yield* Ref.make(0);
|
|
459
|
+
|
|
387
460
|
const next = <A>(decode: (value: string) => A, prefix: string) =>
|
|
388
461
|
Ref.getAndUpdate(counter, (value) => value + 1).pipe(
|
|
389
462
|
Effect.map((value) => decode(`${prefix}-${value}`)),
|
|
390
463
|
);
|
|
464
|
+
|
|
391
465
|
return {
|
|
392
|
-
|
|
466
|
+
nextThreadId: next(decodeThreadId, "certify-fixture-thread"),
|
|
393
467
|
nextRunId: next(Schema.decodeSync(RunId), "certify-fixture-run"),
|
|
394
468
|
nextTurnId: next(Schema.decodeSync(TurnId), "certify-fixture-turn"),
|
|
395
469
|
};
|
|
@@ -403,15 +477,15 @@ const delegationSupport = Layer.mergeAll(SubagentReservationsMemoryLive, identif
|
|
|
403
477
|
// ---------------------------------------------------------------------------
|
|
404
478
|
|
|
405
479
|
interface CertificationCell {
|
|
406
|
-
readonly
|
|
480
|
+
readonly bindings: ReadonlyArray<ResolvedBinding>;
|
|
407
481
|
/** Idempotent submission batch — safe to replay verbatim after a submit-boundary fault. */
|
|
408
482
|
readonly submit: Effect.Effect<ReadonlyArray<Receipt>, DurableSubmitFailure, DurableAgentRuntime>;
|
|
409
483
|
/** All lanes of the cell in drive order, computed from the (possibly replayed) receipts. */
|
|
410
|
-
readonly lanes: (receipts: ReadonlyArray<Receipt>) => ReadonlyArray<
|
|
484
|
+
readonly lanes: (receipts: ReadonlyArray<Receipt>) => ReadonlyArray<ThreadId>;
|
|
411
485
|
}
|
|
412
486
|
|
|
413
|
-
const submitOptionsFor = (slug: string,
|
|
414
|
-
|
|
487
|
+
const submitOptionsFor = (slug: string, threadId: ThreadId): DurableSubmitOptions => ({
|
|
488
|
+
threadId,
|
|
415
489
|
principal: PRINCIPAL,
|
|
416
490
|
idempotencyKey: decodeIdempotencyKey(`certify-key-${slug}`),
|
|
417
491
|
definitions: DIGESTS,
|
|
@@ -423,20 +497,24 @@ const makeSingleAgentCell = (
|
|
|
423
497
|
resolved: ResolvedBinding,
|
|
424
498
|
slug: string,
|
|
425
499
|
): CertificationCell => {
|
|
426
|
-
const
|
|
500
|
+
const threadId = decodeThreadId(`certify-${slug}`);
|
|
501
|
+
|
|
427
502
|
const submit = Effect.gen(function* () {
|
|
428
503
|
const runtime = yield* DurableAgentRuntime;
|
|
504
|
+
|
|
429
505
|
const receipt = yield* runtime.submit(
|
|
430
506
|
{ definition: { id: definition.id, input: definition.input } },
|
|
431
507
|
{ question: `certify ${slug}` },
|
|
432
|
-
submitOptionsFor(slug,
|
|
508
|
+
submitOptionsFor(slug, threadId),
|
|
433
509
|
);
|
|
510
|
+
|
|
434
511
|
return [receipt];
|
|
435
512
|
});
|
|
513
|
+
|
|
436
514
|
return {
|
|
437
|
-
|
|
515
|
+
bindings: [resolved],
|
|
438
516
|
submit,
|
|
439
|
-
lanes: () => [
|
|
517
|
+
lanes: () => [threadId],
|
|
440
518
|
};
|
|
441
519
|
};
|
|
442
520
|
|
|
@@ -450,7 +528,9 @@ const makeCell = Effect.fn("Certification.makeCell")(function* (
|
|
|
450
528
|
plainDefinition,
|
|
451
529
|
promptShapeModel("certify-plain", '{"answer":"done"}'),
|
|
452
530
|
);
|
|
531
|
+
|
|
453
532
|
const resolved = yield* DurableWorkerBinding.make(binding, DIGESTS);
|
|
533
|
+
|
|
454
534
|
return makeSingleAgentCell(plainDefinition, resolved, slug);
|
|
455
535
|
}
|
|
456
536
|
case "uncertain-tool": {
|
|
@@ -462,12 +542,15 @@ const makeCell = Effect.fn("Certification.makeCell")(function* (
|
|
|
462
542
|
toolTurn(toolCallPart("book-1", "book", { ref: `r-${slug}` })),
|
|
463
543
|
),
|
|
464
544
|
);
|
|
545
|
+
|
|
465
546
|
const toolLayer = bookToolkit.toLayer({
|
|
466
547
|
book: ({ ref }) => Effect.succeed({ confirmation: `confirmed-${ref}` }),
|
|
467
548
|
});
|
|
549
|
+
|
|
468
550
|
const resolved = yield* DurableWorkerBinding.make(binding, DIGESTS).pipe(
|
|
469
551
|
Effect.provide(toolLayer),
|
|
470
552
|
);
|
|
553
|
+
|
|
471
554
|
return makeSingleAgentCell(uncertainDefinition, resolved, slug);
|
|
472
555
|
}
|
|
473
556
|
case "durable-steps": {
|
|
@@ -479,26 +562,32 @@ const makeCell = Effect.fn("Certification.makeCell")(function* (
|
|
|
479
562
|
toolTurn(toolCallPart("itinerary-1", "itinerary", { ref: `trip-${slug}` })),
|
|
480
563
|
),
|
|
481
564
|
);
|
|
565
|
+
|
|
482
566
|
const toolLayer = itineraryToolkit.toLayer({
|
|
483
567
|
itinerary: ({ ref }) =>
|
|
484
568
|
Effect.gen(function* () {
|
|
485
569
|
const step = yield* DurableStep;
|
|
570
|
+
|
|
486
571
|
const flight = yield* step.do(
|
|
487
572
|
"reserve-flight",
|
|
488
573
|
Schema.String,
|
|
489
574
|
Effect.succeed(`flight-${ref}`),
|
|
490
575
|
);
|
|
576
|
+
|
|
491
577
|
const lodging = yield* step.do(
|
|
492
578
|
"reserve-lodging",
|
|
493
579
|
Schema.String,
|
|
494
580
|
Effect.succeed(`lodging-${ref}`),
|
|
495
581
|
);
|
|
582
|
+
|
|
496
583
|
return { state: `${flight}+${lodging}` };
|
|
497
584
|
}),
|
|
498
585
|
});
|
|
586
|
+
|
|
499
587
|
const resolved = yield* DurableWorkerBinding.make(binding, DIGESTS).pipe(
|
|
500
588
|
Effect.provide(toolLayer),
|
|
501
589
|
);
|
|
590
|
+
|
|
502
591
|
return makeSingleAgentCell(stepsDefinition, resolved, slug);
|
|
503
592
|
}
|
|
504
593
|
case "approval": {
|
|
@@ -510,12 +599,15 @@ const makeCell = Effect.fn("Certification.makeCell")(function* (
|
|
|
510
599
|
toolTurn(toolCallPart("book-1", "book", { ref: `r-${slug}` })),
|
|
511
600
|
),
|
|
512
601
|
);
|
|
602
|
+
|
|
513
603
|
const toolLayer = approvalToolkit.toLayer({
|
|
514
604
|
book: ({ ref }) => Effect.succeed({ confirmation: `confirmed-${ref}` }),
|
|
515
605
|
});
|
|
606
|
+
|
|
516
607
|
const resolved = yield* DurableWorkerBinding.make(binding, DIGESTS).pipe(
|
|
517
608
|
Effect.provide(toolLayer),
|
|
518
609
|
);
|
|
610
|
+
|
|
519
611
|
return makeSingleAgentCell(approvalDefinition, resolved, slug);
|
|
520
612
|
}
|
|
521
613
|
case "join": {
|
|
@@ -523,31 +615,39 @@ const makeCell = Effect.fn("Certification.makeCell")(function* (
|
|
|
523
615
|
plainDefinition,
|
|
524
616
|
promptShapeModel("certify-join", '{"answer":"host answer"}'),
|
|
525
617
|
);
|
|
618
|
+
|
|
526
619
|
const resolved = yield* DurableWorkerBinding.make(binding, DIGESTS);
|
|
527
|
-
const
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
620
|
+
const threadId = decodeThreadId(`certify-${slug}`);
|
|
621
|
+
|
|
622
|
+
const submitOne = Effect.fn("Certification.submitOne")(function* (
|
|
623
|
+
key: string,
|
|
624
|
+
question: string,
|
|
625
|
+
) {
|
|
626
|
+
const runtime = yield* DurableAgentRuntime;
|
|
627
|
+
|
|
628
|
+
return yield* runtime.submit(
|
|
629
|
+
{ definition: { id: plainDefinition.id, input: plainDefinition.input } },
|
|
630
|
+
{ question },
|
|
631
|
+
{
|
|
632
|
+
threadId,
|
|
633
|
+
principal: PRINCIPAL,
|
|
634
|
+
idempotencyKey: decodeIdempotencyKey(key),
|
|
635
|
+
definitions: DIGESTS,
|
|
636
|
+
},
|
|
637
|
+
);
|
|
638
|
+
});
|
|
639
|
+
|
|
542
640
|
const cell: CertificationCell = {
|
|
543
|
-
|
|
641
|
+
bindings: [resolved],
|
|
544
642
|
submit: Effect.gen(function* () {
|
|
545
643
|
const host = yield* submitOne(`certify-key-${slug}-host`, "host question");
|
|
546
644
|
const queued = yield* submitOne(`certify-key-${slug}-queued`, "queued question");
|
|
645
|
+
|
|
547
646
|
return [host, queued];
|
|
548
647
|
}),
|
|
549
|
-
lanes: () => [
|
|
648
|
+
lanes: () => [threadId],
|
|
550
649
|
};
|
|
650
|
+
|
|
551
651
|
return cell;
|
|
552
652
|
}
|
|
553
653
|
case "delegation": {
|
|
@@ -555,6 +655,7 @@ const makeCell = Effect.fn("Certification.makeCell")(function* (
|
|
|
555
655
|
childDefinition,
|
|
556
656
|
promptShapeModel("certify-child", '{"answer":"child-answer"}'),
|
|
557
657
|
);
|
|
658
|
+
|
|
558
659
|
const parentBinding = Agent.withModel(
|
|
559
660
|
coordinatorDefinition,
|
|
560
661
|
promptShapeModel(
|
|
@@ -566,36 +667,45 @@ const makeCell = Effect.fn("Certification.makeCell")(function* (
|
|
|
566
667
|
),
|
|
567
668
|
),
|
|
568
669
|
);
|
|
569
|
-
|
|
670
|
+
|
|
671
|
+
const delegationLayer = Subagent.layer(researchDelegation, childBinding, {
|
|
570
672
|
mapChildFailure,
|
|
571
673
|
durable: { targetDigests: CHILD_DIGEST_STRINGS },
|
|
572
674
|
}).pipe(Layer.provide(delegationSupport));
|
|
675
|
+
|
|
573
676
|
const lookupLayer = Toolkit.make(Lookup).toLayer({
|
|
574
677
|
lookup: ({ key }) => Effect.succeed({ value: `found-${key}` }),
|
|
575
678
|
});
|
|
679
|
+
|
|
576
680
|
const parentResolved = yield* DurableWorkerBinding.make(parentBinding, DIGESTS).pipe(
|
|
577
681
|
Effect.provide(Layer.mergeAll(delegationLayer, lookupLayer)),
|
|
578
682
|
);
|
|
683
|
+
|
|
579
684
|
const childResolved = yield* DurableWorkerBinding.make(childBinding, CHILD_DIGESTS);
|
|
580
|
-
const
|
|
685
|
+
const threadId = decodeThreadId(`certify-${slug}`);
|
|
686
|
+
|
|
581
687
|
const cell: CertificationCell = {
|
|
582
|
-
|
|
688
|
+
bindings: [parentResolved, childResolved],
|
|
583
689
|
submit: Effect.gen(function* () {
|
|
584
690
|
const runtime = yield* DurableAgentRuntime;
|
|
691
|
+
|
|
585
692
|
const receipt = yield* runtime.submit(
|
|
586
693
|
{ definition: { id: coordinatorDefinition.id, input: coordinatorDefinition.input } },
|
|
587
694
|
{ mission: "plan" },
|
|
588
|
-
submitOptionsFor(slug,
|
|
695
|
+
submitOptionsFor(slug, threadId),
|
|
589
696
|
);
|
|
697
|
+
|
|
590
698
|
return [receipt];
|
|
591
699
|
}),
|
|
592
700
|
lanes: (receipts) => {
|
|
593
701
|
const parent = receipts.at(0);
|
|
702
|
+
|
|
594
703
|
return parent === undefined
|
|
595
|
-
? [
|
|
596
|
-
: [
|
|
704
|
+
? [threadId]
|
|
705
|
+
: [threadId, childThreadIdFor(parent.submissionId, DELEGATE_CALL)];
|
|
597
706
|
},
|
|
598
707
|
};
|
|
708
|
+
|
|
599
709
|
return cell;
|
|
600
710
|
}
|
|
601
711
|
}
|
|
@@ -611,107 +721,139 @@ const MAX_REDRIVE_ROUNDS = 8;
|
|
|
611
721
|
* the digest chain is fully recomputed instead of skipped.
|
|
612
722
|
*/
|
|
613
723
|
const verifyLane = Effect.fn("Certification.verifyLane")(function* (
|
|
614
|
-
lane:
|
|
724
|
+
lane: ThreadId,
|
|
615
725
|
batchProducers: ReadonlyMap<BatchId, ProducerId>,
|
|
616
726
|
) {
|
|
617
|
-
const store = yield*
|
|
727
|
+
const store = yield* ThreadStore;
|
|
618
728
|
const ledger = yield* SubmissionLedger;
|
|
619
|
-
const exported = yield* store.export(
|
|
729
|
+
const exported = yield* store.export(ThreadExportRequest.make({ threadId: lane }));
|
|
620
730
|
const rows = new Map<SubmissionId, SubmissionSnapshot>();
|
|
621
731
|
const nonterminal = yield* Stream.runCollect(ledger.scanNonterminal);
|
|
732
|
+
|
|
622
733
|
for (const submission of nonterminal) {
|
|
623
|
-
if (submission.
|
|
734
|
+
if (submission.threadId === lane) rows.set(submission.submissionId, submission);
|
|
624
735
|
}
|
|
625
736
|
const named = new Set<SubmissionId>();
|
|
737
|
+
|
|
626
738
|
for (const envelope of exported.records) {
|
|
627
739
|
const payload = envelope.record.payload;
|
|
740
|
+
|
|
628
741
|
if (
|
|
629
742
|
payload._tag === "UserInputRecorded" ||
|
|
630
743
|
payload._tag === "SubmissionSettled" ||
|
|
631
744
|
payload._tag === "AbortRequested"
|
|
632
745
|
) {
|
|
633
|
-
named.add(payload.submissionId);
|
|
746
|
+
if (payload.submissionId !== undefined) named.add(payload.submissionId);
|
|
634
747
|
}
|
|
635
748
|
}
|
|
636
749
|
for (const submissionId of named) {
|
|
637
750
|
if (rows.has(submissionId)) continue;
|
|
638
751
|
const found = yield* ledger.lookup(SubmissionLookupById.make({ submissionId }));
|
|
639
|
-
|
|
752
|
+
|
|
753
|
+
if (Option.isSome(found) && found.value.threadId === lane) {
|
|
640
754
|
rows.set(submissionId, found.value);
|
|
641
755
|
}
|
|
642
756
|
}
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
757
|
+
|
|
758
|
+
const checkpoint =
|
|
759
|
+
store.checkpoints === undefined
|
|
760
|
+
? Option.none()
|
|
761
|
+
: yield* store.checkpoints.load(LoadCheckpointRequest.make({ threadId: lane }));
|
|
762
|
+
|
|
763
|
+
return yield* verifyThreadInvariants({
|
|
647
764
|
export: exported,
|
|
648
765
|
submissions: [...rows.values()],
|
|
649
766
|
batchProducers,
|
|
650
767
|
checkpoint: Option.getOrUndefined(checkpoint),
|
|
768
|
+
checkpointsSupported: store.checkpoints !== undefined,
|
|
651
769
|
requireAllSettled: true,
|
|
652
770
|
});
|
|
653
771
|
});
|
|
654
772
|
|
|
655
773
|
const failureTagOf = <E>(cause: Cause.Cause<E>): string => {
|
|
656
774
|
const failure = Cause.findErrorOption(cause);
|
|
775
|
+
|
|
657
776
|
if (Option.isSome(failure)) {
|
|
658
777
|
const error: unknown = failure.value;
|
|
778
|
+
|
|
659
779
|
if (typeof error === "object" && error !== null && "_tag" in error) {
|
|
660
|
-
return String(
|
|
780
|
+
return String(error._tag);
|
|
661
781
|
}
|
|
782
|
+
|
|
662
783
|
return String(error).slice(0, 256);
|
|
663
784
|
}
|
|
785
|
+
|
|
664
786
|
return "defect";
|
|
665
787
|
};
|
|
666
788
|
|
|
667
|
-
|
|
789
|
+
type SweepOutcome = Pick<
|
|
790
|
+
CertificationSweepResult,
|
|
791
|
+
"failpointFired" | "status" | "digestChainVerified" | "detail"
|
|
792
|
+
>;
|
|
793
|
+
|
|
794
|
+
/** Discover a clean path, or arm one location; both drives converge and verify real storage. */
|
|
668
795
|
const runSweepCell = Effect.fn("Certification.runSweepCell")(function* (
|
|
669
796
|
scenario: CertificationScenario,
|
|
670
|
-
location: DurableRuntimeFailpointLocation,
|
|
797
|
+
location: DurableRuntimeFailpointLocation | undefined,
|
|
671
798
|
batchProducers: ReadonlyMap<BatchId, ProducerId>,
|
|
672
799
|
leaseAdvance: Duration.Duration,
|
|
800
|
+
reached: Set<DurableRuntimeFailpointLocation>,
|
|
673
801
|
) {
|
|
674
|
-
const runtime = yield* DurableAgentRuntime;
|
|
675
802
|
const ledger = yield* SubmissionLedger;
|
|
676
803
|
const control = yield* DurableRuntimeFailpointTestControl;
|
|
677
|
-
const slug = `${scenario}-${location
|
|
678
|
-
|
|
679
|
-
const failed = (detail: string, fired: boolean):
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
digestChainVerified: false,
|
|
686
|
-
detail: detail.slice(0, 4_096),
|
|
687
|
-
});
|
|
804
|
+
const slug = `${scenario}-${location?.replaceAll(":", "-") ?? "discovery"}`;
|
|
805
|
+
|
|
806
|
+
const failed = (detail: string, fired: boolean): SweepOutcome => ({
|
|
807
|
+
failpointFired: fired,
|
|
808
|
+
status: "failed",
|
|
809
|
+
digestChainVerified: false,
|
|
810
|
+
detail: detail.slice(0, 4_096),
|
|
811
|
+
});
|
|
688
812
|
|
|
689
813
|
const cell = yield* makeCell(scenario, slug);
|
|
690
814
|
|
|
815
|
+
const runtime = yield* DurableAgentRuntime.pipe(
|
|
816
|
+
Effect.provide(
|
|
817
|
+
DurableAgentRuntime.layerWithBindings(cell.bindings).pipe(
|
|
818
|
+
Layer.provide(RunToolAuthorization.allowAll),
|
|
819
|
+
),
|
|
820
|
+
),
|
|
821
|
+
);
|
|
822
|
+
|
|
823
|
+
const submit = cell.submit.pipe(Effect.provideService(DurableAgentRuntime, runtime));
|
|
824
|
+
|
|
691
825
|
// One-shot arm: the fault fires at most once anywhere in the cell (initial drive OR a
|
|
692
826
|
// re-drive round's public unblocking operation), modelling one crash at this boundary.
|
|
693
827
|
const fired = yield* Ref.make(false);
|
|
828
|
+
|
|
694
829
|
yield* control.setHandler((hit) =>
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
830
|
+
Effect.suspend(() => {
|
|
831
|
+
reached.add(hit);
|
|
832
|
+
|
|
833
|
+
return hit !== location
|
|
834
|
+
? Effect.void
|
|
835
|
+
: Ref.getAndSet(fired, true).pipe(
|
|
836
|
+
Effect.flatMap((already) =>
|
|
837
|
+
already
|
|
838
|
+
? Effect.void
|
|
839
|
+
: Effect.fail(DurableRuntimeFailpointError.make({ location: hit })),
|
|
840
|
+
),
|
|
841
|
+
);
|
|
842
|
+
}),
|
|
704
843
|
);
|
|
705
844
|
|
|
706
845
|
// Submissions are idempotent (DUR-001): one replay recovers a submit-boundary fault.
|
|
707
846
|
let receipts: ReadonlyArray<Receipt>;
|
|
708
|
-
const firstSubmit = yield* Effect.exit(
|
|
847
|
+
const firstSubmit = yield* Effect.exit(submit);
|
|
848
|
+
|
|
709
849
|
if (Exit.isSuccess(firstSubmit)) {
|
|
710
850
|
receipts = firstSubmit.value;
|
|
711
851
|
} else {
|
|
712
|
-
const secondSubmit = yield* Effect.exit(
|
|
852
|
+
const secondSubmit = yield* Effect.exit(submit);
|
|
853
|
+
|
|
713
854
|
if (Exit.isFailure(secondSubmit)) {
|
|
714
855
|
yield* control.clear;
|
|
856
|
+
|
|
715
857
|
return failed(
|
|
716
858
|
`submission replay did not recover: ${failureTagOf(secondSubmit.cause)}`,
|
|
717
859
|
yield* Ref.get(fired),
|
|
@@ -721,24 +863,24 @@ const runSweepCell = Effect.fn("Certification.runSweepCell")(function* (
|
|
|
721
863
|
}
|
|
722
864
|
const lanes = cell.lanes(receipts);
|
|
723
865
|
|
|
724
|
-
const driveLane = (lane:
|
|
725
|
-
runtime
|
|
726
|
-
.processConversationResolved(lane)
|
|
727
|
-
.pipe(Effect.provideService(AgentBindingResolver, cell.resolver));
|
|
866
|
+
const driveLane = (lane: ThreadId) => runtime.processThreadResolved(lane);
|
|
728
867
|
|
|
729
868
|
const allSettled = Effect.gen(function* () {
|
|
730
869
|
for (const receipt of receipts) {
|
|
731
870
|
const snapshot = yield* ledger.lookup(
|
|
732
871
|
SubmissionLookupById.make({ submissionId: receipt.submissionId }),
|
|
733
872
|
);
|
|
873
|
+
|
|
734
874
|
if (Option.isNone(snapshot) || snapshot.value.state !== "settled") return false;
|
|
735
875
|
}
|
|
876
|
+
|
|
736
877
|
return true;
|
|
737
878
|
});
|
|
738
879
|
|
|
739
880
|
// Re-drive to convergence using ONLY public operations: worker drives, recovery passes,
|
|
740
|
-
// and the authorized DUR-017/approval unblocking paths chosen from `
|
|
881
|
+
// and the authorized DUR-017/approval unblocking paths chosen from `explainThread`.
|
|
741
882
|
let converged = false;
|
|
883
|
+
|
|
742
884
|
for (let round = 0; round < MAX_REDRIVE_ROUNDS && !converged; round++) {
|
|
743
885
|
// Expire any lease a faulted Attempt left behind (D5): virtual time is the
|
|
744
886
|
// adapter-neutral reclaim lever — a live lease may block every new claim.
|
|
@@ -748,7 +890,8 @@ const runSweepCell = Effect.fn("Certification.runSweepCell")(function* (
|
|
|
748
890
|
yield* Effect.exit(driveLane(lane));
|
|
749
891
|
}
|
|
750
892
|
for (const lane of lanes) {
|
|
751
|
-
const explains = yield* Effect.exit(runtime.
|
|
893
|
+
const explains = yield* Effect.exit(runtime.explainThread(lane));
|
|
894
|
+
|
|
752
895
|
if (Exit.isFailure(explains)) continue;
|
|
753
896
|
for (const explanation of explains.value) {
|
|
754
897
|
for (const unknown of explanation.evidence.unknownCalls) {
|
|
@@ -769,6 +912,7 @@ const runSweepCell = Effect.fn("Certification.runSweepCell")(function* (
|
|
|
769
912
|
const decided = explanation.evidence.approvalDecisions.some(
|
|
770
913
|
(decision) => decision.toolCallId === pending.toolCallId,
|
|
771
914
|
);
|
|
915
|
+
|
|
772
916
|
if (decided) continue;
|
|
773
917
|
yield* Effect.exit(
|
|
774
918
|
runtime.resolveApproval(
|
|
@@ -785,6 +929,7 @@ const runSweepCell = Effect.fn("Certification.runSweepCell")(function* (
|
|
|
785
929
|
}
|
|
786
930
|
}
|
|
787
931
|
const settled = yield* Effect.exit(allSettled);
|
|
932
|
+
|
|
788
933
|
converged = Exit.isSuccess(settled) && settled.value;
|
|
789
934
|
}
|
|
790
935
|
yield* control.clear;
|
|
@@ -797,8 +942,10 @@ const runSweepCell = Effect.fn("Certification.runSweepCell")(function* (
|
|
|
797
942
|
// Every lane of the cell must verify in convergence mode with a recomputed digest chain.
|
|
798
943
|
let digestChainVerified = true;
|
|
799
944
|
const failedChecks: Array<string> = [];
|
|
945
|
+
|
|
800
946
|
for (const lane of lanes) {
|
|
801
947
|
const verdict = yield* Effect.exit(verifyLane(lane, batchProducers));
|
|
948
|
+
|
|
802
949
|
if (Exit.isFailure(verdict)) {
|
|
803
950
|
return failed(`lane ${lane} could not be verified: ${failureTagOf(verdict.cause)}`, wasFired);
|
|
804
951
|
}
|
|
@@ -822,13 +969,11 @@ const runSweepCell = Effect.fn("Certification.runSweepCell")(function* (
|
|
|
822
969
|
);
|
|
823
970
|
}
|
|
824
971
|
|
|
825
|
-
return
|
|
826
|
-
scenario,
|
|
827
|
-
location,
|
|
972
|
+
return {
|
|
828
973
|
failpointFired: wasFired,
|
|
829
974
|
status: wasFired ? "converged" : "not-triggered",
|
|
830
975
|
digestChainVerified,
|
|
831
|
-
}
|
|
976
|
+
} satisfies SweepOutcome;
|
|
832
977
|
});
|
|
833
978
|
|
|
834
979
|
// ---------------------------------------------------------------------------
|
|
@@ -859,10 +1004,14 @@ export const resolveTierThree = Effect.fn("Certification.resolveTierThree")(func
|
|
|
859
1004
|
}
|
|
860
1005
|
if (options.crashLever !== undefined) {
|
|
861
1006
|
const cases = yield* options.crashLever;
|
|
1007
|
+
|
|
862
1008
|
return CertificationTierThreeReport.make({
|
|
863
|
-
status: "exercised",
|
|
1009
|
+
status: cases.length === 0 ? "not-exercised" : "exercised",
|
|
864
1010
|
evidence: options.tierThreeEvidence ?? [],
|
|
865
1011
|
cases,
|
|
1012
|
+
...(cases.length === 0
|
|
1013
|
+
? { detail: "the supplied crash lever executed no real-loss cases" }
|
|
1014
|
+
: {}),
|
|
866
1015
|
});
|
|
867
1016
|
}
|
|
868
1017
|
if (options.tierThreeEvidence !== undefined && options.tierThreeEvidence.length > 0) {
|
|
@@ -872,6 +1021,7 @@ export const resolveTierThree = Effect.fn("Certification.resolveTierThree")(func
|
|
|
872
1021
|
cases: [],
|
|
873
1022
|
});
|
|
874
1023
|
}
|
|
1024
|
+
|
|
875
1025
|
return CertificationTierThreeReport.make({
|
|
876
1026
|
status: "not-exercised",
|
|
877
1027
|
evidence: [],
|
|
@@ -900,13 +1050,15 @@ export const certifyDurableAdapters = <LedgerE = never, StoreE = never>(
|
|
|
900
1050
|
options: CertifyDurableAdaptersOptions<LedgerE, StoreE>,
|
|
901
1051
|
): Effect.Effect<CertificationReport, LedgerE | StoreE, Crypto.Crypto> => {
|
|
902
1052
|
const batchProducers = new Map<BatchId, ProducerId>();
|
|
1053
|
+
|
|
903
1054
|
// Interpose the candidate store with an append-time capture of each batch's producer
|
|
904
|
-
// identity — the one value the
|
|
1055
|
+
// identity — the one value the ThreadStore port deliberately does not export — so
|
|
905
1056
|
// Tier 2's invariant verification recomputes the FULL digest chain instead of skipping it.
|
|
906
|
-
const capturingStore = Layer.effect(
|
|
1057
|
+
const capturingStore = Layer.effect(ThreadStore)(
|
|
907
1058
|
Effect.gen(function* () {
|
|
908
|
-
const inner = yield*
|
|
909
|
-
|
|
1059
|
+
const inner = yield* ThreadStore;
|
|
1060
|
+
|
|
1061
|
+
return ThreadStore.of({
|
|
910
1062
|
...inner,
|
|
911
1063
|
append: (request) =>
|
|
912
1064
|
Effect.sync(() => {
|
|
@@ -914,13 +1066,15 @@ export const certifyDurableAdapters = <LedgerE = never, StoreE = never>(
|
|
|
914
1066
|
}).pipe(Effect.andThen(inner.append(request))),
|
|
915
1067
|
});
|
|
916
1068
|
}),
|
|
917
|
-
).pipe(Layer.provide(options.
|
|
1069
|
+
).pipe(Layer.provide(options.threadStore));
|
|
918
1070
|
|
|
919
|
-
|
|
1071
|
+
// RUN-036: certification uses the default-none Tool failure observer. Trusted application
|
|
1072
|
+
// reporting adds no durable transition and is verified separately from adapter certification.
|
|
1073
|
+
const environment = Layer.mergeAll(
|
|
920
1074
|
options.submissionLedger,
|
|
921
1075
|
capturingStore,
|
|
922
1076
|
options.wakeScheduler ?? WakeScheduler.layerNoop,
|
|
923
|
-
|
|
1077
|
+
DurableRuntimeFailpointTestControl.layer,
|
|
924
1078
|
ToolReconciler.uncertain,
|
|
925
1079
|
DurableRuntimeConfig.layer({
|
|
926
1080
|
deploymentId: Schema.decodeSync(DeploymentId)("deployment-certification"),
|
|
@@ -930,7 +1084,6 @@ export const certifyDurableAdapters = <LedgerE = never, StoreE = never>(
|
|
|
930
1084
|
abortPollInterval: Duration.millis(50),
|
|
931
1085
|
}),
|
|
932
1086
|
);
|
|
933
|
-
const environment = DurableAgentRuntime.layer.pipe(Layer.provideMerge(support));
|
|
934
1087
|
|
|
935
1088
|
const leaseAdvance = Duration.millis(
|
|
936
1089
|
Duration.toMillis(options.ownershipLeaseDuration ?? DEFAULT_OWNERSHIP_LEASE_DURATION) + 1_000,
|
|
@@ -939,11 +1092,73 @@ export const certifyDurableAdapters = <LedgerE = never, StoreE = never>(
|
|
|
939
1092
|
const program = Effect.gen(function* () {
|
|
940
1093
|
const ledger = yield* SubmissionLedger;
|
|
941
1094
|
|
|
942
|
-
// Tier 2 —
|
|
1095
|
+
// Tier 2 — discover each deterministic shape without a fault. Unreached locations
|
|
1096
|
+
// would all repeat this same clean drive, including its full digest verification.
|
|
943
1097
|
const tier2: Array<CertificationSweepResult> = [];
|
|
1098
|
+
|
|
1099
|
+
const discoveries: Array<{
|
|
1100
|
+
readonly scenario: CertificationScenario;
|
|
1101
|
+
readonly outcome: SweepOutcome;
|
|
1102
|
+
readonly reached: Set<DurableRuntimeFailpointLocation>;
|
|
1103
|
+
}> = [];
|
|
1104
|
+
|
|
944
1105
|
for (const scenario of CERTIFICATION_SCENARIOS) {
|
|
1106
|
+
const reached = new Set<DurableRuntimeFailpointLocation>();
|
|
1107
|
+
|
|
1108
|
+
const outcome = yield* runSweepCell(
|
|
1109
|
+
scenario,
|
|
1110
|
+
undefined,
|
|
1111
|
+
batchProducers,
|
|
1112
|
+
leaseAdvance,
|
|
1113
|
+
reached,
|
|
1114
|
+
);
|
|
1115
|
+
|
|
1116
|
+
discoveries.push({ scenario, outcome, reached });
|
|
1117
|
+
}
|
|
1118
|
+
const observed = new Set(discoveries.flatMap(({ reached }) => [...reached]));
|
|
1119
|
+
|
|
1120
|
+
// Never silently omit a new location: if neither discovery nor the documented
|
|
1121
|
+
// never-fired set accounts for it, arm it in EVERY shape. Tests pin the fired paths
|
|
1122
|
+
// per shape and the never-fired set, so new or lost routes require explicit review.
|
|
1123
|
+
const unaccounted = new Set(
|
|
1124
|
+
DurableRuntimeFailpointLocation.literals.filter(
|
|
1125
|
+
(location) => !observed.has(location) && !TIER2_UNREACHED_LOCATIONS.includes(location),
|
|
1126
|
+
),
|
|
1127
|
+
);
|
|
1128
|
+
|
|
1129
|
+
for (const discovery of discoveries) {
|
|
1130
|
+
const { scenario, reached } = discovery;
|
|
1131
|
+
const outcomes = new Map<DurableRuntimeFailpointLocation, SweepOutcome>();
|
|
1132
|
+
|
|
1133
|
+
if (discovery.outcome.status !== "failed") {
|
|
1134
|
+
while (true) {
|
|
1135
|
+
// Recovery may expose another route. Discover those too, without re-running a
|
|
1136
|
+
// cell already checked. Schema order bounds this to one drive per location.
|
|
1137
|
+
const location = DurableRuntimeFailpointLocation.literals.find(
|
|
1138
|
+
(candidate) =>
|
|
1139
|
+
!outcomes.has(candidate) && (reached.has(candidate) || unaccounted.has(candidate)),
|
|
1140
|
+
);
|
|
1141
|
+
|
|
1142
|
+
if (location === undefined) break;
|
|
1143
|
+
outcomes.set(
|
|
1144
|
+
location,
|
|
1145
|
+
yield* runSweepCell(scenario, location, batchProducers, leaseAdvance, reached),
|
|
1146
|
+
);
|
|
1147
|
+
}
|
|
1148
|
+
}
|
|
945
1149
|
for (const location of DurableRuntimeFailpointLocation.literals) {
|
|
946
|
-
|
|
1150
|
+
const outcome = outcomes.get(location) ?? discovery.outcome;
|
|
1151
|
+
|
|
1152
|
+
tier2.push(
|
|
1153
|
+
CertificationSweepResult.make({
|
|
1154
|
+
scenario,
|
|
1155
|
+
location,
|
|
1156
|
+
...outcome,
|
|
1157
|
+
...(outcomes.has(location) || outcome.status === "failed"
|
|
1158
|
+
? {}
|
|
1159
|
+
: { detail: "verified clean scenario did not reach this location" }),
|
|
1160
|
+
}),
|
|
1161
|
+
);
|
|
947
1162
|
}
|
|
948
1163
|
}
|
|
949
1164
|
|
|
@@ -955,13 +1170,8 @@ export const certifyDurableAdapters = <LedgerE = never, StoreE = never>(
|
|
|
955
1170
|
const tier3 = yield* resolveTierThree(capabilities.durability, options);
|
|
956
1171
|
|
|
957
1172
|
const generatedAt = yield* nowUtc;
|
|
958
|
-
const ok =
|
|
959
|
-
tier1.every((result) => result.status === "passed") &&
|
|
960
|
-
tier2.every((result) => result.status !== "failed") &&
|
|
961
|
-
tier3.cases.every((result) => result.status === "passed");
|
|
962
1173
|
|
|
963
|
-
return
|
|
964
|
-
format: "effect-agent/certification@1",
|
|
1174
|
+
return makeCertificationReport({
|
|
965
1175
|
adapter: CertifiedAdapterIdentity.make({
|
|
966
1176
|
name: options.adapter.name,
|
|
967
1177
|
...(options.adapter.version === undefined ? {} : { version: options.adapter.version }),
|
|
@@ -971,7 +1181,6 @@ export const certifyDurableAdapters = <LedgerE = never, StoreE = never>(
|
|
|
971
1181
|
tier1,
|
|
972
1182
|
tier2,
|
|
973
1183
|
tier3,
|
|
974
|
-
ok,
|
|
975
1184
|
});
|
|
976
1185
|
});
|
|
977
1186
|
|