@effect-agent/testing 0.1.0-beta.11 → 0.1.0-beta.111
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Certification.d.mts +105 -0
- package/dist/Certification.mjs +637 -0
- package/dist/Certification.mjs.map +1 -0
- package/dist/Chaos.d.mts +127 -0
- package/dist/Chaos.mjs +756 -0
- package/dist/Chaos.mjs.map +1 -0
- package/{src/code-executor-conformance.d.ts → dist/CodeExecutorConformance.d.mts} +12 -10
- package/dist/CodeExecutorConformance.mjs +231 -0
- package/dist/CodeExecutorConformance.mjs.map +1 -0
- package/dist/CodeExecutorSubstitute.d.mts +20 -0
- package/dist/CodeExecutorSubstitute.mjs +368 -0
- package/dist/CodeExecutorSubstitute.mjs.map +1 -0
- package/dist/DocsResearcher.d.mts +276 -0
- package/dist/DocsResearcher.mjs +490 -0
- package/dist/DocsResearcher.mjs.map +1 -0
- package/dist/ScriptedModel-HIvFn-5K.d.mts +220 -0
- package/dist/ScriptedModel.d.mts +2 -0
- package/dist/ScriptedModel.mjs +155 -0
- package/dist/ScriptedModel.mjs.map +1 -0
- package/dist/TravelPlanner.d.mts +1680 -0
- package/dist/TravelPlanner.mjs +1985 -0
- package/dist/TravelPlanner.mjs.map +1 -0
- package/dist/deterministic-layers-D5owIoke.mjs +358 -0
- package/dist/deterministic-layers-D5owIoke.mjs.map +1 -0
- package/dist/index.d.mts +2 -3408
- package/dist/index.mjs +2 -4771
- package/dist/rolldown-runtime-D7D4PA-g.mjs +13 -0
- package/package.json +1 -48
- package/src/{certification.ts → Certification.ts} +375 -171
- package/src/{chaos.ts → Chaos.ts} +290 -191
- package/src/{code-executor-conformance.ts → CodeExecutorConformance.ts} +30 -7
- package/src/{code-executor-substitute.ts → CodeExecutorSubstitute.ts} +113 -46
- package/src/{fixtures/docs-researcher/index.ts → DocsResearcher.ts} +68 -5
- package/src/{scripted-model.ts → ScriptedModel.ts} +25 -29
- package/src/TravelPlanner.ts +232 -0
- package/src/fixtures/docs-researcher/definition.ts +20 -11
- package/src/fixtures/docs-researcher/harness.ts +41 -34
- package/src/fixtures/docs-researcher/mcp.ts +49 -3
- package/src/fixtures/travel-planner/definition.ts +15 -2
- package/src/fixtures/travel-planner/deterministic-layers.ts +47 -4
- package/src/fixtures/travel-planner/phase2.ts +4 -3
- package/src/fixtures/travel-planner/phase3.ts +19 -42
- package/src/fixtures/travel-planner/phase4.ts +24 -37
- package/src/fixtures/travel-planner/phase5.ts +35 -42
- package/src/fixtures/travel-planner/phase6.ts +224 -96
- package/src/fixtures/travel-planner/phase7.ts +4 -102
- package/src/fixtures/travel-planner/scenarios.ts +3 -4
- package/src/fixtures/travel-planner/subagents-durable.ts +35 -61
- package/src/fixtures/travel-planner/subagents.ts +36 -14
- package/src/index.ts +1 -11
- package/src/internal/certification-report.ts +25 -0
- package/dist/index.mjs.map +0 -1
- package/src/fixtures/travel-planner/index.ts +0 -11
- package/src/fixtures/warehouse/index.ts +0 -412
|
@@ -1,80 +1,83 @@
|
|
|
1
|
+
import type { Crypto } from "effect";
|
|
1
2
|
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
Cause,
|
|
4
|
+
Clock,
|
|
5
|
+
DateTime,
|
|
6
|
+
Duration,
|
|
7
|
+
Effect,
|
|
8
|
+
Exit,
|
|
9
|
+
Layer,
|
|
10
|
+
Option,
|
|
11
|
+
Ref,
|
|
12
|
+
Schema,
|
|
13
|
+
Stream,
|
|
14
|
+
} from "effect";
|
|
15
|
+
import * as Agent from "effect-agent/agent";
|
|
16
|
+
import { AgentPolicy } from "effect-agent/agent-policy";
|
|
17
|
+
import { DurableWorkerBinding, type ResolvedBinding } from "effect-agent/agent-registration";
|
|
7
18
|
import {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
19
|
+
DurableAgentRuntime,
|
|
20
|
+
DurableRuntimeConfig,
|
|
21
|
+
type DurableSubmitFailure,
|
|
22
|
+
type DurableSubmitOptions,
|
|
23
|
+
type Receipt,
|
|
24
|
+
} from "effect-agent/durable-agent-runtime";
|
|
25
|
+
import {
|
|
26
|
+
DurableRuntimeFailpointError,
|
|
27
|
+
DurableRuntimeFailpointLocation,
|
|
28
|
+
} from "effect-agent/durable-failpoint";
|
|
29
|
+
import { DurableStep, DurableStepError } from "effect-agent/durable-step";
|
|
30
|
+
import { IdGenerator } from "effect-agent/id-generator";
|
|
31
|
+
import {
|
|
32
|
+
ThreadId,
|
|
12
33
|
RunId,
|
|
13
34
|
ToolCallId,
|
|
14
35
|
TurnId,
|
|
15
36
|
type AgentId,
|
|
16
37
|
type SubmissionId,
|
|
17
|
-
} from "
|
|
18
|
-
import { DurableStep, DurableStepError } from "@effect-agent/engine";
|
|
38
|
+
} from "effect-agent/identifiers";
|
|
19
39
|
import {
|
|
20
|
-
AgentBindingResolver,
|
|
21
|
-
ApprovalDecisionCommand,
|
|
22
|
-
CertificationCaseResult,
|
|
23
|
-
CertificationReport,
|
|
24
|
-
CertificationSweepResult,
|
|
25
|
-
CertificationTierThreeReport,
|
|
26
|
-
CertifiedAdapterIdentity,
|
|
27
|
-
ConversationExportRequest,
|
|
28
|
-
ConversationStore,
|
|
29
40
|
DefinitionDigests,
|
|
30
41
|
DeploymentId,
|
|
31
42
|
Digest,
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
43
|
+
ProducerId,
|
|
44
|
+
type BatchId,
|
|
45
|
+
} from "effect-agent/records";
|
|
46
|
+
import { childThreadIdFor } from "effect-agent/run-journal";
|
|
47
|
+
import { RunToolAuthorization } from "effect-agent/run-options";
|
|
48
|
+
import * as Subagent from "effect-agent/subagent";
|
|
49
|
+
import { SubagentPolicy } from "effect-agent/subagent";
|
|
50
|
+
import { SubagentReservationsMemoryLive } from "effect-agent/subagent-reservations";
|
|
51
|
+
import {
|
|
52
|
+
ApprovalDecisionCommand,
|
|
39
53
|
IdempotencyKey,
|
|
40
|
-
LoadCheckpointRequest,
|
|
41
54
|
Principal,
|
|
42
|
-
ProducerId,
|
|
43
55
|
ResolutionSafeToRetry,
|
|
44
56
|
SubmissionLedger,
|
|
45
57
|
SubmissionLookupById,
|
|
46
|
-
ToolReconciler,
|
|
47
58
|
UnknownResolutionCommand,
|
|
48
|
-
WakeScheduler,
|
|
49
59
|
DEFAULT_OWNERSHIP_LEASE_DURATION,
|
|
50
|
-
certifyPorts,
|
|
51
|
-
childConversationIdFor,
|
|
52
|
-
verifyConversationInvariants,
|
|
53
|
-
type BatchId,
|
|
54
|
-
type CertificationScenario,
|
|
55
|
-
type DurableSubmitFailure,
|
|
56
|
-
type DurableSubmitOptions,
|
|
57
|
-
type Receipt,
|
|
58
|
-
type ResolvedBinding,
|
|
59
60
|
type SubmissionSnapshot,
|
|
60
|
-
} from "
|
|
61
|
+
} from "effect-agent/submission-ledger";
|
|
61
62
|
import {
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
} from "effect";
|
|
63
|
+
type CertificationCaseResult,
|
|
64
|
+
type CertificationReport,
|
|
65
|
+
CertificationSweepResult,
|
|
66
|
+
CertificationTierThreeReport,
|
|
67
|
+
CertifiedAdapterIdentity,
|
|
68
|
+
certifyPorts,
|
|
69
|
+
type CertificationScenario,
|
|
70
|
+
} from "effect-agent/testing/certification";
|
|
71
|
+
import { DurableRuntimeFailpointTestControl } from "effect-agent/testing/durable-failpoint-test-control";
|
|
72
|
+
import { verifyThreadInvariants } from "effect-agent/thread-invariants";
|
|
73
|
+
import { ThreadExportRequest, ThreadStore, LoadCheckpointRequest } from "effect-agent/thread-store";
|
|
74
|
+
import { ToolReconciler } from "effect-agent/tool-reconciler";
|
|
75
|
+
import { WakeScheduler } from "effect-agent/wake-scheduler";
|
|
75
76
|
import { TestClock } from "effect/testing";
|
|
76
77
|
import { LanguageModel, Model, Tool, Toolkit, type Response } from "effect/unstable/ai";
|
|
77
78
|
|
|
79
|
+
import { makeCertificationReport } from "./internal/certification-report.ts";
|
|
80
|
+
|
|
78
81
|
/**
|
|
79
82
|
* P7 WP2 — `certifyDurableAdapters` (plan §1): the one certification entry point a durable
|
|
80
83
|
* adapter pair runs to earn a Schema-encoded certificate.
|
|
@@ -83,12 +86,13 @@ import { LanguageModel, Model, Tool, Toolkit, type Response } from "effect/unsta
|
|
|
83
86
|
* `certifyPorts` (TEST-004/STORE-010).
|
|
84
87
|
* - **Tier 2 — coordinator protocol + failpoint convergence**: the durable coordinator is
|
|
85
88
|
* assembled over the CANDIDATE Layer pair with a scripted deterministic model; every
|
|
86
|
-
* `DurableRuntimeFailpointLocation` is armed one-shot across the six scenario shapes
|
|
87
|
-
* (plain / uncertain-tool / durable-steps / approval / join / delegation).
|
|
88
|
-
*
|
|
89
|
-
*
|
|
90
|
-
*
|
|
91
|
-
*
|
|
89
|
+
* reached `DurableRuntimeFailpointLocation` is armed one-shot across the six scenario shapes
|
|
90
|
+
* (plain / uncertain-tool / durable-steps / approval / join / delegation). Unreached locations
|
|
91
|
+
* share a verified clean run per shape; new unaccounted-for locations get a full sweep.
|
|
92
|
+
* After the injected fault the runner asserts the state stays CLASSIFIABLE (recovery +
|
|
93
|
+
* the public unblocking operations `resolveUnknown`/`resolveApproval` are the only levers
|
|
94
|
+
* used) and that the re-drive CONVERGES to `verifyThreadInvariants` with `requireAllSettled`,
|
|
95
|
+
* including a fully discharged digest-chain check, because the runner captures per-batch producer
|
|
92
96
|
* identity at append time.
|
|
93
97
|
* - **Tier 3 — real loss lever**: recorded honestly. A durable adapter either supplies a
|
|
94
98
|
* `CertificationCrashLever` executed in this run, cites its committed real-loss evidence
|
|
@@ -126,7 +130,7 @@ export interface CertifyDurableAdaptersOptions<LedgerE = never, StoreE = never>
|
|
|
126
130
|
* does); the certification's own environment supplies nothing else.
|
|
127
131
|
*/
|
|
128
132
|
readonly submissionLedger: Layer.Layer<SubmissionLedger, LedgerE, Crypto.Crypto>;
|
|
129
|
-
readonly
|
|
133
|
+
readonly threadStore: Layer.Layer<ThreadStore, StoreE, Crypto.Crypto>;
|
|
130
134
|
/** Defaults to `WakeScheduler.layerNoop`; the runner re-drives lanes explicitly. */
|
|
131
135
|
readonly wakeScheduler?: Layer.Layer<WakeScheduler> | undefined;
|
|
132
136
|
/** Executes Tier 3 in this run; takes precedence over `tierThreeEvidence`. */
|
|
@@ -156,23 +160,67 @@ export const CERTIFICATION_SCENARIOS: ReadonlyArray<CertificationScenario> = [
|
|
|
156
160
|
|
|
157
161
|
/**
|
|
158
162
|
* Coordinator failpoint locations that none of the six scenario shapes can reach, recorded
|
|
159
|
-
* honestly instead of silently claimed
|
|
160
|
-
* not take. They are pinned in-process
|
|
161
|
-
* (`packages/testing/test/durable-tools.test.ts` "resolveUnknown is idempotent across the
|
|
163
|
+
* honestly instead of silently claimed. These require operator, compaction, reservation,
|
|
164
|
+
* background-worker, or Agent-update paths the shapes do not take. They are pinned in-process
|
|
165
|
+
* by the P5/S2 suites (`packages/testing/test/durable-tools.test.ts` "resolveUnknown is idempotent across the
|
|
162
166
|
* intent failpoint", `durable-runtime.test.ts` abort rows,
|
|
163
167
|
* `durable-subagents.test.ts` abort propagation) and by the process-kill/eviction crash
|
|
164
168
|
* matrices. Runner tests assert the observed never-fired set equals EXACTLY this list, so a
|
|
165
169
|
* protocol change that silently stops exercising a location fails the certification.
|
|
166
170
|
*/
|
|
167
171
|
export const TIER2_UNREACHED_LOCATIONS: ReadonlyArray<DurableRuntimeFailpointLocation> = [
|
|
172
|
+
// Active timeout recording is covered before/after append by durable-runtime.test.ts.
|
|
173
|
+
"run:before-duration-append",
|
|
174
|
+
"run:after-duration-append",
|
|
175
|
+
// Tier 2 has no native compaction; dedicated checkpoint process-loss tests cover these.
|
|
176
|
+
"checkpoint:before-save",
|
|
177
|
+
"checkpoint:after-save",
|
|
168
178
|
"abort:after-intent",
|
|
169
179
|
// Compaction requires a `contextTokenLimit` policy plus prior-Run history
|
|
170
180
|
// none of the six scenario shapes carries; pinned in-process by the
|
|
171
181
|
// RUN-026 rows in `packages/testing/test/durable-runtime.test.ts`
|
|
172
182
|
// (compaction failpoint idempotence across re-drive).
|
|
183
|
+
"compaction:before-canonical-append",
|
|
173
184
|
"compaction:after-canonical-append",
|
|
185
|
+
// These shapes use neither programmatic Tools nor grace finalization. Their reservations
|
|
186
|
+
// are exercised before/after append in the public durable-runtime regression suite.
|
|
187
|
+
"policy:before-reservation-append",
|
|
188
|
+
"policy:after-reservation-append",
|
|
174
189
|
"resolve:after-intent",
|
|
190
|
+
// All six shapes retain their original Tool contracts, so none retires an unavailable
|
|
191
|
+
// operation. The deployment continuity matrix in durable-runtime.test.ts exercises the
|
|
192
|
+
// actual before/after-unavailable append boundaries, asserting absent/present canonical
|
|
193
|
+
// ToolUnavailable results at the crash and one original-call settlement after re-drive.
|
|
194
|
+
"tools:before-unavailable-append",
|
|
195
|
+
"tools:after-unavailable-append",
|
|
175
196
|
"subagent:after-child-abort-intent",
|
|
197
|
+
// Background workers use retained delivery, source capacity, and child-origin paths absent
|
|
198
|
+
// from these six attached/ordinary scenarios. The before/after creation and completion
|
|
199
|
+
// boundaries are exercised by packages/effect-agent/test/durable/worker-host.test.ts.
|
|
200
|
+
"worker:before-source-append",
|
|
201
|
+
"worker:after-source-append",
|
|
202
|
+
"worker:before-origin-append",
|
|
203
|
+
"worker:after-origin-append",
|
|
204
|
+
"worker:before-completion-append",
|
|
205
|
+
"worker:after-completion-append",
|
|
206
|
+
// Root attached declarations keep their independent pool; nested shared subtree mutations
|
|
207
|
+
// are covered by the same focused worker-host failpoint suite.
|
|
208
|
+
"worker:before-subtree-append",
|
|
209
|
+
"worker:after-subtree-append",
|
|
210
|
+
// Automatic report decisions and retained delivery insertion have dedicated worker crash tests.
|
|
211
|
+
"worker:before-report-append",
|
|
212
|
+
"worker:after-report-append",
|
|
213
|
+
"worker:before-report-delivery",
|
|
214
|
+
"worker:after-report-delivery",
|
|
215
|
+
// None of the six shapes emits Agent updates. All four boundaries are exercised separately
|
|
216
|
+
// by packages/effect-agent/test/durable/worker-host.test.ts ("repairs an accepted parent update after ...").
|
|
217
|
+
// Node restart/lost-ack coverage is in packages/platform-node/test/worker-updates.test.ts;
|
|
218
|
+
// Cloudflare eviction/alarm recovery is in packages/platform-cloudflare/test/background-workers.test.ts.
|
|
219
|
+
// These suites are not executed by this certification runner; its update rows remain not-triggered.
|
|
220
|
+
"update:before-canonical-append",
|
|
221
|
+
"update:after-canonical-append",
|
|
222
|
+
"update:before-delivery-insert",
|
|
223
|
+
"update:after-delivery-insert",
|
|
176
224
|
];
|
|
177
225
|
|
|
178
226
|
/** Locations of `tier2` rows whose armed fault never fired in ANY scenario, sorted. */
|
|
@@ -180,9 +228,11 @@ export const tier2NeverFiredLocations = (
|
|
|
180
228
|
tier2: ReadonlyArray<CertificationSweepResult>,
|
|
181
229
|
): ReadonlyArray<DurableRuntimeFailpointLocation> => {
|
|
182
230
|
const fired = new Set<DurableRuntimeFailpointLocation>();
|
|
231
|
+
|
|
183
232
|
for (const row of tier2) {
|
|
184
233
|
if (row.failpointFired) fired.add(row.location);
|
|
185
234
|
}
|
|
235
|
+
|
|
186
236
|
return DurableRuntimeFailpointLocation.literals.filter((location) => !fired.has(location)).sort();
|
|
187
237
|
};
|
|
188
238
|
|
|
@@ -192,18 +242,21 @@ export const tier2NeverFiredLocations = (
|
|
|
192
242
|
|
|
193
243
|
const SHA_A = Schema.decodeSync(Digest)("a".repeat(64));
|
|
194
244
|
const DIGESTS = DefinitionDigests.make({ agent: SHA_A, model: SHA_A, tools: SHA_A });
|
|
245
|
+
|
|
195
246
|
const CHILD_DIGEST_STRINGS = {
|
|
196
247
|
agent: "b".repeat(64),
|
|
197
248
|
model: "c".repeat(64),
|
|
198
249
|
tools: "d".repeat(64),
|
|
199
250
|
} as const;
|
|
251
|
+
|
|
200
252
|
const CHILD_DIGESTS = DefinitionDigests.make({
|
|
201
253
|
agent: Schema.decodeSync(Digest)(CHILD_DIGEST_STRINGS.agent),
|
|
202
254
|
model: Schema.decodeSync(Digest)(CHILD_DIGEST_STRINGS.model),
|
|
203
255
|
tools: Schema.decodeSync(Digest)(CHILD_DIGEST_STRINGS.tools),
|
|
204
256
|
});
|
|
257
|
+
|
|
205
258
|
const PRINCIPAL = Schema.decodeSync(Principal)("principal-certification");
|
|
206
|
-
const
|
|
259
|
+
const decodeThreadId = Schema.decodeSync(ThreadId);
|
|
207
260
|
const decodeIdempotencyKey = Schema.decodeSync(IdempotencyKey);
|
|
208
261
|
const decodeToolCallId = Schema.decodeSync(ToolCallId);
|
|
209
262
|
|
|
@@ -253,18 +306,26 @@ const promptShapeModel = (
|
|
|
253
306
|
generateText: () => Effect.succeed([]),
|
|
254
307
|
streamText: (request) => {
|
|
255
308
|
const hasToolResult = request.prompt.content.some((message) => message.role === "tool");
|
|
309
|
+
|
|
256
310
|
const parts =
|
|
257
311
|
toolParts === undefined || hasToolResult ? finalParts(finalText) : toolParts;
|
|
312
|
+
|
|
258
313
|
return Stream.fromIterable(parts);
|
|
259
314
|
},
|
|
260
315
|
}),
|
|
261
316
|
),
|
|
262
317
|
);
|
|
263
318
|
|
|
319
|
+
// Tier-2 intentionally advances virtual time past a 30-second ownership lease on every
|
|
320
|
+
// re-drive round. Keep its business-policy duration above the full eight-round sweep so this
|
|
321
|
+
// fixture certifies crash convergence rather than accidentally relying on a fresh duration
|
|
322
|
+
// allowance per replacement Attempt. RUN-030 owns the dedicated duration-expiry coverage.
|
|
323
|
+
const CERTIFICATION_MAX_DURATION = "10 minutes" as const;
|
|
324
|
+
|
|
264
325
|
const policy = AgentPolicy.make({
|
|
265
326
|
maxTurns: 4,
|
|
266
327
|
maxToolCalls: 4,
|
|
267
|
-
maxDuration:
|
|
328
|
+
maxDuration: CERTIFICATION_MAX_DURATION,
|
|
268
329
|
toolConcurrency: 2,
|
|
269
330
|
});
|
|
270
331
|
|
|
@@ -272,7 +333,7 @@ const QuestionInput = Schema.Struct({ question: Schema.String });
|
|
|
272
333
|
const AnswerOutput = Schema.Struct({ answer: Schema.String });
|
|
273
334
|
|
|
274
335
|
/** plain / join: no tools — the pure Turn/submission/join seams. */
|
|
275
|
-
const plainDefinition = Agent.
|
|
336
|
+
const plainDefinition = Agent.make("certify-plain", {
|
|
276
337
|
input: QuestionInput,
|
|
277
338
|
output: AnswerOutput,
|
|
278
339
|
instructions: "Answer as JSON.",
|
|
@@ -285,8 +346,10 @@ const Book = Tool.make("book", {
|
|
|
285
346
|
parameters: Schema.Struct({ ref: Schema.String }),
|
|
286
347
|
success: Schema.Struct({ confirmation: Schema.String }),
|
|
287
348
|
});
|
|
349
|
+
|
|
288
350
|
const bookToolkit = Toolkit.make(Book);
|
|
289
|
-
|
|
351
|
+
|
|
352
|
+
const uncertainDefinition = Agent.make("certify-uncertain", {
|
|
290
353
|
input: QuestionInput,
|
|
291
354
|
output: AnswerOutput,
|
|
292
355
|
instructions: "Book it.",
|
|
@@ -301,8 +364,10 @@ const Itinerary = Tool.make("itinerary", {
|
|
|
301
364
|
failure: DurableStepError,
|
|
302
365
|
dependencies: [DurableStep],
|
|
303
366
|
});
|
|
367
|
+
|
|
304
368
|
const itineraryToolkit = Toolkit.make(Itinerary);
|
|
305
|
-
|
|
369
|
+
|
|
370
|
+
const stepsDefinition = Agent.make("certify-steps", {
|
|
306
371
|
input: QuestionInput,
|
|
307
372
|
output: AnswerOutput,
|
|
308
373
|
instructions: "Reserve the itinerary.",
|
|
@@ -316,8 +381,10 @@ const BookApproval = Tool.make("book", {
|
|
|
316
381
|
success: Schema.Struct({ confirmation: Schema.String }),
|
|
317
382
|
needsApproval: true,
|
|
318
383
|
});
|
|
384
|
+
|
|
319
385
|
const approvalToolkit = Toolkit.make(BookApproval);
|
|
320
|
-
|
|
386
|
+
|
|
387
|
+
const approvalDefinition = Agent.make("certify-approval", {
|
|
321
388
|
input: QuestionInput,
|
|
322
389
|
output: AnswerOutput,
|
|
323
390
|
instructions: "Book after approval.",
|
|
@@ -326,7 +393,7 @@ const approvalDefinition = Agent.define("certify-approval", {
|
|
|
326
393
|
});
|
|
327
394
|
|
|
328
395
|
/** delegation: durable attached child plus an ordinary uncertain sibling in ONE batch. */
|
|
329
|
-
const childDefinition = Agent.
|
|
396
|
+
const childDefinition = Agent.make("certify-child", {
|
|
330
397
|
input: QuestionInput,
|
|
331
398
|
output: AnswerOutput,
|
|
332
399
|
instructions: "Answer as JSON.",
|
|
@@ -334,7 +401,7 @@ const childDefinition = Agent.define("certify-child", {
|
|
|
334
401
|
policy: AgentPolicy.make({
|
|
335
402
|
maxTurns: 2,
|
|
336
403
|
maxToolCalls: 1,
|
|
337
|
-
maxDuration:
|
|
404
|
+
maxDuration: CERTIFICATION_MAX_DURATION,
|
|
338
405
|
toolConcurrency: 1,
|
|
339
406
|
}),
|
|
340
407
|
});
|
|
@@ -357,7 +424,7 @@ const researchDelegation = Subagent.define("delegate_research", {
|
|
|
357
424
|
maxConcurrency: 2,
|
|
358
425
|
maxTurns: 4,
|
|
359
426
|
maxToolCalls: 4,
|
|
360
|
-
maxDuration:
|
|
427
|
+
maxDuration: CERTIFICATION_MAX_DURATION,
|
|
361
428
|
}),
|
|
362
429
|
});
|
|
363
430
|
|
|
@@ -366,7 +433,7 @@ const Lookup = Tool.make("lookup", {
|
|
|
366
433
|
success: Schema.Struct({ value: Schema.String }),
|
|
367
434
|
});
|
|
368
435
|
|
|
369
|
-
const coordinatorDefinition = Agent.
|
|
436
|
+
const coordinatorDefinition = Agent.make("certify-coordinator", {
|
|
370
437
|
input: Schema.Struct({ mission: Schema.String }),
|
|
371
438
|
output: Schema.Struct({ report: Schema.String }),
|
|
372
439
|
instructions: "Delegate and look up, then answer as JSON.",
|
|
@@ -374,7 +441,7 @@ const coordinatorDefinition = Agent.define("certify-coordinator", {
|
|
|
374
441
|
policy: AgentPolicy.make({
|
|
375
442
|
maxTurns: 4,
|
|
376
443
|
maxToolCalls: 3,
|
|
377
|
-
maxDuration:
|
|
444
|
+
maxDuration: CERTIFICATION_MAX_DURATION,
|
|
378
445
|
toolConcurrency: 2,
|
|
379
446
|
}),
|
|
380
447
|
});
|
|
@@ -389,12 +456,14 @@ const identifiers = Layer.effect(
|
|
|
389
456
|
IdGenerator,
|
|
390
457
|
Effect.gen(function* () {
|
|
391
458
|
const counter = yield* Ref.make(0);
|
|
459
|
+
|
|
392
460
|
const next = <A>(decode: (value: string) => A, prefix: string) =>
|
|
393
461
|
Ref.getAndUpdate(counter, (value) => value + 1).pipe(
|
|
394
462
|
Effect.map((value) => decode(`${prefix}-${value}`)),
|
|
395
463
|
);
|
|
464
|
+
|
|
396
465
|
return {
|
|
397
|
-
|
|
466
|
+
nextThreadId: next(decodeThreadId, "certify-fixture-thread"),
|
|
398
467
|
nextRunId: next(Schema.decodeSync(RunId), "certify-fixture-run"),
|
|
399
468
|
nextTurnId: next(Schema.decodeSync(TurnId), "certify-fixture-turn"),
|
|
400
469
|
};
|
|
@@ -408,15 +477,15 @@ const delegationSupport = Layer.mergeAll(SubagentReservationsMemoryLive, identif
|
|
|
408
477
|
// ---------------------------------------------------------------------------
|
|
409
478
|
|
|
410
479
|
interface CertificationCell {
|
|
411
|
-
readonly
|
|
480
|
+
readonly bindings: ReadonlyArray<ResolvedBinding>;
|
|
412
481
|
/** Idempotent submission batch — safe to replay verbatim after a submit-boundary fault. */
|
|
413
482
|
readonly submit: Effect.Effect<ReadonlyArray<Receipt>, DurableSubmitFailure, DurableAgentRuntime>;
|
|
414
483
|
/** All lanes of the cell in drive order, computed from the (possibly replayed) receipts. */
|
|
415
|
-
readonly lanes: (receipts: ReadonlyArray<Receipt>) => ReadonlyArray<
|
|
484
|
+
readonly lanes: (receipts: ReadonlyArray<Receipt>) => ReadonlyArray<ThreadId>;
|
|
416
485
|
}
|
|
417
486
|
|
|
418
|
-
const submitOptionsFor = (slug: string,
|
|
419
|
-
|
|
487
|
+
const submitOptionsFor = (slug: string, threadId: ThreadId): DurableSubmitOptions => ({
|
|
488
|
+
threadId,
|
|
420
489
|
principal: PRINCIPAL,
|
|
421
490
|
idempotencyKey: decodeIdempotencyKey(`certify-key-${slug}`),
|
|
422
491
|
definitions: DIGESTS,
|
|
@@ -428,20 +497,24 @@ const makeSingleAgentCell = (
|
|
|
428
497
|
resolved: ResolvedBinding,
|
|
429
498
|
slug: string,
|
|
430
499
|
): CertificationCell => {
|
|
431
|
-
const
|
|
500
|
+
const threadId = decodeThreadId(`certify-${slug}`);
|
|
501
|
+
|
|
432
502
|
const submit = Effect.gen(function* () {
|
|
433
503
|
const runtime = yield* DurableAgentRuntime;
|
|
504
|
+
|
|
434
505
|
const receipt = yield* runtime.submit(
|
|
435
506
|
{ definition: { id: definition.id, input: definition.input } },
|
|
436
507
|
{ question: `certify ${slug}` },
|
|
437
|
-
submitOptionsFor(slug,
|
|
508
|
+
submitOptionsFor(slug, threadId),
|
|
438
509
|
);
|
|
510
|
+
|
|
439
511
|
return [receipt];
|
|
440
512
|
});
|
|
513
|
+
|
|
441
514
|
return {
|
|
442
|
-
|
|
515
|
+
bindings: [resolved],
|
|
443
516
|
submit,
|
|
444
|
-
lanes: () => [
|
|
517
|
+
lanes: () => [threadId],
|
|
445
518
|
};
|
|
446
519
|
};
|
|
447
520
|
|
|
@@ -455,7 +528,9 @@ const makeCell = Effect.fn("Certification.makeCell")(function* (
|
|
|
455
528
|
plainDefinition,
|
|
456
529
|
promptShapeModel("certify-plain", '{"answer":"done"}'),
|
|
457
530
|
);
|
|
531
|
+
|
|
458
532
|
const resolved = yield* DurableWorkerBinding.make(binding, DIGESTS);
|
|
533
|
+
|
|
459
534
|
return makeSingleAgentCell(plainDefinition, resolved, slug);
|
|
460
535
|
}
|
|
461
536
|
case "uncertain-tool": {
|
|
@@ -467,12 +542,15 @@ const makeCell = Effect.fn("Certification.makeCell")(function* (
|
|
|
467
542
|
toolTurn(toolCallPart("book-1", "book", { ref: `r-${slug}` })),
|
|
468
543
|
),
|
|
469
544
|
);
|
|
545
|
+
|
|
470
546
|
const toolLayer = bookToolkit.toLayer({
|
|
471
547
|
book: ({ ref }) => Effect.succeed({ confirmation: `confirmed-${ref}` }),
|
|
472
548
|
});
|
|
549
|
+
|
|
473
550
|
const resolved = yield* DurableWorkerBinding.make(binding, DIGESTS).pipe(
|
|
474
551
|
Effect.provide(toolLayer),
|
|
475
552
|
);
|
|
553
|
+
|
|
476
554
|
return makeSingleAgentCell(uncertainDefinition, resolved, slug);
|
|
477
555
|
}
|
|
478
556
|
case "durable-steps": {
|
|
@@ -484,26 +562,32 @@ const makeCell = Effect.fn("Certification.makeCell")(function* (
|
|
|
484
562
|
toolTurn(toolCallPart("itinerary-1", "itinerary", { ref: `trip-${slug}` })),
|
|
485
563
|
),
|
|
486
564
|
);
|
|
565
|
+
|
|
487
566
|
const toolLayer = itineraryToolkit.toLayer({
|
|
488
567
|
itinerary: ({ ref }) =>
|
|
489
568
|
Effect.gen(function* () {
|
|
490
569
|
const step = yield* DurableStep;
|
|
570
|
+
|
|
491
571
|
const flight = yield* step.do(
|
|
492
572
|
"reserve-flight",
|
|
493
573
|
Schema.String,
|
|
494
574
|
Effect.succeed(`flight-${ref}`),
|
|
495
575
|
);
|
|
576
|
+
|
|
496
577
|
const lodging = yield* step.do(
|
|
497
578
|
"reserve-lodging",
|
|
498
579
|
Schema.String,
|
|
499
580
|
Effect.succeed(`lodging-${ref}`),
|
|
500
581
|
);
|
|
582
|
+
|
|
501
583
|
return { state: `${flight}+${lodging}` };
|
|
502
584
|
}),
|
|
503
585
|
});
|
|
586
|
+
|
|
504
587
|
const resolved = yield* DurableWorkerBinding.make(binding, DIGESTS).pipe(
|
|
505
588
|
Effect.provide(toolLayer),
|
|
506
589
|
);
|
|
590
|
+
|
|
507
591
|
return makeSingleAgentCell(stepsDefinition, resolved, slug);
|
|
508
592
|
}
|
|
509
593
|
case "approval": {
|
|
@@ -515,12 +599,15 @@ const makeCell = Effect.fn("Certification.makeCell")(function* (
|
|
|
515
599
|
toolTurn(toolCallPart("book-1", "book", { ref: `r-${slug}` })),
|
|
516
600
|
),
|
|
517
601
|
);
|
|
602
|
+
|
|
518
603
|
const toolLayer = approvalToolkit.toLayer({
|
|
519
604
|
book: ({ ref }) => Effect.succeed({ confirmation: `confirmed-${ref}` }),
|
|
520
605
|
});
|
|
606
|
+
|
|
521
607
|
const resolved = yield* DurableWorkerBinding.make(binding, DIGESTS).pipe(
|
|
522
608
|
Effect.provide(toolLayer),
|
|
523
609
|
);
|
|
610
|
+
|
|
524
611
|
return makeSingleAgentCell(approvalDefinition, resolved, slug);
|
|
525
612
|
}
|
|
526
613
|
case "join": {
|
|
@@ -528,31 +615,39 @@ const makeCell = Effect.fn("Certification.makeCell")(function* (
|
|
|
528
615
|
plainDefinition,
|
|
529
616
|
promptShapeModel("certify-join", '{"answer":"host answer"}'),
|
|
530
617
|
);
|
|
618
|
+
|
|
531
619
|
const resolved = yield* DurableWorkerBinding.make(binding, DIGESTS);
|
|
532
|
-
const
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
620
|
+
const threadId = decodeThreadId(`certify-${slug}`);
|
|
621
|
+
|
|
622
|
+
const submitOne = Effect.fn("Certification.submitOne")(function* (
|
|
623
|
+
key: string,
|
|
624
|
+
question: string,
|
|
625
|
+
) {
|
|
626
|
+
const runtime = yield* DurableAgentRuntime;
|
|
627
|
+
|
|
628
|
+
return yield* runtime.submit(
|
|
629
|
+
{ definition: { id: plainDefinition.id, input: plainDefinition.input } },
|
|
630
|
+
{ question },
|
|
631
|
+
{
|
|
632
|
+
threadId,
|
|
633
|
+
principal: PRINCIPAL,
|
|
634
|
+
idempotencyKey: decodeIdempotencyKey(key),
|
|
635
|
+
definitions: DIGESTS,
|
|
636
|
+
},
|
|
637
|
+
);
|
|
638
|
+
});
|
|
639
|
+
|
|
547
640
|
const cell: CertificationCell = {
|
|
548
|
-
|
|
641
|
+
bindings: [resolved],
|
|
549
642
|
submit: Effect.gen(function* () {
|
|
550
643
|
const host = yield* submitOne(`certify-key-${slug}-host`, "host question");
|
|
551
644
|
const queued = yield* submitOne(`certify-key-${slug}-queued`, "queued question");
|
|
645
|
+
|
|
552
646
|
return [host, queued];
|
|
553
647
|
}),
|
|
554
|
-
lanes: () => [
|
|
648
|
+
lanes: () => [threadId],
|
|
555
649
|
};
|
|
650
|
+
|
|
556
651
|
return cell;
|
|
557
652
|
}
|
|
558
653
|
case "delegation": {
|
|
@@ -560,6 +655,7 @@ const makeCell = Effect.fn("Certification.makeCell")(function* (
|
|
|
560
655
|
childDefinition,
|
|
561
656
|
promptShapeModel("certify-child", '{"answer":"child-answer"}'),
|
|
562
657
|
);
|
|
658
|
+
|
|
563
659
|
const parentBinding = Agent.withModel(
|
|
564
660
|
coordinatorDefinition,
|
|
565
661
|
promptShapeModel(
|
|
@@ -571,36 +667,45 @@ const makeCell = Effect.fn("Certification.makeCell")(function* (
|
|
|
571
667
|
),
|
|
572
668
|
),
|
|
573
669
|
);
|
|
574
|
-
|
|
670
|
+
|
|
671
|
+
const delegationLayer = Subagent.layer(researchDelegation, childBinding, {
|
|
575
672
|
mapChildFailure,
|
|
576
673
|
durable: { targetDigests: CHILD_DIGEST_STRINGS },
|
|
577
674
|
}).pipe(Layer.provide(delegationSupport));
|
|
675
|
+
|
|
578
676
|
const lookupLayer = Toolkit.make(Lookup).toLayer({
|
|
579
677
|
lookup: ({ key }) => Effect.succeed({ value: `found-${key}` }),
|
|
580
678
|
});
|
|
679
|
+
|
|
581
680
|
const parentResolved = yield* DurableWorkerBinding.make(parentBinding, DIGESTS).pipe(
|
|
582
681
|
Effect.provide(Layer.mergeAll(delegationLayer, lookupLayer)),
|
|
583
682
|
);
|
|
683
|
+
|
|
584
684
|
const childResolved = yield* DurableWorkerBinding.make(childBinding, CHILD_DIGESTS);
|
|
585
|
-
const
|
|
685
|
+
const threadId = decodeThreadId(`certify-${slug}`);
|
|
686
|
+
|
|
586
687
|
const cell: CertificationCell = {
|
|
587
|
-
|
|
688
|
+
bindings: [parentResolved, childResolved],
|
|
588
689
|
submit: Effect.gen(function* () {
|
|
589
690
|
const runtime = yield* DurableAgentRuntime;
|
|
691
|
+
|
|
590
692
|
const receipt = yield* runtime.submit(
|
|
591
693
|
{ definition: { id: coordinatorDefinition.id, input: coordinatorDefinition.input } },
|
|
592
694
|
{ mission: "plan" },
|
|
593
|
-
submitOptionsFor(slug,
|
|
695
|
+
submitOptionsFor(slug, threadId),
|
|
594
696
|
);
|
|
697
|
+
|
|
595
698
|
return [receipt];
|
|
596
699
|
}),
|
|
597
700
|
lanes: (receipts) => {
|
|
598
701
|
const parent = receipts.at(0);
|
|
702
|
+
|
|
599
703
|
return parent === undefined
|
|
600
|
-
? [
|
|
601
|
-
: [
|
|
704
|
+
? [threadId]
|
|
705
|
+
: [threadId, childThreadIdFor(parent.submissionId, DELEGATE_CALL)];
|
|
602
706
|
},
|
|
603
707
|
};
|
|
708
|
+
|
|
604
709
|
return cell;
|
|
605
710
|
}
|
|
606
711
|
}
|
|
@@ -616,107 +721,139 @@ const MAX_REDRIVE_ROUNDS = 8;
|
|
|
616
721
|
* the digest chain is fully recomputed instead of skipped.
|
|
617
722
|
*/
|
|
618
723
|
const verifyLane = Effect.fn("Certification.verifyLane")(function* (
|
|
619
|
-
lane:
|
|
724
|
+
lane: ThreadId,
|
|
620
725
|
batchProducers: ReadonlyMap<BatchId, ProducerId>,
|
|
621
726
|
) {
|
|
622
|
-
const store = yield*
|
|
727
|
+
const store = yield* ThreadStore;
|
|
623
728
|
const ledger = yield* SubmissionLedger;
|
|
624
|
-
const exported = yield* store.export(
|
|
729
|
+
const exported = yield* store.export(ThreadExportRequest.make({ threadId: lane }));
|
|
625
730
|
const rows = new Map<SubmissionId, SubmissionSnapshot>();
|
|
626
731
|
const nonterminal = yield* Stream.runCollect(ledger.scanNonterminal);
|
|
732
|
+
|
|
627
733
|
for (const submission of nonterminal) {
|
|
628
|
-
if (submission.
|
|
734
|
+
if (submission.threadId === lane) rows.set(submission.submissionId, submission);
|
|
629
735
|
}
|
|
630
736
|
const named = new Set<SubmissionId>();
|
|
737
|
+
|
|
631
738
|
for (const envelope of exported.records) {
|
|
632
739
|
const payload = envelope.record.payload;
|
|
740
|
+
|
|
633
741
|
if (
|
|
634
742
|
payload._tag === "UserInputRecorded" ||
|
|
635
743
|
payload._tag === "SubmissionSettled" ||
|
|
636
744
|
payload._tag === "AbortRequested"
|
|
637
745
|
) {
|
|
638
|
-
named.add(payload.submissionId);
|
|
746
|
+
if (payload.submissionId !== undefined) named.add(payload.submissionId);
|
|
639
747
|
}
|
|
640
748
|
}
|
|
641
749
|
for (const submissionId of named) {
|
|
642
750
|
if (rows.has(submissionId)) continue;
|
|
643
751
|
const found = yield* ledger.lookup(SubmissionLookupById.make({ submissionId }));
|
|
644
|
-
|
|
752
|
+
|
|
753
|
+
if (Option.isSome(found) && found.value.threadId === lane) {
|
|
645
754
|
rows.set(submissionId, found.value);
|
|
646
755
|
}
|
|
647
756
|
}
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
757
|
+
|
|
758
|
+
const checkpoint =
|
|
759
|
+
store.checkpoints === undefined
|
|
760
|
+
? Option.none()
|
|
761
|
+
: yield* store.checkpoints.load(LoadCheckpointRequest.make({ threadId: lane }));
|
|
762
|
+
|
|
763
|
+
return yield* verifyThreadInvariants({
|
|
652
764
|
export: exported,
|
|
653
765
|
submissions: [...rows.values()],
|
|
654
766
|
batchProducers,
|
|
655
767
|
checkpoint: Option.getOrUndefined(checkpoint),
|
|
768
|
+
checkpointsSupported: store.checkpoints !== undefined,
|
|
656
769
|
requireAllSettled: true,
|
|
657
770
|
});
|
|
658
771
|
});
|
|
659
772
|
|
|
660
773
|
const failureTagOf = <E>(cause: Cause.Cause<E>): string => {
|
|
661
774
|
const failure = Cause.findErrorOption(cause);
|
|
775
|
+
|
|
662
776
|
if (Option.isSome(failure)) {
|
|
663
777
|
const error: unknown = failure.value;
|
|
778
|
+
|
|
664
779
|
if (typeof error === "object" && error !== null && "_tag" in error) {
|
|
665
|
-
return String(
|
|
780
|
+
return String(error._tag);
|
|
666
781
|
}
|
|
782
|
+
|
|
667
783
|
return String(error).slice(0, 256);
|
|
668
784
|
}
|
|
785
|
+
|
|
669
786
|
return "defect";
|
|
670
787
|
};
|
|
671
788
|
|
|
672
|
-
|
|
789
|
+
type SweepOutcome = Pick<
|
|
790
|
+
CertificationSweepResult,
|
|
791
|
+
"failpointFired" | "status" | "digestChainVerified" | "detail"
|
|
792
|
+
>;
|
|
793
|
+
|
|
794
|
+
/** Discover a clean path, or arm one location; both drives converge and verify real storage. */
|
|
673
795
|
const runSweepCell = Effect.fn("Certification.runSweepCell")(function* (
|
|
674
796
|
scenario: CertificationScenario,
|
|
675
|
-
location: DurableRuntimeFailpointLocation,
|
|
797
|
+
location: DurableRuntimeFailpointLocation | undefined,
|
|
676
798
|
batchProducers: ReadonlyMap<BatchId, ProducerId>,
|
|
677
799
|
leaseAdvance: Duration.Duration,
|
|
800
|
+
reached: Set<DurableRuntimeFailpointLocation>,
|
|
678
801
|
) {
|
|
679
|
-
const runtime = yield* DurableAgentRuntime;
|
|
680
802
|
const ledger = yield* SubmissionLedger;
|
|
681
803
|
const control = yield* DurableRuntimeFailpointTestControl;
|
|
682
|
-
const slug = `${scenario}-${location
|
|
683
|
-
|
|
684
|
-
const failed = (detail: string, fired: boolean):
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
digestChainVerified: false,
|
|
691
|
-
detail: detail.slice(0, 4_096),
|
|
692
|
-
});
|
|
804
|
+
const slug = `${scenario}-${location?.replaceAll(":", "-") ?? "discovery"}`;
|
|
805
|
+
|
|
806
|
+
const failed = (detail: string, fired: boolean): SweepOutcome => ({
|
|
807
|
+
failpointFired: fired,
|
|
808
|
+
status: "failed",
|
|
809
|
+
digestChainVerified: false,
|
|
810
|
+
detail: detail.slice(0, 4_096),
|
|
811
|
+
});
|
|
693
812
|
|
|
694
813
|
const cell = yield* makeCell(scenario, slug);
|
|
695
814
|
|
|
815
|
+
const runtime = yield* DurableAgentRuntime.pipe(
|
|
816
|
+
Effect.provide(
|
|
817
|
+
DurableAgentRuntime.layerWithBindings(cell.bindings).pipe(
|
|
818
|
+
Layer.provide(RunToolAuthorization.allowAll),
|
|
819
|
+
),
|
|
820
|
+
),
|
|
821
|
+
);
|
|
822
|
+
|
|
823
|
+
const submit = cell.submit.pipe(Effect.provideService(DurableAgentRuntime, runtime));
|
|
824
|
+
|
|
696
825
|
// One-shot arm: the fault fires at most once anywhere in the cell (initial drive OR a
|
|
697
826
|
// re-drive round's public unblocking operation), modelling one crash at this boundary.
|
|
698
827
|
const fired = yield* Ref.make(false);
|
|
828
|
+
|
|
699
829
|
yield* control.setHandler((hit) =>
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
830
|
+
Effect.suspend(() => {
|
|
831
|
+
reached.add(hit);
|
|
832
|
+
|
|
833
|
+
return hit !== location
|
|
834
|
+
? Effect.void
|
|
835
|
+
: Ref.getAndSet(fired, true).pipe(
|
|
836
|
+
Effect.flatMap((already) =>
|
|
837
|
+
already
|
|
838
|
+
? Effect.void
|
|
839
|
+
: Effect.fail(DurableRuntimeFailpointError.make({ location: hit })),
|
|
840
|
+
),
|
|
841
|
+
);
|
|
842
|
+
}),
|
|
709
843
|
);
|
|
710
844
|
|
|
711
845
|
// Submissions are idempotent (DUR-001): one replay recovers a submit-boundary fault.
|
|
712
846
|
let receipts: ReadonlyArray<Receipt>;
|
|
713
|
-
const firstSubmit = yield* Effect.exit(
|
|
847
|
+
const firstSubmit = yield* Effect.exit(submit);
|
|
848
|
+
|
|
714
849
|
if (Exit.isSuccess(firstSubmit)) {
|
|
715
850
|
receipts = firstSubmit.value;
|
|
716
851
|
} else {
|
|
717
|
-
const secondSubmit = yield* Effect.exit(
|
|
852
|
+
const secondSubmit = yield* Effect.exit(submit);
|
|
853
|
+
|
|
718
854
|
if (Exit.isFailure(secondSubmit)) {
|
|
719
855
|
yield* control.clear;
|
|
856
|
+
|
|
720
857
|
return failed(
|
|
721
858
|
`submission replay did not recover: ${failureTagOf(secondSubmit.cause)}`,
|
|
722
859
|
yield* Ref.get(fired),
|
|
@@ -726,24 +863,24 @@ const runSweepCell = Effect.fn("Certification.runSweepCell")(function* (
|
|
|
726
863
|
}
|
|
727
864
|
const lanes = cell.lanes(receipts);
|
|
728
865
|
|
|
729
|
-
const driveLane = (lane:
|
|
730
|
-
runtime
|
|
731
|
-
.processConversationResolved(lane)
|
|
732
|
-
.pipe(Effect.provideService(AgentBindingResolver, cell.resolver));
|
|
866
|
+
const driveLane = (lane: ThreadId) => runtime.processThreadResolved(lane);
|
|
733
867
|
|
|
734
868
|
const allSettled = Effect.gen(function* () {
|
|
735
869
|
for (const receipt of receipts) {
|
|
736
870
|
const snapshot = yield* ledger.lookup(
|
|
737
871
|
SubmissionLookupById.make({ submissionId: receipt.submissionId }),
|
|
738
872
|
);
|
|
873
|
+
|
|
739
874
|
if (Option.isNone(snapshot) || snapshot.value.state !== "settled") return false;
|
|
740
875
|
}
|
|
876
|
+
|
|
741
877
|
return true;
|
|
742
878
|
});
|
|
743
879
|
|
|
744
880
|
// Re-drive to convergence using ONLY public operations: worker drives, recovery passes,
|
|
745
|
-
// and the authorized DUR-017/approval unblocking paths chosen from `
|
|
881
|
+
// and the authorized DUR-017/approval unblocking paths chosen from `explainThread`.
|
|
746
882
|
let converged = false;
|
|
883
|
+
|
|
747
884
|
for (let round = 0; round < MAX_REDRIVE_ROUNDS && !converged; round++) {
|
|
748
885
|
// Expire any lease a faulted Attempt left behind (D5): virtual time is the
|
|
749
886
|
// adapter-neutral reclaim lever — a live lease may block every new claim.
|
|
@@ -753,7 +890,8 @@ const runSweepCell = Effect.fn("Certification.runSweepCell")(function* (
|
|
|
753
890
|
yield* Effect.exit(driveLane(lane));
|
|
754
891
|
}
|
|
755
892
|
for (const lane of lanes) {
|
|
756
|
-
const explains = yield* Effect.exit(runtime.
|
|
893
|
+
const explains = yield* Effect.exit(runtime.explainThread(lane));
|
|
894
|
+
|
|
757
895
|
if (Exit.isFailure(explains)) continue;
|
|
758
896
|
for (const explanation of explains.value) {
|
|
759
897
|
for (const unknown of explanation.evidence.unknownCalls) {
|
|
@@ -774,6 +912,7 @@ const runSweepCell = Effect.fn("Certification.runSweepCell")(function* (
|
|
|
774
912
|
const decided = explanation.evidence.approvalDecisions.some(
|
|
775
913
|
(decision) => decision.toolCallId === pending.toolCallId,
|
|
776
914
|
);
|
|
915
|
+
|
|
777
916
|
if (decided) continue;
|
|
778
917
|
yield* Effect.exit(
|
|
779
918
|
runtime.resolveApproval(
|
|
@@ -790,6 +929,7 @@ const runSweepCell = Effect.fn("Certification.runSweepCell")(function* (
|
|
|
790
929
|
}
|
|
791
930
|
}
|
|
792
931
|
const settled = yield* Effect.exit(allSettled);
|
|
932
|
+
|
|
793
933
|
converged = Exit.isSuccess(settled) && settled.value;
|
|
794
934
|
}
|
|
795
935
|
yield* control.clear;
|
|
@@ -802,8 +942,10 @@ const runSweepCell = Effect.fn("Certification.runSweepCell")(function* (
|
|
|
802
942
|
// Every lane of the cell must verify in convergence mode with a recomputed digest chain.
|
|
803
943
|
let digestChainVerified = true;
|
|
804
944
|
const failedChecks: Array<string> = [];
|
|
945
|
+
|
|
805
946
|
for (const lane of lanes) {
|
|
806
947
|
const verdict = yield* Effect.exit(verifyLane(lane, batchProducers));
|
|
948
|
+
|
|
807
949
|
if (Exit.isFailure(verdict)) {
|
|
808
950
|
return failed(`lane ${lane} could not be verified: ${failureTagOf(verdict.cause)}`, wasFired);
|
|
809
951
|
}
|
|
@@ -827,13 +969,11 @@ const runSweepCell = Effect.fn("Certification.runSweepCell")(function* (
|
|
|
827
969
|
);
|
|
828
970
|
}
|
|
829
971
|
|
|
830
|
-
return
|
|
831
|
-
scenario,
|
|
832
|
-
location,
|
|
972
|
+
return {
|
|
833
973
|
failpointFired: wasFired,
|
|
834
974
|
status: wasFired ? "converged" : "not-triggered",
|
|
835
975
|
digestChainVerified,
|
|
836
|
-
}
|
|
976
|
+
} satisfies SweepOutcome;
|
|
837
977
|
});
|
|
838
978
|
|
|
839
979
|
// ---------------------------------------------------------------------------
|
|
@@ -864,10 +1004,14 @@ export const resolveTierThree = Effect.fn("Certification.resolveTierThree")(func
|
|
|
864
1004
|
}
|
|
865
1005
|
if (options.crashLever !== undefined) {
|
|
866
1006
|
const cases = yield* options.crashLever;
|
|
1007
|
+
|
|
867
1008
|
return CertificationTierThreeReport.make({
|
|
868
|
-
status: "exercised",
|
|
1009
|
+
status: cases.length === 0 ? "not-exercised" : "exercised",
|
|
869
1010
|
evidence: options.tierThreeEvidence ?? [],
|
|
870
1011
|
cases,
|
|
1012
|
+
...(cases.length === 0
|
|
1013
|
+
? { detail: "the supplied crash lever executed no real-loss cases" }
|
|
1014
|
+
: {}),
|
|
871
1015
|
});
|
|
872
1016
|
}
|
|
873
1017
|
if (options.tierThreeEvidence !== undefined && options.tierThreeEvidence.length > 0) {
|
|
@@ -877,6 +1021,7 @@ export const resolveTierThree = Effect.fn("Certification.resolveTierThree")(func
|
|
|
877
1021
|
cases: [],
|
|
878
1022
|
});
|
|
879
1023
|
}
|
|
1024
|
+
|
|
880
1025
|
return CertificationTierThreeReport.make({
|
|
881
1026
|
status: "not-exercised",
|
|
882
1027
|
evidence: [],
|
|
@@ -905,13 +1050,15 @@ export const certifyDurableAdapters = <LedgerE = never, StoreE = never>(
|
|
|
905
1050
|
options: CertifyDurableAdaptersOptions<LedgerE, StoreE>,
|
|
906
1051
|
): Effect.Effect<CertificationReport, LedgerE | StoreE, Crypto.Crypto> => {
|
|
907
1052
|
const batchProducers = new Map<BatchId, ProducerId>();
|
|
1053
|
+
|
|
908
1054
|
// Interpose the candidate store with an append-time capture of each batch's producer
|
|
909
|
-
// identity — the one value the
|
|
1055
|
+
// identity — the one value the ThreadStore port deliberately does not export — so
|
|
910
1056
|
// Tier 2's invariant verification recomputes the FULL digest chain instead of skipping it.
|
|
911
|
-
const capturingStore = Layer.effect(
|
|
1057
|
+
const capturingStore = Layer.effect(ThreadStore)(
|
|
912
1058
|
Effect.gen(function* () {
|
|
913
|
-
const inner = yield*
|
|
914
|
-
|
|
1059
|
+
const inner = yield* ThreadStore;
|
|
1060
|
+
|
|
1061
|
+
return ThreadStore.of({
|
|
915
1062
|
...inner,
|
|
916
1063
|
append: (request) =>
|
|
917
1064
|
Effect.sync(() => {
|
|
@@ -919,13 +1066,15 @@ export const certifyDurableAdapters = <LedgerE = never, StoreE = never>(
|
|
|
919
1066
|
}).pipe(Effect.andThen(inner.append(request))),
|
|
920
1067
|
});
|
|
921
1068
|
}),
|
|
922
|
-
).pipe(Layer.provide(options.
|
|
1069
|
+
).pipe(Layer.provide(options.threadStore));
|
|
923
1070
|
|
|
924
|
-
|
|
1071
|
+
// RUN-036: certification uses the default-none Tool failure observer. Trusted application
|
|
1072
|
+
// reporting adds no durable transition and is verified separately from adapter certification.
|
|
1073
|
+
const environment = Layer.mergeAll(
|
|
925
1074
|
options.submissionLedger,
|
|
926
1075
|
capturingStore,
|
|
927
1076
|
options.wakeScheduler ?? WakeScheduler.layerNoop,
|
|
928
|
-
|
|
1077
|
+
DurableRuntimeFailpointTestControl.layer,
|
|
929
1078
|
ToolReconciler.uncertain,
|
|
930
1079
|
DurableRuntimeConfig.layer({
|
|
931
1080
|
deploymentId: Schema.decodeSync(DeploymentId)("deployment-certification"),
|
|
@@ -935,7 +1084,6 @@ export const certifyDurableAdapters = <LedgerE = never, StoreE = never>(
|
|
|
935
1084
|
abortPollInterval: Duration.millis(50),
|
|
936
1085
|
}),
|
|
937
1086
|
);
|
|
938
|
-
const environment = DurableAgentRuntime.layer.pipe(Layer.provideMerge(support));
|
|
939
1087
|
|
|
940
1088
|
const leaseAdvance = Duration.millis(
|
|
941
1089
|
Duration.toMillis(options.ownershipLeaseDuration ?? DEFAULT_OWNERSHIP_LEASE_DURATION) + 1_000,
|
|
@@ -944,11 +1092,73 @@ export const certifyDurableAdapters = <LedgerE = never, StoreE = never>(
|
|
|
944
1092
|
const program = Effect.gen(function* () {
|
|
945
1093
|
const ledger = yield* SubmissionLedger;
|
|
946
1094
|
|
|
947
|
-
// Tier 2 —
|
|
1095
|
+
// Tier 2 — discover each deterministic shape without a fault. Unreached locations
|
|
1096
|
+
// would all repeat this same clean drive, including its full digest verification.
|
|
948
1097
|
const tier2: Array<CertificationSweepResult> = [];
|
|
1098
|
+
|
|
1099
|
+
const discoveries: Array<{
|
|
1100
|
+
readonly scenario: CertificationScenario;
|
|
1101
|
+
readonly outcome: SweepOutcome;
|
|
1102
|
+
readonly reached: Set<DurableRuntimeFailpointLocation>;
|
|
1103
|
+
}> = [];
|
|
1104
|
+
|
|
949
1105
|
for (const scenario of CERTIFICATION_SCENARIOS) {
|
|
1106
|
+
const reached = new Set<DurableRuntimeFailpointLocation>();
|
|
1107
|
+
|
|
1108
|
+
const outcome = yield* runSweepCell(
|
|
1109
|
+
scenario,
|
|
1110
|
+
undefined,
|
|
1111
|
+
batchProducers,
|
|
1112
|
+
leaseAdvance,
|
|
1113
|
+
reached,
|
|
1114
|
+
);
|
|
1115
|
+
|
|
1116
|
+
discoveries.push({ scenario, outcome, reached });
|
|
1117
|
+
}
|
|
1118
|
+
const observed = new Set(discoveries.flatMap(({ reached }) => [...reached]));
|
|
1119
|
+
|
|
1120
|
+
// Never silently omit a new location: if neither discovery nor the documented
|
|
1121
|
+
// never-fired set accounts for it, arm it in EVERY shape. Tests pin the fired paths
|
|
1122
|
+
// per shape and the never-fired set, so new or lost routes require explicit review.
|
|
1123
|
+
const unaccounted = new Set(
|
|
1124
|
+
DurableRuntimeFailpointLocation.literals.filter(
|
|
1125
|
+
(location) => !observed.has(location) && !TIER2_UNREACHED_LOCATIONS.includes(location),
|
|
1126
|
+
),
|
|
1127
|
+
);
|
|
1128
|
+
|
|
1129
|
+
for (const discovery of discoveries) {
|
|
1130
|
+
const { scenario, reached } = discovery;
|
|
1131
|
+
const outcomes = new Map<DurableRuntimeFailpointLocation, SweepOutcome>();
|
|
1132
|
+
|
|
1133
|
+
if (discovery.outcome.status !== "failed") {
|
|
1134
|
+
while (true) {
|
|
1135
|
+
// Recovery may expose another route. Discover those too, without re-running a
|
|
1136
|
+
// cell already checked. Schema order bounds this to one drive per location.
|
|
1137
|
+
const location = DurableRuntimeFailpointLocation.literals.find(
|
|
1138
|
+
(candidate) =>
|
|
1139
|
+
!outcomes.has(candidate) && (reached.has(candidate) || unaccounted.has(candidate)),
|
|
1140
|
+
);
|
|
1141
|
+
|
|
1142
|
+
if (location === undefined) break;
|
|
1143
|
+
outcomes.set(
|
|
1144
|
+
location,
|
|
1145
|
+
yield* runSweepCell(scenario, location, batchProducers, leaseAdvance, reached),
|
|
1146
|
+
);
|
|
1147
|
+
}
|
|
1148
|
+
}
|
|
950
1149
|
for (const location of DurableRuntimeFailpointLocation.literals) {
|
|
951
|
-
|
|
1150
|
+
const outcome = outcomes.get(location) ?? discovery.outcome;
|
|
1151
|
+
|
|
1152
|
+
tier2.push(
|
|
1153
|
+
CertificationSweepResult.make({
|
|
1154
|
+
scenario,
|
|
1155
|
+
location,
|
|
1156
|
+
...outcome,
|
|
1157
|
+
...(outcomes.has(location) || outcome.status === "failed"
|
|
1158
|
+
? {}
|
|
1159
|
+
: { detail: "verified clean scenario did not reach this location" }),
|
|
1160
|
+
}),
|
|
1161
|
+
);
|
|
952
1162
|
}
|
|
953
1163
|
}
|
|
954
1164
|
|
|
@@ -960,13 +1170,8 @@ export const certifyDurableAdapters = <LedgerE = never, StoreE = never>(
|
|
|
960
1170
|
const tier3 = yield* resolveTierThree(capabilities.durability, options);
|
|
961
1171
|
|
|
962
1172
|
const generatedAt = yield* nowUtc;
|
|
963
|
-
const ok =
|
|
964
|
-
tier1.every((result) => result.status === "passed") &&
|
|
965
|
-
tier2.every((result) => result.status !== "failed") &&
|
|
966
|
-
tier3.cases.every((result) => result.status === "passed");
|
|
967
1173
|
|
|
968
|
-
return
|
|
969
|
-
format: "effect-agent/certification@1",
|
|
1174
|
+
return makeCertificationReport({
|
|
970
1175
|
adapter: CertifiedAdapterIdentity.make({
|
|
971
1176
|
name: options.adapter.name,
|
|
972
1177
|
...(options.adapter.version === undefined ? {} : { version: options.adapter.version }),
|
|
@@ -976,7 +1181,6 @@ export const certifyDurableAdapters = <LedgerE = never, StoreE = never>(
|
|
|
976
1181
|
tier1,
|
|
977
1182
|
tier2,
|
|
978
1183
|
tier3,
|
|
979
|
-
ok,
|
|
980
1184
|
});
|
|
981
1185
|
});
|
|
982
1186
|
|