@effect-agent/testing 0.1.0-beta.8 → 0.1.0-beta.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Certification.d.mts +105 -0
- package/dist/Certification.mjs +592 -0
- package/dist/Certification.mjs.map +1 -0
- package/dist/Chaos.d.mts +126 -0
- package/dist/Chaos.mjs +755 -0
- package/dist/Chaos.mjs.map +1 -0
- package/dist/CodeExecutorConformance.d.mts +33 -0
- package/dist/CodeExecutorConformance.mjs +231 -0
- package/dist/CodeExecutorConformance.mjs.map +1 -0
- package/dist/CodeExecutorSubstitute.d.mts +21 -0
- package/dist/CodeExecutorSubstitute.mjs +368 -0
- package/dist/CodeExecutorSubstitute.mjs.map +1 -0
- package/dist/DocsResearcher.d.mts +270 -0
- package/dist/DocsResearcher.mjs +490 -0
- package/dist/DocsResearcher.mjs.map +1 -0
- package/dist/ScriptedModel-DAvxIiud.d.mts +220 -0
- package/dist/ScriptedModel.d.mts +2 -0
- package/dist/ScriptedModel.mjs +155 -0
- package/dist/ScriptedModel.mjs.map +1 -0
- package/dist/TravelPlanner.d.mts +1665 -0
- package/dist/TravelPlanner.mjs +1963 -0
- package/dist/TravelPlanner.mjs.map +1 -0
- package/dist/deterministic-layers-Eka0fMZq.mjs +358 -0
- package/dist/deterministic-layers-Eka0fMZq.mjs.map +1 -0
- package/dist/index.d.mts +2 -3406
- package/dist/index.mjs +2 -4771
- package/dist/rolldown-runtime-D7D4PA-g.mjs +13 -0
- package/package.json +1 -48
- package/src/{certification.ts → Certification.ts} +251 -117
- package/src/{chaos.ts → Chaos.ts} +246 -122
- package/src/{code-executor-conformance.ts → CodeExecutorConformance.ts} +29 -6
- package/src/{code-executor-substitute.ts → CodeExecutorSubstitute.ts} +112 -45
- package/src/{fixtures/docs-researcher/index.ts → DocsResearcher.ts} +68 -5
- package/src/{scripted-model.ts → ScriptedModel.ts} +22 -25
- package/src/TravelPlanner.ts +232 -0
- package/src/fixtures/docs-researcher/definition.ts +19 -10
- package/src/fixtures/docs-researcher/harness.ts +41 -30
- package/src/fixtures/docs-researcher/mcp.ts +49 -3
- package/src/fixtures/travel-planner/definition.ts +15 -2
- package/src/fixtures/travel-planner/deterministic-layers.ts +47 -4
- package/src/fixtures/travel-planner/phase2.ts +4 -3
- package/src/fixtures/travel-planner/phase3.ts +18 -41
- package/src/fixtures/travel-planner/phase4.ts +23 -36
- package/src/fixtures/travel-planner/phase5.ts +38 -41
- package/src/fixtures/travel-planner/phase6.ts +187 -84
- package/src/fixtures/travel-planner/phase7.ts +4 -102
- package/src/fixtures/travel-planner/scenarios.ts +3 -4
- package/src/fixtures/travel-planner/subagents-durable.ts +33 -57
- package/src/fixtures/travel-planner/subagents.ts +35 -13
- package/src/index.ts +1 -11
- package/dist/index.mjs.map +0 -1
- package/src/code-executor-conformance.d.ts +0 -30
- package/src/fixtures/travel-planner/index.ts +0 -11
- package/src/fixtures/warehouse/index.ts +0 -412
|
@@ -1,16 +1,15 @@
|
|
|
1
|
-
import { SubagentReservationsMemoryLive } from "@effect-agent/capabilities";
|
|
2
|
-
import
|
|
1
|
+
import { SubagentReservationsMemoryLive } from "@effect-agent/capabilities/SubagentReservations";
|
|
2
|
+
import * as Agent from "@effect-agent/core/Agent";
|
|
3
|
+
import { DurableWorkerBinding, type ResolvedBinding } from "@effect-agent/thread/AgentRegistration";
|
|
4
|
+
import { type Receipt } from "@effect-agent/thread/DurableAgentRuntime";
|
|
3
5
|
import {
|
|
4
6
|
DefinitionDigests,
|
|
5
7
|
DeploymentId,
|
|
6
8
|
Digest,
|
|
7
|
-
DurableWorkerBinding,
|
|
8
9
|
ProducerId,
|
|
9
10
|
type CanonicalRecordEnvelope,
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
type ToolReconciler,
|
|
13
|
-
} from "@effect-agent/session";
|
|
11
|
+
} from "@effect-agent/thread/Records";
|
|
12
|
+
import { type ToolReconciler } from "@effect-agent/thread/ToolReconciler";
|
|
14
13
|
import { Duration, Effect, Layer, Schema, Stream } from "effect";
|
|
15
14
|
import { LanguageModel, Model, type Response } from "effect/unstable/ai";
|
|
16
15
|
|
|
@@ -54,58 +53,17 @@ import {
|
|
|
54
53
|
encodedDestinationReport,
|
|
55
54
|
} from "./subagents.ts";
|
|
56
55
|
|
|
57
|
-
|
|
58
|
-
// Phase 6 (P6): the SAME cumulative Travel Planner on the Cloudflare Durable
|
|
59
|
-
// Object runtime, deployment class DC. This module is deliberately
|
|
60
|
-
// platform-neutral (it imports no Cloudflare types): the worker Bindings it
|
|
61
|
-
// builds are plain `ResolvedBinding` values a Conversation Object registers,
|
|
62
|
-
// and the cross-platform evidence normal form is shared by the DN and DC
|
|
63
|
-
// halves of the equivalence suite (plan §1.8, D-P6-6).
|
|
64
|
-
// ---------------------------------------------------------------------------
|
|
65
|
-
|
|
66
|
-
/**
|
|
67
|
-
* The Phase 6 profile: the P4/P5/S2 Travel Planner claims re-earned on the Cloudflare Durable
|
|
68
|
-
* Object runtime (deployment class `DC`), where eviction and alarm redelivery replace process
|
|
69
|
-
* kill and restart as the exercised recovery path. `cloudflareEquivalence` is the claim the S2
|
|
70
|
-
* fixture explicitly deferred to P6 (`TravelPlannerSubagentDurabilityProfile` pins it `false`
|
|
71
|
-
* for `DN`): it flips to `true` here ONLY because the phase-6 suites assert byte-equal
|
|
72
|
-
* cross-platform normalized canonical evidence against one committed golden. Exactly-once
|
|
73
|
-
* EXTERNAL effects remain — deliberately — unclaimed on every platform (DUR-003).
|
|
74
|
-
*/
|
|
75
|
-
export class TravelPlannerCloudflareProfile extends Schema.Class<TravelPlannerCloudflareProfile>(
|
|
76
|
-
"@effect-agent/testing/travel-planner/TravelPlannerCloudflareProfile",
|
|
77
|
-
)({
|
|
78
|
-
deploymentClass: Schema.Literal("DC"),
|
|
79
|
-
durableAcceptedWork: Schema.Literal(true),
|
|
80
|
-
canonicalSchemaVersion: Schema.Literal(1),
|
|
81
|
-
/** P5 semantics under DC recovery: prepared/settled records, Unknown Outcomes, approvals. */
|
|
82
|
-
supplierBookingUncertaintyProtocol: Schema.Literal(true),
|
|
83
|
-
/** S2 semantics under DC recovery: cross-Object establishment/join, completed child never re-runs. */
|
|
84
|
-
durableAttachedSubagents: Schema.Literal(true),
|
|
85
|
-
/** DN and DC produce byte-equal cross-platform normalized canonical evidence (one golden). */
|
|
86
|
-
cloudflareEquivalence: Schema.Literal(true),
|
|
87
|
-
/** Never claimed at any phase on any platform (DUR-003). */
|
|
88
|
-
exactlyOnceExternalEffects: Schema.Literal(false),
|
|
89
|
-
}) {}
|
|
90
|
-
|
|
91
|
-
export const phase6TravelPlannerProfile = TravelPlannerCloudflareProfile.make({
|
|
92
|
-
deploymentClass: "DC",
|
|
93
|
-
durableAcceptedWork: true,
|
|
94
|
-
canonicalSchemaVersion: 1,
|
|
95
|
-
supplierBookingUncertaintyProtocol: true,
|
|
96
|
-
durableAttachedSubagents: true,
|
|
97
|
-
cloudflareEquivalence: true,
|
|
98
|
-
exactlyOnceExternalEffects: false,
|
|
99
|
-
});
|
|
100
|
-
|
|
56
|
+
/** Platform-neutral bindings and canonical evidence shared by Node and Cloudflare tests. */
|
|
101
57
|
export const phase6TravelPlannerDeploymentId = Schema.decodeSync(DeploymentId)(
|
|
102
58
|
"travel-planner-p6-deployment",
|
|
103
59
|
);
|
|
104
|
-
|
|
60
|
+
|
|
61
|
+
/** Producer prefix of the DC host; each Object mints `{prefix}:{threadId}`. */
|
|
105
62
|
export const phase6TravelPlannerProducerPrefix = "travel-planner-p6-producer";
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
63
|
+
|
|
64
|
+
/** The full producer identity one DC Thread Object mints for itself. */
|
|
65
|
+
export const phase6TravelPlannerProducerId = (threadId: string): ProducerId =>
|
|
66
|
+
Schema.decodeSync(ProducerId)(`${phase6TravelPlannerProducerPrefix}:${threadId}`);
|
|
109
67
|
|
|
110
68
|
const digestOf = (character: string) => Schema.decodeSync(Digest)(character.repeat(64));
|
|
111
69
|
|
|
@@ -127,11 +85,11 @@ export const phase6GatedPlannerDefinitionDigests = DefinitionDigests.make({
|
|
|
127
85
|
|
|
128
86
|
/** The run-specific identities the cross-platform normal form scrubs. */
|
|
129
87
|
export interface CrossPlatformEvidenceIdentity {
|
|
130
|
-
/** The
|
|
131
|
-
readonly
|
|
88
|
+
/** The Thread lane the evidence came from. */
|
|
89
|
+
readonly threadId: string;
|
|
132
90
|
/** The host's deployment identity (`DeploymentId` on every record envelope). */
|
|
133
91
|
readonly deploymentId: string;
|
|
134
|
-
/** The full producer identity of the run (DN: configured; DC: `{prefix}:{
|
|
92
|
+
/** The full producer identity of the run (DN: configured; DC: `{prefix}:{threadId}`). */
|
|
135
93
|
readonly producerId: string;
|
|
136
94
|
}
|
|
137
95
|
|
|
@@ -143,12 +101,13 @@ const ComparableEnvelope = Schema.Struct({
|
|
|
143
101
|
sequence: Schema.Number,
|
|
144
102
|
record: Schema.Json,
|
|
145
103
|
});
|
|
104
|
+
|
|
146
105
|
const decodeComparableEnvelopes = Schema.decodeUnknownEffect(Schema.Array(ComparableEnvelope));
|
|
147
106
|
|
|
148
107
|
/**
|
|
149
108
|
* The CROSS-PLATFORM extension of `normalizeDurableTravelPlannerEvidence` (D-P6-6): after the
|
|
150
109
|
* base normalization replaces the two ledger-minted identities (which also normalizes the
|
|
151
|
-
* DC-format routable `{uuidv7}:{
|
|
110
|
+
* DC-format routable `{uuidv7}:{threadId}` Submission identities and everything derived
|
|
152
111
|
* from them), this form additionally scrubs everything that legitimately differs between a DN
|
|
153
112
|
* process and a DC Durable Object over the same scenario:
|
|
154
113
|
*
|
|
@@ -158,7 +117,7 @@ const decodeComparableEnvelopes = Schema.decodeUnknownEffect(Schema.Array(Compar
|
|
|
158
117
|
* control (on DC even a CLEAN run carries one, because every pass reconciles before it
|
|
159
118
|
* claims, so the ready lane's input is applied through the recovery path). Canonical ORDER
|
|
160
119
|
* is the durability §5 claim; sequence contiguity is a platform artifact of who appended;
|
|
161
|
-
* - the
|
|
120
|
+
* - the Thread identity (DC lanes mint unique names per test run);
|
|
162
121
|
* - the deployment and producer identities (host configuration, not canonical semantics);
|
|
163
122
|
* - `createdAt` commit timestamps (wall clock);
|
|
164
123
|
* - 64-hex digests (they hash RAW content that legally embeds run-specific identity, so they
|
|
@@ -180,15 +139,18 @@ export const normalizeCrossPlatformTravelPlannerEvidence = Effect.fn(
|
|
|
180
139
|
const canonical = records.filter(
|
|
181
140
|
(envelope) => envelope.record.payload._tag !== "RepairAnnotated",
|
|
182
141
|
);
|
|
142
|
+
|
|
183
143
|
const base = yield* normalizeDurableTravelPlannerEvidence(canonical, receipt);
|
|
144
|
+
|
|
184
145
|
const scrubbed: unknown = JSON.parse(
|
|
185
146
|
JSON.stringify(base)
|
|
186
147
|
.replaceAll(identity.producerId, "{producerId}")
|
|
187
148
|
.replaceAll(identity.deploymentId, "{deploymentId}")
|
|
188
|
-
.replaceAll(identity.
|
|
149
|
+
.replaceAll(identity.threadId, "{threadId}")
|
|
189
150
|
.replaceAll(/\d{4}-\d{2}-\d{2}T[0-9:.]+Z/g, "{timestamp}")
|
|
190
151
|
.replaceAll(/"[0-9a-f]{64}"/g, '"{digest}"'),
|
|
191
152
|
);
|
|
153
|
+
|
|
192
154
|
const comparable = yield* decodeComparableEnvelopes(scrubbed).pipe(
|
|
193
155
|
Effect.mapError((error) =>
|
|
194
156
|
TravelPlannerDurableEvidenceError.make({
|
|
@@ -196,11 +158,13 @@ export const normalizeCrossPlatformTravelPlannerEvidence = Effect.fn(
|
|
|
196
158
|
}),
|
|
197
159
|
),
|
|
198
160
|
);
|
|
161
|
+
|
|
199
162
|
const renumbered = comparable.map((entry, index) => ({
|
|
200
163
|
batchId: entry.batchId,
|
|
201
164
|
sequence: index + 1,
|
|
202
165
|
record: entry.record,
|
|
203
166
|
}));
|
|
167
|
+
|
|
204
168
|
return yield* decodeComparableJson(renumbered).pipe(
|
|
205
169
|
Effect.mapError((error) =>
|
|
206
170
|
TravelPlannerDurableEvidenceError.make({
|
|
@@ -291,7 +255,7 @@ export const phase6PlannerModel = promptAwareModel("travel-planner-phase-4", pla
|
|
|
291
255
|
// ---------------------------------------------------------------------------
|
|
292
256
|
// Deterministic test gate for the admission-limits rows. Module state is
|
|
293
257
|
// intentionally NOT durable: it plays the external world's role (a slow
|
|
294
|
-
// upstream model), never
|
|
258
|
+
// upstream model), never Thread state.
|
|
295
259
|
// ---------------------------------------------------------------------------
|
|
296
260
|
|
|
297
261
|
const releasedPlannerGates = new Set<string>();
|
|
@@ -345,6 +309,7 @@ export const phase6GatedPlannerModel = Model.make(
|
|
|
345
309
|
Stream.unwrap(
|
|
346
310
|
Effect.sync(() => {
|
|
347
311
|
const promptJson = JSON.stringify(options.prompt);
|
|
312
|
+
|
|
348
313
|
return promptJson.includes(phase6FlightCallId)
|
|
349
314
|
? plannerDecide(promptJson)
|
|
350
315
|
: Stream.fromEffectDrain(awaitPlannerGate(gateMarkerFromPrompt(promptJson))).pipe(
|
|
@@ -381,7 +346,7 @@ export const phase6SupplierDeskLayer: Layer.Layer<SupplierBookingDesk> = Layer.s
|
|
|
381
346
|
|
|
382
347
|
/**
|
|
383
348
|
* The REAL P5 supplier reconciliation policy over the shared desk, closed to no requirements
|
|
384
|
-
* so a
|
|
349
|
+
* so a Thread Object can install it directly: `book_flight` recovers only from supplier
|
|
385
350
|
* truth (absence stays fail-closed `Uncertain` → durable Unknown Outcome), keyed Steps are
|
|
386
351
|
* provably re-enterable.
|
|
387
352
|
*/
|
|
@@ -448,6 +413,7 @@ const bookingReportParts = (marker: string): ReadonlyArray<Response.StreamPartEn
|
|
|
448
413
|
*/
|
|
449
414
|
export const phase6BookingModel = promptAwareModel("travel-planner-phase-5", (promptJson) => {
|
|
450
415
|
const marker = bookingMarkerFromPrompt(promptJson);
|
|
416
|
+
|
|
451
417
|
return promptJson.includes(phase6BookingToolCallId(marker))
|
|
452
418
|
? Stream.fromIterable(bookingReportParts(marker))
|
|
453
419
|
: Stream.fromIterable(bookingCallParts(marker));
|
|
@@ -471,6 +437,7 @@ const countingGuideLayer = Layer.succeed(
|
|
|
471
437
|
lookup: (query) =>
|
|
472
438
|
Effect.suspend(() => {
|
|
473
439
|
guideInvocations += 1;
|
|
440
|
+
|
|
474
441
|
return destinationLookup(query);
|
|
475
442
|
}),
|
|
476
443
|
}),
|
|
@@ -576,14 +543,14 @@ export const phase6ResearcherModel = promptAwareModel("destination-researcher-p6
|
|
|
576
543
|
);
|
|
577
544
|
|
|
578
545
|
// ---------------------------------------------------------------------------
|
|
579
|
-
// Worker Binding registrations for the DC
|
|
546
|
+
// Worker Binding registrations for the DC Thread Object
|
|
580
547
|
// ---------------------------------------------------------------------------
|
|
581
548
|
|
|
582
549
|
/**
|
|
583
|
-
* Every phase-6 Travel Planner worker Binding, captured with its requirement Contexts
|
|
584
|
-
*
|
|
550
|
+
* Every phase-6 Travel Planner worker Binding, captured with its requirement Contexts:
|
|
551
|
+
* the P4 planner and its gated twin, the P5 booking agent over the
|
|
585
552
|
* shared supplier desk, and the S2 coordinator/researcher pair wired through the durable
|
|
586
|
-
* delegation Layer. A
|
|
553
|
+
* delegation Layer. A Thread Object registers these via its `bindings` option; the
|
|
587
554
|
* capture runs once per incarnation, and everything stateful the assertions rely on (desk,
|
|
588
555
|
* guide counter, gates) lives at module level so it survives incarnation loss.
|
|
589
556
|
*/
|
|
@@ -609,6 +576,7 @@ export const makePhase6TravelPlannerBindings: Effect.Effect<ReadonlyArray<Resolv
|
|
|
609
576
|
);
|
|
610
577
|
|
|
611
578
|
const researcherBinding = Agent.withModel(DestinationResearcher, phase6ResearcherModel);
|
|
579
|
+
|
|
612
580
|
const childToolkitLayer = DestinationResearcherToolkitLayer.pipe(
|
|
613
581
|
Layer.provideMerge(countingGuideLayer),
|
|
614
582
|
);
|
|
@@ -656,16 +624,16 @@ export const makePhase6TravelPlannerBindings: Effect.Effect<ReadonlyArray<Resolv
|
|
|
656
624
|
*/
|
|
657
625
|
export const phase6TravelPlannerGoldenEvidence: Schema.Json = [
|
|
658
626
|
{
|
|
659
|
-
batchId: "
|
|
627
|
+
batchId: "thread-created:{threadId}",
|
|
660
628
|
sequence: 1,
|
|
661
629
|
record: {
|
|
662
|
-
recordId: "
|
|
663
|
-
family: "
|
|
630
|
+
recordId: "thread-created:{threadId}",
|
|
631
|
+
family: "thread",
|
|
664
632
|
schemaVersion: 1,
|
|
665
633
|
createdAt: "{timestamp}",
|
|
666
634
|
deploymentId: "{deploymentId}",
|
|
667
635
|
payload: {
|
|
668
|
-
_tag: "
|
|
636
|
+
_tag: "ThreadCreated",
|
|
669
637
|
agentId: "travel-planner-phase-4",
|
|
670
638
|
definitions: {
|
|
671
639
|
agent: "{digest}",
|
|
@@ -680,7 +648,7 @@ export const phase6TravelPlannerGoldenEvidence: Schema.Json = [
|
|
|
680
648
|
sequence: 2,
|
|
681
649
|
record: {
|
|
682
650
|
recordId: "input:{submissionId}",
|
|
683
|
-
family: "
|
|
651
|
+
family: "thread",
|
|
684
652
|
schemaVersion: 1,
|
|
685
653
|
createdAt: "{timestamp}",
|
|
686
654
|
deploymentId: "{deploymentId}",
|
|
@@ -704,11 +672,28 @@ export const phase6TravelPlannerGoldenEvidence: Schema.Json = [
|
|
|
704
672
|
},
|
|
705
673
|
},
|
|
706
674
|
{
|
|
707
|
-
batchId: "
|
|
675
|
+
batchId: "run-start:run:{submissionId}",
|
|
708
676
|
sequence: 3,
|
|
677
|
+
record: {
|
|
678
|
+
recordId: "run-start:run:{submissionId}",
|
|
679
|
+
family: "thread",
|
|
680
|
+
schemaVersion: 1,
|
|
681
|
+
createdAt: "{timestamp}",
|
|
682
|
+
deploymentId: "{deploymentId}",
|
|
683
|
+
payload: {
|
|
684
|
+
_tag: "RunStarted",
|
|
685
|
+
runId: "run:{submissionId}",
|
|
686
|
+
policyAccountingVersion: 1,
|
|
687
|
+
maxDurationMillis: 30_000,
|
|
688
|
+
},
|
|
689
|
+
},
|
|
690
|
+
},
|
|
691
|
+
{
|
|
692
|
+
batchId: "turn-response:run:{submissionId}:1",
|
|
693
|
+
sequence: 4,
|
|
709
694
|
record: {
|
|
710
695
|
recordId: "model-response:run:{submissionId}:1",
|
|
711
|
-
family: "
|
|
696
|
+
family: "thread",
|
|
712
697
|
schemaVersion: 1,
|
|
713
698
|
createdAt: "{timestamp}",
|
|
714
699
|
deploymentId: "{deploymentId}",
|
|
@@ -719,6 +704,28 @@ export const phase6TravelPlannerGoldenEvidence: Schema.Json = [
|
|
|
719
704
|
turn: 1,
|
|
720
705
|
inputTokens: 128,
|
|
721
706
|
outputTokens: 96,
|
|
707
|
+
runScopedPrefixLength: 2,
|
|
708
|
+
modelUsage: [
|
|
709
|
+
{
|
|
710
|
+
provider: "scripted",
|
|
711
|
+
model: "travel-planner-phase-4",
|
|
712
|
+
purpose: "turn",
|
|
713
|
+
usageStatus: "partial",
|
|
714
|
+
pricingStatus: "unknown",
|
|
715
|
+
inputTokens: {
|
|
716
|
+
total: 128,
|
|
717
|
+
uncached: 128,
|
|
718
|
+
cacheRead: 0,
|
|
719
|
+
cacheWrite: 0,
|
|
720
|
+
},
|
|
721
|
+
outputTokens: {
|
|
722
|
+
total: 96,
|
|
723
|
+
text: 96,
|
|
724
|
+
reasoning: 0,
|
|
725
|
+
},
|
|
726
|
+
costMicrousd: 0,
|
|
727
|
+
},
|
|
728
|
+
],
|
|
722
729
|
messages: {
|
|
723
730
|
content: [
|
|
724
731
|
{
|
|
@@ -786,10 +793,10 @@ export const phase6TravelPlannerGoldenEvidence: Schema.Json = [
|
|
|
786
793
|
},
|
|
787
794
|
{
|
|
788
795
|
batchId: "turn-results:run:{submissionId}:1",
|
|
789
|
-
sequence:
|
|
796
|
+
sequence: 5,
|
|
790
797
|
record: {
|
|
791
798
|
recordId: "tool-settled:run:{submissionId}:1:flight-call-1",
|
|
792
|
-
family: "
|
|
799
|
+
family: "thread",
|
|
793
800
|
schemaVersion: 1,
|
|
794
801
|
createdAt: "{timestamp}",
|
|
795
802
|
deploymentId: "{deploymentId}",
|
|
@@ -810,10 +817,10 @@ export const phase6TravelPlannerGoldenEvidence: Schema.Json = [
|
|
|
810
817
|
},
|
|
811
818
|
{
|
|
812
819
|
batchId: "turn-results:run:{submissionId}:1",
|
|
813
|
-
sequence:
|
|
820
|
+
sequence: 6,
|
|
814
821
|
record: {
|
|
815
822
|
recordId: "tool-settled:run:{submissionId}:1:lodging-call-1",
|
|
816
|
-
family: "
|
|
823
|
+
family: "thread",
|
|
817
824
|
schemaVersion: 1,
|
|
818
825
|
createdAt: "{timestamp}",
|
|
819
826
|
deploymentId: "{deploymentId}",
|
|
@@ -833,10 +840,10 @@ export const phase6TravelPlannerGoldenEvidence: Schema.Json = [
|
|
|
833
840
|
},
|
|
834
841
|
{
|
|
835
842
|
batchId: "turn-results:run:{submissionId}:1",
|
|
836
|
-
sequence:
|
|
843
|
+
sequence: 7,
|
|
837
844
|
record: {
|
|
838
845
|
recordId: "tool-settled:run:{submissionId}:1:activity-call-1",
|
|
839
|
-
family: "
|
|
846
|
+
family: "thread",
|
|
840
847
|
schemaVersion: 1,
|
|
841
848
|
createdAt: "{timestamp}",
|
|
842
849
|
deploymentId: "{deploymentId}",
|
|
@@ -854,10 +861,10 @@ export const phase6TravelPlannerGoldenEvidence: Schema.Json = [
|
|
|
854
861
|
},
|
|
855
862
|
{
|
|
856
863
|
batchId: "turn:run:{submissionId}:2",
|
|
857
|
-
sequence:
|
|
864
|
+
sequence: 8,
|
|
858
865
|
record: {
|
|
859
866
|
recordId: "model-response:run:{submissionId}:2",
|
|
860
|
-
family: "
|
|
867
|
+
family: "thread",
|
|
861
868
|
schemaVersion: 1,
|
|
862
869
|
createdAt: "{timestamp}",
|
|
863
870
|
deploymentId: "{deploymentId}",
|
|
@@ -868,6 +875,27 @@ export const phase6TravelPlannerGoldenEvidence: Schema.Json = [
|
|
|
868
875
|
turn: 2,
|
|
869
876
|
inputTokens: 128,
|
|
870
877
|
outputTokens: 96,
|
|
878
|
+
modelUsage: [
|
|
879
|
+
{
|
|
880
|
+
provider: "scripted",
|
|
881
|
+
model: "travel-planner-phase-4",
|
|
882
|
+
purpose: "turn",
|
|
883
|
+
usageStatus: "partial",
|
|
884
|
+
pricingStatus: "unknown",
|
|
885
|
+
inputTokens: {
|
|
886
|
+
total: 128,
|
|
887
|
+
uncached: 128,
|
|
888
|
+
cacheRead: 0,
|
|
889
|
+
cacheWrite: 0,
|
|
890
|
+
},
|
|
891
|
+
outputTokens: {
|
|
892
|
+
total: 96,
|
|
893
|
+
text: 96,
|
|
894
|
+
reasoning: 0,
|
|
895
|
+
},
|
|
896
|
+
costMicrousd: 0,
|
|
897
|
+
},
|
|
898
|
+
],
|
|
871
899
|
messages: {
|
|
872
900
|
content: [
|
|
873
901
|
{
|
|
@@ -882,12 +910,50 @@ export const phase6TravelPlannerGoldenEvidence: Schema.Json = [
|
|
|
882
910
|
},
|
|
883
911
|
},
|
|
884
912
|
},
|
|
913
|
+
{
|
|
914
|
+
batchId: "turn:run:{submissionId}:2",
|
|
915
|
+
sequence: 9,
|
|
916
|
+
record: {
|
|
917
|
+
recordId: "run-completed:run:{submissionId}",
|
|
918
|
+
family: "thread",
|
|
919
|
+
schemaVersion: 1,
|
|
920
|
+
createdAt: "{timestamp}",
|
|
921
|
+
deploymentId: "{deploymentId}",
|
|
922
|
+
payload: {
|
|
923
|
+
_tag: "RunCompleted",
|
|
924
|
+
runId: "run:{submissionId}",
|
|
925
|
+
output: {
|
|
926
|
+
itineraries: [
|
|
927
|
+
{
|
|
928
|
+
title: "Westward light, eastbound overnight",
|
|
929
|
+
route: "San Francisco → London",
|
|
930
|
+
dates: "14–19 September 2026",
|
|
931
|
+
flight: "EA 218 · nonstop · SFO 18:40 → LHR 13:05+1",
|
|
932
|
+
lodging: "Bloomsbury House · refundable studio · 4 nights",
|
|
933
|
+
activities: ["British Museum timed entry", "Thames evening walk"],
|
|
934
|
+
estimatedTotalCents: 284000,
|
|
935
|
+
currency: "USD",
|
|
936
|
+
quoteId: "quote-sfo-lhr-001",
|
|
937
|
+
assumptions: [
|
|
938
|
+
"Two travelers sharing one studio",
|
|
939
|
+
"Quote is read-only availability, not a reservation",
|
|
940
|
+
],
|
|
941
|
+
unresolvedConstraints: [
|
|
942
|
+
"Traveler names and accessibility requests are intentionally omitted",
|
|
943
|
+
],
|
|
944
|
+
nextAction: "review",
|
|
945
|
+
},
|
|
946
|
+
],
|
|
947
|
+
},
|
|
948
|
+
},
|
|
949
|
+
},
|
|
950
|
+
},
|
|
885
951
|
{
|
|
886
952
|
batchId: "submission-settlement:{submissionId}",
|
|
887
|
-
sequence:
|
|
953
|
+
sequence: 10,
|
|
888
954
|
record: {
|
|
889
955
|
recordId: "settlement:{submissionId}",
|
|
890
|
-
family: "
|
|
956
|
+
family: "thread",
|
|
891
957
|
schemaVersion: 1,
|
|
892
958
|
createdAt: "{timestamp}",
|
|
893
959
|
deploymentId: "{deploymentId}",
|
|
@@ -921,6 +987,43 @@ export const phase6TravelPlannerGoldenEvidence: Schema.Json = [
|
|
|
921
987
|
},
|
|
922
988
|
],
|
|
923
989
|
},
|
|
990
|
+
usageSummary: {
|
|
991
|
+
usageStatus: "partial",
|
|
992
|
+
pricingStatus: "unknown",
|
|
993
|
+
unobservedModelCalls: 0,
|
|
994
|
+
modelCalls: 2,
|
|
995
|
+
inputTokens: {
|
|
996
|
+
total: 256,
|
|
997
|
+
uncached: 256,
|
|
998
|
+
cacheRead: 0,
|
|
999
|
+
cacheWrite: 0,
|
|
1000
|
+
},
|
|
1001
|
+
outputTokens: {
|
|
1002
|
+
total: 192,
|
|
1003
|
+
text: 192,
|
|
1004
|
+
reasoning: 0,
|
|
1005
|
+
},
|
|
1006
|
+
costMicrousd: 0,
|
|
1007
|
+
byModel: [
|
|
1008
|
+
{
|
|
1009
|
+
provider: "scripted",
|
|
1010
|
+
model: "travel-planner-phase-4",
|
|
1011
|
+
modelCalls: 2,
|
|
1012
|
+
inputTokens: {
|
|
1013
|
+
total: 256,
|
|
1014
|
+
uncached: 256,
|
|
1015
|
+
cacheRead: 0,
|
|
1016
|
+
cacheWrite: 0,
|
|
1017
|
+
},
|
|
1018
|
+
outputTokens: {
|
|
1019
|
+
total: 192,
|
|
1020
|
+
text: 192,
|
|
1021
|
+
reasoning: 0,
|
|
1022
|
+
},
|
|
1023
|
+
costMicrousd: 0,
|
|
1024
|
+
},
|
|
1025
|
+
],
|
|
1026
|
+
},
|
|
924
1027
|
},
|
|
925
1028
|
},
|
|
926
1029
|
},
|
|
@@ -1,112 +1,14 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
// ---------------------------------------------------------------------------
|
|
4
|
-
// Phase 7 (P7): the Travel Planner as an INTERNAL agent with two explicit
|
|
5
|
-
// profiles (ROADMAP P7 exit gate: "Travel Planner ... retains a deterministic
|
|
6
|
-
// offline conformance profile alongside its live integration profiles").
|
|
7
|
-
//
|
|
8
|
-
// - The OFFLINE profile is the cumulative P1–P6/S1/S2 suite set that already
|
|
9
|
-
// exists: scripted `LanguageModel`, deterministic supplier Layers,
|
|
10
|
-
// controllable time and IDs, no network, no credentials. Nothing in P7
|
|
11
|
-
// weakens it; this module only pins the claim as a Schema value so evidence
|
|
12
|
-
// can cite one committed fact instead of prose.
|
|
13
|
-
// - The LIVE profile is opt-in and test-side: a suite gates itself with
|
|
14
|
-
// `describe.skipIf(...)` on the environment predicate below, so ordinary
|
|
15
|
-
// `bun run test` never makes a network request and never needs a credential.
|
|
16
|
-
// Live profiles bind live MODEL Layers only. No real travel supplier exists
|
|
17
|
-
// to integrate, so the deterministic supplier desk is retained deliberately
|
|
18
|
-
// and `liveSupplierLayers` is pinned `false` — the roadmap's "selected
|
|
19
|
-
// supplier Layers" is honestly scoped to live-model-only (P7 plan decision
|
|
20
|
-
// 9), recorded here rather than silently claimed.
|
|
21
|
-
// ---------------------------------------------------------------------------
|
|
22
|
-
|
|
23
|
-
/**
|
|
24
|
-
* The P7 dual-profile claim, schema-first so the exact scope of "live
|
|
25
|
-
* integration profiles" is a committed, decodable value:
|
|
26
|
-
*
|
|
27
|
-
* - `offlineConformanceDeterministic` / `offlineRequiresCredentials`: the
|
|
28
|
-
* cumulative conformance suites stay deterministic and credential-free.
|
|
29
|
-
* - `liveProfileOptIn`: live suites are excluded from ordinary gates by the
|
|
30
|
-
* environment predicate (`phase7LiveProfileEnabled`), never by test-runner
|
|
31
|
-
* configuration that could silently drift.
|
|
32
|
-
* - `liveModelLayers` / `liveSupplierLayers`: live profiles exercise real
|
|
33
|
-
* model Layers over the SAME deterministic supplier desk — no claim of a
|
|
34
|
-
* live supplier integration is made anywhere (decision 9).
|
|
35
|
-
* - `structurallyRedactedTranscripts`: transcript evidence a live profile
|
|
36
|
-
* emits passes through the structural `Redactor` first (SEC-008,
|
|
37
|
-
* testing.md §12: "live model and supplier profiles are opt-in smoke or
|
|
38
|
-
* release tests, rate-limited and structurally redacted").
|
|
39
|
-
* - `exactlyOnceExternalEffects`: never claimed at any phase (DUR-003).
|
|
40
|
-
*/
|
|
41
|
-
export class TravelPlannerPhase7Profile extends Schema.Class<TravelPlannerPhase7Profile>(
|
|
42
|
-
"@effect-agent/testing/travel-planner/TravelPlannerPhase7Profile",
|
|
43
|
-
)({
|
|
44
|
-
phase: Schema.Literal("P7"),
|
|
45
|
-
offlineConformanceDeterministic: Schema.Literal(true),
|
|
46
|
-
offlineRequiresCredentials: Schema.Literal(false),
|
|
47
|
-
liveProfileOptIn: Schema.Literal(true),
|
|
48
|
-
liveModelLayers: Schema.Literal(true),
|
|
49
|
-
liveSupplierLayers: Schema.Literal(false),
|
|
50
|
-
structurallyRedactedTranscripts: Schema.Literal(true),
|
|
51
|
-
exactlyOnceExternalEffects: Schema.Literal(false),
|
|
52
|
-
}) {}
|
|
53
|
-
|
|
54
|
-
export const phase7TravelPlannerProfile = TravelPlannerPhase7Profile.make({
|
|
55
|
-
phase: "P7",
|
|
56
|
-
offlineConformanceDeterministic: true,
|
|
57
|
-
offlineRequiresCredentials: false,
|
|
58
|
-
liveProfileOptIn: true,
|
|
59
|
-
liveModelLayers: true,
|
|
60
|
-
liveSupplierLayers: false,
|
|
61
|
-
structurallyRedactedTranscripts: true,
|
|
62
|
-
exactlyOnceExternalEffects: false,
|
|
63
|
-
});
|
|
64
|
-
|
|
65
|
-
/**
|
|
66
|
-
* The one opt-in switch for EVERY live profile in this repository. `"1"` is
|
|
67
|
-
* the only enabling value: an unset, empty, or differently-truthy value keeps
|
|
68
|
-
* the suite skipped, so CI and ordinary developer runs stay offline.
|
|
69
|
-
*/
|
|
1
|
+
/** Explicit opt-in switch for live integration tests. */
|
|
70
2
|
export const PHASE7_LIVE_GATE_ENV = "EFFECT_AGENT_LIVE";
|
|
71
3
|
|
|
72
|
-
/** The credential
|
|
4
|
+
/** The credential required by the Travel Planner live-model tests. */
|
|
73
5
|
export const PHASE7_LIVE_CREDENTIAL_ENV = "OPENAI_API_KEY";
|
|
74
6
|
|
|
75
|
-
/** Structural shape of
|
|
7
|
+
/** Structural shape of an environment without importing Node types. */
|
|
76
8
|
export interface Phase7LiveGateEnvironment {
|
|
77
9
|
readonly [name: string]: string | undefined;
|
|
78
10
|
}
|
|
79
11
|
|
|
80
|
-
/**
|
|
81
|
-
* The test-side live gate (P7 plan §6: no test-side live-gating pattern
|
|
82
|
-
* existed before this — the demo gates at serve time via
|
|
83
|
-
* `Config.redacted("OPENAI_API_KEY")`). Suites use it as
|
|
84
|
-
* `describe.skipIf(!phase7LiveProfileEnabled(process.env))`, which keeps the
|
|
85
|
-
* live block out of ordinary gates while the SAME file's ungated tests keep
|
|
86
|
-
* pinning the profile schema on every run.
|
|
87
|
-
*/
|
|
12
|
+
/** Enable live tests only when both the opt-in flag and credential are present. */
|
|
88
13
|
export const phase7LiveProfileEnabled = (env: Phase7LiveGateEnvironment): boolean =>
|
|
89
14
|
env[PHASE7_LIVE_GATE_ENV] === "1" && (env[PHASE7_LIVE_CREDENTIAL_ENV] ?? "") !== "";
|
|
90
|
-
|
|
91
|
-
// ---------------------------------------------------------------------------
|
|
92
|
-
// Authoring friction note (WP7 input; real observations from wiring the P7
|
|
93
|
-
// live profile onto the existing Travel Planner):
|
|
94
|
-
//
|
|
95
|
-
// 1. There was no framework-owned place to put a test-side live gate: every
|
|
96
|
-
// earlier profile was either always-deterministic or gated at serve time
|
|
97
|
-
// inside an application. The predicate had to be invented here as a plain
|
|
98
|
-
// exported function because `packages/testing/src` must stay
|
|
99
|
-
// platform-neutral and cannot read `process.env` itself — fine, but the
|
|
100
|
-
// split (fixture exports the predicate, the example test applies it to
|
|
101
|
-
// `process.env`) is a convention a future author has to discover by
|
|
102
|
-
// reading this file rather than a typed seam.
|
|
103
|
-
// 2. Capturing a structurally redacted transcript from a live Run takes
|
|
104
|
-
// manual assembly: collect `RunEvent`s, `Schema.encode` each one, pass the
|
|
105
|
-
// encoded value through `Redactor.redact`. Nothing composes those three
|
|
106
|
-
// steps, and nothing type-level stops a test from logging the RAW event by
|
|
107
|
-
// accident. A `redactedTranscript(events)` helper in capabilities (or an
|
|
108
|
-
// engine stream combinator) would make the safe path the short path.
|
|
109
|
-
// 3. Binding a live model to the existing definition was pleasantly trivial
|
|
110
|
-
// (`Agent.withModel` + the upstream client Layer): no friction to report
|
|
111
|
-
// on the authoring surface itself for plain ephemeral runs.
|
|
112
|
-
// ---------------------------------------------------------------------------
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { Schema } from "effect";
|
|
2
2
|
|
|
3
|
-
import type { ScriptedTurnInput } from "../../
|
|
3
|
+
import type { ScriptedTurnInput } from "../../ScriptedModel.ts";
|
|
4
4
|
import { TravelPlan, TripRequest, type TravelPlan as TravelPlanValue } from "./definition.ts";
|
|
5
5
|
|
|
6
6
|
const usage = { inputTokens: { total: 128 }, outputTokens: { total: 96 } };
|
|
7
|
+
|
|
7
8
|
export const phase1Trip = Schema.decodeSync(TripRequest)({
|
|
8
9
|
request:
|
|
9
10
|
"Plan a review-only London trip using the deterministic flight, lodging, and activity searches.",
|
|
@@ -15,8 +16,7 @@ export const phase1Trip = Schema.decodeSync(TripRequest)({
|
|
|
15
16
|
budgetCents: 350_000,
|
|
16
17
|
currency: "USD",
|
|
17
18
|
});
|
|
18
|
-
|
|
19
|
-
export const phase0Trip = phase1Trip;
|
|
19
|
+
|
|
20
20
|
export const expectedTravelPlan: TravelPlanValue = Schema.decodeSync(TravelPlan)({
|
|
21
21
|
itineraries: [
|
|
22
22
|
{
|
|
@@ -82,4 +82,3 @@ export const phase1HappyPathTurns = [
|
|
|
82
82
|
termination: { _tag: "Complete" },
|
|
83
83
|
},
|
|
84
84
|
] satisfies readonly [ScriptedTurnInput, ScriptedTurnInput];
|
|
85
|
-
export const phase0HappyPathTurns = phase1HappyPathTurns;
|