@effect-agent/testing 0.1.0-beta.41 → 0.1.0-beta.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/certification.mjs.map +1 -1
- package/dist/chaos.mjs.map +1 -1
- package/dist/code-executor.mjs.map +1 -1
- package/dist/deterministic-layers-CKyYxBhN.mjs.map +1 -1
- package/dist/docs-researcher.mjs +3 -2
- package/dist/docs-researcher.mjs.map +1 -1
- package/dist/scripted-model-C2y0ztuj.mjs.map +1 -1
- package/dist/travel-planner.mjs.map +1 -1
- package/package.json +1 -71
- package/src/certification.ts +78 -0
- package/src/chaos.ts +96 -0
- package/src/code-executor-conformance.ts +19 -0
- package/src/code-executor-substitute.ts +32 -0
- package/src/fixtures/docs-researcher/definition.ts +7 -0
- package/src/fixtures/docs-researcher/harness.ts +19 -0
- package/src/fixtures/docs-researcher/mcp.ts +14 -2
- package/src/fixtures/travel-planner/definition.ts +12 -0
- package/src/fixtures/travel-planner/deterministic-layers.ts +38 -0
- package/src/fixtures/travel-planner/phase3.ts +4 -0
- package/src/fixtures/travel-planner/phase4.ts +10 -0
- package/src/fixtures/travel-planner/phase5.ts +20 -0
- package/src/fixtures/travel-planner/phase6.ts +12 -0
- package/src/fixtures/travel-planner/scenarios.ts +2 -0
- package/src/fixtures/travel-planner/subagents-durable.ts +13 -0
- package/src/fixtures/travel-planner/subagents.ts +21 -0
- package/src/scripted-model.ts +18 -0
|
@@ -50,6 +50,7 @@ export const LookupDestination = Tool.make("lookup_destination", {
|
|
|
50
50
|
});
|
|
51
51
|
|
|
52
52
|
export const DestinationResearcherToolkit = Toolkit.make(LookupDestination);
|
|
53
|
+
|
|
53
54
|
export const DestinationResearcherToolkitLayer = DestinationResearcherToolkit.toLayer({
|
|
54
55
|
lookup_destination: (query) => Effect.flatMap(DestinationGuide, (guide) => guide.lookup(query)),
|
|
55
56
|
});
|
|
@@ -111,6 +112,7 @@ export const destinationLookup = (
|
|
|
111
112
|
query: DestinationQuery,
|
|
112
113
|
): Effect.Effect<DestinationFacts, DestinationGuideUnavailable> => {
|
|
113
114
|
const facts = guideFacts.get(query.destination);
|
|
115
|
+
|
|
114
116
|
return facts === undefined
|
|
115
117
|
? Effect.fail(
|
|
116
118
|
DestinationGuideUnavailable.make({
|
|
@@ -123,15 +125,18 @@ export const destinationLookup = (
|
|
|
123
125
|
|
|
124
126
|
const requireDestinationFacts = (destination: string): DestinationFacts => {
|
|
125
127
|
const facts = guideFacts.get(destination);
|
|
128
|
+
|
|
126
129
|
if (facts === undefined) {
|
|
127
130
|
throw new Error(`No deterministic guide entry exists for destination ${destination}`);
|
|
128
131
|
}
|
|
132
|
+
|
|
129
133
|
return facts;
|
|
130
134
|
};
|
|
131
135
|
|
|
132
136
|
/** The report the scripted researcher writes after consulting the guide. */
|
|
133
137
|
export const destinationReportFor = (destination: string): DestinationReport => {
|
|
134
138
|
const facts = requireDestinationFacts(destination);
|
|
139
|
+
|
|
135
140
|
return DestinationReport.make({
|
|
136
141
|
destination: facts.destination,
|
|
137
142
|
highlights: facts.highlights,
|
|
@@ -146,7 +151,9 @@ export const DestinationGuideLayer = Layer.effect(
|
|
|
146
151
|
DestinationGuide,
|
|
147
152
|
Effect.gen(function* () {
|
|
148
153
|
const lifecycle = yield* CatalogLifecycle;
|
|
154
|
+
|
|
149
155
|
yield* Effect.acquireRelease(lifecycle.markAcquired, () => lifecycle.markFinalized);
|
|
156
|
+
|
|
150
157
|
return DestinationGuide.of({ lookup: destinationLookup });
|
|
151
158
|
}),
|
|
152
159
|
);
|
|
@@ -223,7 +230,9 @@ export const destinationResearchDelegation = Subagent.define("delegate_destinati
|
|
|
223
230
|
prepareInput: (request) =>
|
|
224
231
|
Effect.gen(function* () {
|
|
225
232
|
const gate = yield* ResearchDispatchGate;
|
|
233
|
+
|
|
226
234
|
yield* gate.awaitDispatch(request.destination);
|
|
235
|
+
|
|
227
236
|
return DestinationBrief.make({
|
|
228
237
|
destination: request.destination,
|
|
229
238
|
focus: `research:${request.focus}`,
|
|
@@ -460,19 +469,23 @@ export const makeDestinationResearcherModel = (destinations: ReadonlyArray<strin
|
|
|
460
469
|
const lifecycle = yield* CatalogLifecycle;
|
|
461
470
|
const prompts = yield* Ref.make<ReadonlyArray<string>>([]);
|
|
462
471
|
const gates = new Map<string, ResearcherGates>();
|
|
472
|
+
|
|
463
473
|
for (const destination of destinations) {
|
|
464
474
|
gates.set(destination, {
|
|
465
475
|
started: yield* Deferred.make<void>(),
|
|
466
476
|
release: yield* Deferred.make<void>(),
|
|
467
477
|
});
|
|
468
478
|
}
|
|
479
|
+
|
|
469
480
|
const gatesFor = (destination: string): Effect.Effect<ResearcherGates> =>
|
|
470
481
|
Effect.suspend(() => {
|
|
471
482
|
const entry = gates.get(destination);
|
|
483
|
+
|
|
472
484
|
return entry === undefined
|
|
473
485
|
? Effect.die(new Error(`No researcher gates exist for destination ${destination}`))
|
|
474
486
|
: Effect.succeed(entry);
|
|
475
487
|
});
|
|
488
|
+
|
|
476
489
|
const controls: DestinationResearcherControls = {
|
|
477
490
|
awaitStarted: (destination) =>
|
|
478
491
|
gatesFor(destination).pipe(Effect.flatMap((entry) => Deferred.await(entry.started))),
|
|
@@ -483,6 +496,7 @@ export const makeDestinationResearcherModel = (destinations: ReadonlyArray<strin
|
|
|
483
496
|
),
|
|
484
497
|
prompts: Ref.get(prompts),
|
|
485
498
|
};
|
|
499
|
+
|
|
486
500
|
const model = Model.make(
|
|
487
501
|
"scripted",
|
|
488
502
|
"destination-researcher-scripted",
|
|
@@ -491,15 +505,18 @@ export const makeDestinationResearcherModel = (destinations: ReadonlyArray<strin
|
|
|
491
505
|
Effect.gen(function* () {
|
|
492
506
|
yield* Effect.acquireRelease(lifecycle.markAcquired, () => lifecycle.markFinalized);
|
|
493
507
|
const turn = yield* Ref.make(0);
|
|
508
|
+
|
|
494
509
|
return yield* LanguageModel.make({
|
|
495
510
|
generateText: () => Effect.succeed([]),
|
|
496
511
|
streamText: (options) =>
|
|
497
512
|
Stream.unwrap(
|
|
498
513
|
Effect.gen(function* () {
|
|
499
514
|
const promptJson = JSON.stringify(options.prompt.content);
|
|
515
|
+
|
|
500
516
|
const destination = destinations.find((candidate) =>
|
|
501
517
|
promptJson.includes(candidate),
|
|
502
518
|
);
|
|
519
|
+
|
|
503
520
|
if (destination === undefined) {
|
|
504
521
|
return yield* Effect.die(
|
|
505
522
|
new Error("The researcher prompt names no scripted destination"),
|
|
@@ -507,12 +524,15 @@ export const makeDestinationResearcherModel = (destinations: ReadonlyArray<strin
|
|
|
507
524
|
}
|
|
508
525
|
const entry = yield* gatesFor(destination);
|
|
509
526
|
const index = yield* Ref.getAndUpdate(turn, (value) => value + 1);
|
|
527
|
+
|
|
510
528
|
if (index === 0) {
|
|
511
529
|
yield* Ref.update(prompts, (previous) => [...previous, promptJson]);
|
|
512
530
|
yield* Deferred.succeed(entry.started, undefined);
|
|
531
|
+
|
|
513
532
|
return Stream.fromIterable(researcherLookupParts(destination));
|
|
514
533
|
}
|
|
515
534
|
yield* Deferred.await(entry.release);
|
|
535
|
+
|
|
516
536
|
return Stream.fromIterable(researcherReportParts(destination));
|
|
517
537
|
}),
|
|
518
538
|
),
|
|
@@ -520,5 +540,6 @@ export const makeDestinationResearcherModel = (destinations: ReadonlyArray<strin
|
|
|
520
540
|
}),
|
|
521
541
|
),
|
|
522
542
|
);
|
|
543
|
+
|
|
523
544
|
return { controls, model };
|
|
524
545
|
});
|
package/src/scripted-model.ts
CHANGED
|
@@ -2,9 +2,11 @@ import { Context, Effect, Layer, Ref, Schema, Stream } from "effect";
|
|
|
2
2
|
import { AiError, LanguageModel, Response } from "effect/unstable/ai";
|
|
3
3
|
|
|
4
4
|
const ScriptedPartMetadata = Schema.Record(Schema.String, Schema.NullOr(Schema.Json));
|
|
5
|
+
|
|
5
6
|
const ScriptedPartBase = {
|
|
6
7
|
metadata: Schema.optionalKey(ScriptedPartMetadata),
|
|
7
8
|
};
|
|
9
|
+
|
|
8
10
|
const ScriptedToolCallPart = Schema.Struct({
|
|
9
11
|
...ScriptedPartBase,
|
|
10
12
|
type: Schema.Literal("tool-call"),
|
|
@@ -13,6 +15,7 @@ const ScriptedToolCallPart = Schema.Struct({
|
|
|
13
15
|
params: Schema.Unknown,
|
|
14
16
|
providerExecuted: Schema.optionalKey(Schema.Boolean),
|
|
15
17
|
});
|
|
18
|
+
|
|
16
19
|
const ScriptedToolResultPart = Schema.Struct({
|
|
17
20
|
...ScriptedPartBase,
|
|
18
21
|
type: Schema.Literal("tool-result"),
|
|
@@ -44,6 +47,7 @@ export const ScriptedGeneratePart = Schema.Union([
|
|
|
44
47
|
Schema.toEncoded(Response.ResponseMetadataPart),
|
|
45
48
|
Schema.toEncoded(Response.FinishPart),
|
|
46
49
|
]).annotate({ identifier: "ScriptedGeneratePart" });
|
|
50
|
+
|
|
47
51
|
export type ScriptedGeneratePart = typeof ScriptedGeneratePart.Type;
|
|
48
52
|
|
|
49
53
|
/**
|
|
@@ -69,6 +73,7 @@ export const ScriptedStreamPart = Schema.Union([
|
|
|
69
73
|
Schema.toEncoded(Response.FinishPart),
|
|
70
74
|
Schema.toEncoded(Response.ErrorPart),
|
|
71
75
|
]).annotate({ identifier: "ScriptedStreamPart" });
|
|
76
|
+
|
|
72
77
|
export type ScriptedStreamPart = typeof ScriptedStreamPart.Type;
|
|
73
78
|
|
|
74
79
|
/** Controls whether a scripted stream completes, fails, or waits for interruption. */
|
|
@@ -79,12 +84,14 @@ export const ScriptedStreamTermination = Schema.Union([
|
|
|
79
84
|
}),
|
|
80
85
|
Schema.TaggedStruct("Hang", {}),
|
|
81
86
|
]);
|
|
87
|
+
|
|
82
88
|
export type ScriptedStreamTermination = typeof ScriptedStreamTermination.Type;
|
|
83
89
|
|
|
84
90
|
/** One non-streaming invocation and the encoded response parts it returns. */
|
|
85
91
|
export const ScriptedGenerateTurn = Schema.TaggedStruct("Generate", {
|
|
86
92
|
parts: Schema.Array(ScriptedGeneratePart),
|
|
87
93
|
});
|
|
94
|
+
|
|
88
95
|
export type ScriptedGenerateTurn = typeof ScriptedGenerateTurn.Type;
|
|
89
96
|
|
|
90
97
|
/** One streaming invocation with its encoded parts and terminal behavior. */
|
|
@@ -92,6 +99,7 @@ export const ScriptedStreamTurn = Schema.TaggedStruct("Stream", {
|
|
|
92
99
|
parts: Schema.Array(ScriptedStreamPart),
|
|
93
100
|
termination: ScriptedStreamTermination,
|
|
94
101
|
});
|
|
102
|
+
|
|
95
103
|
export type ScriptedStreamTurn = typeof ScriptedStreamTurn.Type;
|
|
96
104
|
|
|
97
105
|
/**
|
|
@@ -147,8 +155,10 @@ const runAssertion = Effect.fn("ScriptedModel.runAssertion")((
|
|
|
147
155
|
if (assertion === undefined) {
|
|
148
156
|
return Effect.void;
|
|
149
157
|
}
|
|
158
|
+
|
|
150
159
|
return Effect.suspend(() => {
|
|
151
160
|
const result = assertion(request);
|
|
161
|
+
|
|
152
162
|
return Effect.isEffect(result) ? result : Effect.void;
|
|
153
163
|
});
|
|
154
164
|
});
|
|
@@ -161,6 +171,7 @@ const takeTurn = Effect.fn("ScriptedModel.takeTurn")(
|
|
|
161
171
|
): Effect.Effect<ScriptedTurnInput, AiError.AiError> =>
|
|
162
172
|
Ref.modify(state, (current) => {
|
|
163
173
|
const turn = current.remaining[0];
|
|
174
|
+
|
|
164
175
|
if (turn === undefined) {
|
|
165
176
|
return [
|
|
166
177
|
undefined,
|
|
@@ -170,6 +181,7 @@ const takeTurn = Effect.fn("ScriptedModel.takeTurn")(
|
|
|
170
181
|
},
|
|
171
182
|
] as const;
|
|
172
183
|
}
|
|
184
|
+
|
|
173
185
|
return [
|
|
174
186
|
turn,
|
|
175
187
|
{
|
|
@@ -206,6 +218,7 @@ const streamForTurn = (
|
|
|
206
218
|
let stream: Stream.Stream<Response.StreamPartEncoded, AiError.AiError> = Stream.fromIterable(
|
|
207
219
|
turn.parts,
|
|
208
220
|
);
|
|
221
|
+
|
|
209
222
|
switch (turn.termination._tag) {
|
|
210
223
|
case "Complete": {
|
|
211
224
|
break;
|
|
@@ -227,6 +240,7 @@ const streamForTurn = (
|
|
|
227
240
|
if (turn.onStreamFinalize !== undefined) {
|
|
228
241
|
stream = stream.pipe(Stream.ensuring(turn.onStreamFinalize));
|
|
229
242
|
}
|
|
243
|
+
|
|
230
244
|
return stream;
|
|
231
245
|
};
|
|
232
246
|
|
|
@@ -262,16 +276,20 @@ export class ScriptedModel extends Context.Service<
|
|
|
262
276
|
generateText: (options) =>
|
|
263
277
|
Effect.gen(function* () {
|
|
264
278
|
const turn = yield* takeTurn(state, "generate", options);
|
|
279
|
+
|
|
265
280
|
yield* runAssertion(turn.assertRequest, options);
|
|
266
281
|
const generateTurn = yield* requireGenerateTurn(turn);
|
|
282
|
+
|
|
267
283
|
return [...generateTurn.parts];
|
|
268
284
|
}),
|
|
269
285
|
streamText: (options) =>
|
|
270
286
|
Stream.unwrap(
|
|
271
287
|
Effect.gen(function* () {
|
|
272
288
|
const turn = yield* takeTurn(state, "stream", options);
|
|
289
|
+
|
|
273
290
|
yield* runAssertion(turn.assertRequest, options);
|
|
274
291
|
const streamTurn = yield* requireStreamTurn(turn);
|
|
292
|
+
|
|
275
293
|
return streamForTurn(streamTurn);
|
|
276
294
|
}),
|
|
277
295
|
),
|