effect-assistant 0.8.3 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/AssistantTest.d.ts +94 -0
- package/dist/AssistantTest.d.ts.map +1 -0
- package/dist/AssistantTest.js +146 -0
- package/dist/AssistantTest.js.map +1 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/internal/loop.js +2 -2
- package/dist/internal/loop.js.map +1 -1
- package/dist/internal/telemetry.js +18 -26
- package/dist/internal/telemetry.js.map +1 -1
- package/package.json +1 -1
- package/src/AssistantTest.ts +254 -0
- package/src/index.ts +5 -0
- package/src/internal/loop.ts +1 -1
- package/src/internal/telemetry.ts +34 -40
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { ChatId } from "./ChatId.js";
|
|
2
|
+
import { AnyEntry, ApprovalRequested, AssistantEvent_d_exports, Input } from "./AssistantEvent.js";
|
|
3
|
+
import { Block, Block_d_exports, Codec } from "./Block.js";
|
|
4
|
+
import { Assistant_d_exports, Service } from "./Assistant.js";
|
|
5
|
+
import * as Effect from "effect/Effect";
|
|
6
|
+
import * as Duration from "effect/Duration";
|
|
7
|
+
declare namespace AssistantTest_d_exports {
|
|
8
|
+
export { Log, TestChat, TestChatError, chat, log };
|
|
9
|
+
}
|
|
10
|
+
declare const TestChatError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
|
|
11
|
+
readonly _tag: "TestChatError";
|
|
12
|
+
} & Readonly<A>;
|
|
13
|
+
/**
|
|
14
|
+
* Represents a step that couldn't complete: no turn handled the input in time, the turn failed,
|
|
15
|
+
* or there was nothing to approve or answer.
|
|
16
|
+
*
|
|
17
|
+
* @category errors
|
|
18
|
+
* @since 0.9.0
|
|
19
|
+
*/
|
|
20
|
+
export declare class TestChatError extends TestChatError_base<{
|
|
21
|
+
readonly reason: "TimedOut" | "TurnFailed" | "NothingToApprove" | "NothingToAnswer";
|
|
22
|
+
readonly message: string;
|
|
23
|
+
}> {}
|
|
24
|
+
/**
|
|
25
|
+
* Represents what the assistant did in some of a chat's entries.
|
|
26
|
+
*
|
|
27
|
+
* @category models
|
|
28
|
+
* @since 0.9.0
|
|
29
|
+
*/
|
|
30
|
+
export interface Log {
|
|
31
|
+
readonly entries: ReadonlyArray<AnyEntry>;
|
|
32
|
+
/**
|
|
33
|
+
* The app's tools that ran, each once, in the order they first ran. A call that failed or that
|
|
34
|
+
* the library refused didn't run, and the assistant's own tools (`send_user_message`, `wait`,
|
|
35
|
+
* `load_skill`) aren't listed.
|
|
36
|
+
*/
|
|
37
|
+
readonly toolsRun: ReadonlyArray<string>;
|
|
38
|
+
/** How many times the model called this tool, whether or not it ran. */
|
|
39
|
+
readonly toolCalls: (tool: string) => number;
|
|
40
|
+
/** The kind of each block the assistant showed, in order; one per message. */
|
|
41
|
+
readonly blocks: ReadonlyArray<string>;
|
|
42
|
+
/** The calls waiting for the user's approval, in order. */
|
|
43
|
+
readonly approvals: ReadonlyArray<ApprovalRequested>;
|
|
44
|
+
/** The tools of the calls waiting for the user's approval. */
|
|
45
|
+
readonly waitingFor: ReadonlyArray<string>;
|
|
46
|
+
/** The messages the assistant sent, in order. */
|
|
47
|
+
readonly messages: ReadonlyArray<string>;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Reads what the assistant did in these entries.
|
|
51
|
+
*
|
|
52
|
+
* @category constructors
|
|
53
|
+
* @since 0.9.0
|
|
54
|
+
*/
|
|
55
|
+
export declare const log: (entries: ReadonlyArray<AnyEntry>) => Log;
|
|
56
|
+
/**
|
|
57
|
+
* Represents a chat driven like a user would.
|
|
58
|
+
*
|
|
59
|
+
* @category models
|
|
60
|
+
* @since 0.9.0
|
|
61
|
+
*/
|
|
62
|
+
export interface TestChat extends Log {
|
|
63
|
+
readonly chatId: ChatId;
|
|
64
|
+
/** What the assistant did for the last step alone. */
|
|
65
|
+
readonly step: Log;
|
|
66
|
+
/** Submits an input, and returns the chat once a turn has handled it. */
|
|
67
|
+
readonly send: (input: Input) => Effect.Effect<TestChat, TestChatError>;
|
|
68
|
+
/** Sends the user's message. */
|
|
69
|
+
readonly say: (text: string) => Effect.Effect<TestChat, TestChatError>;
|
|
70
|
+
/** Approves the last call waiting for approval. */
|
|
71
|
+
readonly approve: Effect.Effect<TestChat, TestChatError>;
|
|
72
|
+
/** Rejects the last call waiting for approval. */
|
|
73
|
+
readonly reject: (options?: {
|
|
74
|
+
readonly reason?: string | undefined;
|
|
75
|
+
}) => Effect.Effect<TestChat, TestChatError>;
|
|
76
|
+
/** Answers the last block of this kind the assistant showed. */
|
|
77
|
+
readonly answer: <Answer extends Codec>(block: Block<string, Codec, Answer>, answer: Answer["Type"]) => Effect.Effect<TestChat, TestChatError>;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Creates a chat to drive the assistant with, for a chat the app already created. A step fails
|
|
81
|
+
* when no turn has handled its input within `timeout` (3 minutes by default), or when a turn failed
|
|
82
|
+
* while handling it.
|
|
83
|
+
*
|
|
84
|
+
* @category constructors
|
|
85
|
+
* @since 0.9.0
|
|
86
|
+
*/
|
|
87
|
+
export declare const chat: <Events extends {
|
|
88
|
+
readonly _tag: string;
|
|
89
|
+
} = never>(assistant: Service<Events>, chatId: ChatId, options?: {
|
|
90
|
+
readonly timeout?: Duration.Input | undefined;
|
|
91
|
+
}) => TestChat;
|
|
92
|
+
//#endregion
|
|
93
|
+
export { AssistantTest_d_exports };
|
|
94
|
+
//# sourceMappingURL=AssistantTest.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AssistantTest.d.ts","names":[],"sources":["../src/AssistantTest.ts"],"mappings":";;;;;;;;;;;;;;;;;;;qBAmDa,sBAAsB;WACxB;WACA;;;;;;;;iBAYM;WACN,SAAS,cAAc;;;;;;WAMvB,UAAU;;WAEV,YAAY;;WAEZ,QAAQ;;WAER,WAAW,cAAc;;WAEzB,YAAY;;WAEZ,UAAU;;;;;;;;qBASR,MAAG,SAAa,cAAc,cAA2B;;;;;;;iBA4BrD,iBAAiB;WACvB,QAAQ;;WAER,MAAM;;WAEN,OAAO,OAAO,UAAyB,OAAO,OAAO,UAAU;;WAE/D,MAAM,iBAAiB,OAAO,OAAO,UAAU;;WAE/C,SAAS,OAAO,OAAO,UAAU;;WAEjC,SAAS;aAAqB;QAAkC,OAAO,OAAO,UAAU;;WAExF,SAAS,eAAe,OAC/B,OAAO,cAAoB,OAAa,SACxC,QAAQ,mBACL,OAAO,OAAO,UAAU;;;;;;;;;;qBAoBlB,OAAQ;WAA0B;WAAsB,WACxD,QAAkB,SAAO,QAC5B,QAAM;WACO,UAAU,SAAS;MACvC"}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { __exportAll } from "./_virtual/_rolldown/runtime.js";
|
|
2
|
+
import { fromKey } from "./InputId.js";
|
|
3
|
+
import { BlockAnswer, UserMessage, assistantTexts, isInput, isLibraryEvent, isTurnEnd } from "./AssistantEvent.js";
|
|
4
|
+
import { approve, reject } from "./ToolApproval.js";
|
|
5
|
+
import { sendUserMessageName, waitToolName } from "./internal/tools.js";
|
|
6
|
+
import * as Effect from "effect/Effect";
|
|
7
|
+
import * as Predicate from "effect/Predicate";
|
|
8
|
+
import * as Schema from "effect/Schema";
|
|
9
|
+
import * as Stream from "effect/Stream";
|
|
10
|
+
import * as Data from "effect/Data";
|
|
11
|
+
import * as Option from "effect/Option";
|
|
12
|
+
//#region src/AssistantTest.ts
|
|
13
|
+
/**
|
|
14
|
+
* This module provides a chat to drive an assistant like a user would, for tests and evals.
|
|
15
|
+
*
|
|
16
|
+
* {@link chat} wraps a chat the app created. Each step (`say`, `approve`, `reject`, `answer`, or
|
|
17
|
+
* `send` for any input) submits the input, waits until a turn has handled it, and returns the chat
|
|
18
|
+
* as it is then. A {@link Log} reads what the assistant did, for the whole chat or for the last
|
|
19
|
+
* step alone (`step`): the tools that ran, the blocks it showed, the calls waiting for approval,
|
|
20
|
+
* and the messages it sent.
|
|
21
|
+
*
|
|
22
|
+
* **Example** (An assistant asks before it emails)
|
|
23
|
+
*
|
|
24
|
+
* ```ts
|
|
25
|
+
* import { AssistantTest } from "effect-assistant"
|
|
26
|
+
* import * as Effect from "effect/Effect"
|
|
27
|
+
*
|
|
28
|
+
* const program = Effect.gen(function* () {
|
|
29
|
+
* const chat = AssistantTest.chat(yield* MyAssistant, chatId)
|
|
30
|
+
* const asked = yield* chat.say("Email the report to ana@example.com")
|
|
31
|
+
* asked.step.waitingFor // ["send_report"]
|
|
32
|
+
* const sent = yield* asked.approve
|
|
33
|
+
* sent.step.toolsRun // ["send_report"]
|
|
34
|
+
* })
|
|
35
|
+
* ```
|
|
36
|
+
*
|
|
37
|
+
* @since 0.9.0
|
|
38
|
+
*/
|
|
39
|
+
var AssistantTest_exports = /* @__PURE__ */ __exportAll({
|
|
40
|
+
TestChatError: () => TestChatError,
|
|
41
|
+
chat: () => chat,
|
|
42
|
+
log: () => log
|
|
43
|
+
});
|
|
44
|
+
const ownTools = /* @__PURE__ */ new Set([
|
|
45
|
+
sendUserMessageName,
|
|
46
|
+
waitToolName,
|
|
47
|
+
"load_skill"
|
|
48
|
+
]);
|
|
49
|
+
/**
|
|
50
|
+
* Represents a step that couldn't complete: no turn handled the input in time, the turn failed,
|
|
51
|
+
* or there was nothing to approve or answer.
|
|
52
|
+
*
|
|
53
|
+
* @category errors
|
|
54
|
+
* @since 0.9.0
|
|
55
|
+
*/
|
|
56
|
+
var TestChatError = class extends Data.TaggedError("TestChatError") {};
|
|
57
|
+
const libraryEvents = (entries) => entries.flatMap(({ event }) => isLibraryEvent(event) ? [event] : []);
|
|
58
|
+
/**
|
|
59
|
+
* Reads what the assistant did in these entries.
|
|
60
|
+
*
|
|
61
|
+
* @category constructors
|
|
62
|
+
* @since 0.9.0
|
|
63
|
+
*/
|
|
64
|
+
const log = (entries) => {
|
|
65
|
+
const events = libraryEvents(entries);
|
|
66
|
+
const results = events.flatMap((event) => Predicate.isTagged(event, "ToolResult") ? [event] : []);
|
|
67
|
+
const approvals = events.flatMap((event) => Predicate.isTagged(event, "ApprovalRequested") ? [event] : []);
|
|
68
|
+
return {
|
|
69
|
+
entries,
|
|
70
|
+
toolsRun: [...new Set(results.filter(({ outcome, tool }) => outcome === "success" && !ownTools.has(tool)).map(({ tool }) => tool))],
|
|
71
|
+
toolCalls: (tool) => results.filter((result) => result.tool === tool).length,
|
|
72
|
+
blocks: events.flatMap((event) => Predicate.isTagged(event, "AssistantMessage") && event.block !== void 0 ? [event.block.kind] : []),
|
|
73
|
+
approvals,
|
|
74
|
+
waitingFor: approvals.map(({ tool }) => tool),
|
|
75
|
+
messages: assistantTexts(entries)
|
|
76
|
+
};
|
|
77
|
+
};
|
|
78
|
+
const handled = (entries, inputId) => {
|
|
79
|
+
const input = entries.find(({ event }) => isInput(event) && event.id === inputId);
|
|
80
|
+
return input !== void 0 && libraryEvents(entries).some((event) => isTurnEnd(event) && event.throughSequence >= input.sequence);
|
|
81
|
+
};
|
|
82
|
+
/**
|
|
83
|
+
* Creates a chat to drive the assistant with, for a chat the app already created. A step fails
|
|
84
|
+
* when no turn has handled its input within `timeout` (3 minutes by default), or when a turn failed
|
|
85
|
+
* while handling it.
|
|
86
|
+
*
|
|
87
|
+
* @category constructors
|
|
88
|
+
* @since 0.9.0
|
|
89
|
+
*/
|
|
90
|
+
const chat = (assistant, chatId, options) => at(assistant, chatId, options?.timeout ?? "3 minutes", [], []);
|
|
91
|
+
const at = (assistant, chatId, timeout, entries, step) => {
|
|
92
|
+
const all = log(entries);
|
|
93
|
+
const send = (input) => Effect.gen(function* () {
|
|
94
|
+
const before = entries.at(-1)?.sequence ?? 0;
|
|
95
|
+
let seen = [];
|
|
96
|
+
yield* assistant.submit(chatId, input).pipe(Effect.orDie);
|
|
97
|
+
const after = yield* assistant.subscribe(chatId).pipe(Stream.scan(() => [], (acc, entry) => [...acc, entry]), Stream.tap((acc) => Effect.sync(() => seen = acc)), Stream.takeUntil((acc) => handled(acc, input.id)), Stream.runLast, Effect.map(Option.getOrElse(() => [])), Effect.timeoutOrElse({
|
|
98
|
+
duration: timeout,
|
|
99
|
+
orElse: () => Effect.fail(new TestChatError({
|
|
100
|
+
reason: "TimedOut",
|
|
101
|
+
message: `no turn handled the input; the assistant said: ${log(seen).messages.join(" | ")}`
|
|
102
|
+
}))
|
|
103
|
+
}));
|
|
104
|
+
const added = after.filter(({ sequence }) => sequence > before);
|
|
105
|
+
const failed = libraryEvents(added).flatMap((event) => Predicate.isTagged(event, "TurnFailed") ? [event.message] : []);
|
|
106
|
+
if (failed.length > 0) return yield* new TestChatError({
|
|
107
|
+
reason: "TurnFailed",
|
|
108
|
+
message: failed.join(" | ")
|
|
109
|
+
});
|
|
110
|
+
return at(assistant, chatId, timeout, after, added);
|
|
111
|
+
});
|
|
112
|
+
const decide = (decision) => {
|
|
113
|
+
const waiting = all.approvals.at(-1);
|
|
114
|
+
return waiting === void 0 ? Effect.fail(new TestChatError({
|
|
115
|
+
reason: "NothingToApprove",
|
|
116
|
+
message: "no call waits for approval"
|
|
117
|
+
})) : send(decision({ id: waiting.callId }));
|
|
118
|
+
};
|
|
119
|
+
const answer = (block, value) => {
|
|
120
|
+
const asked = libraryEvents(entries).findLast((event) => Predicate.isTagged(event, "AssistantMessage") && event.block?.kind === block.kind);
|
|
121
|
+
if (asked === void 0 || block.answer === void 0) return Effect.fail(new TestChatError({
|
|
122
|
+
reason: "NothingToAnswer",
|
|
123
|
+
message: `no ${block.kind} block to answer`
|
|
124
|
+
}));
|
|
125
|
+
return send(BlockAnswer.make({
|
|
126
|
+
id: fromKey(`${asked.id}:answer`),
|
|
127
|
+
messageId: asked.id,
|
|
128
|
+
kind: block.kind,
|
|
129
|
+
value: Schema.encodeUnknownSync(Schema.toCodecJson(block.answer))(value)
|
|
130
|
+
}));
|
|
131
|
+
};
|
|
132
|
+
return {
|
|
133
|
+
...all,
|
|
134
|
+
chatId,
|
|
135
|
+
step: log(step),
|
|
136
|
+
send,
|
|
137
|
+
say: (text) => send(UserMessage.make({ text })),
|
|
138
|
+
approve: decide(approve),
|
|
139
|
+
reject: (options) => decide((approval) => reject(approval, options)),
|
|
140
|
+
answer
|
|
141
|
+
};
|
|
142
|
+
};
|
|
143
|
+
//#endregion
|
|
144
|
+
export { AssistantTest_exports, TestChatError, chat, log };
|
|
145
|
+
|
|
146
|
+
//# sourceMappingURL=AssistantTest.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AssistantTest.js","names":["AssistantEvent.isLibraryEvent","AssistantEvent.assistantTexts","AssistantEvent.isInput","AssistantEvent.isTurnEnd","InputId.fromKey","ToolApproval.approve","ToolApproval.reject"],"sources":["../src/AssistantTest.ts"],"sourcesContent":["/**\n * This module provides a chat to drive an assistant like a user would, for tests and evals.\n *\n * {@link chat} wraps a chat the app created. Each step (`say`, `approve`, `reject`, `answer`, or\n * `send` for any input) submits the input, waits until a turn has handled it, and returns the chat\n * as it is then. A {@link Log} reads what the assistant did, for the whole chat or for the last\n * step alone (`step`): the tools that ran, the blocks it showed, the calls waiting for approval,\n * and the messages it sent.\n *\n * **Example** (An assistant asks before it emails)\n *\n * ```ts\n * import { AssistantTest } from \"effect-assistant\"\n * import * as Effect from \"effect/Effect\"\n *\n * const program = Effect.gen(function* () {\n * const chat = AssistantTest.chat(yield* MyAssistant, chatId)\n * const asked = yield* chat.say(\"Email the report to ana@example.com\")\n * asked.step.waitingFor // [\"send_report\"]\n * const sent = yield* asked.approve\n * sent.step.toolsRun // [\"send_report\"]\n * })\n * ```\n *\n * @since 0.9.0\n */\nimport * as Data from \"effect/Data\"\nimport type * as Duration from \"effect/Duration\"\nimport * as Effect from \"effect/Effect\"\nimport * as Option from \"effect/Option\"\nimport * as Predicate from \"effect/Predicate\"\nimport * as Schema from \"effect/Schema\"\nimport * as Stream from \"effect/Stream\"\nimport type * as Assistant from \"./Assistant.ts\"\nimport type * as Block from \"./Block.ts\"\nimport * as AssistantEvent from \"./AssistantEvent.ts\"\nimport type { ChatId } from \"./ChatId.ts\"\nimport * as InputId from \"./InputId.ts\"\nimport { sendUserMessageName, waitToolName } from \"./internal/tools.ts\"\nimport * as ToolApproval from \"./ToolApproval.ts\"\n\n// The assistant's own tools: how it talks, waits and reads its skills, not work it does for the user.\nconst ownTools: ReadonlySet<string> = new Set([sendUserMessageName, waitToolName, \"load_skill\"])\n\n/**\n * Represents a step that couldn't complete: no turn handled the input in time, the turn failed,\n * or there was nothing to approve or answer.\n *\n * @category errors\n * @since 0.9.0\n */\nexport class TestChatError extends Data.TaggedError(\"TestChatError\")<{\n readonly reason: \"TimedOut\" | \"TurnFailed\" | \"NothingToApprove\" | \"NothingToAnswer\"\n readonly message: string\n}> {}\n\nconst libraryEvents = (entries: ReadonlyArray<AssistantEvent.AnyEntry>): ReadonlyArray<AssistantEvent.AssistantEvent> =>\n entries.flatMap(({ event }) => (AssistantEvent.isLibraryEvent(event) ? [event] : []))\n\n/**\n * Represents what the assistant did in some of a chat's entries.\n *\n * @category models\n * @since 0.9.0\n */\nexport interface Log {\n readonly entries: ReadonlyArray<AssistantEvent.AnyEntry>\n /**\n * The app's tools that ran, each once, in the order they first ran. A call that failed or that\n * the library refused didn't run, and the assistant's own tools (`send_user_message`, `wait`,\n * `load_skill`) aren't listed.\n */\n readonly toolsRun: ReadonlyArray<string>\n /** How many times the model called this tool, whether or not it ran. */\n readonly toolCalls: (tool: string) => number\n /** The kind of each block the assistant showed, in order; one per message. */\n readonly blocks: ReadonlyArray<string>\n /** The calls waiting for the user's approval, in order. */\n readonly approvals: ReadonlyArray<AssistantEvent.ApprovalRequested>\n /** The tools of the calls waiting for the user's approval. */\n readonly waitingFor: ReadonlyArray<string>\n /** The messages the assistant sent, in order. */\n readonly messages: ReadonlyArray<string>\n}\n\n/**\n * Reads what the assistant did in these entries.\n *\n * @category constructors\n * @since 0.9.0\n */\nexport const log = (entries: ReadonlyArray<AssistantEvent.AnyEntry>): Log => {\n const events = libraryEvents(entries)\n const results = events.flatMap((event) => (Predicate.isTagged(event, \"ToolResult\") ? [event] : []))\n const approvals = events.flatMap((event) => (Predicate.isTagged(event, \"ApprovalRequested\") ? [event] : []))\n\n return {\n entries,\n toolsRun: [\n ...new Set(\n results.filter(({ outcome, tool }) => outcome === \"success\" && !ownTools.has(tool)).map(({ tool }) => tool)\n )\n ],\n toolCalls: (tool) => results.filter((result) => result.tool === tool).length,\n blocks: events.flatMap((event) =>\n Predicate.isTagged(event, \"AssistantMessage\") && event.block !== undefined ? [event.block.kind] : []\n ),\n approvals,\n waitingFor: approvals.map(({ tool }) => tool),\n messages: AssistantEvent.assistantTexts(entries)\n }\n}\n\n/**\n * Represents a chat driven like a user would.\n *\n * @category models\n * @since 0.9.0\n */\nexport interface TestChat extends Log {\n readonly chatId: ChatId\n /** What the assistant did for the last step alone. */\n readonly step: Log\n /** Submits an input, and returns the chat once a turn has handled it. */\n readonly send: (input: AssistantEvent.Input) => Effect.Effect<TestChat, TestChatError>\n /** Sends the user's message. */\n readonly say: (text: string) => Effect.Effect<TestChat, TestChatError>\n /** Approves the last call waiting for approval. */\n readonly approve: Effect.Effect<TestChat, TestChatError>\n /** Rejects the last call waiting for approval. */\n readonly reject: (options?: { readonly reason?: string | undefined }) => Effect.Effect<TestChat, TestChatError>\n /** Answers the last block of this kind the assistant showed. */\n readonly answer: <Answer extends Block.Codec>(\n block: Block.Block<string, Block.Codec, Answer>,\n answer: Answer[\"Type\"]\n ) => Effect.Effect<TestChat, TestChatError>\n}\n\nconst handled = (entries: ReadonlyArray<AssistantEvent.AnyEntry>, inputId: string): boolean => {\n const input = entries.find(({ event }) => AssistantEvent.isInput(event) && event.id === inputId)\n\n return (\n input !== undefined &&\n libraryEvents(entries).some((event) => AssistantEvent.isTurnEnd(event) && event.throughSequence >= input.sequence)\n )\n}\n\n/**\n * Creates a chat to drive the assistant with, for a chat the app already created. A step fails\n * when no turn has handled its input within `timeout` (3 minutes by default), or when a turn failed\n * while handling it.\n *\n * @category constructors\n * @since 0.9.0\n */\nexport const chat = <Events extends { readonly _tag: string } = never>(\n assistant: Assistant.Service<Events>,\n chatId: ChatId,\n options?: { readonly timeout?: Duration.Input | undefined }\n): TestChat => at(assistant, chatId, options?.timeout ?? \"3 minutes\", [], [])\n\nconst at = <Events extends { readonly _tag: string }>(\n assistant: Assistant.Service<Events>,\n chatId: ChatId,\n timeout: Duration.Input,\n entries: ReadonlyArray<AssistantEvent.AnyEntry>,\n step: ReadonlyArray<AssistantEvent.AnyEntry>\n): TestChat => {\n const all = log(entries)\n\n const send = (input: AssistantEvent.Input): Effect.Effect<TestChat, TestChatError> =>\n Effect.gen(function* () {\n const before = entries.at(-1)?.sequence ?? 0\n let seen: ReadonlyArray<AssistantEvent.AnyEntry> = []\n\n yield* assistant.submit(chatId, input).pipe(Effect.orDie)\n\n const after = yield* assistant.subscribe(chatId).pipe(\n Stream.scan(\n (): ReadonlyArray<AssistantEvent.AnyEntry> => [],\n (acc, entry) => [...acc, entry]\n ),\n Stream.tap((acc) => Effect.sync(() => (seen = acc))),\n Stream.takeUntil((acc) => handled(acc, input.id)),\n Stream.runLast,\n Effect.map(Option.getOrElse((): ReadonlyArray<AssistantEvent.AnyEntry> => [])),\n Effect.timeoutOrElse({\n duration: timeout,\n orElse: () =>\n Effect.fail(\n new TestChatError({\n reason: \"TimedOut\",\n message: `no turn handled the input; the assistant said: ${log(seen).messages.join(\" | \")}`\n })\n )\n })\n )\n\n const added = after.filter(({ sequence }) => sequence > before)\n\n const failed = libraryEvents(added).flatMap((event) =>\n Predicate.isTagged(event, \"TurnFailed\") ? [event.message] : []\n )\n\n if (failed.length > 0) {\n return yield* new TestChatError({ reason: \"TurnFailed\", message: failed.join(\" | \") })\n }\n\n return at(assistant, chatId, timeout, after, added)\n })\n\n const decide = (decision: (approval: { readonly id: string }) => AssistantEvent.ApprovalDecision) => {\n const waiting = all.approvals.at(-1)\n\n return waiting === undefined\n ? Effect.fail(new TestChatError({ reason: \"NothingToApprove\", message: \"no call waits for approval\" }))\n : send(decision({ id: waiting.callId }))\n }\n\n const answer = <Answer extends Block.Codec>(\n block: Block.Block<string, Block.Codec, Answer>,\n value: Answer[\"Type\"]\n ): Effect.Effect<TestChat, TestChatError> => {\n const asked = libraryEvents(entries).findLast(\n (event): event is AssistantEvent.AssistantMessage =>\n Predicate.isTagged(event, \"AssistantMessage\") && event.block?.kind === block.kind\n )\n\n if (asked === undefined || block.answer === undefined) {\n return Effect.fail(new TestChatError({ reason: \"NothingToAnswer\", message: `no ${block.kind} block to answer` }))\n }\n\n return send(\n AssistantEvent.BlockAnswer.make({\n // As the client sends it: the same id every time, so a block takes one answer.\n id: InputId.fromKey(`${asked.id}:answer`),\n messageId: asked.id,\n kind: block.kind,\n value: Schema.encodeUnknownSync(Schema.toCodecJson(block.answer))(value)\n })\n )\n }\n\n return {\n ...all,\n chatId,\n step: log(step),\n send,\n say: (text) => send(AssistantEvent.UserMessage.make({ text })),\n approve: decide(ToolApproval.approve),\n reject: (options) => decide((approval) => ToolApproval.reject(approval, options)),\n answer\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0CA,MAAM,2BAAgC,IAAI,IAAI;CAAC;CAAqB;CAAc;AAAY,CAAC;;;;;;;;AAS/F,IAAa,gBAAb,cAAmC,KAAK,YAAY,eAAe,CAAC,CAGjE,CAAC;AAEJ,MAAM,iBAAiB,YACrB,QAAQ,SAAS,EAAE,YAAaA,eAA8B,KAAK,IAAI,CAAC,KAAK,IAAI,CAAC,CAAE;;;;;;;AAkCtF,MAAa,OAAO,YAAyD;CAC3E,MAAM,SAAS,cAAc,OAAO;CACpC,MAAM,UAAU,OAAO,SAAS,UAAW,UAAU,SAAS,OAAO,YAAY,IAAI,CAAC,KAAK,IAAI,CAAC,CAAE;CAClG,MAAM,YAAY,OAAO,SAAS,UAAW,UAAU,SAAS,OAAO,mBAAmB,IAAI,CAAC,KAAK,IAAI,CAAC,CAAE;CAE3G,OAAO;EACL;EACA,UAAU,CACR,GAAG,IAAI,IACL,QAAQ,QAAQ,EAAE,SAAS,WAAW,YAAY,aAAa,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,EAAE,WAAW,IAAI,CAC5G,CACF;EACA,YAAY,SAAS,QAAQ,QAAQ,WAAW,OAAO,SAAS,IAAI,CAAC,CAAC;EACtE,QAAQ,OAAO,SAAS,UACtB,UAAU,SAAS,OAAO,kBAAkB,KAAK,MAAM,UAAU,KAAA,IAAY,CAAC,MAAM,MAAM,IAAI,IAAI,CAAC,CACrG;EACA;EACA,YAAY,UAAU,KAAK,EAAE,WAAW,IAAI;EAC5C,UAAUC,eAA8B,OAAO;CACjD;AACF;AA2BA,MAAM,WAAW,SAAiD,YAA6B;CAC7F,MAAM,QAAQ,QAAQ,MAAM,EAAE,YAAYC,QAAuB,KAAK,KAAK,MAAM,OAAO,OAAO;CAE/F,OACE,UAAU,KAAA,KACV,cAAc,OAAO,CAAC,CAAC,MAAM,UAAUC,UAAyB,KAAK,KAAK,MAAM,mBAAmB,MAAM,QAAQ;AAErH;;;;;;;;;AAUA,MAAa,QACX,WACA,QACA,YACa,GAAG,WAAW,QAAQ,SAAS,WAAW,aAAa,CAAC,GAAG,CAAC,CAAC;AAE5E,MAAM,MACJ,WACA,QACA,SACA,SACA,SACa;CACb,MAAM,MAAM,IAAI,OAAO;CAEvB,MAAM,QAAQ,UACZ,OAAO,IAAI,aAAa;EACtB,MAAM,SAAS,QAAQ,GAAG,EAAE,CAAC,EAAE,YAAY;EAC3C,IAAI,OAA+C,CAAC;EAEpD,OAAO,UAAU,OAAO,QAAQ,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK;EAExD,MAAM,QAAQ,OAAO,UAAU,UAAU,MAAM,CAAC,CAAC,KAC/C,OAAO,WACyC,CAAC,IAC9C,KAAK,UAAU,CAAC,GAAG,KAAK,KAAK,CAChC,GACA,OAAO,KAAK,QAAQ,OAAO,WAAY,OAAO,GAAI,CAAC,GACnD,OAAO,WAAW,QAAQ,QAAQ,KAAK,MAAM,EAAE,CAAC,GAChD,OAAO,SACP,OAAO,IAAI,OAAO,gBAAwD,CAAC,CAAC,CAAC,GAC7E,OAAO,cAAc;GACnB,UAAU;GACV,cACE,OAAO,KACL,IAAI,cAAc;IAChB,QAAQ;IACR,SAAS,kDAAkD,IAAI,IAAI,CAAC,CAAC,SAAS,KAAK,KAAK;GAC1F,CAAC,CACH;EACJ,CAAC,CACH;EAEA,MAAM,QAAQ,MAAM,QAAQ,EAAE,eAAe,WAAW,MAAM;EAE9D,MAAM,SAAS,cAAc,KAAK,CAAC,CAAC,SAAS,UAC3C,UAAU,SAAS,OAAO,YAAY,IAAI,CAAC,MAAM,OAAO,IAAI,CAAC,CAC/D;EAEA,IAAI,OAAO,SAAS,GAClB,OAAO,OAAO,IAAI,cAAc;GAAE,QAAQ;GAAc,SAAS,OAAO,KAAK,KAAK;EAAE,CAAC;EAGvF,OAAO,GAAG,WAAW,QAAQ,SAAS,OAAO,KAAK;CACpD,CAAC;CAEH,MAAM,UAAU,aAAqF;EACnG,MAAM,UAAU,IAAI,UAAU,GAAG,EAAE;EAEnC,OAAO,YAAY,KAAA,IACf,OAAO,KAAK,IAAI,cAAc;GAAE,QAAQ;GAAoB,SAAS;EAA6B,CAAC,CAAC,IACpG,KAAK,SAAS,EAAE,IAAI,QAAQ,OAAO,CAAC,CAAC;CAC3C;CAEA,MAAM,UACJ,OACA,UAC2C;EAC3C,MAAM,QAAQ,cAAc,OAAO,CAAC,CAAC,UAClC,UACC,UAAU,SAAS,OAAO,kBAAkB,KAAK,MAAM,OAAO,SAAS,MAAM,IACjF;EAEA,IAAI,UAAU,KAAA,KAAa,MAAM,WAAW,KAAA,GAC1C,OAAO,OAAO,KAAK,IAAI,cAAc;GAAE,QAAQ;GAAmB,SAAS,MAAM,MAAM,KAAK;EAAkB,CAAC,CAAC;EAGlH,OAAO,KAAA,YACsB,KAAK;GAE9B,IAAIC,QAAgB,GAAG,MAAM,GAAG,QAAQ;GACxC,WAAW,MAAM;GACjB,MAAM,MAAM;GACZ,OAAO,OAAO,kBAAkB,OAAO,YAAY,MAAM,MAAM,CAAC,CAAC,CAAC,KAAK;EACzE,CAAC,CACH;CACF;CAEA,OAAO;EACL,GAAG;EACH;EACA,MAAM,IAAI,IAAI;EACd;EACA,MAAM,SAAS,KAAA,YAAgC,KAAK,EAAE,KAAK,CAAC,CAAC;EAC7D,SAAS,OAAOC,OAAoB;EACpC,SAAS,YAAY,QAAQ,aAAaC,OAAoB,UAAU,OAAO,CAAC;EAChF;CACF;AACF"}
|
package/dist/index.d.ts
CHANGED
|
@@ -14,6 +14,7 @@ import { AssistantEngine_d_exports } from "./AssistantEngine.js";
|
|
|
14
14
|
import { Chats_d_exports } from "./Chats.js";
|
|
15
15
|
import { Assistant_d_exports } from "./Assistant.js";
|
|
16
16
|
import { AssistantActivity_d_exports } from "./AssistantActivity.js";
|
|
17
|
+
import { AssistantTest_d_exports } from "./AssistantTest.js";
|
|
17
18
|
import { Compaction_d_exports } from "./Compaction.js";
|
|
18
19
|
import { InputId_d_exports } from "./InputId.js";
|
|
19
20
|
import { ToolFailure_d_exports } from "./ToolFailure.js";
|
|
@@ -22,4 +23,4 @@ import { Skill_d_exports } from "./Skill.js";
|
|
|
22
23
|
import { ToolApproval_d_exports } from "./ToolApproval.js";
|
|
23
24
|
import { ToolDiscovery_d_exports } from "./ToolDiscovery.js";
|
|
24
25
|
import { ToolResult_d_exports } from "./ToolResult.js";
|
|
25
|
-
export { Assistant_d_exports as Assistant, AssistantActivity_d_exports as AssistantActivity, AssistantEngine_d_exports as AssistantEngine, AssistantEvent_d_exports as AssistantEvent, AssistantLoop_d_exports as AssistantLoop, AssistantToolCall_d_exports as AssistantToolCall, Block_d_exports as Block, BlockKit_d_exports as BlockKit, ChatId_d_exports as ChatId, ChatLog_d_exports as ChatLog, Chats_d_exports as Chats, Compaction_d_exports as Compaction, CurrentChat_d_exports as CurrentChat, InputId_d_exports as InputId, Instructions_d_exports as Instructions, Memory_d_exports as Memory, RunId_d_exports as RunId, Skill_d_exports as Skill, Specialist_d_exports as Specialist, Timeline_d_exports as Timeline, ToolApproval_d_exports as ToolApproval, ToolDiscovery_d_exports as ToolDiscovery, ToolFailure_d_exports as ToolFailure, ToolResult_d_exports as ToolResult };
|
|
26
|
+
export { Assistant_d_exports as Assistant, AssistantActivity_d_exports as AssistantActivity, AssistantEngine_d_exports as AssistantEngine, AssistantEvent_d_exports as AssistantEvent, AssistantLoop_d_exports as AssistantLoop, AssistantTest_d_exports as AssistantTest, AssistantToolCall_d_exports as AssistantToolCall, Block_d_exports as Block, BlockKit_d_exports as BlockKit, ChatId_d_exports as ChatId, ChatLog_d_exports as ChatLog, Chats_d_exports as Chats, Compaction_d_exports as Compaction, CurrentChat_d_exports as CurrentChat, InputId_d_exports as InputId, Instructions_d_exports as Instructions, Memory_d_exports as Memory, RunId_d_exports as RunId, Skill_d_exports as Skill, Specialist_d_exports as Specialist, Timeline_d_exports as Timeline, ToolApproval_d_exports as ToolApproval, ToolDiscovery_d_exports as ToolDiscovery, ToolFailure_d_exports as ToolFailure, ToolResult_d_exports as ToolResult };
|
package/dist/index.js
CHANGED
|
@@ -17,9 +17,10 @@ import { AssistantEngine_exports } from "./AssistantEngine.js";
|
|
|
17
17
|
import { Assistant_exports } from "./Assistant.js";
|
|
18
18
|
import { AssistantActivity_exports } from "./AssistantActivity.js";
|
|
19
19
|
import { AssistantLoop_exports } from "./AssistantLoop.js";
|
|
20
|
+
import { AssistantTest_exports } from "./AssistantTest.js";
|
|
20
21
|
import { Compaction_exports } from "./Compaction.js";
|
|
21
22
|
import { Memory_exports } from "./Memory.js";
|
|
22
23
|
import { Skill_exports } from "./Skill.js";
|
|
23
24
|
import { Timeline_exports } from "./Timeline.js";
|
|
24
25
|
import { ToolDiscovery_exports } from "./ToolDiscovery.js";
|
|
25
|
-
export { Assistant_exports as Assistant, AssistantActivity_exports as AssistantActivity, AssistantEngine_exports as AssistantEngine, AssistantEvent_exports as AssistantEvent, AssistantLoop_exports as AssistantLoop, AssistantToolCall_exports as AssistantToolCall, Block_exports as Block, BlockKit_exports as BlockKit, ChatId_exports as ChatId, ChatLog_exports as ChatLog, Chats_exports as Chats, Compaction_exports as Compaction, CurrentChat_exports as CurrentChat, InputId_exports as InputId, Instructions_exports as Instructions, Memory_exports as Memory, RunId_exports as RunId, Skill_exports as Skill, Specialist_exports as Specialist, Timeline_exports as Timeline, ToolApproval_exports as ToolApproval, ToolDiscovery_exports as ToolDiscovery, ToolFailure_exports as ToolFailure, ToolResult_exports as ToolResult };
|
|
26
|
+
export { Assistant_exports as Assistant, AssistantActivity_exports as AssistantActivity, AssistantEngine_exports as AssistantEngine, AssistantEvent_exports as AssistantEvent, AssistantLoop_exports as AssistantLoop, AssistantTest_exports as AssistantTest, AssistantToolCall_exports as AssistantToolCall, Block_exports as Block, BlockKit_exports as BlockKit, ChatId_exports as ChatId, ChatLog_exports as ChatLog, Chats_exports as Chats, Compaction_exports as Compaction, CurrentChat_exports as CurrentChat, InputId_exports as InputId, Instructions_exports as Instructions, Memory_exports as Memory, RunId_exports as RunId, Skill_exports as Skill, Specialist_exports as Specialist, Timeline_exports as Timeline, ToolApproval_exports as ToolApproval, ToolDiscovery_exports as ToolDiscovery, ToolFailure_exports as ToolFailure, ToolResult_exports as ToolResult };
|
package/dist/internal/loop.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { ApprovalRequested, ModelOutput, ModelPart, ToolResult, ToolStarted } from "../AssistantEvent.js";
|
|
2
2
|
import { unansweredCalls } from "./history.js";
|
|
3
|
-
import {
|
|
3
|
+
import { annotateModelCall, toolOutput, toolSpan } from "./telemetry.js";
|
|
4
4
|
import { empty, failed } from "./toolkit.js";
|
|
5
5
|
import { isTurnEnding, isUserMessage } from "./tools.js";
|
|
6
6
|
import * as Context from "effect/Context";
|
|
@@ -129,7 +129,7 @@ const step = (options, then) => {
|
|
|
129
129
|
const prompt = Prompt.fromMessages([Prompt.systemMessage({ content: instructions.text }), ...history]);
|
|
130
130
|
const name = yield* Effect.serviceOption(Model.ModelName);
|
|
131
131
|
const started = yield* Clock.currentTimeMillis;
|
|
132
|
-
const response = yield* generate(prompt, toolkit.definitions, options.toolChoice).pipe(
|
|
132
|
+
const response = yield* generate(prompt, toolkit.definitions, options.toolChoice).pipe(annotateModelCall(instructions.attributes));
|
|
133
133
|
const metadata = response.content.find((part) => part.type === "response-metadata");
|
|
134
134
|
const output = ModelOutput.make({
|
|
135
135
|
parts: modelParts(response.content),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"loop.js","names":["AssistantEvent.ModelPart","Telemetry.toolOutput","Telemetry.toolSpan","Toolkits.failed","Toolkits.empty","History.unansweredCalls","Telemetry.chatSpan"],"sources":["../../src/internal/loop.ts"],"sourcesContent":["import type * as AiError from \"effect/ai/AiError\"\nimport * as LanguageModel from \"effect/ai/LanguageModel\"\nimport * as Model from \"effect/ai/Model\"\nimport * as Prompt from \"effect/ai/Prompt\"\nimport type * as Response from \"effect/ai/Response\"\nimport type * as Tool from \"effect/ai/Tool\"\nimport type * as Toolkit from \"effect/ai/Toolkit\"\nimport * as Clock from \"effect/Clock\"\nimport * as Context from \"effect/Context\"\nimport * as Data from \"effect/Data\"\nimport * as Effect from \"effect/Effect\"\nimport * as Option from \"effect/Option\"\nimport * as Predicate from \"effect/Predicate\"\nimport * as Schema from \"effect/Schema\"\nimport * as Stream from \"effect/Stream\"\nimport * as AssistantEvent from \"../AssistantEvent.ts\"\nimport type * as Instructions from \"../Instructions.ts\"\nimport * as History from \"./history.ts\"\nimport * as Telemetry from \"./telemetry.ts\"\nimport * as Toolkits from \"./toolkit.ts\"\nimport { isTurnEnding, isUserMessage } from \"./tools.ts\"\n\n/** @internal */\nexport type Event =\n | AssistantEvent.ModelOutput\n | AssistantEvent.ToolStarted\n | AssistantEvent.ToolResult\n | AssistantEvent.ApprovalRequested\n\n// Only a turn's loop can wait for the user's approval, and runs the calls they approved. A turn's specialists inherit\n// its context, so their runs set it back to none.\n/** @internal */\nexport const TurnApprovals = Context.Reference<\n Option.Option<{ readonly approved: Effect.Effect<ReadonlyArray<AssistantEvent.ToolCallPart>> }>\n>(\"effect-assistant/TurnApprovals\", { defaultValue: Option.none })\n\n// The assistant's own tools with the turn's services, so an app's own loop in a turn can message the user. A\n// specialist's run has none.\n/** @internal */\nexport const TurnTools = Context.Reference<Option.Option<Toolkits.Tools>>(\"effect-assistant/TurnTools\", {\n defaultValue: Option.none\n})\n\n/** @internal */\nexport interface ChatHistory {\n readonly messages: ReadonlyArray<Prompt.Message>\n readonly throughSequence: number\n}\n\nconst readHistory = (read: ReadonlyArray<Prompt.Message> | ChatHistory) =>\n \"messages\" in read\n ? { messages: read.messages, throughSequence: read.throughSequence }\n : { messages: read, throughSequence: undefined }\n\n// `AssistantLoop` takes native toolkits; turns and specialist runs come here with their tools already merged.\n/** @internal */\nexport interface StepOptions<R = never> {\n readonly instructions: string | Instructions.Resolved\n readonly toolkit?: Toolkits.Tools<R> | undefined\n readonly history: Effect.Effect<ReadonlyArray<Prompt.Message> | ChatHistory>\n readonly toolChoice?: LanguageModel.ToolChoice<string> | undefined\n}\n\n/** @internal */\nexport interface Options<R = never> extends StepOptions<R> {\n readonly maxSteps?: number | undefined\n}\n\n// Its member docs show through `AssistantLoop.StepResult`.\n/** @internal */\nexport interface StepResult {\n /** The model's output; unset when the step only ran calls a crash cut off. */\n readonly output: AssistantEvent.ModelOutput | undefined\n /**\n * Whether the model is done: it made no tool calls, one of its calls waits for the user's\n * approval, or it ended its turn (`wait`, or a message with `wait: true`) and none of the step's\n * calls failed.\n */\n readonly done: boolean\n /**\n * The history's length with this step's output and results: a longer one got something new meanwhile.\n */\n readonly historyLength: number\n}\n\n// The loop runs the calls itself, after storing each as started, so the model call runs none. It also decides which\n// calls wait for approval: the model call would ask for approvals of its own.\nconst requestToolkit = (definitions: ReadonlyArray<Tool.Any>): Toolkit.WithHandler<Record<string, Tool.Any>> => ({\n tools: Object.fromEntries(\n definitions.map((tool) => [\n tool.name,\n tool.needsApproval === undefined || tool.needsApproval === false ? tool : tool.setNeedsApproval(false)\n ])\n ),\n handle: () => Effect.succeed(Stream.empty)\n})\n\n// Tool call resolution stays on with a toolkit that runs nothing: with it off, a call whose params don't fit its\n// tool's schema fails the whole model call, and here the model reads why instead.\nconst generate = (\n prompt: Prompt.Prompt,\n definitions: ReadonlyArray<Tool.Any>,\n toolChoice: LanguageModel.ToolChoice<string> | undefined\n): Effect.Effect<\n LanguageModel.GenerateTextResponse<Record<string, Tool.Any>, \"opaque\">,\n AiError.AiError,\n LanguageModel.LanguageModel\n> => LanguageModel.generateText({ prompt, toolkit: requestToolkit(definitions), toolChoice })\n\nconst decodePart = Schema.decodeUnknownOption(AssistantEvent.ModelPart)\n\nconst modelParts = (content: ReadonlyArray<Response.AllParts<Record<string, Tool.Any>>>) =>\n content.flatMap((part) =>\n Option.toArray(decodePart(part)).filter((decoded) => !(decoded.type === \"tool-call\" && decoded.providerExecuted))\n )\n\nconst toolCalls = (parts: ReadonlyArray<AssistantEvent.ModelPart>): ReadonlyArray<AssistantEvent.ToolCallPart> =>\n parts.flatMap((part) => (part.type === \"tool-call\" ? [part] : []))\n\nconst runCall = Effect.fnUntraced(function* <R>(tools: Toolkits.Tools<R>, call: AssistantEvent.ToolCallPart) {\n const result = yield* Effect.gen(function* () {\n const result = yield* tools.run(call)\n\n yield* Telemetry.toolOutput(result.result)\n\n return result\n }).pipe(Telemetry.toolSpan(call))\n\n return AssistantEvent.ToolResult.make({ callId: call.id, tool: call.name, ...result })\n})\n\n// A call that needs approval waits for the user in a turn. A specialist's run can't ask them, so its call fails.\nconst awaitApproval = Effect.fnUntraced(function* (call: AssistantEvent.ToolCallPart) {\n if (Option.isNone(yield* TurnApprovals)) {\n return AssistantEvent.ToolResult.make({\n callId: call.id,\n tool: call.name,\n ...Toolkits.failed(\n `${call.name} needs the user's approval, and only the assistant can ask for it. Don't run it: say in your answer what needs approving, with the details.`,\n \"NeedsApproval\"\n )\n })\n }\n\n return AssistantEvent.ApprovalRequested.make({ callId: call.id, tool: call.name, params: call.params })\n})\n\ninterface Pending {\n readonly call: AssistantEvent.ToolCallPart\n /** The user approved it: it runs without asking again. */\n readonly approved: boolean\n}\n\ninterface Ran {\n /** One of the calls failed, so the model has to read the results. */\n readonly failed: boolean\n /** One of the calls waits for the user's approval, so the turn ends. */\n readonly waiting: boolean\n}\n\n// The approval shows the call, not what it does: the user learns that from the assistant's message beside it.\nconst unexplained = (call: AssistantEvent.ToolCallPart) =>\n AssistantEvent.ToolResult.make({\n callId: call.id,\n tool: call.name,\n ...Toolkits.failed(\n `Not sent for approval: the user would see buttons without knowing what they approve. In one step, send them a message with send_user_message saying what ${call.name} will do (what, to whom, how many), and call ${call.name} again.`,\n \"ApprovalNeedsMessage\"\n )\n })\n\n// Every `ToolStarted` is stored before any call runs; a call that waits for approval doesn't start.\nconst runCalls = <R, S, E, R2>(\n tools: Toolkits.Tools<R>,\n pending: ReadonlyArray<Pending>,\n history: ReadonlyArray<Prompt.Message>,\n then: (ran: Ran) => Stream.Stream<Step<never, S>, E, R2>\n): Stream.Stream<Step<Event, S>, E, R | R2> =>\n Stream.unwrap(\n Effect.gen(function* () {\n const waits = yield* Effect.forEach(pending, ({ call, approved }) =>\n approved ? Effect.succeed(false) : tools.needsApproval(call, history)\n )\n\n const started = pending.flatMap(({ call }, i) =>\n waits[i] === true ? [] : [value(AssistantEvent.ToolStarted.make({ callId: call.id, tool: call.name }))]\n )\n\n return Stream.fromIterable(started).pipe(\n Stream.concat(\n Stream.unwrap(\n Effect.gen(function* () {\n const explained = pending.some(({ call }) => isUserMessage(call))\n\n const done = yield* Effect.forEach(\n pending,\n ({ call }, i): Effect.Effect<Event, never, R> =>\n waits[i] !== true\n ? runCall(tools, call)\n : explained\n ? awaitApproval(call)\n : Effect.succeed(unexplained(call)),\n { concurrency: \"unbounded\" }\n )\n\n return Stream.fromIterable(done.map(value)).pipe(\n Stream.concat(\n then({\n failed: done.some(isFailure),\n waiting: done.some((event) => Predicate.isTagged(event, \"ApprovalRequested\"))\n })\n )\n )\n })\n )\n )\n )\n })\n )\n\nconst isFailure = (event: Event) => Predicate.isTagged(event, \"ToolResult\") && event.outcome === \"failure\"\n\n/** @internal */\nexport type Step<A, S> = Data.TaggedEnum<{\n Value: { readonly value: A }\n Next: { readonly state: S }\n Stop: Record<never, never>\n}>\n\ninterface StepDefinition extends Data.TaggedEnum.WithGenerics<2> {\n readonly taggedEnum: Step<this[\"A\"], this[\"B\"]>\n}\n\nconst Steps = Data.taggedEnum<StepDefinition>()\n\n/** @internal */\nexport const value = <A>(value: A): Step<A, never> => Steps.Value({ value })\n\n/** @internal */\nexport const next = <S>(state: S): Stream.Stream<Step<never, S>> => Stream.succeed(Steps.Next({ state }))\n\n/** @internal */\n// oxlint-disable-next-line no-unnecessary-type-parameters -- `stop<number>()` types a loop's state where inference can't.\nexport const stop = <S = never>(): Stream.Stream<Step<never, S>> => Stream.succeed(Steps.Stop())\n\nconst isValue = <A, S>(step: Step<A, S>): step is Data.TaggedEnum.Value<Step<A, S>, \"Value\"> =>\n Predicate.isTagged(step, \"Value\")\n\n// A body that ends without `next` or `stop` stops too.\n/** @internal */\nexport const loop = <S, A, E, R>(\n initial: S,\n body: (state: S) => Stream.Stream<Step<A, S>, E, R>\n): Stream.Stream<A, E, R> => {\n const go = (state: S): Stream.Stream<A, E, R> =>\n Stream.suspend(() => {\n let after: Option.Option<S> = Option.none()\n\n return body(state).pipe(\n Stream.tap((step) =>\n Effect.sync(() => (after = Predicate.isTagged(step, \"Next\") ? Option.some(step.state) : after))\n ),\n Stream.filter(isValue),\n Stream.map((step) => step.value),\n Stream.concat(Stream.suspend(() => Option.match(after, { onNone: () => Stream.empty, onSome: go })))\n )\n })\n\n return go(initial)\n}\n\n/** @internal */\nexport const step = <R, S, E2 = never, R2 = never>(\n options: StepOptions<R>,\n then: (result: StepResult) => Stream.Stream<Step<never, S>, E2, R2>\n): Stream.Stream<Step<Event, S>, AiError.AiError | E2, LanguageModel.LanguageModel | R | R2> => {\n const toolkit = options.toolkit ?? Toolkits.empty\n const instructions = Predicate.isString(options.instructions) ? { text: options.instructions } : options.instructions\n\n const after = (result: StepResult) => Stream.suspend(() => then(result))\n\n return Stream.unwrap(\n Effect.gen(function* () {\n const { messages: history, throughSequence } = readHistory(yield* options.history)\n // Calls a crash cut off, with no result: they run again, with the same ids, before the model goes on.\n const cutOff = History.unansweredCalls(history)\n const approvals = yield* TurnApprovals\n\n // Calls the user approved: they run as the model made them, with the same ids, before the model goes on. Not in\n // a loop without their tools.\n const approved = Option.isNone(approvals)\n ? []\n : (yield* approvals.value.approved).filter(\n (call) =>\n toolkit.definitions.some((tool) => tool.name === call.name) && !cutOff.some((cut) => cut.id === call.id)\n )\n\n if (cutOff.length > 0 || approved.length > 0) {\n const pending = [\n ...cutOff.map((call) => ({ call, approved: false })),\n ...approved.map((call) => ({ call, approved: true }))\n ]\n\n return runCalls(toolkit, pending, history, ({ waiting }) =>\n after({ output: undefined, done: waiting, historyLength: history.length + pending.length })\n )\n }\n\n const prompt = Prompt.fromMessages([Prompt.systemMessage({ content: instructions.text }), ...history])\n const name = yield* Effect.serviceOption(Model.ModelName)\n const started = yield* Clock.currentTimeMillis\n\n const response = yield* generate(prompt, toolkit.definitions, options.toolChoice).pipe(\n Telemetry.chatSpan(prompt, instructions.attributes)\n )\n\n const metadata = response.content.find((part) => part.type === \"response-metadata\")\n\n const output = AssistantEvent.ModelOutput.make({\n parts: modelParts(response.content),\n seenThroughSequence: throughSequence,\n model: metadata?.modelId ?? Option.getOrUndefined(name),\n responseId: metadata?.id,\n usage: response.usage,\n durationMillis: (yield* Clock.currentTimeMillis) - started\n })\n\n const calls = toolCalls(output.parts)\n const historyLength = history.length + Prompt.fromResponseParts(output.parts).content.length + calls.length\n\n // A failed call doesn't end the turn: a message that failed never reached the user, so the model reads why. A call\n // that waits for approval does: the turn waits for the user.\n return Stream.make(value(output)).pipe(\n Stream.concat(\n runCalls(\n toolkit,\n calls.map((call) => ({ call, approved: false })),\n history,\n ({ failed, waiting }) =>\n after({\n output,\n done: calls.length === 0 || waiting || (!failed && calls.some(isTurnEnding)),\n historyLength\n })\n )\n )\n )\n })\n )\n}\n\n/** @internal */\nexport const run = <R = never>(\n options: Options<R>\n): Stream.Stream<Event, AiError.AiError, LanguageModel.LanguageModel | R> => {\n const maxSteps = options.maxSteps ?? 16\n\n // Done: stop, unless something arrived while the model was working; then it reads that first.\n const afterDone = (steps: number, historyLength: number) =>\n Stream.unwrap(\n Effect.gen(function* () {\n const read = yield* options.history\n\n return readHistory(read).messages.length > historyLength ? next(steps) : stop<number>()\n })\n )\n\n return loop(0, (steps: number) =>\n step(options, (result) => {\n const taken = result.output === undefined ? steps : steps + 1\n\n if (taken >= maxSteps) return stop<number>()\n\n return result.done ? afterDone(taken, result.historyLength) : next(taken)\n })\n )\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAgCA,MAAa,gBAAgB,QAAQ,UAEnC,kCAAkC,EAAE,cAAc,OAAO,KAAK,CAAC;;AAKjE,MAAa,YAAY,QAAQ,UAAyC,8BAA8B,EACtG,cAAc,OAAO,KACvB,CAAC;AAQD,MAAM,eAAe,SACnB,cAAc,OACV;CAAE,UAAU,KAAK;CAAU,iBAAiB,KAAK;AAAgB,IACjE;CAAE,UAAU;CAAM,iBAAiB,KAAA;AAAU;AAmCnD,MAAM,kBAAkB,iBAAyF;CAC/G,OAAO,OAAO,YACZ,YAAY,KAAK,SAAS,CACxB,KAAK,MACL,KAAK,kBAAkB,KAAA,KAAa,KAAK,kBAAkB,QAAQ,OAAO,KAAK,iBAAiB,KAAK,CACvG,CAAC,CACH;CACA,cAAc,OAAO,QAAQ,OAAO,KAAK;AAC3C;AAIA,MAAM,YACJ,QACA,aACA,eAKG,cAAc,aAAa;CAAE;CAAQ,SAAS,eAAe,WAAW;CAAG;AAAW,CAAC;AAE5F,MAAM,aAAa,OAAO,oBAAoBA,SAAwB;AAEtE,MAAM,cAAc,YAClB,QAAQ,SAAS,SACf,OAAO,QAAQ,WAAW,IAAI,CAAC,CAAC,CAAC,QAAQ,YAAY,EAAE,QAAQ,SAAS,eAAe,QAAQ,iBAAiB,CAClH;AAEF,MAAM,aAAa,UACjB,MAAM,SAAS,SAAU,KAAK,SAAS,cAAc,CAAC,IAAI,IAAI,CAAC,CAAE;AAEnE,MAAM,UAAU,OAAO,WAAW,WAAc,OAA0B,MAAmC;CAC3G,MAAM,SAAS,OAAO,OAAO,IAAI,aAAa;EAC5C,MAAM,SAAS,OAAO,MAAM,IAAI,IAAI;EAEpC,OAAOC,WAAqB,OAAO,MAAM;EAEzC,OAAO;CACT,CAAC,CAAC,CAAC,KAAKC,SAAmB,IAAI,CAAC;CAEhC,OAAA,WAAiC,KAAK;EAAE,QAAQ,KAAK;EAAI,MAAM,KAAK;EAAM,GAAG;CAAO,CAAC;AACvF,CAAC;AAGD,MAAM,gBAAgB,OAAO,WAAW,WAAW,MAAmC;CACpF,IAAI,OAAO,OAAO,OAAO,aAAa,GACpC,OAAA,WAAiC,KAAK;EACpC,QAAQ,KAAK;EACb,MAAM,KAAK;EACX,GAAGC,OACD,GAAG,KAAK,KAAK,8IACb,eACF;CACF,CAAC;CAGH,OAAA,kBAAwC,KAAK;EAAE,QAAQ,KAAK;EAAI,MAAM,KAAK;EAAM,QAAQ,KAAK;CAAO,CAAC;AACxG,CAAC;AAgBD,MAAM,eAAe,SAAA,WACO,KAAK;CAC7B,QAAQ,KAAK;CACb,MAAM,KAAK;CACX,GAAGA,OACD,4JAA4J,KAAK,KAAK,+CAA+C,KAAK,KAAK,UAC/N,sBACF;AACF,CAAC;AAGH,MAAM,YACJ,OACA,SACA,SACA,SAEA,OAAO,OACL,OAAO,IAAI,aAAa;CACtB,MAAM,QAAQ,OAAO,OAAO,QAAQ,UAAU,EAAE,MAAM,eACpD,WAAW,OAAO,QAAQ,KAAK,IAAI,MAAM,cAAc,MAAM,OAAO,CACtE;CAEA,MAAM,UAAU,QAAQ,SAAS,EAAE,QAAQ,MACzC,MAAM,OAAO,OAAO,CAAC,IAAI,CAAC,MAAA,YAAiC,KAAK;EAAE,QAAQ,KAAK;EAAI,MAAM,KAAK;CAAK,CAAC,CAAC,CAAC,CACxG;CAEA,OAAO,OAAO,aAAa,OAAO,CAAC,CAAC,KAClC,OAAO,OACL,OAAO,OACL,OAAO,IAAI,aAAa;EACtB,MAAM,YAAY,QAAQ,MAAM,EAAE,WAAW,cAAc,IAAI,CAAC;EAEhE,MAAM,OAAO,OAAO,OAAO,QACzB,UACC,EAAE,QAAQ,MACT,MAAM,OAAO,OACT,QAAQ,OAAO,IAAI,IACnB,YACE,cAAc,IAAI,IAClB,OAAO,QAAQ,YAAY,IAAI,CAAC,GACxC,EAAE,aAAa,YAAY,CAC7B;EAEA,OAAO,OAAO,aAAa,KAAK,IAAI,KAAK,CAAC,CAAC,CAAC,KAC1C,OAAO,OACL,KAAK;GACH,QAAQ,KAAK,KAAK,SAAS;GAC3B,SAAS,KAAK,MAAM,UAAU,UAAU,SAAS,OAAO,mBAAmB,CAAC;EAC9E,CAAC,CACH,CACF;CACF,CAAC,CACH,CACF,CACF;AACF,CAAC,CACH;AAEF,MAAM,aAAa,UAAiB,UAAU,SAAS,OAAO,YAAY,KAAK,MAAM,YAAY;AAajG,MAAM,QAAQ,KAAK,WAA2B;;AAG9C,MAAa,SAAY,UAA6B,MAAM,MAAM,EAAE,MAAM,CAAC;;AAG3E,MAAa,QAAW,UAA4C,OAAO,QAAQ,MAAM,KAAK,EAAE,MAAM,CAAC,CAAC;;AAIxG,MAAa,aAAuD,OAAO,QAAQ,MAAM,KAAK,CAAC;AAE/F,MAAM,WAAiB,SACrB,UAAU,SAAS,MAAM,OAAO;;AAIlC,MAAa,QACX,SACA,SAC2B;CAC3B,MAAM,MAAM,UACV,OAAO,cAAc;EACnB,IAAI,QAA0B,OAAO,KAAK;EAE1C,OAAO,KAAK,KAAK,CAAC,CAAC,KACjB,OAAO,KAAK,SACV,OAAO,WAAY,QAAQ,UAAU,SAAS,MAAM,MAAM,IAAI,OAAO,KAAK,KAAK,KAAK,IAAI,KAAM,CAChG,GACA,OAAO,OAAO,OAAO,GACrB,OAAO,KAAK,SAAS,KAAK,KAAK,GAC/B,OAAO,OAAO,OAAO,cAAc,OAAO,MAAM,OAAO;GAAE,cAAc,OAAO;GAAO,QAAQ;EAAG,CAAC,CAAC,CAAC,CACrG;CACF,CAAC;CAEH,OAAO,GAAG,OAAO;AACnB;;AAGA,MAAa,QACX,SACA,SAC8F;CAC9F,MAAM,UAAU,QAAQ,WAAWC;CACnC,MAAM,eAAe,UAAU,SAAS,QAAQ,YAAY,IAAI,EAAE,MAAM,QAAQ,aAAa,IAAI,QAAQ;CAEzG,MAAM,SAAS,WAAuB,OAAO,cAAc,KAAK,MAAM,CAAC;CAEvE,OAAO,OAAO,OACZ,OAAO,IAAI,aAAa;EACtB,MAAM,EAAE,UAAU,SAAS,oBAAoB,YAAY,OAAO,QAAQ,OAAO;EAEjF,MAAM,SAASC,gBAAwB,OAAO;EAC9C,MAAM,YAAY,OAAO;EAIzB,MAAM,WAAW,OAAO,OAAO,SAAS,IACpC,CAAC,KACA,OAAO,UAAU,MAAM,SAAA,CAAU,QAC/B,SACC,QAAQ,YAAY,MAAM,SAAS,KAAK,SAAS,KAAK,IAAI,KAAK,CAAC,OAAO,MAAM,QAAQ,IAAI,OAAO,KAAK,EAAE,CAC3G;EAEJ,IAAI,OAAO,SAAS,KAAK,SAAS,SAAS,GAAG;GAC5C,MAAM,UAAU,CACd,GAAG,OAAO,KAAK,UAAU;IAAE;IAAM,UAAU;GAAM,EAAE,GACnD,GAAG,SAAS,KAAK,UAAU;IAAE;IAAM,UAAU;GAAK,EAAE,CACtD;GAEA,OAAO,SAAS,SAAS,SAAS,UAAU,EAAE,cAC5C,MAAM;IAAE,QAAQ,KAAA;IAAW,MAAM;IAAS,eAAe,QAAQ,SAAS,QAAQ;GAAO,CAAC,CAC5F;EACF;EAEA,MAAM,SAAS,OAAO,aAAa,CAAC,OAAO,cAAc,EAAE,SAAS,aAAa,KAAK,CAAC,GAAG,GAAG,OAAO,CAAC;EACrG,MAAM,OAAO,OAAO,OAAO,cAAc,MAAM,SAAS;EACxD,MAAM,UAAU,OAAO,MAAM;EAE7B,MAAM,WAAW,OAAO,SAAS,QAAQ,QAAQ,aAAa,QAAQ,UAAU,CAAC,CAAC,KAChFC,SAAmB,QAAQ,aAAa,UAAU,CACpD;EAEA,MAAM,WAAW,SAAS,QAAQ,MAAM,SAAS,KAAK,SAAS,mBAAmB;EAElF,MAAM,SAAA,YAAoC,KAAK;GAC7C,OAAO,WAAW,SAAS,OAAO;GAClC,qBAAqB;GACrB,OAAO,UAAU,WAAW,OAAO,eAAe,IAAI;GACtD,YAAY,UAAU;GACtB,OAAO,SAAS;GAChB,iBAAiB,OAAO,MAAM,qBAAqB;EACrD,CAAC;EAED,MAAM,QAAQ,UAAU,OAAO,KAAK;EACpC,MAAM,gBAAgB,QAAQ,SAAS,OAAO,kBAAkB,OAAO,KAAK,CAAC,CAAC,QAAQ,SAAS,MAAM;EAIrG,OAAO,OAAO,KAAK,MAAM,MAAM,CAAC,CAAC,CAAC,KAChC,OAAO,OACL,SACE,SACA,MAAM,KAAK,UAAU;GAAE;GAAM,UAAU;EAAM,EAAE,GAC/C,UACC,EAAE,QAAQ,cACT,MAAM;GACJ;GACA,MAAM,MAAM,WAAW,KAAK,WAAY,CAAC,UAAU,MAAM,KAAK,YAAY;GAC1E;EACF,CAAC,CACL,CACF,CACF;CACF,CAAC,CACH;AACF;;AAGA,MAAa,OACX,YAC2E;CAC3E,MAAM,WAAW,QAAQ,YAAY;CAGrC,MAAM,aAAa,OAAe,kBAChC,OAAO,OACL,OAAO,IAAI,aAAa;EACtB,MAAM,OAAO,OAAO,QAAQ;EAE5B,OAAO,YAAY,IAAI,CAAC,CAAC,SAAS,SAAS,gBAAgB,KAAK,KAAK,IAAI,KAAa;CACxF,CAAC,CACH;CAEF,OAAO,KAAK,IAAI,UACd,KAAK,UAAU,WAAW;EACxB,MAAM,QAAQ,OAAO,WAAW,KAAA,IAAY,QAAQ,QAAQ;EAE5D,IAAI,SAAS,UAAU,OAAO,KAAa;EAE3C,OAAO,OAAO,OAAO,UAAU,OAAO,OAAO,aAAa,IAAI,KAAK,KAAK;CAC1E,CAAC,CACH;AACF"}
|
|
1
|
+
{"version":3,"file":"loop.js","names":["AssistantEvent.ModelPart","Telemetry.toolOutput","Telemetry.toolSpan","Toolkits.failed","Toolkits.empty","History.unansweredCalls","Telemetry.annotateModelCall"],"sources":["../../src/internal/loop.ts"],"sourcesContent":["import type * as AiError from \"effect/ai/AiError\"\nimport * as LanguageModel from \"effect/ai/LanguageModel\"\nimport * as Model from \"effect/ai/Model\"\nimport * as Prompt from \"effect/ai/Prompt\"\nimport type * as Response from \"effect/ai/Response\"\nimport type * as Tool from \"effect/ai/Tool\"\nimport type * as Toolkit from \"effect/ai/Toolkit\"\nimport * as Clock from \"effect/Clock\"\nimport * as Context from \"effect/Context\"\nimport * as Data from \"effect/Data\"\nimport * as Effect from \"effect/Effect\"\nimport * as Option from \"effect/Option\"\nimport * as Predicate from \"effect/Predicate\"\nimport * as Schema from \"effect/Schema\"\nimport * as Stream from \"effect/Stream\"\nimport * as AssistantEvent from \"../AssistantEvent.ts\"\nimport type * as Instructions from \"../Instructions.ts\"\nimport * as History from \"./history.ts\"\nimport * as Telemetry from \"./telemetry.ts\"\nimport * as Toolkits from \"./toolkit.ts\"\nimport { isTurnEnding, isUserMessage } from \"./tools.ts\"\n\n/** @internal */\nexport type Event =\n | AssistantEvent.ModelOutput\n | AssistantEvent.ToolStarted\n | AssistantEvent.ToolResult\n | AssistantEvent.ApprovalRequested\n\n// Only a turn's loop can wait for the user's approval, and runs the calls they approved. A turn's specialists inherit\n// its context, so their runs set it back to none.\n/** @internal */\nexport const TurnApprovals = Context.Reference<\n Option.Option<{ readonly approved: Effect.Effect<ReadonlyArray<AssistantEvent.ToolCallPart>> }>\n>(\"effect-assistant/TurnApprovals\", { defaultValue: Option.none })\n\n// The assistant's own tools with the turn's services, so an app's own loop in a turn can message the user. A\n// specialist's run has none.\n/** @internal */\nexport const TurnTools = Context.Reference<Option.Option<Toolkits.Tools>>(\"effect-assistant/TurnTools\", {\n defaultValue: Option.none\n})\n\n/** @internal */\nexport interface ChatHistory {\n readonly messages: ReadonlyArray<Prompt.Message>\n readonly throughSequence: number\n}\n\nconst readHistory = (read: ReadonlyArray<Prompt.Message> | ChatHistory) =>\n \"messages\" in read\n ? { messages: read.messages, throughSequence: read.throughSequence }\n : { messages: read, throughSequence: undefined }\n\n// `AssistantLoop` takes native toolkits; turns and specialist runs come here with their tools already merged.\n/** @internal */\nexport interface StepOptions<R = never> {\n readonly instructions: string | Instructions.Resolved\n readonly toolkit?: Toolkits.Tools<R> | undefined\n readonly history: Effect.Effect<ReadonlyArray<Prompt.Message> | ChatHistory>\n readonly toolChoice?: LanguageModel.ToolChoice<string> | undefined\n}\n\n/** @internal */\nexport interface Options<R = never> extends StepOptions<R> {\n readonly maxSteps?: number | undefined\n}\n\n// Its member docs show through `AssistantLoop.StepResult`.\n/** @internal */\nexport interface StepResult {\n /** The model's output; unset when the step only ran calls a crash cut off. */\n readonly output: AssistantEvent.ModelOutput | undefined\n /**\n * Whether the model is done: it made no tool calls, one of its calls waits for the user's\n * approval, or it ended its turn (`wait`, or a message with `wait: true`) and none of the step's\n * calls failed.\n */\n readonly done: boolean\n /**\n * The history's length with this step's output and results: a longer one got something new meanwhile.\n */\n readonly historyLength: number\n}\n\n// The loop runs the calls itself, after storing each as started, so the model call runs none. It also decides which\n// calls wait for approval: the model call would ask for approvals of its own.\nconst requestToolkit = (definitions: ReadonlyArray<Tool.Any>): Toolkit.WithHandler<Record<string, Tool.Any>> => ({\n tools: Object.fromEntries(\n definitions.map((tool) => [\n tool.name,\n tool.needsApproval === undefined || tool.needsApproval === false ? tool : tool.setNeedsApproval(false)\n ])\n ),\n handle: () => Effect.succeed(Stream.empty)\n})\n\n// Tool call resolution stays on with a toolkit that runs nothing: with it off, a call whose params don't fit its\n// tool's schema fails the whole model call, and here the model reads why instead.\nconst generate = (\n prompt: Prompt.Prompt,\n definitions: ReadonlyArray<Tool.Any>,\n toolChoice: LanguageModel.ToolChoice<string> | undefined\n): Effect.Effect<\n LanguageModel.GenerateTextResponse<Record<string, Tool.Any>, \"opaque\">,\n AiError.AiError,\n LanguageModel.LanguageModel\n> => LanguageModel.generateText({ prompt, toolkit: requestToolkit(definitions), toolChoice })\n\nconst decodePart = Schema.decodeUnknownOption(AssistantEvent.ModelPart)\n\nconst modelParts = (content: ReadonlyArray<Response.AllParts<Record<string, Tool.Any>>>) =>\n content.flatMap((part) =>\n Option.toArray(decodePart(part)).filter((decoded) => !(decoded.type === \"tool-call\" && decoded.providerExecuted))\n )\n\nconst toolCalls = (parts: ReadonlyArray<AssistantEvent.ModelPart>): ReadonlyArray<AssistantEvent.ToolCallPart> =>\n parts.flatMap((part) => (part.type === \"tool-call\" ? [part] : []))\n\nconst runCall = Effect.fnUntraced(function* <R>(tools: Toolkits.Tools<R>, call: AssistantEvent.ToolCallPart) {\n const result = yield* Effect.gen(function* () {\n const result = yield* tools.run(call)\n\n yield* Telemetry.toolOutput(result.result)\n\n return result\n }).pipe(Telemetry.toolSpan(call))\n\n return AssistantEvent.ToolResult.make({ callId: call.id, tool: call.name, ...result })\n})\n\n// A call that needs approval waits for the user in a turn. A specialist's run can't ask them, so its call fails.\nconst awaitApproval = Effect.fnUntraced(function* (call: AssistantEvent.ToolCallPart) {\n if (Option.isNone(yield* TurnApprovals)) {\n return AssistantEvent.ToolResult.make({\n callId: call.id,\n tool: call.name,\n ...Toolkits.failed(\n `${call.name} needs the user's approval, and only the assistant can ask for it. Don't run it: say in your answer what needs approving, with the details.`,\n \"NeedsApproval\"\n )\n })\n }\n\n return AssistantEvent.ApprovalRequested.make({ callId: call.id, tool: call.name, params: call.params })\n})\n\ninterface Pending {\n readonly call: AssistantEvent.ToolCallPart\n /** The user approved it: it runs without asking again. */\n readonly approved: boolean\n}\n\ninterface Ran {\n /** One of the calls failed, so the model has to read the results. */\n readonly failed: boolean\n /** One of the calls waits for the user's approval, so the turn ends. */\n readonly waiting: boolean\n}\n\n// The approval shows the call, not what it does: the user learns that from the assistant's message beside it.\nconst unexplained = (call: AssistantEvent.ToolCallPart) =>\n AssistantEvent.ToolResult.make({\n callId: call.id,\n tool: call.name,\n ...Toolkits.failed(\n `Not sent for approval: the user would see buttons without knowing what they approve. In one step, send them a message with send_user_message saying what ${call.name} will do (what, to whom, how many), and call ${call.name} again.`,\n \"ApprovalNeedsMessage\"\n )\n })\n\n// Every `ToolStarted` is stored before any call runs; a call that waits for approval doesn't start.\nconst runCalls = <R, S, E, R2>(\n tools: Toolkits.Tools<R>,\n pending: ReadonlyArray<Pending>,\n history: ReadonlyArray<Prompt.Message>,\n then: (ran: Ran) => Stream.Stream<Step<never, S>, E, R2>\n): Stream.Stream<Step<Event, S>, E, R | R2> =>\n Stream.unwrap(\n Effect.gen(function* () {\n const waits = yield* Effect.forEach(pending, ({ call, approved }) =>\n approved ? Effect.succeed(false) : tools.needsApproval(call, history)\n )\n\n const started = pending.flatMap(({ call }, i) =>\n waits[i] === true ? [] : [value(AssistantEvent.ToolStarted.make({ callId: call.id, tool: call.name }))]\n )\n\n return Stream.fromIterable(started).pipe(\n Stream.concat(\n Stream.unwrap(\n Effect.gen(function* () {\n const explained = pending.some(({ call }) => isUserMessage(call))\n\n const done = yield* Effect.forEach(\n pending,\n ({ call }, i): Effect.Effect<Event, never, R> =>\n waits[i] !== true\n ? runCall(tools, call)\n : explained\n ? awaitApproval(call)\n : Effect.succeed(unexplained(call)),\n { concurrency: \"unbounded\" }\n )\n\n return Stream.fromIterable(done.map(value)).pipe(\n Stream.concat(\n then({\n failed: done.some(isFailure),\n waiting: done.some((event) => Predicate.isTagged(event, \"ApprovalRequested\"))\n })\n )\n )\n })\n )\n )\n )\n })\n )\n\nconst isFailure = (event: Event) => Predicate.isTagged(event, \"ToolResult\") && event.outcome === \"failure\"\n\n/** @internal */\nexport type Step<A, S> = Data.TaggedEnum<{\n Value: { readonly value: A }\n Next: { readonly state: S }\n Stop: Record<never, never>\n}>\n\ninterface StepDefinition extends Data.TaggedEnum.WithGenerics<2> {\n readonly taggedEnum: Step<this[\"A\"], this[\"B\"]>\n}\n\nconst Steps = Data.taggedEnum<StepDefinition>()\n\n/** @internal */\nexport const value = <A>(value: A): Step<A, never> => Steps.Value({ value })\n\n/** @internal */\nexport const next = <S>(state: S): Stream.Stream<Step<never, S>> => Stream.succeed(Steps.Next({ state }))\n\n/** @internal */\n// oxlint-disable-next-line no-unnecessary-type-parameters -- `stop<number>()` types a loop's state where inference can't.\nexport const stop = <S = never>(): Stream.Stream<Step<never, S>> => Stream.succeed(Steps.Stop())\n\nconst isValue = <A, S>(step: Step<A, S>): step is Data.TaggedEnum.Value<Step<A, S>, \"Value\"> =>\n Predicate.isTagged(step, \"Value\")\n\n// A body that ends without `next` or `stop` stops too.\n/** @internal */\nexport const loop = <S, A, E, R>(\n initial: S,\n body: (state: S) => Stream.Stream<Step<A, S>, E, R>\n): Stream.Stream<A, E, R> => {\n const go = (state: S): Stream.Stream<A, E, R> =>\n Stream.suspend(() => {\n let after: Option.Option<S> = Option.none()\n\n return body(state).pipe(\n Stream.tap((step) =>\n Effect.sync(() => (after = Predicate.isTagged(step, \"Next\") ? Option.some(step.state) : after))\n ),\n Stream.filter(isValue),\n Stream.map((step) => step.value),\n Stream.concat(Stream.suspend(() => Option.match(after, { onNone: () => Stream.empty, onSome: go })))\n )\n })\n\n return go(initial)\n}\n\n/** @internal */\nexport const step = <R, S, E2 = never, R2 = never>(\n options: StepOptions<R>,\n then: (result: StepResult) => Stream.Stream<Step<never, S>, E2, R2>\n): Stream.Stream<Step<Event, S>, AiError.AiError | E2, LanguageModel.LanguageModel | R | R2> => {\n const toolkit = options.toolkit ?? Toolkits.empty\n const instructions = Predicate.isString(options.instructions) ? { text: options.instructions } : options.instructions\n\n const after = (result: StepResult) => Stream.suspend(() => then(result))\n\n return Stream.unwrap(\n Effect.gen(function* () {\n const { messages: history, throughSequence } = readHistory(yield* options.history)\n // Calls a crash cut off, with no result: they run again, with the same ids, before the model goes on.\n const cutOff = History.unansweredCalls(history)\n const approvals = yield* TurnApprovals\n\n // Calls the user approved: they run as the model made them, with the same ids, before the model goes on. Not in\n // a loop without their tools.\n const approved = Option.isNone(approvals)\n ? []\n : (yield* approvals.value.approved).filter(\n (call) =>\n toolkit.definitions.some((tool) => tool.name === call.name) && !cutOff.some((cut) => cut.id === call.id)\n )\n\n if (cutOff.length > 0 || approved.length > 0) {\n const pending = [\n ...cutOff.map((call) => ({ call, approved: false })),\n ...approved.map((call) => ({ call, approved: true }))\n ]\n\n return runCalls(toolkit, pending, history, ({ waiting }) =>\n after({ output: undefined, done: waiting, historyLength: history.length + pending.length })\n )\n }\n\n const prompt = Prompt.fromMessages([Prompt.systemMessage({ content: instructions.text }), ...history])\n const name = yield* Effect.serviceOption(Model.ModelName)\n const started = yield* Clock.currentTimeMillis\n\n const response = yield* generate(prompt, toolkit.definitions, options.toolChoice).pipe(\n Telemetry.annotateModelCall(instructions.attributes)\n )\n\n const metadata = response.content.find((part) => part.type === \"response-metadata\")\n\n const output = AssistantEvent.ModelOutput.make({\n parts: modelParts(response.content),\n seenThroughSequence: throughSequence,\n model: metadata?.modelId ?? Option.getOrUndefined(name),\n responseId: metadata?.id,\n usage: response.usage,\n durationMillis: (yield* Clock.currentTimeMillis) - started\n })\n\n const calls = toolCalls(output.parts)\n const historyLength = history.length + Prompt.fromResponseParts(output.parts).content.length + calls.length\n\n // A failed call doesn't end the turn: a message that failed never reached the user, so the model reads why. A call\n // that waits for approval does: the turn waits for the user.\n return Stream.make(value(output)).pipe(\n Stream.concat(\n runCalls(\n toolkit,\n calls.map((call) => ({ call, approved: false })),\n history,\n ({ failed, waiting }) =>\n after({\n output,\n done: calls.length === 0 || waiting || (!failed && calls.some(isTurnEnding)),\n historyLength\n })\n )\n )\n )\n })\n )\n}\n\n/** @internal */\nexport const run = <R = never>(\n options: Options<R>\n): Stream.Stream<Event, AiError.AiError, LanguageModel.LanguageModel | R> => {\n const maxSteps = options.maxSteps ?? 16\n\n // Done: stop, unless something arrived while the model was working; then it reads that first.\n const afterDone = (steps: number, historyLength: number) =>\n Stream.unwrap(\n Effect.gen(function* () {\n const read = yield* options.history\n\n return readHistory(read).messages.length > historyLength ? next(steps) : stop<number>()\n })\n )\n\n return loop(0, (steps: number) =>\n step(options, (result) => {\n const taken = result.output === undefined ? steps : steps + 1\n\n if (taken >= maxSteps) return stop<number>()\n\n return result.done ? afterDone(taken, result.historyLength) : next(taken)\n })\n )\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAgCA,MAAa,gBAAgB,QAAQ,UAEnC,kCAAkC,EAAE,cAAc,OAAO,KAAK,CAAC;;AAKjE,MAAa,YAAY,QAAQ,UAAyC,8BAA8B,EACtG,cAAc,OAAO,KACvB,CAAC;AAQD,MAAM,eAAe,SACnB,cAAc,OACV;CAAE,UAAU,KAAK;CAAU,iBAAiB,KAAK;AAAgB,IACjE;CAAE,UAAU;CAAM,iBAAiB,KAAA;AAAU;AAmCnD,MAAM,kBAAkB,iBAAyF;CAC/G,OAAO,OAAO,YACZ,YAAY,KAAK,SAAS,CACxB,KAAK,MACL,KAAK,kBAAkB,KAAA,KAAa,KAAK,kBAAkB,QAAQ,OAAO,KAAK,iBAAiB,KAAK,CACvG,CAAC,CACH;CACA,cAAc,OAAO,QAAQ,OAAO,KAAK;AAC3C;AAIA,MAAM,YACJ,QACA,aACA,eAKG,cAAc,aAAa;CAAE;CAAQ,SAAS,eAAe,WAAW;CAAG;AAAW,CAAC;AAE5F,MAAM,aAAa,OAAO,oBAAoBA,SAAwB;AAEtE,MAAM,cAAc,YAClB,QAAQ,SAAS,SACf,OAAO,QAAQ,WAAW,IAAI,CAAC,CAAC,CAAC,QAAQ,YAAY,EAAE,QAAQ,SAAS,eAAe,QAAQ,iBAAiB,CAClH;AAEF,MAAM,aAAa,UACjB,MAAM,SAAS,SAAU,KAAK,SAAS,cAAc,CAAC,IAAI,IAAI,CAAC,CAAE;AAEnE,MAAM,UAAU,OAAO,WAAW,WAAc,OAA0B,MAAmC;CAC3G,MAAM,SAAS,OAAO,OAAO,IAAI,aAAa;EAC5C,MAAM,SAAS,OAAO,MAAM,IAAI,IAAI;EAEpC,OAAOC,WAAqB,OAAO,MAAM;EAEzC,OAAO;CACT,CAAC,CAAC,CAAC,KAAKC,SAAmB,IAAI,CAAC;CAEhC,OAAA,WAAiC,KAAK;EAAE,QAAQ,KAAK;EAAI,MAAM,KAAK;EAAM,GAAG;CAAO,CAAC;AACvF,CAAC;AAGD,MAAM,gBAAgB,OAAO,WAAW,WAAW,MAAmC;CACpF,IAAI,OAAO,OAAO,OAAO,aAAa,GACpC,OAAA,WAAiC,KAAK;EACpC,QAAQ,KAAK;EACb,MAAM,KAAK;EACX,GAAGC,OACD,GAAG,KAAK,KAAK,8IACb,eACF;CACF,CAAC;CAGH,OAAA,kBAAwC,KAAK;EAAE,QAAQ,KAAK;EAAI,MAAM,KAAK;EAAM,QAAQ,KAAK;CAAO,CAAC;AACxG,CAAC;AAgBD,MAAM,eAAe,SAAA,WACO,KAAK;CAC7B,QAAQ,KAAK;CACb,MAAM,KAAK;CACX,GAAGA,OACD,4JAA4J,KAAK,KAAK,+CAA+C,KAAK,KAAK,UAC/N,sBACF;AACF,CAAC;AAGH,MAAM,YACJ,OACA,SACA,SACA,SAEA,OAAO,OACL,OAAO,IAAI,aAAa;CACtB,MAAM,QAAQ,OAAO,OAAO,QAAQ,UAAU,EAAE,MAAM,eACpD,WAAW,OAAO,QAAQ,KAAK,IAAI,MAAM,cAAc,MAAM,OAAO,CACtE;CAEA,MAAM,UAAU,QAAQ,SAAS,EAAE,QAAQ,MACzC,MAAM,OAAO,OAAO,CAAC,IAAI,CAAC,MAAA,YAAiC,KAAK;EAAE,QAAQ,KAAK;EAAI,MAAM,KAAK;CAAK,CAAC,CAAC,CAAC,CACxG;CAEA,OAAO,OAAO,aAAa,OAAO,CAAC,CAAC,KAClC,OAAO,OACL,OAAO,OACL,OAAO,IAAI,aAAa;EACtB,MAAM,YAAY,QAAQ,MAAM,EAAE,WAAW,cAAc,IAAI,CAAC;EAEhE,MAAM,OAAO,OAAO,OAAO,QACzB,UACC,EAAE,QAAQ,MACT,MAAM,OAAO,OACT,QAAQ,OAAO,IAAI,IACnB,YACE,cAAc,IAAI,IAClB,OAAO,QAAQ,YAAY,IAAI,CAAC,GACxC,EAAE,aAAa,YAAY,CAC7B;EAEA,OAAO,OAAO,aAAa,KAAK,IAAI,KAAK,CAAC,CAAC,CAAC,KAC1C,OAAO,OACL,KAAK;GACH,QAAQ,KAAK,KAAK,SAAS;GAC3B,SAAS,KAAK,MAAM,UAAU,UAAU,SAAS,OAAO,mBAAmB,CAAC;EAC9E,CAAC,CACH,CACF;CACF,CAAC,CACH,CACF,CACF;AACF,CAAC,CACH;AAEF,MAAM,aAAa,UAAiB,UAAU,SAAS,OAAO,YAAY,KAAK,MAAM,YAAY;AAajG,MAAM,QAAQ,KAAK,WAA2B;;AAG9C,MAAa,SAAY,UAA6B,MAAM,MAAM,EAAE,MAAM,CAAC;;AAG3E,MAAa,QAAW,UAA4C,OAAO,QAAQ,MAAM,KAAK,EAAE,MAAM,CAAC,CAAC;;AAIxG,MAAa,aAAuD,OAAO,QAAQ,MAAM,KAAK,CAAC;AAE/F,MAAM,WAAiB,SACrB,UAAU,SAAS,MAAM,OAAO;;AAIlC,MAAa,QACX,SACA,SAC2B;CAC3B,MAAM,MAAM,UACV,OAAO,cAAc;EACnB,IAAI,QAA0B,OAAO,KAAK;EAE1C,OAAO,KAAK,KAAK,CAAC,CAAC,KACjB,OAAO,KAAK,SACV,OAAO,WAAY,QAAQ,UAAU,SAAS,MAAM,MAAM,IAAI,OAAO,KAAK,KAAK,KAAK,IAAI,KAAM,CAChG,GACA,OAAO,OAAO,OAAO,GACrB,OAAO,KAAK,SAAS,KAAK,KAAK,GAC/B,OAAO,OAAO,OAAO,cAAc,OAAO,MAAM,OAAO;GAAE,cAAc,OAAO;GAAO,QAAQ;EAAG,CAAC,CAAC,CAAC,CACrG;CACF,CAAC;CAEH,OAAO,GAAG,OAAO;AACnB;;AAGA,MAAa,QACX,SACA,SAC8F;CAC9F,MAAM,UAAU,QAAQ,WAAWC;CACnC,MAAM,eAAe,UAAU,SAAS,QAAQ,YAAY,IAAI,EAAE,MAAM,QAAQ,aAAa,IAAI,QAAQ;CAEzG,MAAM,SAAS,WAAuB,OAAO,cAAc,KAAK,MAAM,CAAC;CAEvE,OAAO,OAAO,OACZ,OAAO,IAAI,aAAa;EACtB,MAAM,EAAE,UAAU,SAAS,oBAAoB,YAAY,OAAO,QAAQ,OAAO;EAEjF,MAAM,SAASC,gBAAwB,OAAO;EAC9C,MAAM,YAAY,OAAO;EAIzB,MAAM,WAAW,OAAO,OAAO,SAAS,IACpC,CAAC,KACA,OAAO,UAAU,MAAM,SAAA,CAAU,QAC/B,SACC,QAAQ,YAAY,MAAM,SAAS,KAAK,SAAS,KAAK,IAAI,KAAK,CAAC,OAAO,MAAM,QAAQ,IAAI,OAAO,KAAK,EAAE,CAC3G;EAEJ,IAAI,OAAO,SAAS,KAAK,SAAS,SAAS,GAAG;GAC5C,MAAM,UAAU,CACd,GAAG,OAAO,KAAK,UAAU;IAAE;IAAM,UAAU;GAAM,EAAE,GACnD,GAAG,SAAS,KAAK,UAAU;IAAE;IAAM,UAAU;GAAK,EAAE,CACtD;GAEA,OAAO,SAAS,SAAS,SAAS,UAAU,EAAE,cAC5C,MAAM;IAAE,QAAQ,KAAA;IAAW,MAAM;IAAS,eAAe,QAAQ,SAAS,QAAQ;GAAO,CAAC,CAC5F;EACF;EAEA,MAAM,SAAS,OAAO,aAAa,CAAC,OAAO,cAAc,EAAE,SAAS,aAAa,KAAK,CAAC,GAAG,GAAG,OAAO,CAAC;EACrG,MAAM,OAAO,OAAO,OAAO,cAAc,MAAM,SAAS;EACxD,MAAM,UAAU,OAAO,MAAM;EAE7B,MAAM,WAAW,OAAO,SAAS,QAAQ,QAAQ,aAAa,QAAQ,UAAU,CAAC,CAAC,KAChFC,kBAA4B,aAAa,UAAU,CACrD;EAEA,MAAM,WAAW,SAAS,QAAQ,MAAM,SAAS,KAAK,SAAS,mBAAmB;EAElF,MAAM,SAAA,YAAoC,KAAK;GAC7C,OAAO,WAAW,SAAS,OAAO;GAClC,qBAAqB;GACrB,OAAO,UAAU,WAAW,OAAO,eAAe,IAAI;GACtD,YAAY,UAAU;GACtB,OAAO,SAAS;GAChB,iBAAiB,OAAO,MAAM,qBAAqB;EACrD,CAAC;EAED,MAAM,QAAQ,UAAU,OAAO,KAAK;EACpC,MAAM,gBAAgB,QAAQ,SAAS,OAAO,kBAAkB,OAAO,KAAK,CAAC,CAAC,QAAQ,SAAS,MAAM;EAIrG,OAAO,OAAO,KAAK,MAAM,MAAM,CAAC,CAAC,CAAC,KAChC,OAAO,OACL,SACE,SACA,MAAM,KAAK,UAAU;GAAE;GAAM,UAAU;EAAM,EAAE,GAC/C,UACC,EAAE,QAAQ,cACT,MAAM;GACJ;GACA,MAAM,MAAM,WAAW,KAAK,WAAY,CAAC,UAAU,MAAM,KAAK,YAAY;GAC1E;EACF,CAAC,CACL,CACF,CACF;CACF,CAAC,CACH;AACF;;AAGA,MAAa,OACX,YAC2E;CAC3E,MAAM,WAAW,QAAQ,YAAY;CAGrC,MAAM,aAAa,OAAe,kBAChC,OAAO,OACL,OAAO,IAAI,aAAa;EACtB,MAAM,OAAO,OAAO,QAAQ;EAE5B,OAAO,YAAY,IAAI,CAAC,CAAC,SAAS,SAAS,gBAAgB,KAAK,KAAK,IAAI,KAAa;CACxF,CAAC,CACH;CAEF,OAAO,KAAK,IAAI,UACd,KAAK,UAAU,WAAW;EACxB,MAAM,QAAQ,OAAO,WAAW,KAAA,IAAY,QAAQ,QAAQ;EAE5D,IAAI,SAAS,UAAU,OAAO,KAAa;EAE3C,OAAO,OAAO,OAAO,UAAU,OAAO,OAAO,aAAa,IAAI,KAAK,KAAK;CAC1E,CAAC,CACH;AACF"}
|
|
@@ -32,39 +32,31 @@ const agentInput = (input) => Effect.annotateCurrentSpan("input.value", toJsonTe
|
|
|
32
32
|
const agentOutput = (output) => Effect.annotateCurrentSpan("output.value", toJsonText(output));
|
|
33
33
|
const encodePrompt = Schema.encodeOption(Prompt.Prompt);
|
|
34
34
|
const encodeParts = Schema.encodeOption(Schema.Array(ModelPart));
|
|
35
|
-
const
|
|
36
|
-
|
|
37
|
-
|
|
35
|
+
const completionOf = (response) => encodeParts(response.flatMap((part) => Option.toArray(Schema.decodeUnknownOption(ModelPart)(part))));
|
|
36
|
+
const annotateModelSpan = ({ span, prompt, response }, name, attributes) => {
|
|
37
|
+
span.attribute("gen_ai.prompt", toJsonText(Option.map(encodePrompt(prompt), (encoded) => encoded.content)));
|
|
38
|
+
span.attribute("gen_ai.completion", toJsonText(Option.getOrUndefined(completionOf(response))));
|
|
39
|
+
for (const [key, value] of Object.entries(attributes ?? {})) span.attribute(key, value);
|
|
40
|
+
const finish = response.find((part) => part.type === "finish");
|
|
38
41
|
Telemetry.addGenAIAnnotations(span, {
|
|
39
42
|
operation: { name: "chat" },
|
|
40
43
|
request: { model: Option.getOrUndefined(name) },
|
|
41
|
-
response: { finishReasons: [
|
|
44
|
+
response: { finishReasons: finish === void 0 ? void 0 : [finish.reason] },
|
|
42
45
|
usage: {
|
|
43
|
-
inputTokens: usage.inputTokens.total,
|
|
44
|
-
outputTokens: usage.outputTokens.total
|
|
46
|
+
inputTokens: finish?.usage.inputTokens.total,
|
|
47
|
+
outputTokens: finish?.usage.outputTokens.total
|
|
45
48
|
}
|
|
46
49
|
});
|
|
47
|
-
|
|
48
|
-
span.attribute("gen_ai.
|
|
49
|
-
|
|
50
|
-
if (usage.inputTokens.cacheWrite !== void 0) span.attribute("gen_ai.usage.cache_creation_input_tokens", usage.inputTokens.cacheWrite);
|
|
51
|
-
});
|
|
50
|
+
if (finish?.usage.inputTokens.cacheRead !== void 0) span.attribute("gen_ai.usage.cache_read_input_tokens", finish.usage.inputTokens.cacheRead);
|
|
51
|
+
if (finish?.usage.inputTokens.cacheWrite !== void 0) span.attribute("gen_ai.usage.cache_creation_input_tokens", finish.usage.inputTokens.cacheWrite);
|
|
52
|
+
};
|
|
52
53
|
/** @internal */
|
|
53
|
-
const
|
|
54
|
+
const annotateModelCall = (attributes) => (self) => Effect.gen(function* () {
|
|
54
55
|
const name = yield* Effect.serviceOption(Model.ModelName);
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
}).pipe(Effect.withSpan(Option.match(name, {
|
|
60
|
-
onNone: () => "chat",
|
|
61
|
-
onSome: (model) => `chat ${model}`
|
|
62
|
-
}), {
|
|
63
|
-
kind: "client",
|
|
64
|
-
attributes: {
|
|
65
|
-
"gen_ai.prompt": toJsonText(Option.map(encodePrompt(prompt), (encoded) => encoded.content)),
|
|
66
|
-
...attributes
|
|
67
|
-
}
|
|
56
|
+
const outer = yield* Effect.serviceOption(Telemetry.CurrentSpanTransformer);
|
|
57
|
+
return yield* self.pipe(Effect.provideService(Telemetry.CurrentSpanTransformer, (call) => {
|
|
58
|
+
if (Option.isSome(outer)) outer.value(call);
|
|
59
|
+
annotateModelSpan(call, name, attributes);
|
|
68
60
|
}));
|
|
69
61
|
});
|
|
70
62
|
/** @internal */
|
|
@@ -80,6 +72,6 @@ const toolSpan = (call) => Effect.withSpan(`execute_tool ${call.name}`, {
|
|
|
80
72
|
/** @internal */
|
|
81
73
|
const toolOutput = (result) => Effect.annotateCurrentSpan("output.value", Schema.is(Schema.String)(result) ? result : toJsonText(result));
|
|
82
74
|
//#endregion
|
|
83
|
-
export { agentInput, agentOutput, agentSpan,
|
|
75
|
+
export { agentInput, agentOutput, agentSpan, annotateModelCall, currentTraceId, toolOutput, toolSpan };
|
|
84
76
|
|
|
85
77
|
//# sourceMappingURL=telemetry.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"telemetry.js","names":["AssistantEvent.ModelPart"],"sources":["../../src/internal/telemetry.ts"],"sourcesContent":["import
|
|
1
|
+
{"version":3,"file":"telemetry.js","names":["AssistantEvent.ModelPart"],"sources":["../../src/internal/telemetry.ts"],"sourcesContent":["import * as Model from \"effect/ai/Model\"\nimport * as Prompt from \"effect/ai/Prompt\"\nimport * as Telemetry from \"effect/ai/Telemetry\"\nimport * as Effect from \"effect/Effect\"\nimport * as Option from \"effect/Option\"\nimport * as References from \"effect/References\"\nimport * as Schema from \"effect/Schema\"\nimport * as Tracer from \"effect/Tracer\"\nimport * as AssistantEvent from \"../AssistantEvent.ts\"\nimport type { SpanAttributes } from \"../Instructions.ts\"\nimport { toJsonText } from \"./schema.ts\"\n\ntype Traced = string | ReadonlyArray<{ readonly _tag: string }>\n\n// Spans follow the OpenTelemetry GenAI conventions, so LLM tools (Langfuse, Phoenix, Datadog LLM Observability) can\n// pick the agent work out of a trace.\n/** @internal */\nexport const agentSpan = (name: string, options: { readonly chatId: string; readonly root?: boolean }) =>\n Effect.withSpan(`invoke_agent ${name}`, {\n kind: \"internal\",\n root: options.root ?? false,\n attributes: {\n \"gen_ai.operation.name\": \"invoke_agent\",\n \"gen_ai.agent.name\": name,\n \"gen_ai.conversation.id\": options.chatId,\n \"session.id\": options.chatId\n }\n })\n\n// None when tracing is off, or when the tracer is Effect's own, which exports nothing.\n/** @internal */\nexport const currentTraceId: Effect.Effect<Option.Option<string>> = Effect.gen(function* () {\n const tracer = yield* Tracer.Tracer\n\n if (!(yield* References.TracerEnabled) || tracer === Tracer.nativeTracer) return Option.none()\n\n return Option.map(yield* Effect.option(Effect.currentSpan), (span) => span.traceId)\n})\n\n/** @internal */\nexport const agentInput = (input: Traced) => Effect.annotateCurrentSpan(\"input.value\", toJsonText(input))\n\n/** @internal */\nexport const agentOutput = (output: Traced) => Effect.annotateCurrentSpan(\"output.value\", toJsonText(output))\n\nconst encodePrompt = Schema.encodeOption(Prompt.Prompt)\n\nconst encodeParts = Schema.encodeOption(Schema.Array(AssistantEvent.ModelPart))\n\ntype ModelCall = Parameters<Telemetry.SpanTransformer>[0]\n\nconst completionOf = (response: ModelCall[\"response\"]) =>\n encodeParts(response.flatMap((part) => Option.toArray(Schema.decodeUnknownOption(AssistantEvent.ModelPart)(part))))\n\nconst annotateModelSpan = (\n { span, prompt, response }: ModelCall,\n name: Option.Option<string>,\n attributes: SpanAttributes | undefined\n) => {\n span.attribute(\"gen_ai.prompt\", toJsonText(Option.map(encodePrompt(prompt), (encoded) => encoded.content)))\n span.attribute(\"gen_ai.completion\", toJsonText(Option.getOrUndefined(completionOf(response))))\n\n for (const [key, value] of Object.entries(attributes ?? {})) {\n span.attribute(key, value)\n }\n\n const finish = response.find((part) => part.type === \"finish\")\n\n Telemetry.addGenAIAnnotations(span, {\n operation: { name: \"chat\" },\n request: { model: Option.getOrUndefined(name) },\n response: { finishReasons: finish === undefined ? undefined : [finish.reason] },\n usage: {\n inputTokens: finish?.usage.inputTokens.total,\n outputTokens: finish?.usage.outputTokens.total\n }\n })\n\n if (finish?.usage.inputTokens.cacheRead !== undefined) {\n span.attribute(\"gen_ai.usage.cache_read_input_tokens\", finish.usage.inputTokens.cacheRead)\n }\n\n if (finish?.usage.inputTokens.cacheWrite !== undefined) {\n span.attribute(\"gen_ai.usage.cache_creation_input_tokens\", finish.usage.inputTokens.cacheWrite)\n }\n}\n\n/** @internal */\nexport const annotateModelCall =\n (attributes: SpanAttributes | undefined) =>\n <A, E, R>(self: Effect.Effect<A, E, R>): Effect.Effect<A, E, R> =>\n Effect.gen(function* () {\n const name = yield* Effect.serviceOption(Model.ModelName)\n const outer = yield* Effect.serviceOption(Telemetry.CurrentSpanTransformer)\n\n return yield* self.pipe(\n Effect.provideService(Telemetry.CurrentSpanTransformer, (call) => {\n if (Option.isSome(outer)) outer.value(call)\n\n annotateModelSpan(call, name, attributes)\n })\n )\n })\n\n/** @internal */\nexport const toolSpan = (call: AssistantEvent.ToolCallPart) =>\n Effect.withSpan(`execute_tool ${call.name}`, {\n kind: \"internal\",\n attributes: {\n \"gen_ai.operation.name\": \"execute_tool\",\n \"gen_ai.tool.name\": call.name,\n \"gen_ai.tool.call.id\": call.id,\n \"input.value\": toJsonText({ arguments: call.params })\n }\n })\n\n/** @internal */\nexport const toolOutput = (result: Schema.Json) =>\n Effect.annotateCurrentSpan(\"output.value\", Schema.is(Schema.String)(result) ? result : toJsonText(result))\n"],"mappings":";;;;;;;;;;;;AAiBA,MAAa,aAAa,MAAc,YACtC,OAAO,SAAS,gBAAgB,QAAQ;CACtC,MAAM;CACN,MAAM,QAAQ,QAAQ;CACtB,YAAY;EACV,yBAAyB;EACzB,qBAAqB;EACrB,0BAA0B,QAAQ;EAClC,cAAc,QAAQ;CACxB;AACF,CAAC;;AAIH,MAAa,iBAAuD,OAAO,IAAI,aAAa;CAC1F,MAAM,SAAS,OAAO,OAAO;CAE7B,IAAI,EAAE,OAAO,WAAW,kBAAkB,WAAW,OAAO,cAAc,OAAO,OAAO,KAAK;CAE7F,OAAO,OAAO,IAAI,OAAO,OAAO,OAAO,OAAO,WAAW,IAAI,SAAS,KAAK,OAAO;AACpF,CAAC;;AAGD,MAAa,cAAc,UAAkB,OAAO,oBAAoB,eAAe,WAAW,KAAK,CAAC;;AAGxG,MAAa,eAAe,WAAmB,OAAO,oBAAoB,gBAAgB,WAAW,MAAM,CAAC;AAE5G,MAAM,eAAe,OAAO,aAAa,OAAO,MAAM;AAEtD,MAAM,cAAc,OAAO,aAAa,OAAO,MAAMA,SAAwB,CAAC;AAI9E,MAAM,gBAAgB,aACpB,YAAY,SAAS,SAAS,SAAS,OAAO,QAAQ,OAAO,oBAAoBA,SAAwB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;AAEpH,MAAM,qBACJ,EAAE,MAAM,QAAQ,YAChB,MACA,eACG;CACH,KAAK,UAAU,iBAAiB,WAAW,OAAO,IAAI,aAAa,MAAM,IAAI,YAAY,QAAQ,OAAO,CAAC,CAAC;CAC1G,KAAK,UAAU,qBAAqB,WAAW,OAAO,eAAe,aAAa,QAAQ,CAAC,CAAC,CAAC;CAE7F,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,cAAc,CAAC,CAAC,GACxD,KAAK,UAAU,KAAK,KAAK;CAG3B,MAAM,SAAS,SAAS,MAAM,SAAS,KAAK,SAAS,QAAQ;CAE7D,UAAU,oBAAoB,MAAM;EAClC,WAAW,EAAE,MAAM,OAAO;EAC1B,SAAS,EAAE,OAAO,OAAO,eAAe,IAAI,EAAE;EAC9C,UAAU,EAAE,eAAe,WAAW,KAAA,IAAY,KAAA,IAAY,CAAC,OAAO,MAAM,EAAE;EAC9E,OAAO;GACL,aAAa,QAAQ,MAAM,YAAY;GACvC,cAAc,QAAQ,MAAM,aAAa;EAC3C;CACF,CAAC;CAED,IAAI,QAAQ,MAAM,YAAY,cAAc,KAAA,GAC1C,KAAK,UAAU,wCAAwC,OAAO,MAAM,YAAY,SAAS;CAG3F,IAAI,QAAQ,MAAM,YAAY,eAAe,KAAA,GAC3C,KAAK,UAAU,4CAA4C,OAAO,MAAM,YAAY,UAAU;AAElG;;AAGA,MAAa,qBACV,gBACS,SACR,OAAO,IAAI,aAAa;CACtB,MAAM,OAAO,OAAO,OAAO,cAAc,MAAM,SAAS;CACxD,MAAM,QAAQ,OAAO,OAAO,cAAc,UAAU,sBAAsB;CAE1E,OAAO,OAAO,KAAK,KACjB,OAAO,eAAe,UAAU,yBAAyB,SAAS;EAChE,IAAI,OAAO,OAAO,KAAK,GAAG,MAAM,MAAM,IAAI;EAE1C,kBAAkB,MAAM,MAAM,UAAU;CAC1C,CAAC,CACH;AACF,CAAC;;AAGL,MAAa,YAAY,SACvB,OAAO,SAAS,gBAAgB,KAAK,QAAQ;CAC3C,MAAM;CACN,YAAY;EACV,yBAAyB;EACzB,oBAAoB,KAAK;EACzB,uBAAuB,KAAK;EAC5B,eAAe,WAAW,EAAE,WAAW,KAAK,OAAO,CAAC;CACtD;AACF,CAAC;;AAGH,MAAa,cAAc,WACzB,OAAO,oBAAoB,gBAAgB,OAAO,GAAG,OAAO,MAAM,CAAC,CAAC,MAAM,IAAI,SAAS,WAAW,MAAM,CAAC"}
|
package/package.json
CHANGED
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This module provides a chat to drive an assistant like a user would, for tests and evals.
|
|
3
|
+
*
|
|
4
|
+
* {@link chat} wraps a chat the app created. Each step (`say`, `approve`, `reject`, `answer`, or
|
|
5
|
+
* `send` for any input) submits the input, waits until a turn has handled it, and returns the chat
|
|
6
|
+
* as it is then. A {@link Log} reads what the assistant did, for the whole chat or for the last
|
|
7
|
+
* step alone (`step`): the tools that ran, the blocks it showed, the calls waiting for approval,
|
|
8
|
+
* and the messages it sent.
|
|
9
|
+
*
|
|
10
|
+
* **Example** (An assistant asks before it emails)
|
|
11
|
+
*
|
|
12
|
+
* ```ts
|
|
13
|
+
* import { AssistantTest } from "effect-assistant"
|
|
14
|
+
* import * as Effect from "effect/Effect"
|
|
15
|
+
*
|
|
16
|
+
* const program = Effect.gen(function* () {
|
|
17
|
+
* const chat = AssistantTest.chat(yield* MyAssistant, chatId)
|
|
18
|
+
* const asked = yield* chat.say("Email the report to ana@example.com")
|
|
19
|
+
* asked.step.waitingFor // ["send_report"]
|
|
20
|
+
* const sent = yield* asked.approve
|
|
21
|
+
* sent.step.toolsRun // ["send_report"]
|
|
22
|
+
* })
|
|
23
|
+
* ```
|
|
24
|
+
*
|
|
25
|
+
* @since 0.9.0
|
|
26
|
+
*/
|
|
27
|
+
import * as Data from "effect/Data"
|
|
28
|
+
import type * as Duration from "effect/Duration"
|
|
29
|
+
import * as Effect from "effect/Effect"
|
|
30
|
+
import * as Option from "effect/Option"
|
|
31
|
+
import * as Predicate from "effect/Predicate"
|
|
32
|
+
import * as Schema from "effect/Schema"
|
|
33
|
+
import * as Stream from "effect/Stream"
|
|
34
|
+
import type * as Assistant from "./Assistant.ts"
|
|
35
|
+
import type * as Block from "./Block.ts"
|
|
36
|
+
import * as AssistantEvent from "./AssistantEvent.ts"
|
|
37
|
+
import type { ChatId } from "./ChatId.ts"
|
|
38
|
+
import * as InputId from "./InputId.ts"
|
|
39
|
+
import { sendUserMessageName, waitToolName } from "./internal/tools.ts"
|
|
40
|
+
import * as ToolApproval from "./ToolApproval.ts"
|
|
41
|
+
|
|
42
|
+
// The assistant's own tools: how it talks, waits and reads its skills, not work it does for the user.
|
|
43
|
+
const ownTools: ReadonlySet<string> = new Set([sendUserMessageName, waitToolName, "load_skill"])
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Represents a step that couldn't complete: no turn handled the input in time, the turn failed,
|
|
47
|
+
* or there was nothing to approve or answer.
|
|
48
|
+
*
|
|
49
|
+
* @category errors
|
|
50
|
+
* @since 0.9.0
|
|
51
|
+
*/
|
|
52
|
+
export class TestChatError extends Data.TaggedError("TestChatError")<{
|
|
53
|
+
readonly reason: "TimedOut" | "TurnFailed" | "NothingToApprove" | "NothingToAnswer"
|
|
54
|
+
readonly message: string
|
|
55
|
+
}> {}
|
|
56
|
+
|
|
57
|
+
const libraryEvents = (entries: ReadonlyArray<AssistantEvent.AnyEntry>): ReadonlyArray<AssistantEvent.AssistantEvent> =>
|
|
58
|
+
entries.flatMap(({ event }) => (AssistantEvent.isLibraryEvent(event) ? [event] : []))
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Represents what the assistant did in some of a chat's entries.
|
|
62
|
+
*
|
|
63
|
+
* @category models
|
|
64
|
+
* @since 0.9.0
|
|
65
|
+
*/
|
|
66
|
+
export interface Log {
|
|
67
|
+
readonly entries: ReadonlyArray<AssistantEvent.AnyEntry>
|
|
68
|
+
/**
|
|
69
|
+
* The app's tools that ran, each once, in the order they first ran. A call that failed or that
|
|
70
|
+
* the library refused didn't run, and the assistant's own tools (`send_user_message`, `wait`,
|
|
71
|
+
* `load_skill`) aren't listed.
|
|
72
|
+
*/
|
|
73
|
+
readonly toolsRun: ReadonlyArray<string>
|
|
74
|
+
/** How many times the model called this tool, whether or not it ran. */
|
|
75
|
+
readonly toolCalls: (tool: string) => number
|
|
76
|
+
/** The kind of each block the assistant showed, in order; one per message. */
|
|
77
|
+
readonly blocks: ReadonlyArray<string>
|
|
78
|
+
/** The calls waiting for the user's approval, in order. */
|
|
79
|
+
readonly approvals: ReadonlyArray<AssistantEvent.ApprovalRequested>
|
|
80
|
+
/** The tools of the calls waiting for the user's approval. */
|
|
81
|
+
readonly waitingFor: ReadonlyArray<string>
|
|
82
|
+
/** The messages the assistant sent, in order. */
|
|
83
|
+
readonly messages: ReadonlyArray<string>
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Reads what the assistant did in these entries.
|
|
88
|
+
*
|
|
89
|
+
* @category constructors
|
|
90
|
+
* @since 0.9.0
|
|
91
|
+
*/
|
|
92
|
+
export const log = (entries: ReadonlyArray<AssistantEvent.AnyEntry>): Log => {
|
|
93
|
+
const events = libraryEvents(entries)
|
|
94
|
+
const results = events.flatMap((event) => (Predicate.isTagged(event, "ToolResult") ? [event] : []))
|
|
95
|
+
const approvals = events.flatMap((event) => (Predicate.isTagged(event, "ApprovalRequested") ? [event] : []))
|
|
96
|
+
|
|
97
|
+
return {
|
|
98
|
+
entries,
|
|
99
|
+
toolsRun: [
|
|
100
|
+
...new Set(
|
|
101
|
+
results.filter(({ outcome, tool }) => outcome === "success" && !ownTools.has(tool)).map(({ tool }) => tool)
|
|
102
|
+
)
|
|
103
|
+
],
|
|
104
|
+
toolCalls: (tool) => results.filter((result) => result.tool === tool).length,
|
|
105
|
+
blocks: events.flatMap((event) =>
|
|
106
|
+
Predicate.isTagged(event, "AssistantMessage") && event.block !== undefined ? [event.block.kind] : []
|
|
107
|
+
),
|
|
108
|
+
approvals,
|
|
109
|
+
waitingFor: approvals.map(({ tool }) => tool),
|
|
110
|
+
messages: AssistantEvent.assistantTexts(entries)
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Represents a chat driven like a user would.
|
|
116
|
+
*
|
|
117
|
+
* @category models
|
|
118
|
+
* @since 0.9.0
|
|
119
|
+
*/
|
|
120
|
+
export interface TestChat extends Log {
|
|
121
|
+
readonly chatId: ChatId
|
|
122
|
+
/** What the assistant did for the last step alone. */
|
|
123
|
+
readonly step: Log
|
|
124
|
+
/** Submits an input, and returns the chat once a turn has handled it. */
|
|
125
|
+
readonly send: (input: AssistantEvent.Input) => Effect.Effect<TestChat, TestChatError>
|
|
126
|
+
/** Sends the user's message. */
|
|
127
|
+
readonly say: (text: string) => Effect.Effect<TestChat, TestChatError>
|
|
128
|
+
/** Approves the last call waiting for approval. */
|
|
129
|
+
readonly approve: Effect.Effect<TestChat, TestChatError>
|
|
130
|
+
/** Rejects the last call waiting for approval. */
|
|
131
|
+
readonly reject: (options?: { readonly reason?: string | undefined }) => Effect.Effect<TestChat, TestChatError>
|
|
132
|
+
/** Answers the last block of this kind the assistant showed. */
|
|
133
|
+
readonly answer: <Answer extends Block.Codec>(
|
|
134
|
+
block: Block.Block<string, Block.Codec, Answer>,
|
|
135
|
+
answer: Answer["Type"]
|
|
136
|
+
) => Effect.Effect<TestChat, TestChatError>
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
const handled = (entries: ReadonlyArray<AssistantEvent.AnyEntry>, inputId: string): boolean => {
|
|
140
|
+
const input = entries.find(({ event }) => AssistantEvent.isInput(event) && event.id === inputId)
|
|
141
|
+
|
|
142
|
+
return (
|
|
143
|
+
input !== undefined &&
|
|
144
|
+
libraryEvents(entries).some((event) => AssistantEvent.isTurnEnd(event) && event.throughSequence >= input.sequence)
|
|
145
|
+
)
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Creates a chat to drive the assistant with, for a chat the app already created. A step fails
|
|
150
|
+
* when no turn has handled its input within `timeout` (3 minutes by default), or when a turn failed
|
|
151
|
+
* while handling it.
|
|
152
|
+
*
|
|
153
|
+
* @category constructors
|
|
154
|
+
* @since 0.9.0
|
|
155
|
+
*/
|
|
156
|
+
export const chat = <Events extends { readonly _tag: string } = never>(
|
|
157
|
+
assistant: Assistant.Service<Events>,
|
|
158
|
+
chatId: ChatId,
|
|
159
|
+
options?: { readonly timeout?: Duration.Input | undefined }
|
|
160
|
+
): TestChat => at(assistant, chatId, options?.timeout ?? "3 minutes", [], [])
|
|
161
|
+
|
|
162
|
+
const at = <Events extends { readonly _tag: string }>(
|
|
163
|
+
assistant: Assistant.Service<Events>,
|
|
164
|
+
chatId: ChatId,
|
|
165
|
+
timeout: Duration.Input,
|
|
166
|
+
entries: ReadonlyArray<AssistantEvent.AnyEntry>,
|
|
167
|
+
step: ReadonlyArray<AssistantEvent.AnyEntry>
|
|
168
|
+
): TestChat => {
|
|
169
|
+
const all = log(entries)
|
|
170
|
+
|
|
171
|
+
const send = (input: AssistantEvent.Input): Effect.Effect<TestChat, TestChatError> =>
|
|
172
|
+
Effect.gen(function* () {
|
|
173
|
+
const before = entries.at(-1)?.sequence ?? 0
|
|
174
|
+
let seen: ReadonlyArray<AssistantEvent.AnyEntry> = []
|
|
175
|
+
|
|
176
|
+
yield* assistant.submit(chatId, input).pipe(Effect.orDie)
|
|
177
|
+
|
|
178
|
+
const after = yield* assistant.subscribe(chatId).pipe(
|
|
179
|
+
Stream.scan(
|
|
180
|
+
(): ReadonlyArray<AssistantEvent.AnyEntry> => [],
|
|
181
|
+
(acc, entry) => [...acc, entry]
|
|
182
|
+
),
|
|
183
|
+
Stream.tap((acc) => Effect.sync(() => (seen = acc))),
|
|
184
|
+
Stream.takeUntil((acc) => handled(acc, input.id)),
|
|
185
|
+
Stream.runLast,
|
|
186
|
+
Effect.map(Option.getOrElse((): ReadonlyArray<AssistantEvent.AnyEntry> => [])),
|
|
187
|
+
Effect.timeoutOrElse({
|
|
188
|
+
duration: timeout,
|
|
189
|
+
orElse: () =>
|
|
190
|
+
Effect.fail(
|
|
191
|
+
new TestChatError({
|
|
192
|
+
reason: "TimedOut",
|
|
193
|
+
message: `no turn handled the input; the assistant said: ${log(seen).messages.join(" | ")}`
|
|
194
|
+
})
|
|
195
|
+
)
|
|
196
|
+
})
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
const added = after.filter(({ sequence }) => sequence > before)
|
|
200
|
+
|
|
201
|
+
const failed = libraryEvents(added).flatMap((event) =>
|
|
202
|
+
Predicate.isTagged(event, "TurnFailed") ? [event.message] : []
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
if (failed.length > 0) {
|
|
206
|
+
return yield* new TestChatError({ reason: "TurnFailed", message: failed.join(" | ") })
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
return at(assistant, chatId, timeout, after, added)
|
|
210
|
+
})
|
|
211
|
+
|
|
212
|
+
const decide = (decision: (approval: { readonly id: string }) => AssistantEvent.ApprovalDecision) => {
|
|
213
|
+
const waiting = all.approvals.at(-1)
|
|
214
|
+
|
|
215
|
+
return waiting === undefined
|
|
216
|
+
? Effect.fail(new TestChatError({ reason: "NothingToApprove", message: "no call waits for approval" }))
|
|
217
|
+
: send(decision({ id: waiting.callId }))
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
const answer = <Answer extends Block.Codec>(
|
|
221
|
+
block: Block.Block<string, Block.Codec, Answer>,
|
|
222
|
+
value: Answer["Type"]
|
|
223
|
+
): Effect.Effect<TestChat, TestChatError> => {
|
|
224
|
+
const asked = libraryEvents(entries).findLast(
|
|
225
|
+
(event): event is AssistantEvent.AssistantMessage =>
|
|
226
|
+
Predicate.isTagged(event, "AssistantMessage") && event.block?.kind === block.kind
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
if (asked === undefined || block.answer === undefined) {
|
|
230
|
+
return Effect.fail(new TestChatError({ reason: "NothingToAnswer", message: `no ${block.kind} block to answer` }))
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
return send(
|
|
234
|
+
AssistantEvent.BlockAnswer.make({
|
|
235
|
+
// As the client sends it: the same id every time, so a block takes one answer.
|
|
236
|
+
id: InputId.fromKey(`${asked.id}:answer`),
|
|
237
|
+
messageId: asked.id,
|
|
238
|
+
kind: block.kind,
|
|
239
|
+
value: Schema.encodeUnknownSync(Schema.toCodecJson(block.answer))(value)
|
|
240
|
+
})
|
|
241
|
+
)
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
return {
|
|
245
|
+
...all,
|
|
246
|
+
chatId,
|
|
247
|
+
step: log(step),
|
|
248
|
+
send,
|
|
249
|
+
say: (text) => send(AssistantEvent.UserMessage.make({ text })),
|
|
250
|
+
approve: decide(ToolApproval.approve),
|
|
251
|
+
reject: (options) => decide((approval) => ToolApproval.reject(approval, options)),
|
|
252
|
+
answer
|
|
253
|
+
}
|
|
254
|
+
}
|
package/src/index.ts
CHANGED
package/src/internal/loop.ts
CHANGED
|
@@ -311,7 +311,7 @@ export const step = <R, S, E2 = never, R2 = never>(
|
|
|
311
311
|
const started = yield* Clock.currentTimeMillis
|
|
312
312
|
|
|
313
313
|
const response = yield* generate(prompt, toolkit.definitions, options.toolChoice).pipe(
|
|
314
|
-
Telemetry.
|
|
314
|
+
Telemetry.annotateModelCall(instructions.attributes)
|
|
315
315
|
)
|
|
316
316
|
|
|
317
317
|
const metadata = response.content.find((part) => part.type === "response-metadata")
|
|
@@ -1,8 +1,6 @@
|
|
|
1
|
-
import type * as LanguageModel from "effect/ai/LanguageModel"
|
|
2
1
|
import * as Model from "effect/ai/Model"
|
|
3
2
|
import * as Prompt from "effect/ai/Prompt"
|
|
4
3
|
import * as Telemetry from "effect/ai/Telemetry"
|
|
5
|
-
import type * as Tool from "effect/ai/Tool"
|
|
6
4
|
import * as Effect from "effect/Effect"
|
|
7
5
|
import * as Option from "effect/Option"
|
|
8
6
|
import * as References from "effect/References"
|
|
@@ -49,61 +47,57 @@ const encodePrompt = Schema.encodeOption(Prompt.Prompt)
|
|
|
49
47
|
|
|
50
48
|
const encodeParts = Schema.encodeOption(Schema.Array(AssistantEvent.ModelPart))
|
|
51
49
|
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
)
|
|
56
|
-
|
|
57
|
-
|
|
50
|
+
type ModelCall = Parameters<Telemetry.SpanTransformer>[0]
|
|
51
|
+
|
|
52
|
+
const completionOf = (response: ModelCall["response"]) =>
|
|
53
|
+
encodeParts(response.flatMap((part) => Option.toArray(Schema.decodeUnknownOption(AssistantEvent.ModelPart)(part))))
|
|
54
|
+
|
|
55
|
+
const annotateModelSpan = (
|
|
56
|
+
{ span, prompt, response }: ModelCall,
|
|
57
|
+
name: Option.Option<string>,
|
|
58
|
+
attributes: SpanAttributes | undefined
|
|
59
|
+
) => {
|
|
60
|
+
span.attribute("gen_ai.prompt", toJsonText(Option.map(encodePrompt(prompt), (encoded) => encoded.content)))
|
|
61
|
+
span.attribute("gen_ai.completion", toJsonText(Option.getOrUndefined(completionOf(response))))
|
|
62
|
+
|
|
63
|
+
for (const [key, value] of Object.entries(attributes ?? {})) {
|
|
64
|
+
span.attribute(key, value)
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const finish = response.find((part) => part.type === "finish")
|
|
58
68
|
|
|
59
69
|
Telemetry.addGenAIAnnotations(span, {
|
|
60
70
|
operation: { name: "chat" },
|
|
61
71
|
request: { model: Option.getOrUndefined(name) },
|
|
62
|
-
response: { finishReasons: [
|
|
72
|
+
response: { finishReasons: finish === undefined ? undefined : [finish.reason] },
|
|
63
73
|
usage: {
|
|
64
|
-
inputTokens: usage.inputTokens.total,
|
|
65
|
-
outputTokens: usage.outputTokens.total
|
|
74
|
+
inputTokens: finish?.usage.inputTokens.total,
|
|
75
|
+
outputTokens: finish?.usage.outputTokens.total
|
|
66
76
|
}
|
|
67
77
|
})
|
|
68
78
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
)
|
|
72
|
-
|
|
73
|
-
span.attribute("gen_ai.completion", toJsonText(Option.getOrUndefined(completion)))
|
|
74
|
-
|
|
75
|
-
if (usage.inputTokens.cacheRead !== undefined) {
|
|
76
|
-
span.attribute("gen_ai.usage.cache_read_input_tokens", usage.inputTokens.cacheRead)
|
|
79
|
+
if (finish?.usage.inputTokens.cacheRead !== undefined) {
|
|
80
|
+
span.attribute("gen_ai.usage.cache_read_input_tokens", finish.usage.inputTokens.cacheRead)
|
|
77
81
|
}
|
|
78
82
|
|
|
79
|
-
if (usage.inputTokens.cacheWrite !== undefined) {
|
|
80
|
-
span.attribute("gen_ai.usage.cache_creation_input_tokens", usage.inputTokens.cacheWrite)
|
|
83
|
+
if (finish?.usage.inputTokens.cacheWrite !== undefined) {
|
|
84
|
+
span.attribute("gen_ai.usage.cache_creation_input_tokens", finish.usage.inputTokens.cacheWrite)
|
|
81
85
|
}
|
|
82
|
-
}
|
|
86
|
+
}
|
|
83
87
|
|
|
84
|
-
// Named after the model when the context has its name (`AnthropicLanguageModel.model(...)` provides it).
|
|
85
88
|
/** @internal */
|
|
86
|
-
export const
|
|
87
|
-
(
|
|
88
|
-
<
|
|
89
|
-
self: Effect.Effect<LanguageModel.GenerateTextResponse<Tools, "opaque">, E, R>
|
|
90
|
-
): Effect.Effect<LanguageModel.GenerateTextResponse<Tools, "opaque">, E, R> =>
|
|
89
|
+
export const annotateModelCall =
|
|
90
|
+
(attributes: SpanAttributes | undefined) =>
|
|
91
|
+
<A, E, R>(self: Effect.Effect<A, E, R>): Effect.Effect<A, E, R> =>
|
|
91
92
|
Effect.gen(function* () {
|
|
92
93
|
const name = yield* Effect.serviceOption(Model.ModelName)
|
|
94
|
+
const outer = yield* Effect.serviceOption(Telemetry.CurrentSpanTransformer)
|
|
93
95
|
|
|
94
|
-
return yield*
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
yield* annotateChat(response, name).pipe(Effect.ignore)
|
|
96
|
+
return yield* self.pipe(
|
|
97
|
+
Effect.provideService(Telemetry.CurrentSpanTransformer, (call) => {
|
|
98
|
+
if (Option.isSome(outer)) outer.value(call)
|
|
98
99
|
|
|
99
|
-
|
|
100
|
-
}).pipe(
|
|
101
|
-
Effect.withSpan(Option.match(name, { onNone: () => "chat", onSome: (model) => `chat ${model}` }), {
|
|
102
|
-
kind: "client",
|
|
103
|
-
attributes: {
|
|
104
|
-
"gen_ai.prompt": toJsonText(Option.map(encodePrompt(prompt), (encoded) => encoded.content)),
|
|
105
|
-
...attributes
|
|
106
|
-
}
|
|
100
|
+
annotateModelSpan(call, name, attributes)
|
|
107
101
|
})
|
|
108
102
|
)
|
|
109
103
|
})
|