effect-assistant 0.8.3 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,94 @@
1
+ import { ChatId } from "./ChatId.js";
2
+ import { AnyEntry, ApprovalRequested, AssistantEvent_d_exports, Input } from "./AssistantEvent.js";
3
+ import { Block, Block_d_exports, Codec } from "./Block.js";
4
+ import { Assistant_d_exports, Service } from "./Assistant.js";
5
+ import * as Effect from "effect/Effect";
6
+ import * as Duration from "effect/Duration";
7
+ declare namespace AssistantTest_d_exports {
8
+ export { Log, TestChat, TestChatError, chat, log };
9
+ }
10
+ declare const TestChatError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
11
+ readonly _tag: "TestChatError";
12
+ } & Readonly<A>;
13
+ /**
14
+ * Represents a step that couldn't complete: no turn handled the input in time, the turn failed,
15
+ * or there was nothing to approve or answer.
16
+ *
17
+ * @category errors
18
+ * @since 0.9.0
19
+ */
20
+ export declare class TestChatError extends TestChatError_base<{
21
+ readonly reason: "TimedOut" | "TurnFailed" | "NothingToApprove" | "NothingToAnswer";
22
+ readonly message: string;
23
+ }> {}
24
+ /**
25
+ * Represents what the assistant did in some of a chat's entries.
26
+ *
27
+ * @category models
28
+ * @since 0.9.0
29
+ */
30
+ export interface Log {
31
+ readonly entries: ReadonlyArray<AnyEntry>;
32
+ /**
33
+ * The app's tools that ran, each once, in the order they first ran. A call that failed or that
34
+ * the library refused didn't run, and the assistant's own tools (`send_user_message`, `wait`,
35
+ * `load_skill`) aren't listed.
36
+ */
37
+ readonly toolsRun: ReadonlyArray<string>;
38
+ /** How many times the model called this tool, whether or not it ran. */
39
+ readonly toolCalls: (tool: string) => number;
40
+ /** The kind of each block the assistant showed, in order; one per message. */
41
+ readonly blocks: ReadonlyArray<string>;
42
+ /** The calls waiting for the user's approval, in order. */
43
+ readonly approvals: ReadonlyArray<ApprovalRequested>;
44
+ /** The tools of the calls waiting for the user's approval. */
45
+ readonly waitingFor: ReadonlyArray<string>;
46
+ /** The messages the assistant sent, in order. */
47
+ readonly messages: ReadonlyArray<string>;
48
+ }
49
+ /**
50
+ * Reads what the assistant did in these entries.
51
+ *
52
+ * @category constructors
53
+ * @since 0.9.0
54
+ */
55
+ export declare const log: (entries: ReadonlyArray<AnyEntry>) => Log;
56
+ /**
57
+ * Represents a chat driven like a user would.
58
+ *
59
+ * @category models
60
+ * @since 0.9.0
61
+ */
62
+ export interface TestChat extends Log {
63
+ readonly chatId: ChatId;
64
+ /** What the assistant did for the last step alone. */
65
+ readonly step: Log;
66
+ /** Submits an input, and returns the chat once a turn has handled it. */
67
+ readonly send: (input: Input) => Effect.Effect<TestChat, TestChatError>;
68
+ /** Sends the user's message. */
69
+ readonly say: (text: string) => Effect.Effect<TestChat, TestChatError>;
70
+ /** Approves the last call waiting for approval. */
71
+ readonly approve: Effect.Effect<TestChat, TestChatError>;
72
+ /** Rejects the last call waiting for approval. */
73
+ readonly reject: (options?: {
74
+ readonly reason?: string | undefined;
75
+ }) => Effect.Effect<TestChat, TestChatError>;
76
+ /** Answers the last block of this kind the assistant showed. */
77
+ readonly answer: <Answer extends Codec>(block: Block<string, Codec, Answer>, answer: Answer["Type"]) => Effect.Effect<TestChat, TestChatError>;
78
+ }
79
+ /**
80
+ * Creates a chat to drive the assistant with, for a chat the app already created. A step fails
81
+ * when no turn has handled its input within `timeout` (3 minutes by default), or when a turn failed
82
+ * while handling it.
83
+ *
84
+ * @category constructors
85
+ * @since 0.9.0
86
+ */
87
+ export declare const chat: <Events extends {
88
+ readonly _tag: string;
89
+ } = never>(assistant: Service<Events>, chatId: ChatId, options?: {
90
+ readonly timeout?: Duration.Input | undefined;
91
+ }) => TestChat;
92
+ //#endregion
93
+ export { AssistantTest_d_exports };
94
+ //# sourceMappingURL=AssistantTest.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"AssistantTest.d.ts","names":[],"sources":["../src/AssistantTest.ts"],"mappings":";;;;;;;;;;;;;;;;;;;qBAmDa,sBAAsB;WACxB;WACA;;;;;;;;iBAYM;WACN,SAAS,cAAc;;;;;;WAMvB,UAAU;;WAEV,YAAY;;WAEZ,QAAQ;;WAER,WAAW,cAAc;;WAEzB,YAAY;;WAEZ,UAAU;;;;;;;;qBASR,MAAG,SAAa,cAAc,cAA2B;;;;;;;iBA4BrD,iBAAiB;WACvB,QAAQ;;WAER,MAAM;;WAEN,OAAO,OAAO,UAAyB,OAAO,OAAO,UAAU;;WAE/D,MAAM,iBAAiB,OAAO,OAAO,UAAU;;WAE/C,SAAS,OAAO,OAAO,UAAU;;WAEjC,SAAS;aAAqB;QAAkC,OAAO,OAAO,UAAU;;WAExF,SAAS,eAAe,OAC/B,OAAO,cAAoB,OAAa,SACxC,QAAQ,mBACL,OAAO,OAAO,UAAU;;;;;;;;;;qBAoBlB,OAAQ;WAA0B;WAAsB,WACxD,QAAkB,SAAO,QAC5B,QAAM;WACO,UAAU,SAAS;MACvC"}
@@ -0,0 +1,146 @@
1
+ import { __exportAll } from "./_virtual/_rolldown/runtime.js";
2
+ import { fromKey } from "./InputId.js";
3
+ import { BlockAnswer, UserMessage, assistantTexts, isInput, isLibraryEvent, isTurnEnd } from "./AssistantEvent.js";
4
+ import { approve, reject } from "./ToolApproval.js";
5
+ import { sendUserMessageName, waitToolName } from "./internal/tools.js";
6
+ import * as Effect from "effect/Effect";
7
+ import * as Predicate from "effect/Predicate";
8
+ import * as Schema from "effect/Schema";
9
+ import * as Stream from "effect/Stream";
10
+ import * as Data from "effect/Data";
11
+ import * as Option from "effect/Option";
12
+ //#region src/AssistantTest.ts
13
+ /**
14
+ * This module provides a chat to drive an assistant like a user would, for tests and evals.
15
+ *
16
+ * {@link chat} wraps a chat the app created. Each step (`say`, `approve`, `reject`, `answer`, or
17
+ * `send` for any input) submits the input, waits until a turn has handled it, and returns the chat
18
+ * as it is then. A {@link Log} reads what the assistant did, for the whole chat or for the last
19
+ * step alone (`step`): the tools that ran, the blocks it showed, the calls waiting for approval,
20
+ * and the messages it sent.
21
+ *
22
+ * **Example** (An assistant asks before it emails)
23
+ *
24
+ * ```ts
25
+ * import { AssistantTest } from "effect-assistant"
26
+ * import * as Effect from "effect/Effect"
27
+ *
28
+ * const program = Effect.gen(function* () {
29
+ * const chat = AssistantTest.chat(yield* MyAssistant, chatId)
30
+ * const asked = yield* chat.say("Email the report to ana@example.com")
31
+ * asked.step.waitingFor // ["send_report"]
32
+ * const sent = yield* asked.approve
33
+ * sent.step.toolsRun // ["send_report"]
34
+ * })
35
+ * ```
36
+ *
37
+ * @since 0.9.0
38
+ */
39
+ var AssistantTest_exports = /* @__PURE__ */ __exportAll({
40
+ TestChatError: () => TestChatError,
41
+ chat: () => chat,
42
+ log: () => log
43
+ });
44
+ const ownTools = /* @__PURE__ */ new Set([
45
+ sendUserMessageName,
46
+ waitToolName,
47
+ "load_skill"
48
+ ]);
49
+ /**
50
+ * Represents a step that couldn't complete: no turn handled the input in time, the turn failed,
51
+ * or there was nothing to approve or answer.
52
+ *
53
+ * @category errors
54
+ * @since 0.9.0
55
+ */
56
+ var TestChatError = class extends Data.TaggedError("TestChatError") {};
57
+ const libraryEvents = (entries) => entries.flatMap(({ event }) => isLibraryEvent(event) ? [event] : []);
58
+ /**
59
+ * Reads what the assistant did in these entries.
60
+ *
61
+ * @category constructors
62
+ * @since 0.9.0
63
+ */
64
+ const log = (entries) => {
65
+ const events = libraryEvents(entries);
66
+ const results = events.flatMap((event) => Predicate.isTagged(event, "ToolResult") ? [event] : []);
67
+ const approvals = events.flatMap((event) => Predicate.isTagged(event, "ApprovalRequested") ? [event] : []);
68
+ return {
69
+ entries,
70
+ toolsRun: [...new Set(results.filter(({ outcome, tool }) => outcome === "success" && !ownTools.has(tool)).map(({ tool }) => tool))],
71
+ toolCalls: (tool) => results.filter((result) => result.tool === tool).length,
72
+ blocks: events.flatMap((event) => Predicate.isTagged(event, "AssistantMessage") && event.block !== void 0 ? [event.block.kind] : []),
73
+ approvals,
74
+ waitingFor: approvals.map(({ tool }) => tool),
75
+ messages: assistantTexts(entries)
76
+ };
77
+ };
78
+ const handled = (entries, inputId) => {
79
+ const input = entries.find(({ event }) => isInput(event) && event.id === inputId);
80
+ return input !== void 0 && libraryEvents(entries).some((event) => isTurnEnd(event) && event.throughSequence >= input.sequence);
81
+ };
82
+ /**
83
+ * Creates a chat to drive the assistant with, for a chat the app already created. A step fails
84
+ * when no turn has handled its input within `timeout` (3 minutes by default), or when a turn failed
85
+ * while handling it.
86
+ *
87
+ * @category constructors
88
+ * @since 0.9.0
89
+ */
90
+ const chat = (assistant, chatId, options) => at(assistant, chatId, options?.timeout ?? "3 minutes", [], []);
91
+ const at = (assistant, chatId, timeout, entries, step) => {
92
+ const all = log(entries);
93
+ const send = (input) => Effect.gen(function* () {
94
+ const before = entries.at(-1)?.sequence ?? 0;
95
+ let seen = [];
96
+ yield* assistant.submit(chatId, input).pipe(Effect.orDie);
97
+ const after = yield* assistant.subscribe(chatId).pipe(Stream.scan(() => [], (acc, entry) => [...acc, entry]), Stream.tap((acc) => Effect.sync(() => seen = acc)), Stream.takeUntil((acc) => handled(acc, input.id)), Stream.runLast, Effect.map(Option.getOrElse(() => [])), Effect.timeoutOrElse({
98
+ duration: timeout,
99
+ orElse: () => Effect.fail(new TestChatError({
100
+ reason: "TimedOut",
101
+ message: `no turn handled the input; the assistant said: ${log(seen).messages.join(" | ")}`
102
+ }))
103
+ }));
104
+ const added = after.filter(({ sequence }) => sequence > before);
105
+ const failed = libraryEvents(added).flatMap((event) => Predicate.isTagged(event, "TurnFailed") ? [event.message] : []);
106
+ if (failed.length > 0) return yield* new TestChatError({
107
+ reason: "TurnFailed",
108
+ message: failed.join(" | ")
109
+ });
110
+ return at(assistant, chatId, timeout, after, added);
111
+ });
112
+ const decide = (decision) => {
113
+ const waiting = all.approvals.at(-1);
114
+ return waiting === void 0 ? Effect.fail(new TestChatError({
115
+ reason: "NothingToApprove",
116
+ message: "no call waits for approval"
117
+ })) : send(decision({ id: waiting.callId }));
118
+ };
119
+ const answer = (block, value) => {
120
+ const asked = libraryEvents(entries).findLast((event) => Predicate.isTagged(event, "AssistantMessage") && event.block?.kind === block.kind);
121
+ if (asked === void 0 || block.answer === void 0) return Effect.fail(new TestChatError({
122
+ reason: "NothingToAnswer",
123
+ message: `no ${block.kind} block to answer`
124
+ }));
125
+ return send(BlockAnswer.make({
126
+ id: fromKey(`${asked.id}:answer`),
127
+ messageId: asked.id,
128
+ kind: block.kind,
129
+ value: Schema.encodeUnknownSync(Schema.toCodecJson(block.answer))(value)
130
+ }));
131
+ };
132
+ return {
133
+ ...all,
134
+ chatId,
135
+ step: log(step),
136
+ send,
137
+ say: (text) => send(UserMessage.make({ text })),
138
+ approve: decide(approve),
139
+ reject: (options) => decide((approval) => reject(approval, options)),
140
+ answer
141
+ };
142
+ };
143
+ //#endregion
144
+ export { AssistantTest_exports, TestChatError, chat, log };
145
+
146
+ //# sourceMappingURL=AssistantTest.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"AssistantTest.js","names":["AssistantEvent.isLibraryEvent","AssistantEvent.assistantTexts","AssistantEvent.isInput","AssistantEvent.isTurnEnd","InputId.fromKey","ToolApproval.approve","ToolApproval.reject"],"sources":["../src/AssistantTest.ts"],"sourcesContent":["/**\n * This module provides a chat to drive an assistant like a user would, for tests and evals.\n *\n * {@link chat} wraps a chat the app created. Each step (`say`, `approve`, `reject`, `answer`, or\n * `send` for any input) submits the input, waits until a turn has handled it, and returns the chat\n * as it is then. A {@link Log} reads what the assistant did, for the whole chat or for the last\n * step alone (`step`): the tools that ran, the blocks it showed, the calls waiting for approval,\n * and the messages it sent.\n *\n * **Example** (An assistant asks before it emails)\n *\n * ```ts\n * import { AssistantTest } from \"effect-assistant\"\n * import * as Effect from \"effect/Effect\"\n *\n * const program = Effect.gen(function* () {\n * const chat = AssistantTest.chat(yield* MyAssistant, chatId)\n * const asked = yield* chat.say(\"Email the report to ana@example.com\")\n * asked.step.waitingFor // [\"send_report\"]\n * const sent = yield* asked.approve\n * sent.step.toolsRun // [\"send_report\"]\n * })\n * ```\n *\n * @since 0.9.0\n */\nimport * as Data from \"effect/Data\"\nimport type * as Duration from \"effect/Duration\"\nimport * as Effect from \"effect/Effect\"\nimport * as Option from \"effect/Option\"\nimport * as Predicate from \"effect/Predicate\"\nimport * as Schema from \"effect/Schema\"\nimport * as Stream from \"effect/Stream\"\nimport type * as Assistant from \"./Assistant.ts\"\nimport type * as Block from \"./Block.ts\"\nimport * as AssistantEvent from \"./AssistantEvent.ts\"\nimport type { ChatId } from \"./ChatId.ts\"\nimport * as InputId from \"./InputId.ts\"\nimport { sendUserMessageName, waitToolName } from \"./internal/tools.ts\"\nimport * as ToolApproval from \"./ToolApproval.ts\"\n\n// The assistant's own tools: how it talks, waits and reads its skills, not work it does for the user.\nconst ownTools: ReadonlySet<string> = new Set([sendUserMessageName, waitToolName, \"load_skill\"])\n\n/**\n * Represents a step that couldn't complete: no turn handled the input in time, the turn failed,\n * or there was nothing to approve or answer.\n *\n * @category errors\n * @since 0.9.0\n */\nexport class TestChatError extends Data.TaggedError(\"TestChatError\")<{\n readonly reason: \"TimedOut\" | \"TurnFailed\" | \"NothingToApprove\" | \"NothingToAnswer\"\n readonly message: string\n}> {}\n\nconst libraryEvents = (entries: ReadonlyArray<AssistantEvent.AnyEntry>): ReadonlyArray<AssistantEvent.AssistantEvent> =>\n entries.flatMap(({ event }) => (AssistantEvent.isLibraryEvent(event) ? [event] : []))\n\n/**\n * Represents what the assistant did in some of a chat's entries.\n *\n * @category models\n * @since 0.9.0\n */\nexport interface Log {\n readonly entries: ReadonlyArray<AssistantEvent.AnyEntry>\n /**\n * The app's tools that ran, each once, in the order they first ran. A call that failed or that\n * the library refused didn't run, and the assistant's own tools (`send_user_message`, `wait`,\n * `load_skill`) aren't listed.\n */\n readonly toolsRun: ReadonlyArray<string>\n /** How many times the model called this tool, whether or not it ran. */\n readonly toolCalls: (tool: string) => number\n /** The kind of each block the assistant showed, in order; one per message. */\n readonly blocks: ReadonlyArray<string>\n /** The calls waiting for the user's approval, in order. */\n readonly approvals: ReadonlyArray<AssistantEvent.ApprovalRequested>\n /** The tools of the calls waiting for the user's approval. */\n readonly waitingFor: ReadonlyArray<string>\n /** The messages the assistant sent, in order. */\n readonly messages: ReadonlyArray<string>\n}\n\n/**\n * Reads what the assistant did in these entries.\n *\n * @category constructors\n * @since 0.9.0\n */\nexport const log = (entries: ReadonlyArray<AssistantEvent.AnyEntry>): Log => {\n const events = libraryEvents(entries)\n const results = events.flatMap((event) => (Predicate.isTagged(event, \"ToolResult\") ? [event] : []))\n const approvals = events.flatMap((event) => (Predicate.isTagged(event, \"ApprovalRequested\") ? [event] : []))\n\n return {\n entries,\n toolsRun: [\n ...new Set(\n results.filter(({ outcome, tool }) => outcome === \"success\" && !ownTools.has(tool)).map(({ tool }) => tool)\n )\n ],\n toolCalls: (tool) => results.filter((result) => result.tool === tool).length,\n blocks: events.flatMap((event) =>\n Predicate.isTagged(event, \"AssistantMessage\") && event.block !== undefined ? [event.block.kind] : []\n ),\n approvals,\n waitingFor: approvals.map(({ tool }) => tool),\n messages: AssistantEvent.assistantTexts(entries)\n }\n}\n\n/**\n * Represents a chat driven like a user would.\n *\n * @category models\n * @since 0.9.0\n */\nexport interface TestChat extends Log {\n readonly chatId: ChatId\n /** What the assistant did for the last step alone. */\n readonly step: Log\n /** Submits an input, and returns the chat once a turn has handled it. */\n readonly send: (input: AssistantEvent.Input) => Effect.Effect<TestChat, TestChatError>\n /** Sends the user's message. */\n readonly say: (text: string) => Effect.Effect<TestChat, TestChatError>\n /** Approves the last call waiting for approval. */\n readonly approve: Effect.Effect<TestChat, TestChatError>\n /** Rejects the last call waiting for approval. */\n readonly reject: (options?: { readonly reason?: string | undefined }) => Effect.Effect<TestChat, TestChatError>\n /** Answers the last block of this kind the assistant showed. */\n readonly answer: <Answer extends Block.Codec>(\n block: Block.Block<string, Block.Codec, Answer>,\n answer: Answer[\"Type\"]\n ) => Effect.Effect<TestChat, TestChatError>\n}\n\nconst handled = (entries: ReadonlyArray<AssistantEvent.AnyEntry>, inputId: string): boolean => {\n const input = entries.find(({ event }) => AssistantEvent.isInput(event) && event.id === inputId)\n\n return (\n input !== undefined &&\n libraryEvents(entries).some((event) => AssistantEvent.isTurnEnd(event) && event.throughSequence >= input.sequence)\n )\n}\n\n/**\n * Creates a chat to drive the assistant with, for a chat the app already created. A step fails\n * when no turn has handled its input within `timeout` (3 minutes by default), or when a turn failed\n * while handling it.\n *\n * @category constructors\n * @since 0.9.0\n */\nexport const chat = <Events extends { readonly _tag: string } = never>(\n assistant: Assistant.Service<Events>,\n chatId: ChatId,\n options?: { readonly timeout?: Duration.Input | undefined }\n): TestChat => at(assistant, chatId, options?.timeout ?? \"3 minutes\", [], [])\n\nconst at = <Events extends { readonly _tag: string }>(\n assistant: Assistant.Service<Events>,\n chatId: ChatId,\n timeout: Duration.Input,\n entries: ReadonlyArray<AssistantEvent.AnyEntry>,\n step: ReadonlyArray<AssistantEvent.AnyEntry>\n): TestChat => {\n const all = log(entries)\n\n const send = (input: AssistantEvent.Input): Effect.Effect<TestChat, TestChatError> =>\n Effect.gen(function* () {\n const before = entries.at(-1)?.sequence ?? 0\n let seen: ReadonlyArray<AssistantEvent.AnyEntry> = []\n\n yield* assistant.submit(chatId, input).pipe(Effect.orDie)\n\n const after = yield* assistant.subscribe(chatId).pipe(\n Stream.scan(\n (): ReadonlyArray<AssistantEvent.AnyEntry> => [],\n (acc, entry) => [...acc, entry]\n ),\n Stream.tap((acc) => Effect.sync(() => (seen = acc))),\n Stream.takeUntil((acc) => handled(acc, input.id)),\n Stream.runLast,\n Effect.map(Option.getOrElse((): ReadonlyArray<AssistantEvent.AnyEntry> => [])),\n Effect.timeoutOrElse({\n duration: timeout,\n orElse: () =>\n Effect.fail(\n new TestChatError({\n reason: \"TimedOut\",\n message: `no turn handled the input; the assistant said: ${log(seen).messages.join(\" | \")}`\n })\n )\n })\n )\n\n const added = after.filter(({ sequence }) => sequence > before)\n\n const failed = libraryEvents(added).flatMap((event) =>\n Predicate.isTagged(event, \"TurnFailed\") ? [event.message] : []\n )\n\n if (failed.length > 0) {\n return yield* new TestChatError({ reason: \"TurnFailed\", message: failed.join(\" | \") })\n }\n\n return at(assistant, chatId, timeout, after, added)\n })\n\n const decide = (decision: (approval: { readonly id: string }) => AssistantEvent.ApprovalDecision) => {\n const waiting = all.approvals.at(-1)\n\n return waiting === undefined\n ? Effect.fail(new TestChatError({ reason: \"NothingToApprove\", message: \"no call waits for approval\" }))\n : send(decision({ id: waiting.callId }))\n }\n\n const answer = <Answer extends Block.Codec>(\n block: Block.Block<string, Block.Codec, Answer>,\n value: Answer[\"Type\"]\n ): Effect.Effect<TestChat, TestChatError> => {\n const asked = libraryEvents(entries).findLast(\n (event): event is AssistantEvent.AssistantMessage =>\n Predicate.isTagged(event, \"AssistantMessage\") && event.block?.kind === block.kind\n )\n\n if (asked === undefined || block.answer === undefined) {\n return Effect.fail(new TestChatError({ reason: \"NothingToAnswer\", message: `no ${block.kind} block to answer` }))\n }\n\n return send(\n AssistantEvent.BlockAnswer.make({\n // As the client sends it: the same id every time, so a block takes one answer.\n id: InputId.fromKey(`${asked.id}:answer`),\n messageId: asked.id,\n kind: block.kind,\n value: Schema.encodeUnknownSync(Schema.toCodecJson(block.answer))(value)\n })\n )\n }\n\n return {\n ...all,\n chatId,\n step: log(step),\n send,\n say: (text) => send(AssistantEvent.UserMessage.make({ text })),\n approve: decide(ToolApproval.approve),\n reject: (options) => decide((approval) => ToolApproval.reject(approval, options)),\n answer\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0CA,MAAM,2BAAgC,IAAI,IAAI;CAAC;CAAqB;CAAc;AAAY,CAAC;;;;;;;;AAS/F,IAAa,gBAAb,cAAmC,KAAK,YAAY,eAAe,CAAC,CAGjE,CAAC;AAEJ,MAAM,iBAAiB,YACrB,QAAQ,SAAS,EAAE,YAAaA,eAA8B,KAAK,IAAI,CAAC,KAAK,IAAI,CAAC,CAAE;;;;;;;AAkCtF,MAAa,OAAO,YAAyD;CAC3E,MAAM,SAAS,cAAc,OAAO;CACpC,MAAM,UAAU,OAAO,SAAS,UAAW,UAAU,SAAS,OAAO,YAAY,IAAI,CAAC,KAAK,IAAI,CAAC,CAAE;CAClG,MAAM,YAAY,OAAO,SAAS,UAAW,UAAU,SAAS,OAAO,mBAAmB,IAAI,CAAC,KAAK,IAAI,CAAC,CAAE;CAE3G,OAAO;EACL;EACA,UAAU,CACR,GAAG,IAAI,IACL,QAAQ,QAAQ,EAAE,SAAS,WAAW,YAAY,aAAa,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,EAAE,WAAW,IAAI,CAC5G,CACF;EACA,YAAY,SAAS,QAAQ,QAAQ,WAAW,OAAO,SAAS,IAAI,CAAC,CAAC;EACtE,QAAQ,OAAO,SAAS,UACtB,UAAU,SAAS,OAAO,kBAAkB,KAAK,MAAM,UAAU,KAAA,IAAY,CAAC,MAAM,MAAM,IAAI,IAAI,CAAC,CACrG;EACA;EACA,YAAY,UAAU,KAAK,EAAE,WAAW,IAAI;EAC5C,UAAUC,eAA8B,OAAO;CACjD;AACF;AA2BA,MAAM,WAAW,SAAiD,YAA6B;CAC7F,MAAM,QAAQ,QAAQ,MAAM,EAAE,YAAYC,QAAuB,KAAK,KAAK,MAAM,OAAO,OAAO;CAE/F,OACE,UAAU,KAAA,KACV,cAAc,OAAO,CAAC,CAAC,MAAM,UAAUC,UAAyB,KAAK,KAAK,MAAM,mBAAmB,MAAM,QAAQ;AAErH;;;;;;;;;AAUA,MAAa,QACX,WACA,QACA,YACa,GAAG,WAAW,QAAQ,SAAS,WAAW,aAAa,CAAC,GAAG,CAAC,CAAC;AAE5E,MAAM,MACJ,WACA,QACA,SACA,SACA,SACa;CACb,MAAM,MAAM,IAAI,OAAO;CAEvB,MAAM,QAAQ,UACZ,OAAO,IAAI,aAAa;EACtB,MAAM,SAAS,QAAQ,GAAG,EAAE,CAAC,EAAE,YAAY;EAC3C,IAAI,OAA+C,CAAC;EAEpD,OAAO,UAAU,OAAO,QAAQ,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK;EAExD,MAAM,QAAQ,OAAO,UAAU,UAAU,MAAM,CAAC,CAAC,KAC/C,OAAO,WACyC,CAAC,IAC9C,KAAK,UAAU,CAAC,GAAG,KAAK,KAAK,CAChC,GACA,OAAO,KAAK,QAAQ,OAAO,WAAY,OAAO,GAAI,CAAC,GACnD,OAAO,WAAW,QAAQ,QAAQ,KAAK,MAAM,EAAE,CAAC,GAChD,OAAO,SACP,OAAO,IAAI,OAAO,gBAAwD,CAAC,CAAC,CAAC,GAC7E,OAAO,cAAc;GACnB,UAAU;GACV,cACE,OAAO,KACL,IAAI,cAAc;IAChB,QAAQ;IACR,SAAS,kDAAkD,IAAI,IAAI,CAAC,CAAC,SAAS,KAAK,KAAK;GAC1F,CAAC,CACH;EACJ,CAAC,CACH;EAEA,MAAM,QAAQ,MAAM,QAAQ,EAAE,eAAe,WAAW,MAAM;EAE9D,MAAM,SAAS,cAAc,KAAK,CAAC,CAAC,SAAS,UAC3C,UAAU,SAAS,OAAO,YAAY,IAAI,CAAC,MAAM,OAAO,IAAI,CAAC,CAC/D;EAEA,IAAI,OAAO,SAAS,GAClB,OAAO,OAAO,IAAI,cAAc;GAAE,QAAQ;GAAc,SAAS,OAAO,KAAK,KAAK;EAAE,CAAC;EAGvF,OAAO,GAAG,WAAW,QAAQ,SAAS,OAAO,KAAK;CACpD,CAAC;CAEH,MAAM,UAAU,aAAqF;EACnG,MAAM,UAAU,IAAI,UAAU,GAAG,EAAE;EAEnC,OAAO,YAAY,KAAA,IACf,OAAO,KAAK,IAAI,cAAc;GAAE,QAAQ;GAAoB,SAAS;EAA6B,CAAC,CAAC,IACpG,KAAK,SAAS,EAAE,IAAI,QAAQ,OAAO,CAAC,CAAC;CAC3C;CAEA,MAAM,UACJ,OACA,UAC2C;EAC3C,MAAM,QAAQ,cAAc,OAAO,CAAC,CAAC,UAClC,UACC,UAAU,SAAS,OAAO,kBAAkB,KAAK,MAAM,OAAO,SAAS,MAAM,IACjF;EAEA,IAAI,UAAU,KAAA,KAAa,MAAM,WAAW,KAAA,GAC1C,OAAO,OAAO,KAAK,IAAI,cAAc;GAAE,QAAQ;GAAmB,SAAS,MAAM,MAAM,KAAK;EAAkB,CAAC,CAAC;EAGlH,OAAO,KAAA,YACsB,KAAK;GAE9B,IAAIC,QAAgB,GAAG,MAAM,GAAG,QAAQ;GACxC,WAAW,MAAM;GACjB,MAAM,MAAM;GACZ,OAAO,OAAO,kBAAkB,OAAO,YAAY,MAAM,MAAM,CAAC,CAAC,CAAC,KAAK;EACzE,CAAC,CACH;CACF;CAEA,OAAO;EACL,GAAG;EACH;EACA,MAAM,IAAI,IAAI;EACd;EACA,MAAM,SAAS,KAAA,YAAgC,KAAK,EAAE,KAAK,CAAC,CAAC;EAC7D,SAAS,OAAOC,OAAoB;EACpC,SAAS,YAAY,QAAQ,aAAaC,OAAoB,UAAU,OAAO,CAAC;EAChF;CACF;AACF"}
package/dist/index.d.ts CHANGED
@@ -14,6 +14,7 @@ import { AssistantEngine_d_exports } from "./AssistantEngine.js";
14
14
  import { Chats_d_exports } from "./Chats.js";
15
15
  import { Assistant_d_exports } from "./Assistant.js";
16
16
  import { AssistantActivity_d_exports } from "./AssistantActivity.js";
17
+ import { AssistantTest_d_exports } from "./AssistantTest.js";
17
18
  import { Compaction_d_exports } from "./Compaction.js";
18
19
  import { InputId_d_exports } from "./InputId.js";
19
20
  import { ToolFailure_d_exports } from "./ToolFailure.js";
@@ -22,4 +23,4 @@ import { Skill_d_exports } from "./Skill.js";
22
23
  import { ToolApproval_d_exports } from "./ToolApproval.js";
23
24
  import { ToolDiscovery_d_exports } from "./ToolDiscovery.js";
24
25
  import { ToolResult_d_exports } from "./ToolResult.js";
25
- export { Assistant_d_exports as Assistant, AssistantActivity_d_exports as AssistantActivity, AssistantEngine_d_exports as AssistantEngine, AssistantEvent_d_exports as AssistantEvent, AssistantLoop_d_exports as AssistantLoop, AssistantToolCall_d_exports as AssistantToolCall, Block_d_exports as Block, BlockKit_d_exports as BlockKit, ChatId_d_exports as ChatId, ChatLog_d_exports as ChatLog, Chats_d_exports as Chats, Compaction_d_exports as Compaction, CurrentChat_d_exports as CurrentChat, InputId_d_exports as InputId, Instructions_d_exports as Instructions, Memory_d_exports as Memory, RunId_d_exports as RunId, Skill_d_exports as Skill, Specialist_d_exports as Specialist, Timeline_d_exports as Timeline, ToolApproval_d_exports as ToolApproval, ToolDiscovery_d_exports as ToolDiscovery, ToolFailure_d_exports as ToolFailure, ToolResult_d_exports as ToolResult };
26
+ export { Assistant_d_exports as Assistant, AssistantActivity_d_exports as AssistantActivity, AssistantEngine_d_exports as AssistantEngine, AssistantEvent_d_exports as AssistantEvent, AssistantLoop_d_exports as AssistantLoop, AssistantTest_d_exports as AssistantTest, AssistantToolCall_d_exports as AssistantToolCall, Block_d_exports as Block, BlockKit_d_exports as BlockKit, ChatId_d_exports as ChatId, ChatLog_d_exports as ChatLog, Chats_d_exports as Chats, Compaction_d_exports as Compaction, CurrentChat_d_exports as CurrentChat, InputId_d_exports as InputId, Instructions_d_exports as Instructions, Memory_d_exports as Memory, RunId_d_exports as RunId, Skill_d_exports as Skill, Specialist_d_exports as Specialist, Timeline_d_exports as Timeline, ToolApproval_d_exports as ToolApproval, ToolDiscovery_d_exports as ToolDiscovery, ToolFailure_d_exports as ToolFailure, ToolResult_d_exports as ToolResult };
package/dist/index.js CHANGED
@@ -17,9 +17,10 @@ import { AssistantEngine_exports } from "./AssistantEngine.js";
17
17
  import { Assistant_exports } from "./Assistant.js";
18
18
  import { AssistantActivity_exports } from "./AssistantActivity.js";
19
19
  import { AssistantLoop_exports } from "./AssistantLoop.js";
20
+ import { AssistantTest_exports } from "./AssistantTest.js";
20
21
  import { Compaction_exports } from "./Compaction.js";
21
22
  import { Memory_exports } from "./Memory.js";
22
23
  import { Skill_exports } from "./Skill.js";
23
24
  import { Timeline_exports } from "./Timeline.js";
24
25
  import { ToolDiscovery_exports } from "./ToolDiscovery.js";
25
- export { Assistant_exports as Assistant, AssistantActivity_exports as AssistantActivity, AssistantEngine_exports as AssistantEngine, AssistantEvent_exports as AssistantEvent, AssistantLoop_exports as AssistantLoop, AssistantToolCall_exports as AssistantToolCall, Block_exports as Block, BlockKit_exports as BlockKit, ChatId_exports as ChatId, ChatLog_exports as ChatLog, Chats_exports as Chats, Compaction_exports as Compaction, CurrentChat_exports as CurrentChat, InputId_exports as InputId, Instructions_exports as Instructions, Memory_exports as Memory, RunId_exports as RunId, Skill_exports as Skill, Specialist_exports as Specialist, Timeline_exports as Timeline, ToolApproval_exports as ToolApproval, ToolDiscovery_exports as ToolDiscovery, ToolFailure_exports as ToolFailure, ToolResult_exports as ToolResult };
26
+ export { Assistant_exports as Assistant, AssistantActivity_exports as AssistantActivity, AssistantEngine_exports as AssistantEngine, AssistantEvent_exports as AssistantEvent, AssistantLoop_exports as AssistantLoop, AssistantTest_exports as AssistantTest, AssistantToolCall_exports as AssistantToolCall, Block_exports as Block, BlockKit_exports as BlockKit, ChatId_exports as ChatId, ChatLog_exports as ChatLog, Chats_exports as Chats, Compaction_exports as Compaction, CurrentChat_exports as CurrentChat, InputId_exports as InputId, Instructions_exports as Instructions, Memory_exports as Memory, RunId_exports as RunId, Skill_exports as Skill, Specialist_exports as Specialist, Timeline_exports as Timeline, ToolApproval_exports as ToolApproval, ToolDiscovery_exports as ToolDiscovery, ToolFailure_exports as ToolFailure, ToolResult_exports as ToolResult };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "effect-assistant",
3
- "version": "0.8.3",
3
+ "version": "0.9.0",
4
4
  "description": "A colleague assistant on Effect: an append-only event log per chat, an interruptible agent loop, specialists, skills, and UI blocks",
5
5
  "license": "MIT",
6
6
  "repository": {
@@ -0,0 +1,254 @@
1
+ /**
2
+ * This module provides a chat to drive an assistant like a user would, for tests and evals.
3
+ *
4
+ * {@link chat} wraps a chat the app created. Each step (`say`, `approve`, `reject`, `answer`, or
5
+ * `send` for any input) submits the input, waits until a turn has handled it, and returns the chat
6
+ * as it is then. A {@link Log} reads what the assistant did, for the whole chat or for the last
7
+ * step alone (`step`): the tools that ran, the blocks it showed, the calls waiting for approval,
8
+ * and the messages it sent.
9
+ *
10
+ * **Example** (An assistant asks before it emails)
11
+ *
12
+ * ```ts
13
+ * import { AssistantTest } from "effect-assistant"
14
+ * import * as Effect from "effect/Effect"
15
+ *
16
+ * const program = Effect.gen(function* () {
17
+ * const chat = AssistantTest.chat(yield* MyAssistant, chatId)
18
+ * const asked = yield* chat.say("Email the report to ana@example.com")
19
+ * asked.step.waitingFor // ["send_report"]
20
+ * const sent = yield* asked.approve
21
+ * sent.step.toolsRun // ["send_report"]
22
+ * })
23
+ * ```
24
+ *
25
+ * @since 0.9.0
26
+ */
27
+ import * as Data from "effect/Data"
28
+ import type * as Duration from "effect/Duration"
29
+ import * as Effect from "effect/Effect"
30
+ import * as Option from "effect/Option"
31
+ import * as Predicate from "effect/Predicate"
32
+ import * as Schema from "effect/Schema"
33
+ import * as Stream from "effect/Stream"
34
+ import type * as Assistant from "./Assistant.ts"
35
+ import type * as Block from "./Block.ts"
36
+ import * as AssistantEvent from "./AssistantEvent.ts"
37
+ import type { ChatId } from "./ChatId.ts"
38
+ import * as InputId from "./InputId.ts"
39
+ import { sendUserMessageName, waitToolName } from "./internal/tools.ts"
40
+ import * as ToolApproval from "./ToolApproval.ts"
41
+
42
+ // The assistant's own tools: how it talks, waits and reads its skills, not work it does for the user.
43
+ const ownTools: ReadonlySet<string> = new Set([sendUserMessageName, waitToolName, "load_skill"])
44
+
45
+ /**
46
+ * Represents a step that couldn't complete: no turn handled the input in time, the turn failed,
47
+ * or there was nothing to approve or answer.
48
+ *
49
+ * @category errors
50
+ * @since 0.9.0
51
+ */
52
+ export class TestChatError extends Data.TaggedError("TestChatError")<{
53
+ readonly reason: "TimedOut" | "TurnFailed" | "NothingToApprove" | "NothingToAnswer"
54
+ readonly message: string
55
+ }> {}
56
+
57
+ const libraryEvents = (entries: ReadonlyArray<AssistantEvent.AnyEntry>): ReadonlyArray<AssistantEvent.AssistantEvent> =>
58
+ entries.flatMap(({ event }) => (AssistantEvent.isLibraryEvent(event) ? [event] : []))
59
+
60
+ /**
61
+ * Represents what the assistant did in some of a chat's entries.
62
+ *
63
+ * @category models
64
+ * @since 0.9.0
65
+ */
66
+ export interface Log {
67
+ readonly entries: ReadonlyArray<AssistantEvent.AnyEntry>
68
+ /**
69
+ * The app's tools that ran, each once, in the order they first ran. A call that failed or that
70
+ * the library refused didn't run, and the assistant's own tools (`send_user_message`, `wait`,
71
+ * `load_skill`) aren't listed.
72
+ */
73
+ readonly toolsRun: ReadonlyArray<string>
74
+ /** How many times the model called this tool, whether or not it ran. */
75
+ readonly toolCalls: (tool: string) => number
76
+ /** The kind of each block the assistant showed, in order; one per message. */
77
+ readonly blocks: ReadonlyArray<string>
78
+ /** The calls waiting for the user's approval, in order. */
79
+ readonly approvals: ReadonlyArray<AssistantEvent.ApprovalRequested>
80
+ /** The tools of the calls waiting for the user's approval. */
81
+ readonly waitingFor: ReadonlyArray<string>
82
+ /** The messages the assistant sent, in order. */
83
+ readonly messages: ReadonlyArray<string>
84
+ }
85
+
86
+ /**
87
+ * Reads what the assistant did in these entries.
88
+ *
89
+ * @category constructors
90
+ * @since 0.9.0
91
+ */
92
+ export const log = (entries: ReadonlyArray<AssistantEvent.AnyEntry>): Log => {
93
+ const events = libraryEvents(entries)
94
+ const results = events.flatMap((event) => (Predicate.isTagged(event, "ToolResult") ? [event] : []))
95
+ const approvals = events.flatMap((event) => (Predicate.isTagged(event, "ApprovalRequested") ? [event] : []))
96
+
97
+ return {
98
+ entries,
99
+ toolsRun: [
100
+ ...new Set(
101
+ results.filter(({ outcome, tool }) => outcome === "success" && !ownTools.has(tool)).map(({ tool }) => tool)
102
+ )
103
+ ],
104
+ toolCalls: (tool) => results.filter((result) => result.tool === tool).length,
105
+ blocks: events.flatMap((event) =>
106
+ Predicate.isTagged(event, "AssistantMessage") && event.block !== undefined ? [event.block.kind] : []
107
+ ),
108
+ approvals,
109
+ waitingFor: approvals.map(({ tool }) => tool),
110
+ messages: AssistantEvent.assistantTexts(entries)
111
+ }
112
+ }
113
+
114
+ /**
115
+ * Represents a chat driven like a user would.
116
+ *
117
+ * @category models
118
+ * @since 0.9.0
119
+ */
120
+ export interface TestChat extends Log {
121
+ readonly chatId: ChatId
122
+ /** What the assistant did for the last step alone. */
123
+ readonly step: Log
124
+ /** Submits an input, and returns the chat once a turn has handled it. */
125
+ readonly send: (input: AssistantEvent.Input) => Effect.Effect<TestChat, TestChatError>
126
+ /** Sends the user's message. */
127
+ readonly say: (text: string) => Effect.Effect<TestChat, TestChatError>
128
+ /** Approves the last call waiting for approval. */
129
+ readonly approve: Effect.Effect<TestChat, TestChatError>
130
+ /** Rejects the last call waiting for approval. */
131
+ readonly reject: (options?: { readonly reason?: string | undefined }) => Effect.Effect<TestChat, TestChatError>
132
+ /** Answers the last block of this kind the assistant showed. */
133
+ readonly answer: <Answer extends Block.Codec>(
134
+ block: Block.Block<string, Block.Codec, Answer>,
135
+ answer: Answer["Type"]
136
+ ) => Effect.Effect<TestChat, TestChatError>
137
+ }
138
+
139
+ const handled = (entries: ReadonlyArray<AssistantEvent.AnyEntry>, inputId: string): boolean => {
140
+ const input = entries.find(({ event }) => AssistantEvent.isInput(event) && event.id === inputId)
141
+
142
+ return (
143
+ input !== undefined &&
144
+ libraryEvents(entries).some((event) => AssistantEvent.isTurnEnd(event) && event.throughSequence >= input.sequence)
145
+ )
146
+ }
147
+
148
+ /**
149
+ * Creates a chat to drive the assistant with, for a chat the app already created. A step fails
150
+ * when no turn has handled its input within `timeout` (3 minutes by default), or when a turn failed
151
+ * while handling it.
152
+ *
153
+ * @category constructors
154
+ * @since 0.9.0
155
+ */
156
+ export const chat = <Events extends { readonly _tag: string } = never>(
157
+ assistant: Assistant.Service<Events>,
158
+ chatId: ChatId,
159
+ options?: { readonly timeout?: Duration.Input | undefined }
160
+ ): TestChat => at(assistant, chatId, options?.timeout ?? "3 minutes", [], [])
161
+
162
+ const at = <Events extends { readonly _tag: string }>(
163
+ assistant: Assistant.Service<Events>,
164
+ chatId: ChatId,
165
+ timeout: Duration.Input,
166
+ entries: ReadonlyArray<AssistantEvent.AnyEntry>,
167
+ step: ReadonlyArray<AssistantEvent.AnyEntry>
168
+ ): TestChat => {
169
+ const all = log(entries)
170
+
171
+ const send = (input: AssistantEvent.Input): Effect.Effect<TestChat, TestChatError> =>
172
+ Effect.gen(function* () {
173
+ const before = entries.at(-1)?.sequence ?? 0
174
+ let seen: ReadonlyArray<AssistantEvent.AnyEntry> = []
175
+
176
+ yield* assistant.submit(chatId, input).pipe(Effect.orDie)
177
+
178
+ const after = yield* assistant.subscribe(chatId).pipe(
179
+ Stream.scan(
180
+ (): ReadonlyArray<AssistantEvent.AnyEntry> => [],
181
+ (acc, entry) => [...acc, entry]
182
+ ),
183
+ Stream.tap((acc) => Effect.sync(() => (seen = acc))),
184
+ Stream.takeUntil((acc) => handled(acc, input.id)),
185
+ Stream.runLast,
186
+ Effect.map(Option.getOrElse((): ReadonlyArray<AssistantEvent.AnyEntry> => [])),
187
+ Effect.timeoutOrElse({
188
+ duration: timeout,
189
+ orElse: () =>
190
+ Effect.fail(
191
+ new TestChatError({
192
+ reason: "TimedOut",
193
+ message: `no turn handled the input; the assistant said: ${log(seen).messages.join(" | ")}`
194
+ })
195
+ )
196
+ })
197
+ )
198
+
199
+ const added = after.filter(({ sequence }) => sequence > before)
200
+
201
+ const failed = libraryEvents(added).flatMap((event) =>
202
+ Predicate.isTagged(event, "TurnFailed") ? [event.message] : []
203
+ )
204
+
205
+ if (failed.length > 0) {
206
+ return yield* new TestChatError({ reason: "TurnFailed", message: failed.join(" | ") })
207
+ }
208
+
209
+ return at(assistant, chatId, timeout, after, added)
210
+ })
211
+
212
+ const decide = (decision: (approval: { readonly id: string }) => AssistantEvent.ApprovalDecision) => {
213
+ const waiting = all.approvals.at(-1)
214
+
215
+ return waiting === undefined
216
+ ? Effect.fail(new TestChatError({ reason: "NothingToApprove", message: "no call waits for approval" }))
217
+ : send(decision({ id: waiting.callId }))
218
+ }
219
+
220
+ const answer = <Answer extends Block.Codec>(
221
+ block: Block.Block<string, Block.Codec, Answer>,
222
+ value: Answer["Type"]
223
+ ): Effect.Effect<TestChat, TestChatError> => {
224
+ const asked = libraryEvents(entries).findLast(
225
+ (event): event is AssistantEvent.AssistantMessage =>
226
+ Predicate.isTagged(event, "AssistantMessage") && event.block?.kind === block.kind
227
+ )
228
+
229
+ if (asked === undefined || block.answer === undefined) {
230
+ return Effect.fail(new TestChatError({ reason: "NothingToAnswer", message: `no ${block.kind} block to answer` }))
231
+ }
232
+
233
+ return send(
234
+ AssistantEvent.BlockAnswer.make({
235
+ // As the client sends it: the same id every time, so a block takes one answer.
236
+ id: InputId.fromKey(`${asked.id}:answer`),
237
+ messageId: asked.id,
238
+ kind: block.kind,
239
+ value: Schema.encodeUnknownSync(Schema.toCodecJson(block.answer))(value)
240
+ })
241
+ )
242
+ }
243
+
244
+ return {
245
+ ...all,
246
+ chatId,
247
+ step: log(step),
248
+ send,
249
+ say: (text) => send(AssistantEvent.UserMessage.make({ text })),
250
+ approve: decide(ToolApproval.approve),
251
+ reject: (options) => decide((approval) => ToolApproval.reject(approval, options)),
252
+ answer
253
+ }
254
+ }
package/src/index.ts CHANGED
@@ -34,6 +34,11 @@ export * as AssistantLoop from "./AssistantLoop.ts"
34
34
  */
35
35
  export * as AssistantToolCall from "./AssistantToolCall.ts"
36
36
 
37
+ /**
38
+ * @since 0.9.0
39
+ */
40
+ export * as AssistantTest from "./AssistantTest.ts"
41
+
37
42
  /**
38
43
  * @since 0.1.0
39
44
  */