effect-assistant 0.8.2 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +84 -0
- package/dist/AssistantTest.d.ts +94 -0
- package/dist/AssistantTest.d.ts.map +1 -0
- package/dist/AssistantTest.js +146 -0
- package/dist/AssistantTest.js.map +1 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/package.json +1 -1
- package/src/AssistantTest.ts +254 -0
- package/src/index.ts +5 -0
package/README.md
CHANGED
|
@@ -180,6 +180,90 @@ reads the last summary plus what came after; the log keeps everything, so the us
|
|
|
180
180
|
turn.run({ instructions, history: Compaction.since(Schema.is(Summary)) })
|
|
181
181
|
```
|
|
182
182
|
|
|
183
|
+
### Putting it together
|
|
184
|
+
|
|
185
|
+
One assistant with all of it: tools with an approval, a skill, memory, blocks that show a tool's result, a specialist,
|
|
186
|
+
a daily budget, and a summary after the reply once the chat is long. The budget and the summarizing are services of
|
|
187
|
+
your app; the handler decides what they mean for the chat. `MyTools` holds a `search_documents` tool and a
|
|
188
|
+
`send_email` tool with `needsApproval`; `ChooseOne` is a block, `WeeklyReport` a [skill](https://github.com/dbrxnds/effect-assistant/tree/main/apps/docs/guide/skills.md) and
|
|
189
|
+
`Research` a [specialist](https://github.com/dbrxnds/effect-assistant/tree/main/apps/docs/guide/specialists.md).
|
|
190
|
+
|
|
191
|
+
```ts
|
|
192
|
+
class Summary extends Schema.TaggedClass<Summary>()("Summary", { text: Schema.String, throughSequence: Schema.Int }) {}
|
|
193
|
+
class BudgetReached extends Schema.TaggedClass<BudgetReached>()("BudgetReached", {}) {}
|
|
194
|
+
|
|
195
|
+
type MyEvent = Summary | BudgetReached
|
|
196
|
+
|
|
197
|
+
export const MyAssistant = Assistant.make("my-assistant", {
|
|
198
|
+
blocks: BlockKit.make([ChooseOne, Block.fromToolResult(SearchDocuments)]),
|
|
199
|
+
events: [Summary, BudgetReached],
|
|
200
|
+
specialists: [Research]
|
|
201
|
+
})
|
|
202
|
+
|
|
203
|
+
class AiBudget extends Context.Service<
|
|
204
|
+
AiBudget,
|
|
205
|
+
{ readonly exceeded: (chatId: ChatId.ChatId) => Effect.Effect<boolean> }
|
|
206
|
+
>()("AiBudget") {}
|
|
207
|
+
|
|
208
|
+
class ChatCompaction extends Context.Service<
|
|
209
|
+
ChatCompaction,
|
|
210
|
+
{
|
|
211
|
+
readonly summarize: (
|
|
212
|
+
chat: Assistant.Chat<MyEvent>,
|
|
213
|
+
turn: Assistant.Turn<MyEvent>
|
|
214
|
+
) => Effect.Effect<Option.Option<Summary>, AiError.AiError>
|
|
215
|
+
}
|
|
216
|
+
>()("ChatCompaction") {}
|
|
217
|
+
|
|
218
|
+
const Skills = Skill.toolkit([WeeklyReport])
|
|
219
|
+
|
|
220
|
+
export const MyAssistantLive = MyAssistant.toLayer(
|
|
221
|
+
Effect.gen(function* () {
|
|
222
|
+
const budget = yield* AiBudget
|
|
223
|
+
const compaction = yield* ChatCompaction
|
|
224
|
+
const memory = yield* Memory.MemoryStore
|
|
225
|
+
|
|
226
|
+
return Effect.fn("MyAssistant.turn")(function* (turn: Assistant.Turn<MyEvent>) {
|
|
227
|
+
const chat = yield* MyAssistant.Chat
|
|
228
|
+
|
|
229
|
+
if (yield* budget.exceeded(turn.chatId)) {
|
|
230
|
+
yield* chat.append(BudgetReached.make({}))
|
|
231
|
+
|
|
232
|
+
return Stream.empty
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
// After the reply, so the user doesn't wait; a failed summary leaves the chat as it is.
|
|
236
|
+
const summarized = compaction.summarize(chat, turn).pipe(
|
|
237
|
+
Effect.flatMap(Option.match({ onNone: () => Effect.void, onSome: (summary) => chat.append(summary) })),
|
|
238
|
+
Effect.catchCause((cause) => Effect.logWarning("compaction failed", cause))
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
return turn
|
|
242
|
+
.run({
|
|
243
|
+
instructions: `${instructions}\n\n${Memory.render(yield* memory.facts(turn.chatId))}`,
|
|
244
|
+
toolkit: Toolkit.merge(MyTools, Memory.Tools, Skills),
|
|
245
|
+
history: Compaction.since(Schema.is(Summary)),
|
|
246
|
+
maxSteps: 20
|
|
247
|
+
})
|
|
248
|
+
.pipe(
|
|
249
|
+
Stream.concat(Stream.drain(Stream.fromEffect(summarized))),
|
|
250
|
+
Stream.provideService(Memory.MemoryStore, memory)
|
|
251
|
+
)
|
|
252
|
+
})
|
|
253
|
+
})
|
|
254
|
+
).pipe(Layer.provide([MyToolsLive, Memory.layerTools, Skills.layer, ResearchLive]))
|
|
255
|
+
```
|
|
256
|
+
|
|
257
|
+
```text
|
|
258
|
+
AiBudget.exceeded ─► BudgetReached ─► chat log (and no turn)
|
|
259
|
+
chat log ─► Compaction.since(Summary) ─► history ──┐
|
|
260
|
+
memory facts ─► instructions ──────────────────────┼─► turn.run ─► events ─► chat log
|
|
261
|
+
MyTools + Memory.Tools + Skills ─► toolkit ────────┘
|
|
262
|
+
ChatCompaction.summarize, after the reply ─► Summary ─► chat log
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
The app provides `AiBudget`, `ChatCompaction` and a `Memory.MemoryStore`.
|
|
266
|
+
|
|
183
267
|
### And more
|
|
184
268
|
|
|
185
269
|
- [Specialists](https://github.com/dbrxnds/effect-assistant/tree/main/apps/docs/guide/specialists.md): focused agents that do slow work in the background.
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { ChatId } from "./ChatId.js";
|
|
2
|
+
import { AnyEntry, ApprovalRequested, AssistantEvent_d_exports, Input } from "./AssistantEvent.js";
|
|
3
|
+
import { Block, Block_d_exports, Codec } from "./Block.js";
|
|
4
|
+
import { Assistant_d_exports, Service } from "./Assistant.js";
|
|
5
|
+
import * as Effect from "effect/Effect";
|
|
6
|
+
import * as Duration from "effect/Duration";
|
|
7
|
+
declare namespace AssistantTest_d_exports {
|
|
8
|
+
export { Log, TestChat, TestChatError, chat, log };
|
|
9
|
+
}
|
|
10
|
+
declare const TestChatError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
|
|
11
|
+
readonly _tag: "TestChatError";
|
|
12
|
+
} & Readonly<A>;
|
|
13
|
+
/**
|
|
14
|
+
* Represents a step that couldn't complete: no turn handled the input in time, the turn failed,
|
|
15
|
+
* or there was nothing to approve or answer.
|
|
16
|
+
*
|
|
17
|
+
* @category errors
|
|
18
|
+
* @since 0.9.0
|
|
19
|
+
*/
|
|
20
|
+
export declare class TestChatError extends TestChatError_base<{
|
|
21
|
+
readonly reason: "TimedOut" | "TurnFailed" | "NothingToApprove" | "NothingToAnswer";
|
|
22
|
+
readonly message: string;
|
|
23
|
+
}> {}
|
|
24
|
+
/**
|
|
25
|
+
* Represents what the assistant did in some of a chat's entries.
|
|
26
|
+
*
|
|
27
|
+
* @category models
|
|
28
|
+
* @since 0.9.0
|
|
29
|
+
*/
|
|
30
|
+
export interface Log {
|
|
31
|
+
readonly entries: ReadonlyArray<AnyEntry>;
|
|
32
|
+
/**
|
|
33
|
+
* The app's tools that ran, each once, in the order they first ran. A call that failed or that
|
|
34
|
+
* the library refused didn't run, and the assistant's own tools (`send_user_message`, `wait`,
|
|
35
|
+
* `load_skill`) aren't listed.
|
|
36
|
+
*/
|
|
37
|
+
readonly toolsRun: ReadonlyArray<string>;
|
|
38
|
+
/** How many times the model called this tool, whether or not it ran. */
|
|
39
|
+
readonly toolCalls: (tool: string) => number;
|
|
40
|
+
/** The kind of each block the assistant showed, in order; one per message. */
|
|
41
|
+
readonly blocks: ReadonlyArray<string>;
|
|
42
|
+
/** The calls waiting for the user's approval, in order. */
|
|
43
|
+
readonly approvals: ReadonlyArray<ApprovalRequested>;
|
|
44
|
+
/** The tools of the calls waiting for the user's approval. */
|
|
45
|
+
readonly waitingFor: ReadonlyArray<string>;
|
|
46
|
+
/** The messages the assistant sent, in order. */
|
|
47
|
+
readonly messages: ReadonlyArray<string>;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Reads what the assistant did in these entries.
|
|
51
|
+
*
|
|
52
|
+
* @category constructors
|
|
53
|
+
* @since 0.9.0
|
|
54
|
+
*/
|
|
55
|
+
export declare const log: (entries: ReadonlyArray<AnyEntry>) => Log;
|
|
56
|
+
/**
|
|
57
|
+
* Represents a chat driven like a user would.
|
|
58
|
+
*
|
|
59
|
+
* @category models
|
|
60
|
+
* @since 0.9.0
|
|
61
|
+
*/
|
|
62
|
+
export interface TestChat extends Log {
|
|
63
|
+
readonly chatId: ChatId;
|
|
64
|
+
/** What the assistant did for the last step alone. */
|
|
65
|
+
readonly step: Log;
|
|
66
|
+
/** Submits an input, and returns the chat once a turn has handled it. */
|
|
67
|
+
readonly send: (input: Input) => Effect.Effect<TestChat, TestChatError>;
|
|
68
|
+
/** Sends the user's message. */
|
|
69
|
+
readonly say: (text: string) => Effect.Effect<TestChat, TestChatError>;
|
|
70
|
+
/** Approves the last call waiting for approval. */
|
|
71
|
+
readonly approve: Effect.Effect<TestChat, TestChatError>;
|
|
72
|
+
/** Rejects the last call waiting for approval. */
|
|
73
|
+
readonly reject: (options?: {
|
|
74
|
+
readonly reason?: string | undefined;
|
|
75
|
+
}) => Effect.Effect<TestChat, TestChatError>;
|
|
76
|
+
/** Answers the last block of this kind the assistant showed. */
|
|
77
|
+
readonly answer: <Answer extends Codec>(block: Block<string, Codec, Answer>, answer: Answer["Type"]) => Effect.Effect<TestChat, TestChatError>;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Creates a chat to drive the assistant with, for a chat the app already created. A step fails
|
|
81
|
+
* when no turn has handled its input within `timeout` (3 minutes by default), or when a turn failed
|
|
82
|
+
* while handling it.
|
|
83
|
+
*
|
|
84
|
+
* @category constructors
|
|
85
|
+
* @since 0.9.0
|
|
86
|
+
*/
|
|
87
|
+
export declare const chat: <Events extends {
|
|
88
|
+
readonly _tag: string;
|
|
89
|
+
} = never>(assistant: Service<Events>, chatId: ChatId, options?: {
|
|
90
|
+
readonly timeout?: Duration.Input | undefined;
|
|
91
|
+
}) => TestChat;
|
|
92
|
+
//#endregion
|
|
93
|
+
export { AssistantTest_d_exports };
|
|
94
|
+
//# sourceMappingURL=AssistantTest.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AssistantTest.d.ts","names":[],"sources":["../src/AssistantTest.ts"],"mappings":";;;;;;;;;;;;;;;;;;;qBAmDa,sBAAsB;WACxB;WACA;;;;;;;;iBAYM;WACN,SAAS,cAAc;;;;;;WAMvB,UAAU;;WAEV,YAAY;;WAEZ,QAAQ;;WAER,WAAW,cAAc;;WAEzB,YAAY;;WAEZ,UAAU;;;;;;;;qBASR,MAAG,SAAa,cAAc,cAA2B;;;;;;;iBA4BrD,iBAAiB;WACvB,QAAQ;;WAER,MAAM;;WAEN,OAAO,OAAO,UAAyB,OAAO,OAAO,UAAU;;WAE/D,MAAM,iBAAiB,OAAO,OAAO,UAAU;;WAE/C,SAAS,OAAO,OAAO,UAAU;;WAEjC,SAAS;aAAqB;QAAkC,OAAO,OAAO,UAAU;;WAExF,SAAS,eAAe,OAC/B,OAAO,cAAoB,OAAa,SACxC,QAAQ,mBACL,OAAO,OAAO,UAAU;;;;;;;;;;qBAoBlB,OAAQ;WAA0B;WAAsB,WACxD,QAAkB,SAAO,QAC5B,QAAM;WACO,UAAU,SAAS;MACvC"}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { __exportAll } from "./_virtual/_rolldown/runtime.js";
|
|
2
|
+
import { fromKey } from "./InputId.js";
|
|
3
|
+
import { BlockAnswer, UserMessage, assistantTexts, isInput, isLibraryEvent, isTurnEnd } from "./AssistantEvent.js";
|
|
4
|
+
import { approve, reject } from "./ToolApproval.js";
|
|
5
|
+
import { sendUserMessageName, waitToolName } from "./internal/tools.js";
|
|
6
|
+
import * as Effect from "effect/Effect";
|
|
7
|
+
import * as Predicate from "effect/Predicate";
|
|
8
|
+
import * as Schema from "effect/Schema";
|
|
9
|
+
import * as Stream from "effect/Stream";
|
|
10
|
+
import * as Data from "effect/Data";
|
|
11
|
+
import * as Option from "effect/Option";
|
|
12
|
+
//#region src/AssistantTest.ts
|
|
13
|
+
/**
|
|
14
|
+
* This module provides a chat to drive an assistant like a user would, for tests and evals.
|
|
15
|
+
*
|
|
16
|
+
* {@link chat} wraps a chat the app created. Each step (`say`, `approve`, `reject`, `answer`, or
|
|
17
|
+
* `send` for any input) submits the input, waits until a turn has handled it, and returns the chat
|
|
18
|
+
* as it is then. A {@link Log} reads what the assistant did, for the whole chat or for the last
|
|
19
|
+
* step alone (`step`): the tools that ran, the blocks it showed, the calls waiting for approval,
|
|
20
|
+
* and the messages it sent.
|
|
21
|
+
*
|
|
22
|
+
* **Example** (An assistant asks before it emails)
|
|
23
|
+
*
|
|
24
|
+
* ```ts
|
|
25
|
+
* import { AssistantTest } from "effect-assistant"
|
|
26
|
+
* import * as Effect from "effect/Effect"
|
|
27
|
+
*
|
|
28
|
+
* const program = Effect.gen(function* () {
|
|
29
|
+
* const chat = AssistantTest.chat(yield* MyAssistant, chatId)
|
|
30
|
+
* const asked = yield* chat.say("Email the report to ana@example.com")
|
|
31
|
+
* asked.step.waitingFor // ["send_report"]
|
|
32
|
+
* const sent = yield* asked.approve
|
|
33
|
+
* sent.step.toolsRun // ["send_report"]
|
|
34
|
+
* })
|
|
35
|
+
* ```
|
|
36
|
+
*
|
|
37
|
+
* @since 0.9.0
|
|
38
|
+
*/
|
|
39
|
+
var AssistantTest_exports = /* @__PURE__ */ __exportAll({
|
|
40
|
+
TestChatError: () => TestChatError,
|
|
41
|
+
chat: () => chat,
|
|
42
|
+
log: () => log
|
|
43
|
+
});
|
|
44
|
+
const ownTools = /* @__PURE__ */ new Set([
|
|
45
|
+
sendUserMessageName,
|
|
46
|
+
waitToolName,
|
|
47
|
+
"load_skill"
|
|
48
|
+
]);
|
|
49
|
+
/**
|
|
50
|
+
* Represents a step that couldn't complete: no turn handled the input in time, the turn failed,
|
|
51
|
+
* or there was nothing to approve or answer.
|
|
52
|
+
*
|
|
53
|
+
* @category errors
|
|
54
|
+
* @since 0.9.0
|
|
55
|
+
*/
|
|
56
|
+
var TestChatError = class extends Data.TaggedError("TestChatError") {};
|
|
57
|
+
const libraryEvents = (entries) => entries.flatMap(({ event }) => isLibraryEvent(event) ? [event] : []);
|
|
58
|
+
/**
|
|
59
|
+
* Reads what the assistant did in these entries.
|
|
60
|
+
*
|
|
61
|
+
* @category constructors
|
|
62
|
+
* @since 0.9.0
|
|
63
|
+
*/
|
|
64
|
+
const log = (entries) => {
|
|
65
|
+
const events = libraryEvents(entries);
|
|
66
|
+
const results = events.flatMap((event) => Predicate.isTagged(event, "ToolResult") ? [event] : []);
|
|
67
|
+
const approvals = events.flatMap((event) => Predicate.isTagged(event, "ApprovalRequested") ? [event] : []);
|
|
68
|
+
return {
|
|
69
|
+
entries,
|
|
70
|
+
toolsRun: [...new Set(results.filter(({ outcome, tool }) => outcome === "success" && !ownTools.has(tool)).map(({ tool }) => tool))],
|
|
71
|
+
toolCalls: (tool) => results.filter((result) => result.tool === tool).length,
|
|
72
|
+
blocks: events.flatMap((event) => Predicate.isTagged(event, "AssistantMessage") && event.block !== void 0 ? [event.block.kind] : []),
|
|
73
|
+
approvals,
|
|
74
|
+
waitingFor: approvals.map(({ tool }) => tool),
|
|
75
|
+
messages: assistantTexts(entries)
|
|
76
|
+
};
|
|
77
|
+
};
|
|
78
|
+
const handled = (entries, inputId) => {
|
|
79
|
+
const input = entries.find(({ event }) => isInput(event) && event.id === inputId);
|
|
80
|
+
return input !== void 0 && libraryEvents(entries).some((event) => isTurnEnd(event) && event.throughSequence >= input.sequence);
|
|
81
|
+
};
|
|
82
|
+
/**
|
|
83
|
+
* Creates a chat to drive the assistant with, for a chat the app already created. A step fails
|
|
84
|
+
* when no turn has handled its input within `timeout` (3 minutes by default), or when a turn failed
|
|
85
|
+
* while handling it.
|
|
86
|
+
*
|
|
87
|
+
* @category constructors
|
|
88
|
+
* @since 0.9.0
|
|
89
|
+
*/
|
|
90
|
+
const chat = (assistant, chatId, options) => at(assistant, chatId, options?.timeout ?? "3 minutes", [], []);
|
|
91
|
+
const at = (assistant, chatId, timeout, entries, step) => {
|
|
92
|
+
const all = log(entries);
|
|
93
|
+
const send = (input) => Effect.gen(function* () {
|
|
94
|
+
const before = entries.at(-1)?.sequence ?? 0;
|
|
95
|
+
let seen = [];
|
|
96
|
+
yield* assistant.submit(chatId, input).pipe(Effect.orDie);
|
|
97
|
+
const after = yield* assistant.subscribe(chatId).pipe(Stream.scan(() => [], (acc, entry) => [...acc, entry]), Stream.tap((acc) => Effect.sync(() => seen = acc)), Stream.takeUntil((acc) => handled(acc, input.id)), Stream.runLast, Effect.map(Option.getOrElse(() => [])), Effect.timeoutOrElse({
|
|
98
|
+
duration: timeout,
|
|
99
|
+
orElse: () => Effect.fail(new TestChatError({
|
|
100
|
+
reason: "TimedOut",
|
|
101
|
+
message: `no turn handled the input; the assistant said: ${log(seen).messages.join(" | ")}`
|
|
102
|
+
}))
|
|
103
|
+
}));
|
|
104
|
+
const added = after.filter(({ sequence }) => sequence > before);
|
|
105
|
+
const failed = libraryEvents(added).flatMap((event) => Predicate.isTagged(event, "TurnFailed") ? [event.message] : []);
|
|
106
|
+
if (failed.length > 0) return yield* new TestChatError({
|
|
107
|
+
reason: "TurnFailed",
|
|
108
|
+
message: failed.join(" | ")
|
|
109
|
+
});
|
|
110
|
+
return at(assistant, chatId, timeout, after, added);
|
|
111
|
+
});
|
|
112
|
+
const decide = (decision) => {
|
|
113
|
+
const waiting = all.approvals.at(-1);
|
|
114
|
+
return waiting === void 0 ? Effect.fail(new TestChatError({
|
|
115
|
+
reason: "NothingToApprove",
|
|
116
|
+
message: "no call waits for approval"
|
|
117
|
+
})) : send(decision({ id: waiting.callId }));
|
|
118
|
+
};
|
|
119
|
+
const answer = (block, value) => {
|
|
120
|
+
const asked = libraryEvents(entries).findLast((event) => Predicate.isTagged(event, "AssistantMessage") && event.block?.kind === block.kind);
|
|
121
|
+
if (asked === void 0 || block.answer === void 0) return Effect.fail(new TestChatError({
|
|
122
|
+
reason: "NothingToAnswer",
|
|
123
|
+
message: `no ${block.kind} block to answer`
|
|
124
|
+
}));
|
|
125
|
+
return send(BlockAnswer.make({
|
|
126
|
+
id: fromKey(`${asked.id}:answer`),
|
|
127
|
+
messageId: asked.id,
|
|
128
|
+
kind: block.kind,
|
|
129
|
+
value: Schema.encodeUnknownSync(Schema.toCodecJson(block.answer))(value)
|
|
130
|
+
}));
|
|
131
|
+
};
|
|
132
|
+
return {
|
|
133
|
+
...all,
|
|
134
|
+
chatId,
|
|
135
|
+
step: log(step),
|
|
136
|
+
send,
|
|
137
|
+
say: (text) => send(UserMessage.make({ text })),
|
|
138
|
+
approve: decide(approve),
|
|
139
|
+
reject: (options) => decide((approval) => reject(approval, options)),
|
|
140
|
+
answer
|
|
141
|
+
};
|
|
142
|
+
};
|
|
143
|
+
//#endregion
|
|
144
|
+
export { AssistantTest_exports, TestChatError, chat, log };
|
|
145
|
+
|
|
146
|
+
//# sourceMappingURL=AssistantTest.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AssistantTest.js","names":["AssistantEvent.isLibraryEvent","AssistantEvent.assistantTexts","AssistantEvent.isInput","AssistantEvent.isTurnEnd","InputId.fromKey","ToolApproval.approve","ToolApproval.reject"],"sources":["../src/AssistantTest.ts"],"sourcesContent":["/**\n * This module provides a chat to drive an assistant like a user would, for tests and evals.\n *\n * {@link chat} wraps a chat the app created. Each step (`say`, `approve`, `reject`, `answer`, or\n * `send` for any input) submits the input, waits until a turn has handled it, and returns the chat\n * as it is then. A {@link Log} reads what the assistant did, for the whole chat or for the last\n * step alone (`step`): the tools that ran, the blocks it showed, the calls waiting for approval,\n * and the messages it sent.\n *\n * **Example** (An assistant asks before it emails)\n *\n * ```ts\n * import { AssistantTest } from \"effect-assistant\"\n * import * as Effect from \"effect/Effect\"\n *\n * const program = Effect.gen(function* () {\n * const chat = AssistantTest.chat(yield* MyAssistant, chatId)\n * const asked = yield* chat.say(\"Email the report to ana@example.com\")\n * asked.step.waitingFor // [\"send_report\"]\n * const sent = yield* asked.approve\n * sent.step.toolsRun // [\"send_report\"]\n * })\n * ```\n *\n * @since 0.9.0\n */\nimport * as Data from \"effect/Data\"\nimport type * as Duration from \"effect/Duration\"\nimport * as Effect from \"effect/Effect\"\nimport * as Option from \"effect/Option\"\nimport * as Predicate from \"effect/Predicate\"\nimport * as Schema from \"effect/Schema\"\nimport * as Stream from \"effect/Stream\"\nimport type * as Assistant from \"./Assistant.ts\"\nimport type * as Block from \"./Block.ts\"\nimport * as AssistantEvent from \"./AssistantEvent.ts\"\nimport type { ChatId } from \"./ChatId.ts\"\nimport * as InputId from \"./InputId.ts\"\nimport { sendUserMessageName, waitToolName } from \"./internal/tools.ts\"\nimport * as ToolApproval from \"./ToolApproval.ts\"\n\n// The assistant's own tools: how it talks, waits and reads its skills, not work it does for the user.\nconst ownTools: ReadonlySet<string> = new Set([sendUserMessageName, waitToolName, \"load_skill\"])\n\n/**\n * Represents a step that couldn't complete: no turn handled the input in time, the turn failed,\n * or there was nothing to approve or answer.\n *\n * @category errors\n * @since 0.9.0\n */\nexport class TestChatError extends Data.TaggedError(\"TestChatError\")<{\n readonly reason: \"TimedOut\" | \"TurnFailed\" | \"NothingToApprove\" | \"NothingToAnswer\"\n readonly message: string\n}> {}\n\nconst libraryEvents = (entries: ReadonlyArray<AssistantEvent.AnyEntry>): ReadonlyArray<AssistantEvent.AssistantEvent> =>\n entries.flatMap(({ event }) => (AssistantEvent.isLibraryEvent(event) ? [event] : []))\n\n/**\n * Represents what the assistant did in some of a chat's entries.\n *\n * @category models\n * @since 0.9.0\n */\nexport interface Log {\n readonly entries: ReadonlyArray<AssistantEvent.AnyEntry>\n /**\n * The app's tools that ran, each once, in the order they first ran. A call that failed or that\n * the library refused didn't run, and the assistant's own tools (`send_user_message`, `wait`,\n * `load_skill`) aren't listed.\n */\n readonly toolsRun: ReadonlyArray<string>\n /** How many times the model called this tool, whether or not it ran. */\n readonly toolCalls: (tool: string) => number\n /** The kind of each block the assistant showed, in order; one per message. */\n readonly blocks: ReadonlyArray<string>\n /** The calls waiting for the user's approval, in order. */\n readonly approvals: ReadonlyArray<AssistantEvent.ApprovalRequested>\n /** The tools of the calls waiting for the user's approval. */\n readonly waitingFor: ReadonlyArray<string>\n /** The messages the assistant sent, in order. */\n readonly messages: ReadonlyArray<string>\n}\n\n/**\n * Reads what the assistant did in these entries.\n *\n * @category constructors\n * @since 0.9.0\n */\nexport const log = (entries: ReadonlyArray<AssistantEvent.AnyEntry>): Log => {\n const events = libraryEvents(entries)\n const results = events.flatMap((event) => (Predicate.isTagged(event, \"ToolResult\") ? [event] : []))\n const approvals = events.flatMap((event) => (Predicate.isTagged(event, \"ApprovalRequested\") ? [event] : []))\n\n return {\n entries,\n toolsRun: [\n ...new Set(\n results.filter(({ outcome, tool }) => outcome === \"success\" && !ownTools.has(tool)).map(({ tool }) => tool)\n )\n ],\n toolCalls: (tool) => results.filter((result) => result.tool === tool).length,\n blocks: events.flatMap((event) =>\n Predicate.isTagged(event, \"AssistantMessage\") && event.block !== undefined ? [event.block.kind] : []\n ),\n approvals,\n waitingFor: approvals.map(({ tool }) => tool),\n messages: AssistantEvent.assistantTexts(entries)\n }\n}\n\n/**\n * Represents a chat driven like a user would.\n *\n * @category models\n * @since 0.9.0\n */\nexport interface TestChat extends Log {\n readonly chatId: ChatId\n /** What the assistant did for the last step alone. */\n readonly step: Log\n /** Submits an input, and returns the chat once a turn has handled it. */\n readonly send: (input: AssistantEvent.Input) => Effect.Effect<TestChat, TestChatError>\n /** Sends the user's message. */\n readonly say: (text: string) => Effect.Effect<TestChat, TestChatError>\n /** Approves the last call waiting for approval. */\n readonly approve: Effect.Effect<TestChat, TestChatError>\n /** Rejects the last call waiting for approval. */\n readonly reject: (options?: { readonly reason?: string | undefined }) => Effect.Effect<TestChat, TestChatError>\n /** Answers the last block of this kind the assistant showed. */\n readonly answer: <Answer extends Block.Codec>(\n block: Block.Block<string, Block.Codec, Answer>,\n answer: Answer[\"Type\"]\n ) => Effect.Effect<TestChat, TestChatError>\n}\n\nconst handled = (entries: ReadonlyArray<AssistantEvent.AnyEntry>, inputId: string): boolean => {\n const input = entries.find(({ event }) => AssistantEvent.isInput(event) && event.id === inputId)\n\n return (\n input !== undefined &&\n libraryEvents(entries).some((event) => AssistantEvent.isTurnEnd(event) && event.throughSequence >= input.sequence)\n )\n}\n\n/**\n * Creates a chat to drive the assistant with, for a chat the app already created. A step fails\n * when no turn has handled its input within `timeout` (3 minutes by default), or when a turn failed\n * while handling it.\n *\n * @category constructors\n * @since 0.9.0\n */\nexport const chat = <Events extends { readonly _tag: string } = never>(\n assistant: Assistant.Service<Events>,\n chatId: ChatId,\n options?: { readonly timeout?: Duration.Input | undefined }\n): TestChat => at(assistant, chatId, options?.timeout ?? \"3 minutes\", [], [])\n\nconst at = <Events extends { readonly _tag: string }>(\n assistant: Assistant.Service<Events>,\n chatId: ChatId,\n timeout: Duration.Input,\n entries: ReadonlyArray<AssistantEvent.AnyEntry>,\n step: ReadonlyArray<AssistantEvent.AnyEntry>\n): TestChat => {\n const all = log(entries)\n\n const send = (input: AssistantEvent.Input): Effect.Effect<TestChat, TestChatError> =>\n Effect.gen(function* () {\n const before = entries.at(-1)?.sequence ?? 0\n let seen: ReadonlyArray<AssistantEvent.AnyEntry> = []\n\n yield* assistant.submit(chatId, input).pipe(Effect.orDie)\n\n const after = yield* assistant.subscribe(chatId).pipe(\n Stream.scan(\n (): ReadonlyArray<AssistantEvent.AnyEntry> => [],\n (acc, entry) => [...acc, entry]\n ),\n Stream.tap((acc) => Effect.sync(() => (seen = acc))),\n Stream.takeUntil((acc) => handled(acc, input.id)),\n Stream.runLast,\n Effect.map(Option.getOrElse((): ReadonlyArray<AssistantEvent.AnyEntry> => [])),\n Effect.timeoutOrElse({\n duration: timeout,\n orElse: () =>\n Effect.fail(\n new TestChatError({\n reason: \"TimedOut\",\n message: `no turn handled the input; the assistant said: ${log(seen).messages.join(\" | \")}`\n })\n )\n })\n )\n\n const added = after.filter(({ sequence }) => sequence > before)\n\n const failed = libraryEvents(added).flatMap((event) =>\n Predicate.isTagged(event, \"TurnFailed\") ? [event.message] : []\n )\n\n if (failed.length > 0) {\n return yield* new TestChatError({ reason: \"TurnFailed\", message: failed.join(\" | \") })\n }\n\n return at(assistant, chatId, timeout, after, added)\n })\n\n const decide = (decision: (approval: { readonly id: string }) => AssistantEvent.ApprovalDecision) => {\n const waiting = all.approvals.at(-1)\n\n return waiting === undefined\n ? Effect.fail(new TestChatError({ reason: \"NothingToApprove\", message: \"no call waits for approval\" }))\n : send(decision({ id: waiting.callId }))\n }\n\n const answer = <Answer extends Block.Codec>(\n block: Block.Block<string, Block.Codec, Answer>,\n value: Answer[\"Type\"]\n ): Effect.Effect<TestChat, TestChatError> => {\n const asked = libraryEvents(entries).findLast(\n (event): event is AssistantEvent.AssistantMessage =>\n Predicate.isTagged(event, \"AssistantMessage\") && event.block?.kind === block.kind\n )\n\n if (asked === undefined || block.answer === undefined) {\n return Effect.fail(new TestChatError({ reason: \"NothingToAnswer\", message: `no ${block.kind} block to answer` }))\n }\n\n return send(\n AssistantEvent.BlockAnswer.make({\n // As the client sends it: the same id every time, so a block takes one answer.\n id: InputId.fromKey(`${asked.id}:answer`),\n messageId: asked.id,\n kind: block.kind,\n value: Schema.encodeUnknownSync(Schema.toCodecJson(block.answer))(value)\n })\n )\n }\n\n return {\n ...all,\n chatId,\n step: log(step),\n send,\n say: (text) => send(AssistantEvent.UserMessage.make({ text })),\n approve: decide(ToolApproval.approve),\n reject: (options) => decide((approval) => ToolApproval.reject(approval, options)),\n answer\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0CA,MAAM,2BAAgC,IAAI,IAAI;CAAC;CAAqB;CAAc;AAAY,CAAC;;;;;;;;AAS/F,IAAa,gBAAb,cAAmC,KAAK,YAAY,eAAe,CAAC,CAGjE,CAAC;AAEJ,MAAM,iBAAiB,YACrB,QAAQ,SAAS,EAAE,YAAaA,eAA8B,KAAK,IAAI,CAAC,KAAK,IAAI,CAAC,CAAE;;;;;;;AAkCtF,MAAa,OAAO,YAAyD;CAC3E,MAAM,SAAS,cAAc,OAAO;CACpC,MAAM,UAAU,OAAO,SAAS,UAAW,UAAU,SAAS,OAAO,YAAY,IAAI,CAAC,KAAK,IAAI,CAAC,CAAE;CAClG,MAAM,YAAY,OAAO,SAAS,UAAW,UAAU,SAAS,OAAO,mBAAmB,IAAI,CAAC,KAAK,IAAI,CAAC,CAAE;CAE3G,OAAO;EACL;EACA,UAAU,CACR,GAAG,IAAI,IACL,QAAQ,QAAQ,EAAE,SAAS,WAAW,YAAY,aAAa,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,EAAE,WAAW,IAAI,CAC5G,CACF;EACA,YAAY,SAAS,QAAQ,QAAQ,WAAW,OAAO,SAAS,IAAI,CAAC,CAAC;EACtE,QAAQ,OAAO,SAAS,UACtB,UAAU,SAAS,OAAO,kBAAkB,KAAK,MAAM,UAAU,KAAA,IAAY,CAAC,MAAM,MAAM,IAAI,IAAI,CAAC,CACrG;EACA;EACA,YAAY,UAAU,KAAK,EAAE,WAAW,IAAI;EAC5C,UAAUC,eAA8B,OAAO;CACjD;AACF;AA2BA,MAAM,WAAW,SAAiD,YAA6B;CAC7F,MAAM,QAAQ,QAAQ,MAAM,EAAE,YAAYC,QAAuB,KAAK,KAAK,MAAM,OAAO,OAAO;CAE/F,OACE,UAAU,KAAA,KACV,cAAc,OAAO,CAAC,CAAC,MAAM,UAAUC,UAAyB,KAAK,KAAK,MAAM,mBAAmB,MAAM,QAAQ;AAErH;;;;;;;;;AAUA,MAAa,QACX,WACA,QACA,YACa,GAAG,WAAW,QAAQ,SAAS,WAAW,aAAa,CAAC,GAAG,CAAC,CAAC;AAE5E,MAAM,MACJ,WACA,QACA,SACA,SACA,SACa;CACb,MAAM,MAAM,IAAI,OAAO;CAEvB,MAAM,QAAQ,UACZ,OAAO,IAAI,aAAa;EACtB,MAAM,SAAS,QAAQ,GAAG,EAAE,CAAC,EAAE,YAAY;EAC3C,IAAI,OAA+C,CAAC;EAEpD,OAAO,UAAU,OAAO,QAAQ,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK;EAExD,MAAM,QAAQ,OAAO,UAAU,UAAU,MAAM,CAAC,CAAC,KAC/C,OAAO,WACyC,CAAC,IAC9C,KAAK,UAAU,CAAC,GAAG,KAAK,KAAK,CAChC,GACA,OAAO,KAAK,QAAQ,OAAO,WAAY,OAAO,GAAI,CAAC,GACnD,OAAO,WAAW,QAAQ,QAAQ,KAAK,MAAM,EAAE,CAAC,GAChD,OAAO,SACP,OAAO,IAAI,OAAO,gBAAwD,CAAC,CAAC,CAAC,GAC7E,OAAO,cAAc;GACnB,UAAU;GACV,cACE,OAAO,KACL,IAAI,cAAc;IAChB,QAAQ;IACR,SAAS,kDAAkD,IAAI,IAAI,CAAC,CAAC,SAAS,KAAK,KAAK;GAC1F,CAAC,CACH;EACJ,CAAC,CACH;EAEA,MAAM,QAAQ,MAAM,QAAQ,EAAE,eAAe,WAAW,MAAM;EAE9D,MAAM,SAAS,cAAc,KAAK,CAAC,CAAC,SAAS,UAC3C,UAAU,SAAS,OAAO,YAAY,IAAI,CAAC,MAAM,OAAO,IAAI,CAAC,CAC/D;EAEA,IAAI,OAAO,SAAS,GAClB,OAAO,OAAO,IAAI,cAAc;GAAE,QAAQ;GAAc,SAAS,OAAO,KAAK,KAAK;EAAE,CAAC;EAGvF,OAAO,GAAG,WAAW,QAAQ,SAAS,OAAO,KAAK;CACpD,CAAC;CAEH,MAAM,UAAU,aAAqF;EACnG,MAAM,UAAU,IAAI,UAAU,GAAG,EAAE;EAEnC,OAAO,YAAY,KAAA,IACf,OAAO,KAAK,IAAI,cAAc;GAAE,QAAQ;GAAoB,SAAS;EAA6B,CAAC,CAAC,IACpG,KAAK,SAAS,EAAE,IAAI,QAAQ,OAAO,CAAC,CAAC;CAC3C;CAEA,MAAM,UACJ,OACA,UAC2C;EAC3C,MAAM,QAAQ,cAAc,OAAO,CAAC,CAAC,UAClC,UACC,UAAU,SAAS,OAAO,kBAAkB,KAAK,MAAM,OAAO,SAAS,MAAM,IACjF;EAEA,IAAI,UAAU,KAAA,KAAa,MAAM,WAAW,KAAA,GAC1C,OAAO,OAAO,KAAK,IAAI,cAAc;GAAE,QAAQ;GAAmB,SAAS,MAAM,MAAM,KAAK;EAAkB,CAAC,CAAC;EAGlH,OAAO,KAAA,YACsB,KAAK;GAE9B,IAAIC,QAAgB,GAAG,MAAM,GAAG,QAAQ;GACxC,WAAW,MAAM;GACjB,MAAM,MAAM;GACZ,OAAO,OAAO,kBAAkB,OAAO,YAAY,MAAM,MAAM,CAAC,CAAC,CAAC,KAAK;EACzE,CAAC,CACH;CACF;CAEA,OAAO;EACL,GAAG;EACH;EACA,MAAM,IAAI,IAAI;EACd;EACA,MAAM,SAAS,KAAA,YAAgC,KAAK,EAAE,KAAK,CAAC,CAAC;EAC7D,SAAS,OAAOC,OAAoB;EACpC,SAAS,YAAY,QAAQ,aAAaC,OAAoB,UAAU,OAAO,CAAC;EAChF;CACF;AACF"}
|
package/dist/index.d.ts
CHANGED
|
@@ -14,6 +14,7 @@ import { AssistantEngine_d_exports } from "./AssistantEngine.js";
|
|
|
14
14
|
import { Chats_d_exports } from "./Chats.js";
|
|
15
15
|
import { Assistant_d_exports } from "./Assistant.js";
|
|
16
16
|
import { AssistantActivity_d_exports } from "./AssistantActivity.js";
|
|
17
|
+
import { AssistantTest_d_exports } from "./AssistantTest.js";
|
|
17
18
|
import { Compaction_d_exports } from "./Compaction.js";
|
|
18
19
|
import { InputId_d_exports } from "./InputId.js";
|
|
19
20
|
import { ToolFailure_d_exports } from "./ToolFailure.js";
|
|
@@ -22,4 +23,4 @@ import { Skill_d_exports } from "./Skill.js";
|
|
|
22
23
|
import { ToolApproval_d_exports } from "./ToolApproval.js";
|
|
23
24
|
import { ToolDiscovery_d_exports } from "./ToolDiscovery.js";
|
|
24
25
|
import { ToolResult_d_exports } from "./ToolResult.js";
|
|
25
|
-
export { Assistant_d_exports as Assistant, AssistantActivity_d_exports as AssistantActivity, AssistantEngine_d_exports as AssistantEngine, AssistantEvent_d_exports as AssistantEvent, AssistantLoop_d_exports as AssistantLoop, AssistantToolCall_d_exports as AssistantToolCall, Block_d_exports as Block, BlockKit_d_exports as BlockKit, ChatId_d_exports as ChatId, ChatLog_d_exports as ChatLog, Chats_d_exports as Chats, Compaction_d_exports as Compaction, CurrentChat_d_exports as CurrentChat, InputId_d_exports as InputId, Instructions_d_exports as Instructions, Memory_d_exports as Memory, RunId_d_exports as RunId, Skill_d_exports as Skill, Specialist_d_exports as Specialist, Timeline_d_exports as Timeline, ToolApproval_d_exports as ToolApproval, ToolDiscovery_d_exports as ToolDiscovery, ToolFailure_d_exports as ToolFailure, ToolResult_d_exports as ToolResult };
|
|
26
|
+
export { Assistant_d_exports as Assistant, AssistantActivity_d_exports as AssistantActivity, AssistantEngine_d_exports as AssistantEngine, AssistantEvent_d_exports as AssistantEvent, AssistantLoop_d_exports as AssistantLoop, AssistantTest_d_exports as AssistantTest, AssistantToolCall_d_exports as AssistantToolCall, Block_d_exports as Block, BlockKit_d_exports as BlockKit, ChatId_d_exports as ChatId, ChatLog_d_exports as ChatLog, Chats_d_exports as Chats, Compaction_d_exports as Compaction, CurrentChat_d_exports as CurrentChat, InputId_d_exports as InputId, Instructions_d_exports as Instructions, Memory_d_exports as Memory, RunId_d_exports as RunId, Skill_d_exports as Skill, Specialist_d_exports as Specialist, Timeline_d_exports as Timeline, ToolApproval_d_exports as ToolApproval, ToolDiscovery_d_exports as ToolDiscovery, ToolFailure_d_exports as ToolFailure, ToolResult_d_exports as ToolResult };
|
package/dist/index.js
CHANGED
|
@@ -17,9 +17,10 @@ import { AssistantEngine_exports } from "./AssistantEngine.js";
|
|
|
17
17
|
import { Assistant_exports } from "./Assistant.js";
|
|
18
18
|
import { AssistantActivity_exports } from "./AssistantActivity.js";
|
|
19
19
|
import { AssistantLoop_exports } from "./AssistantLoop.js";
|
|
20
|
+
import { AssistantTest_exports } from "./AssistantTest.js";
|
|
20
21
|
import { Compaction_exports } from "./Compaction.js";
|
|
21
22
|
import { Memory_exports } from "./Memory.js";
|
|
22
23
|
import { Skill_exports } from "./Skill.js";
|
|
23
24
|
import { Timeline_exports } from "./Timeline.js";
|
|
24
25
|
import { ToolDiscovery_exports } from "./ToolDiscovery.js";
|
|
25
|
-
export { Assistant_exports as Assistant, AssistantActivity_exports as AssistantActivity, AssistantEngine_exports as AssistantEngine, AssistantEvent_exports as AssistantEvent, AssistantLoop_exports as AssistantLoop, AssistantToolCall_exports as AssistantToolCall, Block_exports as Block, BlockKit_exports as BlockKit, ChatId_exports as ChatId, ChatLog_exports as ChatLog, Chats_exports as Chats, Compaction_exports as Compaction, CurrentChat_exports as CurrentChat, InputId_exports as InputId, Instructions_exports as Instructions, Memory_exports as Memory, RunId_exports as RunId, Skill_exports as Skill, Specialist_exports as Specialist, Timeline_exports as Timeline, ToolApproval_exports as ToolApproval, ToolDiscovery_exports as ToolDiscovery, ToolFailure_exports as ToolFailure, ToolResult_exports as ToolResult };
|
|
26
|
+
export { Assistant_exports as Assistant, AssistantActivity_exports as AssistantActivity, AssistantEngine_exports as AssistantEngine, AssistantEvent_exports as AssistantEvent, AssistantLoop_exports as AssistantLoop, AssistantTest_exports as AssistantTest, AssistantToolCall_exports as AssistantToolCall, Block_exports as Block, BlockKit_exports as BlockKit, ChatId_exports as ChatId, ChatLog_exports as ChatLog, Chats_exports as Chats, Compaction_exports as Compaction, CurrentChat_exports as CurrentChat, InputId_exports as InputId, Instructions_exports as Instructions, Memory_exports as Memory, RunId_exports as RunId, Skill_exports as Skill, Specialist_exports as Specialist, Timeline_exports as Timeline, ToolApproval_exports as ToolApproval, ToolDiscovery_exports as ToolDiscovery, ToolFailure_exports as ToolFailure, ToolResult_exports as ToolResult };
|
package/package.json
CHANGED
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This module provides a chat to drive an assistant like a user would, for tests and evals.
|
|
3
|
+
*
|
|
4
|
+
* {@link chat} wraps a chat the app created. Each step (`say`, `approve`, `reject`, `answer`, or
|
|
5
|
+
* `send` for any input) submits the input, waits until a turn has handled it, and returns the chat
|
|
6
|
+
* as it is then. A {@link Log} reads what the assistant did, for the whole chat or for the last
|
|
7
|
+
* step alone (`step`): the tools that ran, the blocks it showed, the calls waiting for approval,
|
|
8
|
+
* and the messages it sent.
|
|
9
|
+
*
|
|
10
|
+
* **Example** (An assistant asks before it emails)
|
|
11
|
+
*
|
|
12
|
+
* ```ts
|
|
13
|
+
* import { AssistantTest } from "effect-assistant"
|
|
14
|
+
* import * as Effect from "effect/Effect"
|
|
15
|
+
*
|
|
16
|
+
* const program = Effect.gen(function* () {
|
|
17
|
+
* const chat = AssistantTest.chat(yield* MyAssistant, chatId)
|
|
18
|
+
* const asked = yield* chat.say("Email the report to ana@example.com")
|
|
19
|
+
* asked.step.waitingFor // ["send_report"]
|
|
20
|
+
* const sent = yield* asked.approve
|
|
21
|
+
* sent.step.toolsRun // ["send_report"]
|
|
22
|
+
* })
|
|
23
|
+
* ```
|
|
24
|
+
*
|
|
25
|
+
* @since 0.9.0
|
|
26
|
+
*/
|
|
27
|
+
import * as Data from "effect/Data"
|
|
28
|
+
import type * as Duration from "effect/Duration"
|
|
29
|
+
import * as Effect from "effect/Effect"
|
|
30
|
+
import * as Option from "effect/Option"
|
|
31
|
+
import * as Predicate from "effect/Predicate"
|
|
32
|
+
import * as Schema from "effect/Schema"
|
|
33
|
+
import * as Stream from "effect/Stream"
|
|
34
|
+
import type * as Assistant from "./Assistant.ts"
|
|
35
|
+
import type * as Block from "./Block.ts"
|
|
36
|
+
import * as AssistantEvent from "./AssistantEvent.ts"
|
|
37
|
+
import type { ChatId } from "./ChatId.ts"
|
|
38
|
+
import * as InputId from "./InputId.ts"
|
|
39
|
+
import { sendUserMessageName, waitToolName } from "./internal/tools.ts"
|
|
40
|
+
import * as ToolApproval from "./ToolApproval.ts"
|
|
41
|
+
|
|
42
|
+
// The assistant's own tools: how it talks, waits and reads its skills, not work it does for the user.
|
|
43
|
+
const ownTools: ReadonlySet<string> = new Set([sendUserMessageName, waitToolName, "load_skill"])
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Represents a step that couldn't complete: no turn handled the input in time, the turn failed,
|
|
47
|
+
* or there was nothing to approve or answer.
|
|
48
|
+
*
|
|
49
|
+
* @category errors
|
|
50
|
+
* @since 0.9.0
|
|
51
|
+
*/
|
|
52
|
+
export class TestChatError extends Data.TaggedError("TestChatError")<{
|
|
53
|
+
readonly reason: "TimedOut" | "TurnFailed" | "NothingToApprove" | "NothingToAnswer"
|
|
54
|
+
readonly message: string
|
|
55
|
+
}> {}
|
|
56
|
+
|
|
57
|
+
const libraryEvents = (entries: ReadonlyArray<AssistantEvent.AnyEntry>): ReadonlyArray<AssistantEvent.AssistantEvent> =>
|
|
58
|
+
entries.flatMap(({ event }) => (AssistantEvent.isLibraryEvent(event) ? [event] : []))
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Represents what the assistant did in some of a chat's entries.
|
|
62
|
+
*
|
|
63
|
+
* @category models
|
|
64
|
+
* @since 0.9.0
|
|
65
|
+
*/
|
|
66
|
+
export interface Log {
|
|
67
|
+
readonly entries: ReadonlyArray<AssistantEvent.AnyEntry>
|
|
68
|
+
/**
|
|
69
|
+
* The app's tools that ran, each once, in the order they first ran. A call that failed or that
|
|
70
|
+
* the library refused didn't run, and the assistant's own tools (`send_user_message`, `wait`,
|
|
71
|
+
* `load_skill`) aren't listed.
|
|
72
|
+
*/
|
|
73
|
+
readonly toolsRun: ReadonlyArray<string>
|
|
74
|
+
/** How many times the model called this tool, whether or not it ran. */
|
|
75
|
+
readonly toolCalls: (tool: string) => number
|
|
76
|
+
/** The kind of each block the assistant showed, in order; one per message. */
|
|
77
|
+
readonly blocks: ReadonlyArray<string>
|
|
78
|
+
/** The calls waiting for the user's approval, in order. */
|
|
79
|
+
readonly approvals: ReadonlyArray<AssistantEvent.ApprovalRequested>
|
|
80
|
+
/** The tools of the calls waiting for the user's approval. */
|
|
81
|
+
readonly waitingFor: ReadonlyArray<string>
|
|
82
|
+
/** The messages the assistant sent, in order. */
|
|
83
|
+
readonly messages: ReadonlyArray<string>
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Reads what the assistant did in these entries.
|
|
88
|
+
*
|
|
89
|
+
* @category constructors
|
|
90
|
+
* @since 0.9.0
|
|
91
|
+
*/
|
|
92
|
+
export const log = (entries: ReadonlyArray<AssistantEvent.AnyEntry>): Log => {
|
|
93
|
+
const events = libraryEvents(entries)
|
|
94
|
+
const results = events.flatMap((event) => (Predicate.isTagged(event, "ToolResult") ? [event] : []))
|
|
95
|
+
const approvals = events.flatMap((event) => (Predicate.isTagged(event, "ApprovalRequested") ? [event] : []))
|
|
96
|
+
|
|
97
|
+
return {
|
|
98
|
+
entries,
|
|
99
|
+
toolsRun: [
|
|
100
|
+
...new Set(
|
|
101
|
+
results.filter(({ outcome, tool }) => outcome === "success" && !ownTools.has(tool)).map(({ tool }) => tool)
|
|
102
|
+
)
|
|
103
|
+
],
|
|
104
|
+
toolCalls: (tool) => results.filter((result) => result.tool === tool).length,
|
|
105
|
+
blocks: events.flatMap((event) =>
|
|
106
|
+
Predicate.isTagged(event, "AssistantMessage") && event.block !== undefined ? [event.block.kind] : []
|
|
107
|
+
),
|
|
108
|
+
approvals,
|
|
109
|
+
waitingFor: approvals.map(({ tool }) => tool),
|
|
110
|
+
messages: AssistantEvent.assistantTexts(entries)
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Represents a chat driven like a user would.
|
|
116
|
+
*
|
|
117
|
+
* @category models
|
|
118
|
+
* @since 0.9.0
|
|
119
|
+
*/
|
|
120
|
+
export interface TestChat extends Log {
|
|
121
|
+
readonly chatId: ChatId
|
|
122
|
+
/** What the assistant did for the last step alone. */
|
|
123
|
+
readonly step: Log
|
|
124
|
+
/** Submits an input, and returns the chat once a turn has handled it. */
|
|
125
|
+
readonly send: (input: AssistantEvent.Input) => Effect.Effect<TestChat, TestChatError>
|
|
126
|
+
/** Sends the user's message. */
|
|
127
|
+
readonly say: (text: string) => Effect.Effect<TestChat, TestChatError>
|
|
128
|
+
/** Approves the last call waiting for approval. */
|
|
129
|
+
readonly approve: Effect.Effect<TestChat, TestChatError>
|
|
130
|
+
/** Rejects the last call waiting for approval. */
|
|
131
|
+
readonly reject: (options?: { readonly reason?: string | undefined }) => Effect.Effect<TestChat, TestChatError>
|
|
132
|
+
/** Answers the last block of this kind the assistant showed. */
|
|
133
|
+
readonly answer: <Answer extends Block.Codec>(
|
|
134
|
+
block: Block.Block<string, Block.Codec, Answer>,
|
|
135
|
+
answer: Answer["Type"]
|
|
136
|
+
) => Effect.Effect<TestChat, TestChatError>
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
const handled = (entries: ReadonlyArray<AssistantEvent.AnyEntry>, inputId: string): boolean => {
|
|
140
|
+
const input = entries.find(({ event }) => AssistantEvent.isInput(event) && event.id === inputId)
|
|
141
|
+
|
|
142
|
+
return (
|
|
143
|
+
input !== undefined &&
|
|
144
|
+
libraryEvents(entries).some((event) => AssistantEvent.isTurnEnd(event) && event.throughSequence >= input.sequence)
|
|
145
|
+
)
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Creates a chat to drive the assistant with, for a chat the app already created. A step fails
|
|
150
|
+
* when no turn has handled its input within `timeout` (3 minutes by default), or when a turn failed
|
|
151
|
+
* while handling it.
|
|
152
|
+
*
|
|
153
|
+
* @category constructors
|
|
154
|
+
* @since 0.9.0
|
|
155
|
+
*/
|
|
156
|
+
export const chat = <Events extends { readonly _tag: string } = never>(
|
|
157
|
+
assistant: Assistant.Service<Events>,
|
|
158
|
+
chatId: ChatId,
|
|
159
|
+
options?: { readonly timeout?: Duration.Input | undefined }
|
|
160
|
+
): TestChat => at(assistant, chatId, options?.timeout ?? "3 minutes", [], [])
|
|
161
|
+
|
|
162
|
+
const at = <Events extends { readonly _tag: string }>(
|
|
163
|
+
assistant: Assistant.Service<Events>,
|
|
164
|
+
chatId: ChatId,
|
|
165
|
+
timeout: Duration.Input,
|
|
166
|
+
entries: ReadonlyArray<AssistantEvent.AnyEntry>,
|
|
167
|
+
step: ReadonlyArray<AssistantEvent.AnyEntry>
|
|
168
|
+
): TestChat => {
|
|
169
|
+
const all = log(entries)
|
|
170
|
+
|
|
171
|
+
const send = (input: AssistantEvent.Input): Effect.Effect<TestChat, TestChatError> =>
|
|
172
|
+
Effect.gen(function* () {
|
|
173
|
+
const before = entries.at(-1)?.sequence ?? 0
|
|
174
|
+
let seen: ReadonlyArray<AssistantEvent.AnyEntry> = []
|
|
175
|
+
|
|
176
|
+
yield* assistant.submit(chatId, input).pipe(Effect.orDie)
|
|
177
|
+
|
|
178
|
+
const after = yield* assistant.subscribe(chatId).pipe(
|
|
179
|
+
Stream.scan(
|
|
180
|
+
(): ReadonlyArray<AssistantEvent.AnyEntry> => [],
|
|
181
|
+
(acc, entry) => [...acc, entry]
|
|
182
|
+
),
|
|
183
|
+
Stream.tap((acc) => Effect.sync(() => (seen = acc))),
|
|
184
|
+
Stream.takeUntil((acc) => handled(acc, input.id)),
|
|
185
|
+
Stream.runLast,
|
|
186
|
+
Effect.map(Option.getOrElse((): ReadonlyArray<AssistantEvent.AnyEntry> => [])),
|
|
187
|
+
Effect.timeoutOrElse({
|
|
188
|
+
duration: timeout,
|
|
189
|
+
orElse: () =>
|
|
190
|
+
Effect.fail(
|
|
191
|
+
new TestChatError({
|
|
192
|
+
reason: "TimedOut",
|
|
193
|
+
message: `no turn handled the input; the assistant said: ${log(seen).messages.join(" | ")}`
|
|
194
|
+
})
|
|
195
|
+
)
|
|
196
|
+
})
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
const added = after.filter(({ sequence }) => sequence > before)
|
|
200
|
+
|
|
201
|
+
const failed = libraryEvents(added).flatMap((event) =>
|
|
202
|
+
Predicate.isTagged(event, "TurnFailed") ? [event.message] : []
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
if (failed.length > 0) {
|
|
206
|
+
return yield* new TestChatError({ reason: "TurnFailed", message: failed.join(" | ") })
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
return at(assistant, chatId, timeout, after, added)
|
|
210
|
+
})
|
|
211
|
+
|
|
212
|
+
const decide = (decision: (approval: { readonly id: string }) => AssistantEvent.ApprovalDecision) => {
|
|
213
|
+
const waiting = all.approvals.at(-1)
|
|
214
|
+
|
|
215
|
+
return waiting === undefined
|
|
216
|
+
? Effect.fail(new TestChatError({ reason: "NothingToApprove", message: "no call waits for approval" }))
|
|
217
|
+
: send(decision({ id: waiting.callId }))
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
const answer = <Answer extends Block.Codec>(
|
|
221
|
+
block: Block.Block<string, Block.Codec, Answer>,
|
|
222
|
+
value: Answer["Type"]
|
|
223
|
+
): Effect.Effect<TestChat, TestChatError> => {
|
|
224
|
+
const asked = libraryEvents(entries).findLast(
|
|
225
|
+
(event): event is AssistantEvent.AssistantMessage =>
|
|
226
|
+
Predicate.isTagged(event, "AssistantMessage") && event.block?.kind === block.kind
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
if (asked === undefined || block.answer === undefined) {
|
|
230
|
+
return Effect.fail(new TestChatError({ reason: "NothingToAnswer", message: `no ${block.kind} block to answer` }))
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
return send(
|
|
234
|
+
AssistantEvent.BlockAnswer.make({
|
|
235
|
+
// As the client sends it: the same id every time, so a block takes one answer.
|
|
236
|
+
id: InputId.fromKey(`${asked.id}:answer`),
|
|
237
|
+
messageId: asked.id,
|
|
238
|
+
kind: block.kind,
|
|
239
|
+
value: Schema.encodeUnknownSync(Schema.toCodecJson(block.answer))(value)
|
|
240
|
+
})
|
|
241
|
+
)
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
return {
|
|
245
|
+
...all,
|
|
246
|
+
chatId,
|
|
247
|
+
step: log(step),
|
|
248
|
+
send,
|
|
249
|
+
say: (text) => send(AssistantEvent.UserMessage.make({ text })),
|
|
250
|
+
approve: decide(ToolApproval.approve),
|
|
251
|
+
reject: (options) => decide((approval) => ToolApproval.reject(approval, options)),
|
|
252
|
+
answer
|
|
253
|
+
}
|
|
254
|
+
}
|
package/src/index.ts
CHANGED