@node-llm/orm 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +130 -0
- package/dist/BaseChat.d.ts +2 -2
- package/dist/BaseChat.d.ts.map +1 -1
- package/dist/Store.d.ts +28 -1
- package/dist/Store.d.ts.map +1 -1
- package/dist/adapters/prisma/AgentSession.d.ts +42 -9
- package/dist/adapters/prisma/AgentSession.d.ts.map +1 -1
- package/dist/adapters/prisma/AgentSession.js +69 -17
- package/dist/adapters/prisma/Chat.d.ts +5 -0
- package/dist/adapters/prisma/Chat.d.ts.map +1 -1
- package/dist/adapters/prisma/Chat.js +14 -4
- package/dist/adapters/prisma/PrismaStore.d.ts +4 -2
- package/dist/adapters/prisma/PrismaStore.d.ts.map +1 -1
- package/dist/adapters/prisma/PrismaStore.js +15 -2
- package/dist/adapters/prisma/index.d.ts +1 -1
- package/dist/adapters/prisma/index.d.ts.map +1 -1
- package/dist/errors.d.ts +14 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +15 -0
- package/dist/history.d.ts +10 -0
- package/dist/history.d.ts.map +1 -0
- package/dist/history.js +42 -0
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/redact.d.ts +20 -0
- package/dist/redact.d.ts.map +1 -0
- package/dist/redact.js +118 -0
- package/package.json +2 -2
- package/src/BaseChat.ts +2 -2
- package/src/Store.ts +30 -1
- package/src/adapters/prisma/AgentSession.ts +130 -26
- package/src/adapters/prisma/Chat.ts +23 -5
- package/src/adapters/prisma/PrismaStore.ts +21 -3
- package/src/adapters/prisma/index.ts +3 -0
- package/src/errors.ts +15 -0
- package/src/history.ts +48 -0
- package/src/index.ts +4 -0
- package/src/redact.ts +157 -0
- package/test/AgentSession.test.ts +12 -2
- package/test/CountCall.test.ts +147 -0
- package/test/Redact.test.ts +228 -0
- package/test/Resume.test.ts +232 -0
- package/test/SessionData.test.ts +109 -0
- package/test/SessionOverrides.test.ts +130 -0
- package/test/StoreStops.test.ts +163 -0
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import { describe, it, expect } from "vitest";
|
|
2
|
+
import { Agent, CostLimitError, createLLM } from "@node-llm/core";
|
|
3
|
+
import type { ChatRequest, ChatResponse, Message, Provider } from "@node-llm/core";
|
|
4
|
+
import { createAgentSession, type AgentSession } from "../src/adapters/prisma/AgentSession.js";
|
|
5
|
+
import type { ChatStore, StoredMessage } from "../src/Store.js";
|
|
6
|
+
|
|
7
|
+
class MemoryStore implements ChatStore {
|
|
8
|
+
messages: StoredMessage[] = [];
|
|
9
|
+
requests: any[] = [];
|
|
10
|
+
sessions = new Map<string, any>();
|
|
11
|
+
private next = 0;
|
|
12
|
+
private id(prefix: string) {
|
|
13
|
+
this.next += 1;
|
|
14
|
+
return `${prefix}-${this.next}`;
|
|
15
|
+
}
|
|
16
|
+
async createChat(chat: any) {
|
|
17
|
+
return { ...chat, id: this.id("chat"), createdAt: new Date(), updatedAt: new Date() };
|
|
18
|
+
}
|
|
19
|
+
async findChat() {
|
|
20
|
+
return null;
|
|
21
|
+
}
|
|
22
|
+
async deleteChat() {}
|
|
23
|
+
async createMessage(message: { chatId: string; role: string; content: string | null }) {
|
|
24
|
+
const record = { ...message, id: this.id("message"), createdAt: new Date() };
|
|
25
|
+
this.messages.push(record);
|
|
26
|
+
return record;
|
|
27
|
+
}
|
|
28
|
+
async updateMessage(id: string, changes: any) {
|
|
29
|
+
return Object.assign(this.messages.find((m) => m.id === id)!, changes);
|
|
30
|
+
}
|
|
31
|
+
async deleteMessage(id: string) {
|
|
32
|
+
this.messages = this.messages.filter((m) => m.id !== id);
|
|
33
|
+
}
|
|
34
|
+
async listMessages(chatId: string) {
|
|
35
|
+
return this.messages.filter((m) => m.chatId === chatId);
|
|
36
|
+
}
|
|
37
|
+
async createToolCall() {}
|
|
38
|
+
async finishToolCall() {}
|
|
39
|
+
async createRequest(request: any) {
|
|
40
|
+
this.requests.push(request);
|
|
41
|
+
}
|
|
42
|
+
async requestTotals(chatId: string) {
|
|
43
|
+
const rows = this.requests.filter((r) => r.chatId === chatId);
|
|
44
|
+
return {
|
|
45
|
+
inputTokens: rows.reduce((sum, r) => sum + r.inputTokens, 0),
|
|
46
|
+
outputTokens: rows.reduce((sum, r) => sum + r.outputTokens, 0),
|
|
47
|
+
cost: rows.reduce((sum, r) => sum + r.cost, 0)
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
async createAgentSession(session: any) {
|
|
51
|
+
const record = {
|
|
52
|
+
...session,
|
|
53
|
+
id: this.id("session"),
|
|
54
|
+
createdAt: new Date(),
|
|
55
|
+
updatedAt: new Date()
|
|
56
|
+
};
|
|
57
|
+
this.sessions.set(record.id, record);
|
|
58
|
+
return record;
|
|
59
|
+
}
|
|
60
|
+
async findAgentSession(id: string) {
|
|
61
|
+
return this.sessions.get(id) ?? null;
|
|
62
|
+
}
|
|
63
|
+
async updateAgentSession() {}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// One look at the page, then an answer; each request costs $0.01.
|
|
67
|
+
class Looking implements Provider {
|
|
68
|
+
id = "scripted";
|
|
69
|
+
requests = 0;
|
|
70
|
+
constructor(private looks = 1) {}
|
|
71
|
+
defaultModel() {
|
|
72
|
+
return "test-model";
|
|
73
|
+
}
|
|
74
|
+
async chat(_request: ChatRequest): Promise<ChatResponse> {
|
|
75
|
+
this.requests++;
|
|
76
|
+
const usage = { input_tokens: 10, output_tokens: 5, total_tokens: 15, cost: 0.01 };
|
|
77
|
+
if (this.requests > this.looks) return { content: "Found it.", usage };
|
|
78
|
+
return {
|
|
79
|
+
content: null,
|
|
80
|
+
usage,
|
|
81
|
+
tool_calls: [
|
|
82
|
+
{ id: `c${this.requests}`, type: "function", function: { name: "look", arguments: "{}" } }
|
|
83
|
+
]
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
formatToolResultMessage(toolCallId: string, content: string): Message {
|
|
87
|
+
return { role: "tool", tool_call_id: toolCallId, content };
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// The answer of another model asked about a picture.
|
|
92
|
+
const pictureAnswer = {
|
|
93
|
+
usage: { input_tokens: 1000, output_tokens: 20, total_tokens: 1020, cost: 0.05 },
|
|
94
|
+
model: "vision-model",
|
|
95
|
+
provider: "scripted",
|
|
96
|
+
latency: 300
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
let session: AgentSession;
|
|
100
|
+
const look = {
|
|
101
|
+
type: "function" as const,
|
|
102
|
+
function: { name: "look", description: "Look", parameters: { type: "object", properties: {} } },
|
|
103
|
+
handler: async () => {
|
|
104
|
+
await session.countCall(pictureAnswer);
|
|
105
|
+
return "a button";
|
|
106
|
+
}
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
class Explorer extends Agent {
|
|
110
|
+
static model = "test-model";
|
|
111
|
+
static tools = [look];
|
|
112
|
+
static assumeModelExists = true;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
describe("countCall", () => {
|
|
116
|
+
it("records a side call as a request of the answer being written, and counts it", async () => {
|
|
117
|
+
const store = new MemoryStore();
|
|
118
|
+
session = await createAgentSession(store, createLLM({ provider: new Looking() }), Explorer);
|
|
119
|
+
|
|
120
|
+
const answer = await session.ask("Find the button");
|
|
121
|
+
|
|
122
|
+
expect(store.requests).toContainEqual(
|
|
123
|
+
expect.objectContaining({
|
|
124
|
+
chatId: session.chatId,
|
|
125
|
+
messageId: answer.id,
|
|
126
|
+
model: "vision-model",
|
|
127
|
+
duration: 300,
|
|
128
|
+
inputTokens: 1000,
|
|
129
|
+
cost: 0.05
|
|
130
|
+
})
|
|
131
|
+
);
|
|
132
|
+
expect((await session.stats()).cost).toBeCloseTo(0.07);
|
|
133
|
+
expect(session.totalUsage.cost).toBeCloseTo(0.07);
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
it("counts toward the agent's maxCost", async () => {
|
|
137
|
+
const provider = new Looking(10);
|
|
138
|
+
class BudgetExplorer extends Explorer {
|
|
139
|
+
static maxCost = 0.1;
|
|
140
|
+
static maxToolCalls = 20;
|
|
141
|
+
}
|
|
142
|
+
session = await createAgentSession(new MemoryStore(), createLLM({ provider }), BudgetExplorer);
|
|
143
|
+
|
|
144
|
+
await expect(session.ask("Find the button")).rejects.toBeInstanceOf(CostLimitError);
|
|
145
|
+
expect(provider.requests).toBe(3);
|
|
146
|
+
});
|
|
147
|
+
});
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
import { describe, it, expect } from "vitest";
|
|
2
|
+
import { Agent, createLLM } from "@node-llm/core";
|
|
3
|
+
import type { ChatRequest, ChatResponse, Message, Provider } from "@node-llm/core";
|
|
4
|
+
import { createAgentSession } from "../src/adapters/prisma/AgentSession.js";
|
|
5
|
+
import { createChat } from "../src/adapters/prisma/Chat.js";
|
|
6
|
+
import { maskSecrets, redactingStore } from "../src/redact.js";
|
|
7
|
+
import type { ChatStore, StoredMessage } from "../src/Store.js";
|
|
8
|
+
|
|
9
|
+
class MemoryStore implements ChatStore {
|
|
10
|
+
messages: StoredMessage[] = [];
|
|
11
|
+
toolCalls: any[] = [];
|
|
12
|
+
sessions = new Map<string, any>();
|
|
13
|
+
private next = 0;
|
|
14
|
+
private id(prefix: string) {
|
|
15
|
+
this.next += 1;
|
|
16
|
+
return `${prefix}-${this.next}`;
|
|
17
|
+
}
|
|
18
|
+
async createChat(chat: any) {
|
|
19
|
+
return { ...chat, id: this.id("chat"), createdAt: new Date(), updatedAt: new Date() };
|
|
20
|
+
}
|
|
21
|
+
async findChat() {
|
|
22
|
+
return null;
|
|
23
|
+
}
|
|
24
|
+
async deleteChat() {}
|
|
25
|
+
async createMessage(message: { chatId: string; role: string; content: string | null }) {
|
|
26
|
+
const record = { ...message, id: this.id("message"), createdAt: new Date() };
|
|
27
|
+
this.messages.push(record);
|
|
28
|
+
return record;
|
|
29
|
+
}
|
|
30
|
+
async updateMessage(id: string, changes: any) {
|
|
31
|
+
return Object.assign(this.messages.find((m) => m.id === id)!, changes);
|
|
32
|
+
}
|
|
33
|
+
async deleteMessage(id: string) {
|
|
34
|
+
this.messages = this.messages.filter((m) => m.id !== id);
|
|
35
|
+
}
|
|
36
|
+
async listMessages(chatId: string, options: { excludeIds?: string[] } = {}) {
|
|
37
|
+
return this.messages.filter((m) => m.chatId === chatId && !options.excludeIds?.includes(m.id));
|
|
38
|
+
}
|
|
39
|
+
async createToolCall(toolCall: any) {
|
|
40
|
+
this.toolCalls.push({ ...toolCall });
|
|
41
|
+
}
|
|
42
|
+
async finishToolCall(messageId: string, toolCallId: string, finished: any) {
|
|
43
|
+
Object.assign(
|
|
44
|
+
this.toolCalls.find((c) => c.messageId === messageId && c.toolCallId === toolCallId),
|
|
45
|
+
finished
|
|
46
|
+
);
|
|
47
|
+
}
|
|
48
|
+
async createRequest() {}
|
|
49
|
+
async requestTotals() {
|
|
50
|
+
return { inputTokens: 0, outputTokens: 0, cost: 0 };
|
|
51
|
+
}
|
|
52
|
+
async createAgentSession(session: any) {
|
|
53
|
+
const record = {
|
|
54
|
+
...session,
|
|
55
|
+
id: this.id("session"),
|
|
56
|
+
createdAt: new Date(),
|
|
57
|
+
updatedAt: new Date()
|
|
58
|
+
};
|
|
59
|
+
this.sessions.set(record.id, record);
|
|
60
|
+
return record;
|
|
61
|
+
}
|
|
62
|
+
async findAgentSession(id: string) {
|
|
63
|
+
return this.sessions.get(id) ?? null;
|
|
64
|
+
}
|
|
65
|
+
async updateAgentSession() {}
|
|
66
|
+
/** A method of the app's own, beyond ChatStore */
|
|
67
|
+
countMessages() {
|
|
68
|
+
return this.messages.length;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const KEY = "sk-proj-abcdefghijklmnopqrstuvwx";
|
|
73
|
+
const usage = { input_tokens: 1, output_tokens: 1, total_tokens: 2 };
|
|
74
|
+
|
|
75
|
+
class Scripted implements Provider {
|
|
76
|
+
id = "scripted";
|
|
77
|
+
constructor(private replies: ChatResponse[]) {}
|
|
78
|
+
defaultModel() {
|
|
79
|
+
return "test-model";
|
|
80
|
+
}
|
|
81
|
+
async chat(_request: ChatRequest): Promise<ChatResponse> {
|
|
82
|
+
return this.replies.shift()!;
|
|
83
|
+
}
|
|
84
|
+
formatToolResultMessage(toolCallId: string, content: string): Message {
|
|
85
|
+
return { role: "tool", tool_call_id: toolCallId, content };
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const connect = {
|
|
90
|
+
type: "function" as const,
|
|
91
|
+
function: {
|
|
92
|
+
name: "connect",
|
|
93
|
+
description: "Connect",
|
|
94
|
+
parameters: { type: "object", properties: { url: { type: "string" } } }
|
|
95
|
+
},
|
|
96
|
+
handler: async () => `connected with ${KEY}`
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
class Ops extends Agent {
|
|
100
|
+
static model = "test-model";
|
|
101
|
+
static tools = [connect];
|
|
102
|
+
static assumeModelExists = true;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
describe("saving conversations", () => {
|
|
106
|
+
it("masks keys and URL passwords in messages, tool arguments and results", async () => {
|
|
107
|
+
const store = new MemoryStore();
|
|
108
|
+
const provider = new Scripted([
|
|
109
|
+
{
|
|
110
|
+
content: null,
|
|
111
|
+
usage,
|
|
112
|
+
tool_calls: [
|
|
113
|
+
{
|
|
114
|
+
id: "c1",
|
|
115
|
+
type: "function",
|
|
116
|
+
function: {
|
|
117
|
+
name: "connect",
|
|
118
|
+
arguments: JSON.stringify({ url: "postgres://admin:hunter2@db/app" })
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
]
|
|
122
|
+
},
|
|
123
|
+
{ content: `Done. Your key is ${KEY}.`, usage }
|
|
124
|
+
]);
|
|
125
|
+
const session = await createAgentSession(store, createLLM({ provider }), Ops);
|
|
126
|
+
|
|
127
|
+
await session.ask(`Connect using ${KEY}`);
|
|
128
|
+
|
|
129
|
+
expect(store.messages.map((m) => m.content)).toEqual([
|
|
130
|
+
"Connect using [REDACTED]",
|
|
131
|
+
"Done. Your key is [REDACTED]."
|
|
132
|
+
]);
|
|
133
|
+
expect(store.toolCalls[0]).toMatchObject({
|
|
134
|
+
arguments: '{"url":"postgres://admin:[REDACTED]@db/app"}',
|
|
135
|
+
result: "connected with [REDACTED]"
|
|
136
|
+
});
|
|
137
|
+
// The tool call arguments stay valid JSON.
|
|
138
|
+
expect(JSON.parse(store.toolCalls[0].arguments).url).toBe("postgres://admin:[REDACTED]@db/app");
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
it("applies the app's own redact after masking keys", async () => {
|
|
142
|
+
const store = new MemoryStore();
|
|
143
|
+
const provider = new Scripted([{ content: "Call Jane on 555-0100.", usage }]);
|
|
144
|
+
const chat = await createChat(store, createLLM({ provider }), {
|
|
145
|
+
model: "test-model",
|
|
146
|
+
assumeModelExists: true,
|
|
147
|
+
redact: (text: string) => text.replace(/Jane/g, "[NAME]")
|
|
148
|
+
} as any);
|
|
149
|
+
|
|
150
|
+
await chat.ask("Who do I call?");
|
|
151
|
+
|
|
152
|
+
expect(store.messages.map((m) => m.content)).toEqual([
|
|
153
|
+
"Who do I call?",
|
|
154
|
+
"Call [NAME] on 555-0100."
|
|
155
|
+
]);
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
it("leaves ordinary text, reads and the store's own methods as they were", async () => {
|
|
159
|
+
const store = new MemoryStore();
|
|
160
|
+
const wrapped = redactingStore(store) as MemoryStore;
|
|
161
|
+
await wrapped.createMessage({ chatId: "c", role: "user", content: "Order 42 shipped" });
|
|
162
|
+
|
|
163
|
+
expect((await wrapped.listMessages("c"))[0]!.content).toBe("Order 42 shipped");
|
|
164
|
+
expect(wrapped.countMessages()).toBe(1);
|
|
165
|
+
expect(redactingStore(wrapped)).toBe(wrapped);
|
|
166
|
+
});
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
describe("maskSecrets", () => {
|
|
170
|
+
it("masks common key formats", () => {
|
|
171
|
+
expect(
|
|
172
|
+
maskSecrets(
|
|
173
|
+
`a ${KEY} b AIzaSyA1234567890abcdefghijklmnopqrstu c Bearer abcdefghijklmnopqrstuvwxyz`
|
|
174
|
+
)
|
|
175
|
+
).toBe("a [REDACTED] b [REDACTED] c Bearer [REDACTED]");
|
|
176
|
+
});
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
describe("review follow-ups", () => {
|
|
180
|
+
it("masks a chat's instructions and metadata, a session's inputs and reasoning", async () => {
|
|
181
|
+
const store = new MemoryStore();
|
|
182
|
+
const created: any[] = [];
|
|
183
|
+
const sessions: any[] = [];
|
|
184
|
+
const wrapped = redactingStore(
|
|
185
|
+
Object.assign(Object.create(store), {
|
|
186
|
+
createChat: async (chat: any) => (created.push(chat), store.createChat(chat)),
|
|
187
|
+
createAgentSession: async (session: any) => (
|
|
188
|
+
sessions.push(session),
|
|
189
|
+
store.createAgentSession(session)
|
|
190
|
+
)
|
|
191
|
+
})
|
|
192
|
+
);
|
|
193
|
+
await wrapped.createChat({
|
|
194
|
+
instructions: `Use ${KEY}`,
|
|
195
|
+
metadata: { db: "postgres://app:hunter2@db/x", count: 3 }
|
|
196
|
+
});
|
|
197
|
+
await wrapped.createAgentSession({
|
|
198
|
+
agentClass: "A",
|
|
199
|
+
chatId: "c",
|
|
200
|
+
metadata: { token: `Bearer abcdefghijklmnopqrstuvwxyz` }
|
|
201
|
+
});
|
|
202
|
+
const message = await wrapped.createMessage({ chatId: "c", role: "assistant", content: null });
|
|
203
|
+
await wrapped.updateMessage(message.id, { reasoning: `thinking about ${KEY}` });
|
|
204
|
+
|
|
205
|
+
expect(created[0]).toMatchObject({
|
|
206
|
+
instructions: "Use [REDACTED]",
|
|
207
|
+
metadata: { db: "postgres://app:[REDACTED]@db/x", count: 3 }
|
|
208
|
+
});
|
|
209
|
+
expect(sessions[0].metadata).toEqual({ token: "Bearer [REDACTED]" });
|
|
210
|
+
expect((store.messages.at(-1) as any).reasoning).toBe("thinking about [REDACTED]");
|
|
211
|
+
});
|
|
212
|
+
|
|
213
|
+
it("masks personal data on save when the agent uses PIIMaskMiddleware", async () => {
|
|
214
|
+
const { PIIMaskMiddleware } = await import("@node-llm/core");
|
|
215
|
+
class Masked extends Agent {
|
|
216
|
+
static model = "test-model";
|
|
217
|
+
static assumeModelExists = true;
|
|
218
|
+
static middlewares = [new PIIMaskMiddleware()];
|
|
219
|
+
}
|
|
220
|
+
const store = new MemoryStore();
|
|
221
|
+
const provider = new Scripted([{ content: "Noted.", usage }]);
|
|
222
|
+
const session = await createAgentSession(store, createLLM({ provider }), Masked);
|
|
223
|
+
|
|
224
|
+
await session.ask("My email is jane@example.com");
|
|
225
|
+
|
|
226
|
+
expect(store.messages[0]!.content).toBe("My email is [REDACTED]");
|
|
227
|
+
});
|
|
228
|
+
});
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
import { describe, it, expect } from "vitest";
|
|
2
|
+
import { Agent, createLLM, Tool, z } from "@node-llm/core";
|
|
3
|
+
import type { ChatRequest, ChatResponse, Message, Provider } from "@node-llm/core";
|
|
4
|
+
import { createAgentSession, loadAgentSession } from "../src/adapters/prisma/AgentSession.js";
|
|
5
|
+
import { storedHistory } from "../src/history.js";
|
|
6
|
+
import type { ChatStore, StoredMessage, StoredToolCall } from "../src/Store.js";
|
|
7
|
+
|
|
8
|
+
// A store kept in memory, as an app might keep its records behind its own API.
|
|
9
|
+
class MemoryStore implements ChatStore {
|
|
10
|
+
messages: StoredMessage[] = [];
|
|
11
|
+
toolCalls: StoredToolCall[] = [];
|
|
12
|
+
sessions = new Map<string, any>();
|
|
13
|
+
private next = 0;
|
|
14
|
+
private id(prefix: string) {
|
|
15
|
+
this.next += 1;
|
|
16
|
+
return `${prefix}-${this.next}`;
|
|
17
|
+
}
|
|
18
|
+
async createChat(chat: any) {
|
|
19
|
+
return { ...chat, id: this.id("chat"), createdAt: new Date(), updatedAt: new Date() };
|
|
20
|
+
}
|
|
21
|
+
async findChat() {
|
|
22
|
+
return null;
|
|
23
|
+
}
|
|
24
|
+
async deleteChat() {}
|
|
25
|
+
async createMessage(message: { chatId: string; role: string; content: string | null }) {
|
|
26
|
+
const record = { ...message, id: this.id("message"), createdAt: new Date() };
|
|
27
|
+
this.messages.push(record);
|
|
28
|
+
return record;
|
|
29
|
+
}
|
|
30
|
+
async updateMessage(id: string, changes: any) {
|
|
31
|
+
return Object.assign(this.messages.find((m) => m.id === id)!, changes);
|
|
32
|
+
}
|
|
33
|
+
async deleteMessage(id: string) {
|
|
34
|
+
this.messages = this.messages.filter((m) => m.id !== id);
|
|
35
|
+
}
|
|
36
|
+
async listMessages(chatId: string) {
|
|
37
|
+
return this.messages.filter((m) => m.chatId === chatId);
|
|
38
|
+
}
|
|
39
|
+
async createToolCall(toolCall: any) {
|
|
40
|
+
this.toolCalls.push({ ...toolCall, result: null });
|
|
41
|
+
}
|
|
42
|
+
async finishToolCall(messageId: string, toolCallId: string, finished: any) {
|
|
43
|
+
Object.assign(
|
|
44
|
+
this.toolCalls.find((c) => c.messageId === messageId && c.toolCallId === toolCallId)!,
|
|
45
|
+
finished
|
|
46
|
+
);
|
|
47
|
+
}
|
|
48
|
+
async listToolCalls(messageIds: string[]) {
|
|
49
|
+
return this.toolCalls.filter((c) => messageIds.includes(c.messageId));
|
|
50
|
+
}
|
|
51
|
+
async createRequest() {}
|
|
52
|
+
async requestTotals() {
|
|
53
|
+
return { inputTokens: 0, outputTokens: 0, cost: 0 };
|
|
54
|
+
}
|
|
55
|
+
async createAgentSession(session: any) {
|
|
56
|
+
const record = {
|
|
57
|
+
...session,
|
|
58
|
+
id: this.id("session"),
|
|
59
|
+
createdAt: new Date(),
|
|
60
|
+
updatedAt: new Date()
|
|
61
|
+
};
|
|
62
|
+
this.sessions.set(record.id, record);
|
|
63
|
+
return record;
|
|
64
|
+
}
|
|
65
|
+
async findAgentSession(id: string) {
|
|
66
|
+
return this.sessions.get(id) ?? null;
|
|
67
|
+
}
|
|
68
|
+
async updateAgentSession() {}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const usage = { input_tokens: 10, output_tokens: 5, total_tokens: 15 };
|
|
72
|
+
|
|
73
|
+
class ScriptedProvider implements Provider {
|
|
74
|
+
id = "scripted";
|
|
75
|
+
requests: ChatRequest[] = [];
|
|
76
|
+
constructor(private responses: ChatResponse[]) {}
|
|
77
|
+
defaultModel() {
|
|
78
|
+
return "test-model";
|
|
79
|
+
}
|
|
80
|
+
async chat(request: ChatRequest): Promise<ChatResponse> {
|
|
81
|
+
this.requests.push({ ...request, messages: [...request.messages] });
|
|
82
|
+
const response = this.responses.shift();
|
|
83
|
+
if (!response) throw new Error("No more responses");
|
|
84
|
+
return response;
|
|
85
|
+
}
|
|
86
|
+
formatToolResultMessage(toolCallId: string, content: string): Message {
|
|
87
|
+
return { role: "tool", tool_call_id: toolCallId, content };
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const call = (id: string, name: string, args: string): ChatResponse => ({
|
|
92
|
+
content: null,
|
|
93
|
+
usage,
|
|
94
|
+
tool_calls: [{ id, type: "function", function: { name, arguments: args } }]
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
// The service "restarts" while the second page is being read: the tool never returns.
|
|
98
|
+
let crash = false;
|
|
99
|
+
class ReadPage extends Tool<{ url: string }> {
|
|
100
|
+
name = "read_page";
|
|
101
|
+
description = "Read a page";
|
|
102
|
+
schema = z.object({ url: z.string() });
|
|
103
|
+
async execute({ url }: { url: string }) {
|
|
104
|
+
if (crash && url === "/pricing") return new Promise<string>(() => {});
|
|
105
|
+
return `contents of ${url}`;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
class Researcher extends Agent {
|
|
110
|
+
static model = "test-model";
|
|
111
|
+
static instructions = "Research the site.";
|
|
112
|
+
static tools = [ReadPage];
|
|
113
|
+
static assumeModelExists = true;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
const roles = (messages: Message[]) =>
|
|
117
|
+
messages.map((m) => (m.tool_calls ? `assistant:${m.tool_calls.map((c) => c.id)}` : m.role));
|
|
118
|
+
|
|
119
|
+
describe("resuming an agent session after its service restarts", () => {
|
|
120
|
+
it("gives the model the tool calls and results it already had, without the unfinished call", async () => {
|
|
121
|
+
const store = new MemoryStore();
|
|
122
|
+
crash = true;
|
|
123
|
+
const first = new ScriptedProvider([
|
|
124
|
+
call("c1", "read_page", '{"url":"/home"}'),
|
|
125
|
+
call("c2", "read_page", '{"url":"/pricing"}')
|
|
126
|
+
]);
|
|
127
|
+
const session = await createAgentSession(store, createLLM({ provider: first }), Researcher);
|
|
128
|
+
void session.ask("Summarise the site");
|
|
129
|
+
await vi_waitFor(() => store.toolCalls.length === 2);
|
|
130
|
+
|
|
131
|
+
// A new process loads the session from the store and asks it to carry on.
|
|
132
|
+
crash = false;
|
|
133
|
+
const second = new ScriptedProvider([
|
|
134
|
+
call("c3", "read_page", '{"url":"/pricing"}'),
|
|
135
|
+
{ content: "A pen shop.", usage }
|
|
136
|
+
]);
|
|
137
|
+
const resumed = await loadAgentSession(
|
|
138
|
+
store,
|
|
139
|
+
createLLM({ provider: second }),
|
|
140
|
+
Researcher,
|
|
141
|
+
session.id
|
|
142
|
+
);
|
|
143
|
+
const answer = await resumed!.ask("Continue where you left off.");
|
|
144
|
+
|
|
145
|
+
expect(answer.content).toBe("A pen shop.");
|
|
146
|
+
const sent = second.requests[0].messages;
|
|
147
|
+
expect(roles(sent)).toEqual(["system", "user", "assistant:c1", "tool", "user"]);
|
|
148
|
+
expect(sent[3]).toMatchObject({ tool_call_id: "c1", content: "contents of /home" });
|
|
149
|
+
expect(sent[2].tool_calls![0]).toMatchObject({
|
|
150
|
+
type: "function",
|
|
151
|
+
function: { name: "read_page", arguments: '{"url":"/home"}' }
|
|
152
|
+
});
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
it("keeps finished turns' tool calls before their answers", () => {
|
|
156
|
+
const at = new Date();
|
|
157
|
+
const messages: StoredMessage[] = [
|
|
158
|
+
{ id: "m1", chatId: "c", role: "user", content: "How many pens?", createdAt: at },
|
|
159
|
+
{ id: "m2", chatId: "c", role: "assistant", content: "4 pens.", createdAt: at },
|
|
160
|
+
{ id: "m3", chatId: "c", role: "user", content: "And pencils?", createdAt: at },
|
|
161
|
+
{ id: "m4", chatId: "c", role: "assistant", content: "No pencils.", createdAt: at }
|
|
162
|
+
];
|
|
163
|
+
const toolCalls: StoredToolCall[] = [
|
|
164
|
+
{ messageId: "m2", toolCallId: "a", name: "stock", arguments: '{"sku":"PEN"}', result: "4" },
|
|
165
|
+
{
|
|
166
|
+
messageId: "m2",
|
|
167
|
+
toolCallId: "b",
|
|
168
|
+
name: "stock",
|
|
169
|
+
arguments: '{"sku":"INK"}',
|
|
170
|
+
result: "0",
|
|
171
|
+
thoughtSignature: "sig"
|
|
172
|
+
},
|
|
173
|
+
{
|
|
174
|
+
messageId: "m4",
|
|
175
|
+
toolCallId: "c",
|
|
176
|
+
name: "stock",
|
|
177
|
+
arguments: '{"sku":"PENCIL"}',
|
|
178
|
+
result: "0"
|
|
179
|
+
}
|
|
180
|
+
];
|
|
181
|
+
|
|
182
|
+
const history = storedHistory(messages, toolCalls);
|
|
183
|
+
|
|
184
|
+
expect(roles(history)).toEqual([
|
|
185
|
+
"user",
|
|
186
|
+
"assistant:a,b",
|
|
187
|
+
"tool",
|
|
188
|
+
"tool",
|
|
189
|
+
"assistant",
|
|
190
|
+
"user",
|
|
191
|
+
"assistant:c",
|
|
192
|
+
"tool",
|
|
193
|
+
"assistant"
|
|
194
|
+
]);
|
|
195
|
+
expect(history[1].tool_calls![1]).toMatchObject({ id: "b", thought_signature: "sig" });
|
|
196
|
+
expect(history[4]).toEqual({ role: "assistant", content: "4 pens." });
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
it("leaves out an assistant message with no text and no finished calls", () => {
|
|
200
|
+
const at = new Date();
|
|
201
|
+
const history = storedHistory(
|
|
202
|
+
[
|
|
203
|
+
{ id: "m1", chatId: "c", role: "user", content: "Hi", createdAt: at },
|
|
204
|
+
{ id: "m2", chatId: "c", role: "assistant", content: null, createdAt: at }
|
|
205
|
+
],
|
|
206
|
+
[{ messageId: "m2", toolCallId: "x", name: "slow", arguments: "{}", result: null }]
|
|
207
|
+
);
|
|
208
|
+
|
|
209
|
+
expect(history).toEqual([{ role: "user", content: "Hi" }]);
|
|
210
|
+
});
|
|
211
|
+
|
|
212
|
+
it("gives only the messages' text when the store can't list tool calls", async () => {
|
|
213
|
+
const store = new MemoryStore();
|
|
214
|
+
(store as Partial<MemoryStore>).listToolCalls = undefined;
|
|
215
|
+
const provider = new ScriptedProvider([{ content: "Hello again.", usage }]);
|
|
216
|
+
const session = await createAgentSession(store, createLLM({ provider }), Researcher);
|
|
217
|
+
store.messages.push(
|
|
218
|
+
{ id: "u", chatId: session.chatId, role: "user", content: "Hi", createdAt: new Date() },
|
|
219
|
+
{ id: "a", chatId: session.chatId, role: "assistant", content: null, createdAt: new Date() }
|
|
220
|
+
);
|
|
221
|
+
|
|
222
|
+
const resumed = await loadAgentSession(store, createLLM({ provider }), Researcher, session.id);
|
|
223
|
+
await resumed!.ask("Still there?");
|
|
224
|
+
|
|
225
|
+
expect(roles(provider.requests[0].messages)).toEqual(["system", "user", "user"]);
|
|
226
|
+
});
|
|
227
|
+
});
|
|
228
|
+
|
|
229
|
+
async function vi_waitFor(condition: () => boolean) {
|
|
230
|
+
for (let i = 0; i < 100 && !condition(); i++) await new Promise((r) => setTimeout(r, 5));
|
|
231
|
+
if (!condition()) throw new Error("timed out");
|
|
232
|
+
}
|