@terreno/ai 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bunfig.toml +5 -0
- package/dist/routes/projects.js +11 -11
- package/package.json +6 -5
- package/src/aiApp.test.ts +150 -0
- package/src/isolated/langfuseClient.isolated.ts +64 -0
- package/src/langfuseApp.test.ts +135 -0
- package/src/langfuseCache.test.ts +62 -76
- package/src/langfusePrompts.test.ts +59 -114
- package/src/langfuseRoutes.test.ts +140 -28
- package/src/langfuseTracing.test.ts +68 -0
- package/src/langfuseVercelAi.test.ts +83 -0
- package/src/models/aiRequest.test.ts +81 -0
- package/src/models/fileAttachment.test.ts +3 -1
- package/src/routes/files.test.ts +126 -0
- package/src/routes/mcp.test.ts +132 -0
- package/src/routes/projects.test.ts +205 -0
- package/src/routes/projects.ts +25 -25
- package/src/routes/routes.test.ts +999 -12
- package/src/service/aiService.test.ts +238 -13
- package/src/service/fileStorage.test.ts +90 -0
- package/src/service/mcpService.test.ts +139 -0
- package/tsconfig.json +8 -1
|
@@ -1,25 +1,8 @@
|
|
|
1
1
|
import {beforeEach, describe, expect, it, mock} from "bun:test";
|
|
2
2
|
|
|
3
|
+
import {LangfuseCache} from "./langfuseCache";
|
|
3
4
|
import type {LangfuseCachedPrompt} from "./langfuseTypes";
|
|
4
5
|
|
|
5
|
-
const cacheValues = new Map<string, LangfuseCachedPrompt>();
|
|
6
|
-
|
|
7
|
-
const mockLoggerDebug = mock(() => {});
|
|
8
|
-
const mockLoggerInfo = mock(() => {});
|
|
9
|
-
const mockGetCached = mock(async (key: string) => {
|
|
10
|
-
return cacheValues.get(key) ?? null;
|
|
11
|
-
});
|
|
12
|
-
const mockSetCached = mock(async (key: string, value: LangfuseCachedPrompt) => {
|
|
13
|
-
cacheValues.set(key, value);
|
|
14
|
-
});
|
|
15
|
-
const mockInvalidateCache = mock(async (prefix: string) => {
|
|
16
|
-
const keys = [...cacheValues.keys()];
|
|
17
|
-
for (const key of keys) {
|
|
18
|
-
if (key.startsWith(prefix)) {
|
|
19
|
-
cacheValues.delete(key);
|
|
20
|
-
}
|
|
21
|
-
}
|
|
22
|
-
});
|
|
23
6
|
const mockPromptCreate = mock(async (_params: Record<string, unknown>) => {});
|
|
24
7
|
const mockPromptGet = mock(async (_name: string, _options?: Record<string, unknown>) => ({
|
|
25
8
|
config: {},
|
|
@@ -31,20 +14,9 @@ const mockPromptGet = mock(async (_name: string, _options?: Record<string, unkno
|
|
|
31
14
|
version: 1,
|
|
32
15
|
}));
|
|
33
16
|
|
|
34
|
-
|
|
35
|
-
logger: {
|
|
36
|
-
debug: mockLoggerDebug,
|
|
37
|
-
info: mockLoggerInfo,
|
|
38
|
-
},
|
|
39
|
-
}));
|
|
40
|
-
|
|
41
|
-
mock.module("./langfuseCache", () => ({
|
|
42
|
-
getCached: mockGetCached,
|
|
43
|
-
invalidateCache: mockInvalidateCache,
|
|
44
|
-
setCached: mockSetCached,
|
|
45
|
-
}));
|
|
46
|
-
|
|
17
|
+
const realLangfuseClient = await import("./langfuseClient");
|
|
47
18
|
mock.module("./langfuseClient", () => ({
|
|
19
|
+
...realLangfuseClient,
|
|
48
20
|
getLangfuseClient: () => ({prompt: {create: mockPromptCreate, get: mockPromptGet}}),
|
|
49
21
|
}));
|
|
50
22
|
|
|
@@ -75,13 +47,8 @@ const chatPrompt: LangfuseCachedPrompt = {
|
|
|
75
47
|
version: 1,
|
|
76
48
|
};
|
|
77
49
|
|
|
78
|
-
beforeEach(() => {
|
|
79
|
-
|
|
80
|
-
mockLoggerDebug.mockClear();
|
|
81
|
-
mockLoggerInfo.mockClear();
|
|
82
|
-
mockGetCached.mockClear();
|
|
83
|
-
mockSetCached.mockClear();
|
|
84
|
-
mockInvalidateCache.mockClear();
|
|
50
|
+
beforeEach(async () => {
|
|
51
|
+
await LangfuseCache.deleteMany({});
|
|
85
52
|
mockPromptCreate.mockClear();
|
|
86
53
|
mockPromptGet.mockClear();
|
|
87
54
|
mockPromptGet.mockImplementation(async () => ({
|
|
@@ -119,15 +86,18 @@ describe("compilePrompt", () => {
|
|
|
119
86
|
|
|
120
87
|
describe("getPrompt", () => {
|
|
121
88
|
it("returns cached prompts without calling Langfuse", async () => {
|
|
122
|
-
|
|
89
|
+
await LangfuseCache.create({
|
|
90
|
+
expiresAt: new Date(Date.now() + 60_000),
|
|
91
|
+
key: "prompt:test-prompt:production",
|
|
92
|
+
value: textPrompt,
|
|
93
|
+
});
|
|
123
94
|
|
|
124
95
|
const result = await getPrompt("test-prompt");
|
|
125
96
|
|
|
126
|
-
expect(result).
|
|
97
|
+
expect(result.name).toBe(textPrompt.name);
|
|
98
|
+
expect(result.prompt).toBe(textPrompt.prompt);
|
|
99
|
+
expect(result.version).toBe(textPrompt.version);
|
|
127
100
|
expect(mockPromptGet).toHaveBeenCalledTimes(0);
|
|
128
|
-
expect(mockLoggerDebug).toHaveBeenCalledWith(
|
|
129
|
-
'Langfuse prompt cache hit: "test-prompt" v1 (label: production)'
|
|
130
|
-
);
|
|
131
101
|
});
|
|
132
102
|
|
|
133
103
|
it("fetches and caches prompts with default ttl/label", async () => {
|
|
@@ -137,8 +107,8 @@ describe("getPrompt", () => {
|
|
|
137
107
|
cacheTtlSeconds: 0,
|
|
138
108
|
label: "production",
|
|
139
109
|
});
|
|
140
|
-
|
|
141
|
-
expect(
|
|
110
|
+
const cached = await LangfuseCache.findOne({key: "prompt:test-prompt:production"});
|
|
111
|
+
expect(cached?.value).toEqual(result as unknown as LangfuseCachedPrompt);
|
|
142
112
|
});
|
|
143
113
|
|
|
144
114
|
it("fetches and caches prompts with custom ttl/label", async () => {
|
|
@@ -148,95 +118,70 @@ describe("getPrompt", () => {
|
|
|
148
118
|
cacheTtlSeconds: 0,
|
|
149
119
|
label: "staging",
|
|
150
120
|
});
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
prompt: "Hello world",
|
|
158
|
-
tags: [],
|
|
159
|
-
type: "text",
|
|
160
|
-
version: 1,
|
|
161
|
-
},
|
|
162
|
-
120
|
|
163
|
-
);
|
|
121
|
+
const cached = await LangfuseCache.findOne({key: "prompt:test-prompt:staging"});
|
|
122
|
+
expect(cached).toBeTruthy();
|
|
123
|
+
const expectedExpiryMs = Date.now() + 120_000;
|
|
124
|
+
// Allow some slack for test execution time
|
|
125
|
+
expect(cached!.expiresAt.getTime()).toBeGreaterThan(expectedExpiryMs - 5000);
|
|
126
|
+
expect(cached!.expiresAt.getTime()).toBeLessThan(expectedExpiryMs + 5000);
|
|
164
127
|
});
|
|
165
128
|
|
|
166
129
|
it("throws when Langfuse fetch fails", async () => {
|
|
167
130
|
mockPromptGet.mockImplementation(async () => {
|
|
168
|
-
throw new Error("
|
|
131
|
+
throw new Error("boom");
|
|
169
132
|
});
|
|
170
|
-
|
|
171
|
-
await expect(getPrompt("missing-prompt")).rejects.toThrow("Prompt not found");
|
|
133
|
+
await expect(getPrompt("test-prompt")).rejects.toThrow("boom");
|
|
172
134
|
});
|
|
173
135
|
});
|
|
174
136
|
|
|
175
137
|
describe("createPrompt", () => {
|
|
176
138
|
it("creates text prompts with default labels/tags", async () => {
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
expect(mockPromptCreate).toHaveBeenCalledWith({
|
|
184
|
-
config: undefined,
|
|
185
|
-
labels: ["production"],
|
|
186
|
-
name: "created-prompt",
|
|
187
|
-
prompt: "Text prompt body",
|
|
188
|
-
tags: [],
|
|
189
|
-
type: "text",
|
|
190
|
-
});
|
|
191
|
-
expect(mockInvalidateCache).toHaveBeenCalledWith("prompt:created-prompt:");
|
|
192
|
-
expect(mockPromptGet).toHaveBeenCalledWith("created-prompt", {
|
|
193
|
-
cacheTtlSeconds: 0,
|
|
194
|
-
label: "production",
|
|
195
|
-
});
|
|
196
|
-
expect(result.name).toBe("test-prompt");
|
|
139
|
+
await createPrompt({name: "new-prompt", prompt: "Hello", type: "text"});
|
|
140
|
+
expect(mockPromptCreate).toHaveBeenCalled();
|
|
141
|
+
const call = (mockPromptCreate.mock.calls[0] as Array<Record<string, unknown>>)[0];
|
|
142
|
+
expect(call.name).toBe("new-prompt");
|
|
143
|
+
expect(call.type).toBe("text");
|
|
197
144
|
});
|
|
198
145
|
|
|
199
146
|
it("creates chat prompts preserving role and content", async () => {
|
|
200
147
|
await createPrompt({
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
prompt: [
|
|
205
|
-
{content: "System says hello", role: "system"},
|
|
206
|
-
{content: "User asks question", role: "user"},
|
|
207
|
-
],
|
|
208
|
-
tags: ["support"],
|
|
209
|
-
type: "chat",
|
|
210
|
-
});
|
|
211
|
-
|
|
212
|
-
expect(mockPromptCreate).toHaveBeenCalledWith({
|
|
213
|
-
config: {temperature: 0.3},
|
|
214
|
-
labels: ["staging"],
|
|
215
|
-
name: "chat-prompt",
|
|
216
|
-
prompt: [
|
|
217
|
-
{content: "System says hello", role: "system"},
|
|
218
|
-
{content: "User asks question", role: "user"},
|
|
219
|
-
],
|
|
220
|
-
tags: ["support"],
|
|
148
|
+
name: "chat-new",
|
|
149
|
+
prompt: [{content: "hi", role: "user"}],
|
|
150
|
+
tags: ["x"],
|
|
221
151
|
type: "chat",
|
|
222
152
|
});
|
|
153
|
+
expect(mockPromptCreate).toHaveBeenCalled();
|
|
154
|
+
const call = (mockPromptCreate.mock.calls[0] as Array<Record<string, unknown>>)[0];
|
|
155
|
+
expect(call.type).toBe("chat");
|
|
156
|
+
expect(Array.isArray(call.prompt)).toBe(true);
|
|
157
|
+
expect(call.tags).toEqual(["x"]);
|
|
223
158
|
});
|
|
224
159
|
});
|
|
225
160
|
|
|
226
161
|
describe("invalidatePromptCache", () => {
|
|
227
162
|
it("invalidates cache entries for a prompt prefix", async () => {
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
163
|
+
await LangfuseCache.create({
|
|
164
|
+
expiresAt: new Date(Date.now() + 60_000),
|
|
165
|
+
key: "prompt:my-prompt:production",
|
|
166
|
+
value: textPrompt,
|
|
167
|
+
});
|
|
168
|
+
await LangfuseCache.create({
|
|
169
|
+
expiresAt: new Date(Date.now() + 60_000),
|
|
170
|
+
key: "prompt:my-prompt:staging",
|
|
171
|
+
value: textPrompt,
|
|
172
|
+
});
|
|
173
|
+
await LangfuseCache.create({
|
|
174
|
+
expiresAt: new Date(Date.now() + 60_000),
|
|
175
|
+
key: "prompt:other-prompt:production",
|
|
176
|
+
value: textPrompt,
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
await invalidatePromptCache("my-prompt");
|
|
180
|
+
|
|
181
|
+
const remaining = await LangfuseCache.find({});
|
|
182
|
+
const keys = remaining.map((r) => r.key);
|
|
183
|
+
expect(keys).toContain("prompt:other-prompt:production");
|
|
184
|
+
expect(keys).not.toContain("prompt:my-prompt:production");
|
|
185
|
+
expect(keys).not.toContain("prompt:my-prompt:staging");
|
|
241
186
|
});
|
|
242
187
|
});
|
|
@@ -4,30 +4,39 @@ import mongoose from "mongoose";
|
|
|
4
4
|
import passportLocalMongoose from "passport-local-mongoose";
|
|
5
5
|
import supertest from "supertest";
|
|
6
6
|
|
|
7
|
+
const scoreCreate = mock(() => {});
|
|
8
|
+
const promptCreate = mock(async (_params: Record<string, unknown>) => ({}));
|
|
9
|
+
const promptGet = mock(async (name: string) => ({
|
|
10
|
+
config: {temperature: 0.5},
|
|
11
|
+
labels: ["production"],
|
|
12
|
+
name,
|
|
13
|
+
prompt: "Hello {{name}}, welcome to {{place}}",
|
|
14
|
+
tags: ["tag"],
|
|
15
|
+
type: "text",
|
|
16
|
+
version: 1,
|
|
17
|
+
}));
|
|
18
|
+
const promptsList = mock(async () => ({
|
|
19
|
+
data: [{name: "p1", versions: [1]}],
|
|
20
|
+
meta: {limit: 20, page: 1, total: 1, totalPages: 1},
|
|
21
|
+
}));
|
|
22
|
+
const traceList = mock(async () => ({
|
|
23
|
+
data: [{id: "t1"}],
|
|
24
|
+
meta: {limit: 20, page: 1, total: 1, totalPages: 1},
|
|
25
|
+
}));
|
|
26
|
+
const traceGet = mock(async (traceId: string) => ({id: traceId, name: "Trace"}));
|
|
27
|
+
const flush = mock(async () => {});
|
|
28
|
+
|
|
29
|
+
const realLangfuseClient = await import("./langfuseClient");
|
|
7
30
|
mock.module("./langfuseClient", () => ({
|
|
31
|
+
...realLangfuseClient,
|
|
8
32
|
getLangfuseClient: () => ({
|
|
9
33
|
api: {
|
|
10
|
-
prompts: {
|
|
11
|
-
|
|
12
|
-
},
|
|
13
|
-
trace: {
|
|
14
|
-
get: async () => null,
|
|
15
|
-
list: async () => ({data: [], meta: {limit: 20, page: 1, total: 0, totalPages: 0}}),
|
|
16
|
-
},
|
|
34
|
+
prompts: {list: promptsList},
|
|
35
|
+
trace: {get: traceGet, list: traceList},
|
|
17
36
|
},
|
|
18
|
-
flush
|
|
19
|
-
prompt: {
|
|
20
|
-
|
|
21
|
-
config: {},
|
|
22
|
-
labels: [],
|
|
23
|
-
name: "test",
|
|
24
|
-
prompt: "",
|
|
25
|
-
tags: [],
|
|
26
|
-
type: "text",
|
|
27
|
-
version: 1,
|
|
28
|
-
}),
|
|
29
|
-
},
|
|
30
|
-
score: {create: () => {}},
|
|
37
|
+
flush,
|
|
38
|
+
prompt: {create: promptCreate, get: promptGet},
|
|
39
|
+
score: {create: scoreCreate},
|
|
31
40
|
}),
|
|
32
41
|
}));
|
|
33
42
|
|
|
@@ -99,10 +108,53 @@ describe("Langfuse routes", () => {
|
|
|
99
108
|
expect(res.status).toBe(403);
|
|
100
109
|
});
|
|
101
110
|
|
|
102
|
-
it("
|
|
111
|
+
it("lists prompts for admin users", async () => {
|
|
103
112
|
const agent = await authAsUser(app, "admin");
|
|
104
|
-
const res = await agent.get("/admin/langfuse/prompts");
|
|
105
|
-
expect(res.status).
|
|
113
|
+
const res = await agent.get("/admin/langfuse/prompts?page=2&limit=5");
|
|
114
|
+
expect(res.status).toBe(200);
|
|
115
|
+
expect(res.body.data).toBeDefined();
|
|
116
|
+
expect(promptsList).toHaveBeenCalled();
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
it("returns a prompt by name for admin users", async () => {
|
|
120
|
+
const agent = await authAsUser(app, "admin");
|
|
121
|
+
const res = await agent.get("/admin/langfuse/prompts/mine?version=2&label=staging");
|
|
122
|
+
expect(res.status).toBe(200);
|
|
123
|
+
expect(res.body.name).toBe("mine");
|
|
124
|
+
expect(res.body.version).toBe(1);
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
it("creates a text prompt", async () => {
|
|
128
|
+
const agent = await authAsUser(app, "admin");
|
|
129
|
+
const res = await agent
|
|
130
|
+
.post("/admin/langfuse/prompts")
|
|
131
|
+
.send({name: "new", prompt: "Hello", type: "text"});
|
|
132
|
+
expect(res.status).toBe(201);
|
|
133
|
+
expect(promptCreate).toHaveBeenCalled();
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
it("creates a chat prompt", async () => {
|
|
137
|
+
const agent = await authAsUser(app, "admin");
|
|
138
|
+
const res = await agent.post("/admin/langfuse/prompts").send({
|
|
139
|
+
name: "chat",
|
|
140
|
+
prompt: [{content: "hi", role: "user"}],
|
|
141
|
+
tags: ["x"],
|
|
142
|
+
type: "chat",
|
|
143
|
+
});
|
|
144
|
+
expect(res.status).toBe(201);
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
it("rejects prompt create when required fields are missing", async () => {
|
|
148
|
+
const agent = await authAsUser(app, "admin");
|
|
149
|
+
const res = await agent.post("/admin/langfuse/prompts").send({name: "x"});
|
|
150
|
+
expect(res.status).toBe(400);
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
it("invalidates prompt cache", async () => {
|
|
154
|
+
const agent = await authAsUser(app, "admin");
|
|
155
|
+
const res = await agent.delete("/admin/langfuse/prompts/mine/cache");
|
|
156
|
+
expect(res.status).toBe(200);
|
|
157
|
+
expect(res.body.invalidated).toBe(true);
|
|
106
158
|
});
|
|
107
159
|
});
|
|
108
160
|
|
|
@@ -113,10 +165,21 @@ describe("Langfuse routes", () => {
|
|
|
113
165
|
expect(res.status).toBe(403);
|
|
114
166
|
});
|
|
115
167
|
|
|
116
|
-
it("
|
|
168
|
+
it("lists traces for admin users with filters", async () => {
|
|
117
169
|
const agent = await authAsUser(app, "admin");
|
|
118
|
-
const res = await agent.get(
|
|
119
|
-
|
|
170
|
+
const res = await agent.get(
|
|
171
|
+
"/admin/langfuse/traces?page=3&limit=7&userId=u1&from=2024-01-01&to=2024-12-31"
|
|
172
|
+
);
|
|
173
|
+
expect(res.status).toBe(200);
|
|
174
|
+
expect(traceList).toHaveBeenCalled();
|
|
175
|
+
expect(res.body.data).toBeDefined();
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
it("returns a single trace by id", async () => {
|
|
179
|
+
const agent = await authAsUser(app, "admin");
|
|
180
|
+
const res = await agent.get("/admin/langfuse/traces/trace-1");
|
|
181
|
+
expect(res.status).toBe(200);
|
|
182
|
+
expect(res.body.id).toBe("trace-1");
|
|
120
183
|
});
|
|
121
184
|
});
|
|
122
185
|
|
|
@@ -127,10 +190,43 @@ describe("Langfuse routes", () => {
|
|
|
127
190
|
expect(res.status).toBe(403);
|
|
128
191
|
});
|
|
129
192
|
|
|
130
|
-
it("returns 400
|
|
193
|
+
it("returns 400 when promptName is missing", async () => {
|
|
131
194
|
const agent = await authAsUser(app, "admin");
|
|
132
195
|
const res = await agent.post("/admin/langfuse/playground").send({});
|
|
133
|
-
expect(res.status).
|
|
196
|
+
expect(res.status).toBe(400);
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
it("compiles a prompt for admin users", async () => {
|
|
200
|
+
const agent = await authAsUser(app, "admin");
|
|
201
|
+
const res = await agent
|
|
202
|
+
.post("/admin/langfuse/playground")
|
|
203
|
+
.send({promptName: "welcome", variables: {name: "Sam", place: "Earth"}});
|
|
204
|
+
expect(res.status).toBe(200);
|
|
205
|
+
expect(res.body.compiled).toBe("Hello Sam, welcome to Earth");
|
|
206
|
+
expect(res.body.variables.sort()).toEqual(["name", "place"]);
|
|
207
|
+
});
|
|
208
|
+
|
|
209
|
+
it("compiles a chat prompt and extracts variables from chat messages", async () => {
|
|
210
|
+
promptGet.mockImplementationOnce(async (name: string) => ({
|
|
211
|
+
config: {},
|
|
212
|
+
labels: ["production"],
|
|
213
|
+
name,
|
|
214
|
+
prompt: [
|
|
215
|
+
{content: "You are {{role}}.", role: "system"},
|
|
216
|
+
{content: "Greet {{user}}.", role: "user"},
|
|
217
|
+
],
|
|
218
|
+
tags: [],
|
|
219
|
+
type: "chat",
|
|
220
|
+
version: 2,
|
|
221
|
+
}));
|
|
222
|
+
const agent = await authAsUser(app, "admin");
|
|
223
|
+
const res = await agent
|
|
224
|
+
.post("/admin/langfuse/playground")
|
|
225
|
+
.send({promptName: "chatter", variables: {role: "bot", user: "Alex"}});
|
|
226
|
+
expect(res.status).toBe(200);
|
|
227
|
+
expect(res.body.type).toBe("chat");
|
|
228
|
+
expect(Array.isArray(res.body.compiled)).toBe(true);
|
|
229
|
+
expect(res.body.variables.sort()).toEqual(["role", "user"]);
|
|
134
230
|
});
|
|
135
231
|
});
|
|
136
232
|
|
|
@@ -141,6 +237,22 @@ describe("Langfuse routes", () => {
|
|
|
141
237
|
expect(res.status).toBe(403);
|
|
142
238
|
});
|
|
143
239
|
|
|
240
|
+
it("rejects evaluation submissions missing required fields", async () => {
|
|
241
|
+
const agent = await authAsUser(app, "admin");
|
|
242
|
+
const res = await agent.post("/admin/langfuse/evaluations").send({name: "x"});
|
|
243
|
+
expect(res.status).toBe(400);
|
|
244
|
+
});
|
|
245
|
+
|
|
246
|
+
it("creates an evaluation score", async () => {
|
|
247
|
+
const agent = await authAsUser(app, "admin");
|
|
248
|
+
const res = await agent
|
|
249
|
+
.post("/admin/langfuse/evaluations")
|
|
250
|
+
.send({dataType: "NUMERIC", name: "quality", traceId: "t1", value: 5});
|
|
251
|
+
expect(res.status).toBe(201);
|
|
252
|
+
expect(scoreCreate).toHaveBeenCalled();
|
|
253
|
+
expect(flush).toHaveBeenCalled();
|
|
254
|
+
});
|
|
255
|
+
|
|
144
256
|
it("returns non-403 for admin users on config", async () => {
|
|
145
257
|
const agent = await authAsUser(app, "admin");
|
|
146
258
|
const res = await agent.get("/admin/langfuse/evaluations/config");
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import {beforeEach, describe, expect, it, mock} from "bun:test";
|
|
2
|
+
|
|
3
|
+
interface FakeSdkOptions {
|
|
4
|
+
serviceName?: string;
|
|
5
|
+
spanProcessors?: unknown[];
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
const sdkStart = mock(() => {});
|
|
9
|
+
const sdkShutdown = mock(async () => {});
|
|
10
|
+
let lastSdkOptions: FakeSdkOptions | null = null;
|
|
11
|
+
|
|
12
|
+
class FakeNodeSDK {
|
|
13
|
+
constructor(public opts: FakeSdkOptions) {
|
|
14
|
+
lastSdkOptions = opts;
|
|
15
|
+
}
|
|
16
|
+
start() {
|
|
17
|
+
sdkStart();
|
|
18
|
+
}
|
|
19
|
+
async shutdown() {
|
|
20
|
+
await sdkShutdown();
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
class FakeLangfuseSpanProcessor {
|
|
25
|
+
constructor(public opts: Record<string, unknown>) {}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
mock.module("@opentelemetry/sdk-node", () => ({NodeSDK: FakeNodeSDK}));
|
|
29
|
+
mock.module("@langfuse/otel", () => ({LangfuseSpanProcessor: FakeLangfuseSpanProcessor}));
|
|
30
|
+
|
|
31
|
+
const {initTracing, shutdownTracing} = await import("./langfuseTracing");
|
|
32
|
+
|
|
33
|
+
describe("langfuseTracing", () => {
|
|
34
|
+
beforeEach(async () => {
|
|
35
|
+
await shutdownTracing();
|
|
36
|
+
sdkStart.mockClear();
|
|
37
|
+
sdkShutdown.mockClear();
|
|
38
|
+
lastSdkOptions = null;
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
it("initializes a NodeSDK with default service name", () => {
|
|
42
|
+
initTracing({publicKey: "pk", secretKey: "sk"});
|
|
43
|
+
expect(sdkStart).toHaveBeenCalled();
|
|
44
|
+
expect(lastSdkOptions?.serviceName).toBe("terreno-app");
|
|
45
|
+
expect(lastSdkOptions?.spanProcessors).toHaveLength(1);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
it("respects serviceName and baseUrl", () => {
|
|
49
|
+
initTracing({
|
|
50
|
+
baseUrl: "https://lf.example.com",
|
|
51
|
+
publicKey: "pk",
|
|
52
|
+
secretKey: "sk",
|
|
53
|
+
serviceName: "my-service",
|
|
54
|
+
});
|
|
55
|
+
expect(lastSdkOptions?.serviceName).toBe("my-service");
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
it("shuts down the SDK", async () => {
|
|
59
|
+
initTracing({publicKey: "pk", secretKey: "sk"});
|
|
60
|
+
await shutdownTracing();
|
|
61
|
+
expect(sdkShutdown).toHaveBeenCalled();
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
it("no-ops shutdown when not initialized", async () => {
|
|
65
|
+
await shutdownTracing();
|
|
66
|
+
expect(sdkShutdown).not.toHaveBeenCalled();
|
|
67
|
+
});
|
|
68
|
+
});
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import {beforeEach, describe, expect, it, mock} from "bun:test";
|
|
2
|
+
|
|
3
|
+
import {LangfuseCache} from "./langfuseCache";
|
|
4
|
+
|
|
5
|
+
const mockPromptGet = mock(async (_name: string, _options?: Record<string, unknown>) => ({
|
|
6
|
+
config: {temperature: 0.5},
|
|
7
|
+
labels: [],
|
|
8
|
+
name: "welcome",
|
|
9
|
+
prompt: "Hi {{name}}",
|
|
10
|
+
tags: [],
|
|
11
|
+
type: "text" as const,
|
|
12
|
+
version: 3,
|
|
13
|
+
}));
|
|
14
|
+
|
|
15
|
+
// Mock the langfuse SDK client so the real `getPrompt`/`compilePrompt` from
|
|
16
|
+
// `./langfusePrompts` can be exercised without network access. We intentionally
|
|
17
|
+
// do NOT mock `./langfusePrompts` directly because that would leak into other
|
|
18
|
+
// test suites that share the module cache in the same bun process.
|
|
19
|
+
const realLangfuseClient = await import("./langfuseClient");
|
|
20
|
+
mock.module("./langfuseClient", () => ({
|
|
21
|
+
...realLangfuseClient,
|
|
22
|
+
getLangfuseClient: () => ({prompt: {get: mockPromptGet}}),
|
|
23
|
+
}));
|
|
24
|
+
|
|
25
|
+
const {createTelemetryConfig, preparePromptForAI} = await import("./langfuseVercelAi");
|
|
26
|
+
|
|
27
|
+
describe("langfuseVercelAi", () => {
|
|
28
|
+
beforeEach(async () => {
|
|
29
|
+
mockPromptGet.mockClear();
|
|
30
|
+
await LangfuseCache.deleteMany({});
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
describe("preparePromptForAI", () => {
|
|
34
|
+
it("returns a compiled text prompt with telemetry", async () => {
|
|
35
|
+
const result = await preparePromptForAI({
|
|
36
|
+
promptName: "welcome",
|
|
37
|
+
userId: "user-1",
|
|
38
|
+
variables: {name: "Sam"},
|
|
39
|
+
});
|
|
40
|
+
expect(result.prompt).toBe("Hi Sam");
|
|
41
|
+
expect(result.telemetry.functionId).toBe("prompt:welcome");
|
|
42
|
+
expect(result.telemetry.metadata?.userId).toBe("user-1");
|
|
43
|
+
expect(result.telemetry.metadata?.langfusePromptVersion).toBe(3);
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
it("omits userId metadata when not provided", async () => {
|
|
47
|
+
const result = await preparePromptForAI({promptName: "welcome"});
|
|
48
|
+
expect(result.telemetry.metadata?.userId).toBeUndefined();
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
it("returns chat messages for chat prompts", async () => {
|
|
52
|
+
mockPromptGet.mockImplementationOnce(async (name: string) => ({
|
|
53
|
+
config: {},
|
|
54
|
+
labels: [],
|
|
55
|
+
name,
|
|
56
|
+
prompt: [{content: "Hello", role: "user" as const}],
|
|
57
|
+
tags: [],
|
|
58
|
+
type: "chat" as const,
|
|
59
|
+
version: 1,
|
|
60
|
+
}));
|
|
61
|
+
const result = await preparePromptForAI({promptName: "chat"});
|
|
62
|
+
expect(result.messages).toEqual([{content: "Hello", role: "user"}]);
|
|
63
|
+
});
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
describe("createTelemetryConfig", () => {
|
|
67
|
+
it("builds a telemetry config with userId and metadata", () => {
|
|
68
|
+
const t = createTelemetryConfig({
|
|
69
|
+
functionId: "fn-1",
|
|
70
|
+
metadata: {foo: "bar"},
|
|
71
|
+
userId: "user-1",
|
|
72
|
+
});
|
|
73
|
+
expect(t.functionId).toBe("fn-1");
|
|
74
|
+
expect(t.isEnabled).toBe(true);
|
|
75
|
+
expect(t.metadata).toEqual({foo: "bar", userId: "user-1"});
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
it("omits userId when not provided", () => {
|
|
79
|
+
const t = createTelemetryConfig({functionId: "fn-2"});
|
|
80
|
+
expect(t.metadata).toEqual({});
|
|
81
|
+
});
|
|
82
|
+
});
|
|
83
|
+
});
|
|
@@ -79,6 +79,87 @@ describe("AIRequest Model", () => {
|
|
|
79
79
|
});
|
|
80
80
|
});
|
|
81
81
|
|
|
82
|
+
describe("logMultiAgentRequest static", () => {
|
|
83
|
+
it("should create a parent request with sub-request references", async () => {
|
|
84
|
+
const sub1 = await AIRequest.create({
|
|
85
|
+
aiModel: "gpt-4",
|
|
86
|
+
prompt: "sub task 1",
|
|
87
|
+
requestType: "general",
|
|
88
|
+
response: "result 1",
|
|
89
|
+
responseTime: 100,
|
|
90
|
+
tokensUsed: 10,
|
|
91
|
+
});
|
|
92
|
+
const sub2 = await AIRequest.create({
|
|
93
|
+
aiModel: "gpt-4",
|
|
94
|
+
prompt: "sub task 2",
|
|
95
|
+
requestType: "general",
|
|
96
|
+
response: "result 2",
|
|
97
|
+
responseTime: 200,
|
|
98
|
+
tokensUsed: 20,
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
const parent = await AIRequest.logMultiAgentRequest({
|
|
102
|
+
aiModel: "gpt-4",
|
|
103
|
+
requestType: "multi-agent",
|
|
104
|
+
subRequestIds: [sub1._id, sub2._id],
|
|
105
|
+
totalResponseTime: 300,
|
|
106
|
+
totalTokensUsed: 30,
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
expect(parent._id).toBeDefined();
|
|
110
|
+
expect(parent.prompt).toBe("[multi-agent parent request]");
|
|
111
|
+
expect(parent.requestType).toBe("multi-agent");
|
|
112
|
+
expect(parent.totalResponseTime).toBe(300);
|
|
113
|
+
expect(parent.totalTokensUsed).toBe(30);
|
|
114
|
+
expect(parent.subRequestIds).toHaveLength(2);
|
|
115
|
+
|
|
116
|
+
// Verify sub-requests were updated with parentRequestId
|
|
117
|
+
const updatedSub1 = await AIRequest.findById(sub1._id);
|
|
118
|
+
const updatedSub2 = await AIRequest.findById(sub2._id);
|
|
119
|
+
expect(updatedSub1?.parentRequestId?.toString()).toBe(parent._id.toString());
|
|
120
|
+
expect(updatedSub2?.parentRequestId?.toString()).toBe(parent._id.toString());
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
it("should create a parent request with metadata", async () => {
|
|
124
|
+
const sub = await AIRequest.create({
|
|
125
|
+
aiModel: "gpt-4",
|
|
126
|
+
prompt: "sub",
|
|
127
|
+
requestType: "general",
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
const parent = await AIRequest.logMultiAgentRequest({
|
|
131
|
+
aiModel: "gpt-4",
|
|
132
|
+
metadata: {workflow: "test"},
|
|
133
|
+
requestType: "multi-agent",
|
|
134
|
+
subRequestIds: [sub._id],
|
|
135
|
+
totalResponseTime: 100,
|
|
136
|
+
totalTokensUsed: 10,
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
expect(parent.metadata).toEqual({workflow: "test"});
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
it("should create a parent request with userId", async () => {
|
|
143
|
+
const userId = new mongoose.Types.ObjectId();
|
|
144
|
+
const sub = await AIRequest.create({
|
|
145
|
+
aiModel: "gpt-4",
|
|
146
|
+
prompt: "sub",
|
|
147
|
+
requestType: "general",
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
const parent = await AIRequest.logMultiAgentRequest({
|
|
151
|
+
aiModel: "gpt-4",
|
|
152
|
+
requestType: "multi-agent",
|
|
153
|
+
subRequestIds: [sub._id],
|
|
154
|
+
totalResponseTime: 50,
|
|
155
|
+
totalTokensUsed: 5,
|
|
156
|
+
userId,
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
expect(parent.userId?.toString()).toBe(userId.toString());
|
|
160
|
+
});
|
|
161
|
+
});
|
|
162
|
+
|
|
82
163
|
describe("soft delete", () => {
|
|
83
164
|
it("should filter out deleted records by default", async () => {
|
|
84
165
|
await AIRequest.create({
|