@terreno/ai 0.9.3 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bunfig.toml +5 -0
- package/bunfig.unit.toml +2 -0
- package/dist/routes/projects.js +11 -11
- package/package.json +6 -5
- package/src/aiApp.test.ts +150 -0
- package/src/isolated/langfuseClient.isolated.ts +64 -0
- package/src/langfuseApp.test.ts +135 -0
- package/src/langfuseCache.test.ts +59 -67
- package/src/langfusePrompts.test.ts +122 -67
- package/src/langfuseRoutes.test.ts +140 -28
- package/src/langfuseTracing.test.ts +68 -0
- package/src/langfuseVercelAi.test.ts +83 -0
- package/src/models/aiRequest.test.ts +81 -0
- package/src/models/fileAttachment.test.ts +3 -1
- package/src/routes/files.test.ts +126 -0
- package/src/routes/mcp.test.ts +132 -0
- package/src/routes/projects.test.ts +205 -0
- package/src/routes/projects.ts +25 -25
- package/src/routes/routes.test.ts +999 -12
- package/src/service/aiService.test.ts +238 -13
- package/src/service/fileStorage.test.ts +90 -0
- package/src/service/mcpService.test.ts +139 -0
- package/src/tests/bunSetup.ts +16 -10
- package/tsconfig.json +8 -1
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {beforeEach, describe, expect, it, mock} from "bun:test";
|
|
2
2
|
|
|
3
3
|
import {LangfuseCache} from "./langfuseCache";
|
|
4
4
|
import type {LangfuseCachedPrompt} from "./langfuseTypes";
|
|
5
5
|
|
|
6
|
+
const mockPromptCreate = mock(async (_params: Record<string, unknown>) => {});
|
|
6
7
|
const mockPromptGet = mock(async (_name: string, _options?: Record<string, unknown>) => ({
|
|
7
8
|
config: {},
|
|
8
9
|
labels: ["production"],
|
|
@@ -13,11 +14,15 @@ const mockPromptGet = mock(async (_name: string, _options?: Record<string, unkno
|
|
|
13
14
|
version: 1,
|
|
14
15
|
}));
|
|
15
16
|
|
|
17
|
+
const realLangfuseClient = await import("./langfuseClient");
|
|
16
18
|
mock.module("./langfuseClient", () => ({
|
|
17
|
-
|
|
19
|
+
...realLangfuseClient,
|
|
20
|
+
getLangfuseClient: () => ({prompt: {create: mockPromptCreate, get: mockPromptGet}}),
|
|
18
21
|
}));
|
|
19
22
|
|
|
20
|
-
const {compilePrompt, getPrompt} = await import(
|
|
23
|
+
const {compilePrompt, createPrompt, getPrompt, invalidatePromptCache} = await import(
|
|
24
|
+
"./langfusePrompts"
|
|
25
|
+
);
|
|
21
26
|
|
|
22
27
|
const textPrompt: LangfuseCachedPrompt = {
|
|
23
28
|
config: {},
|
|
@@ -42,91 +47,141 @@ const chatPrompt: LangfuseCachedPrompt = {
|
|
|
42
47
|
version: 1,
|
|
43
48
|
};
|
|
44
49
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
}
|
|
50
|
+
beforeEach(async () => {
|
|
51
|
+
await LangfuseCache.deleteMany({});
|
|
52
|
+
mockPromptCreate.mockClear();
|
|
53
|
+
mockPromptGet.mockClear();
|
|
54
|
+
mockPromptGet.mockImplementation(async () => ({
|
|
55
|
+
config: {},
|
|
56
|
+
labels: ["production"],
|
|
57
|
+
name: "test-prompt",
|
|
58
|
+
prompt: "Hello world",
|
|
59
|
+
tags: [],
|
|
60
|
+
type: "text" as const,
|
|
61
|
+
version: 1,
|
|
62
|
+
}));
|
|
63
|
+
});
|
|
51
64
|
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
65
|
+
describe("compilePrompt", () => {
|
|
66
|
+
it("replaces variables in text prompts", () => {
|
|
67
|
+
const result = compilePrompt(textPrompt, {age: "30", name: "Alice"});
|
|
68
|
+
expect(result).toBe("Hello Alice, you are 30 years old.");
|
|
69
|
+
});
|
|
56
70
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
prompt: "Hello world.",
|
|
61
|
-
};
|
|
62
|
-
const result = compilePrompt(noVarPrompt);
|
|
63
|
-
expect(result).toBe("Hello world.");
|
|
64
|
-
});
|
|
71
|
+
it("keeps missing text variables unreplaced", () => {
|
|
72
|
+
const result = compilePrompt(textPrompt, {name: "Alice"});
|
|
73
|
+
expect(result).toBe("Hello Alice, you are {{age}} years old.");
|
|
65
74
|
});
|
|
66
75
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
expect(messages[0].content).toBe("You are a helpful assistant for Acme.");
|
|
73
|
-
expect(messages[1].content).toBe("My name is Bob.");
|
|
74
|
-
});
|
|
76
|
+
it("replaces variables in chat prompts", () => {
|
|
77
|
+
const result = compilePrompt(chatPrompt, {company: "Acme", name: "Bob"}) as Array<{
|
|
78
|
+
content: string;
|
|
79
|
+
role: string;
|
|
80
|
+
}>;
|
|
75
81
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
role: string;
|
|
79
|
-
content: string;
|
|
80
|
-
}>;
|
|
81
|
-
expect(result[0].role).toBe("system");
|
|
82
|
-
expect(result[1].role).toBe("user");
|
|
83
|
-
});
|
|
82
|
+
expect(result[0]).toEqual({content: "You are a helpful assistant for Acme.", role: "system"});
|
|
83
|
+
expect(result[1]).toEqual({content: "My name is Bob.", role: "user"});
|
|
84
84
|
});
|
|
85
85
|
});
|
|
86
86
|
|
|
87
87
|
describe("getPrompt", () => {
|
|
88
|
-
|
|
89
|
-
await LangfuseCache.
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
name: "test-prompt",
|
|
95
|
-
prompt: "Hello world",
|
|
96
|
-
tags: [],
|
|
97
|
-
type: "text" as const,
|
|
98
|
-
version: 1,
|
|
99
|
-
}));
|
|
100
|
-
});
|
|
88
|
+
it("returns cached prompts without calling Langfuse", async () => {
|
|
89
|
+
await LangfuseCache.create({
|
|
90
|
+
expiresAt: new Date(Date.now() + 60_000),
|
|
91
|
+
key: "prompt:test-prompt:production",
|
|
92
|
+
value: textPrompt,
|
|
93
|
+
});
|
|
101
94
|
|
|
102
|
-
|
|
103
|
-
|
|
95
|
+
const result = await getPrompt("test-prompt");
|
|
96
|
+
|
|
97
|
+
expect(result.name).toBe(textPrompt.name);
|
|
98
|
+
expect(result.prompt).toBe(textPrompt.prompt);
|
|
99
|
+
expect(result.version).toBe(textPrompt.version);
|
|
100
|
+
expect(mockPromptGet).toHaveBeenCalledTimes(0);
|
|
104
101
|
});
|
|
105
102
|
|
|
106
|
-
it("fetches
|
|
103
|
+
it("fetches and caches prompts with default ttl/label", async () => {
|
|
107
104
|
const result = await getPrompt("test-prompt");
|
|
108
|
-
expect(result.name).toBe("test-prompt");
|
|
109
|
-
expect(result.type).toBe("text");
|
|
110
|
-
expect(mockPromptGet).toHaveBeenCalledTimes(1);
|
|
111
|
-
});
|
|
112
105
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
106
|
+
expect(mockPromptGet).toHaveBeenCalledWith("test-prompt", {
|
|
107
|
+
cacheTtlSeconds: 0,
|
|
108
|
+
label: "production",
|
|
109
|
+
});
|
|
110
|
+
const cached = await LangfuseCache.findOne({key: "prompt:test-prompt:production"});
|
|
111
|
+
expect(cached?.value).toEqual(result as unknown as LangfuseCachedPrompt);
|
|
117
112
|
});
|
|
118
113
|
|
|
119
|
-
it("
|
|
120
|
-
await getPrompt("test-prompt");
|
|
121
|
-
|
|
122
|
-
expect(
|
|
114
|
+
it("fetches and caches prompts with custom ttl/label", async () => {
|
|
115
|
+
await getPrompt("test-prompt", {label: "staging"}, {cache: {promptTtlSeconds: 120}});
|
|
116
|
+
|
|
117
|
+
expect(mockPromptGet).toHaveBeenCalledWith("test-prompt", {
|
|
118
|
+
cacheTtlSeconds: 0,
|
|
119
|
+
label: "staging",
|
|
120
|
+
});
|
|
121
|
+
const cached = await LangfuseCache.findOne({key: "prompt:test-prompt:staging"});
|
|
122
|
+
expect(cached).toBeTruthy();
|
|
123
|
+
const expectedExpiryMs = Date.now() + 120_000;
|
|
124
|
+
// Allow some slack for test execution time
|
|
125
|
+
expect(cached!.expiresAt.getTime()).toBeGreaterThan(expectedExpiryMs - 5000);
|
|
126
|
+
expect(cached!.expiresAt.getTime()).toBeLessThan(expectedExpiryMs + 5000);
|
|
123
127
|
});
|
|
124
128
|
|
|
125
129
|
it("throws when Langfuse fetch fails", async () => {
|
|
126
130
|
mockPromptGet.mockImplementation(async () => {
|
|
127
|
-
throw new Error("
|
|
131
|
+
throw new Error("boom");
|
|
132
|
+
});
|
|
133
|
+
await expect(getPrompt("test-prompt")).rejects.toThrow("boom");
|
|
134
|
+
});
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
describe("createPrompt", () => {
|
|
138
|
+
it("creates text prompts with default labels/tags", async () => {
|
|
139
|
+
await createPrompt({name: "new-prompt", prompt: "Hello", type: "text"});
|
|
140
|
+
expect(mockPromptCreate).toHaveBeenCalled();
|
|
141
|
+
const call = (mockPromptCreate.mock.calls[0] as Array<Record<string, unknown>>)[0];
|
|
142
|
+
expect(call.name).toBe("new-prompt");
|
|
143
|
+
expect(call.type).toBe("text");
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
it("creates chat prompts preserving role and content", async () => {
|
|
147
|
+
await createPrompt({
|
|
148
|
+
name: "chat-new",
|
|
149
|
+
prompt: [{content: "hi", role: "user"}],
|
|
150
|
+
tags: ["x"],
|
|
151
|
+
type: "chat",
|
|
128
152
|
});
|
|
153
|
+
expect(mockPromptCreate).toHaveBeenCalled();
|
|
154
|
+
const call = (mockPromptCreate.mock.calls[0] as Array<Record<string, unknown>>)[0];
|
|
155
|
+
expect(call.type).toBe("chat");
|
|
156
|
+
expect(Array.isArray(call.prompt)).toBe(true);
|
|
157
|
+
expect(call.tags).toEqual(["x"]);
|
|
158
|
+
});
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
describe("invalidatePromptCache", () => {
|
|
162
|
+
it("invalidates cache entries for a prompt prefix", async () => {
|
|
163
|
+
await LangfuseCache.create({
|
|
164
|
+
expiresAt: new Date(Date.now() + 60_000),
|
|
165
|
+
key: "prompt:my-prompt:production",
|
|
166
|
+
value: textPrompt,
|
|
167
|
+
});
|
|
168
|
+
await LangfuseCache.create({
|
|
169
|
+
expiresAt: new Date(Date.now() + 60_000),
|
|
170
|
+
key: "prompt:my-prompt:staging",
|
|
171
|
+
value: textPrompt,
|
|
172
|
+
});
|
|
173
|
+
await LangfuseCache.create({
|
|
174
|
+
expiresAt: new Date(Date.now() + 60_000),
|
|
175
|
+
key: "prompt:other-prompt:production",
|
|
176
|
+
value: textPrompt,
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
await invalidatePromptCache("my-prompt");
|
|
129
180
|
|
|
130
|
-
await
|
|
181
|
+
const remaining = await LangfuseCache.find({});
|
|
182
|
+
const keys = remaining.map((r) => r.key);
|
|
183
|
+
expect(keys).toContain("prompt:other-prompt:production");
|
|
184
|
+
expect(keys).not.toContain("prompt:my-prompt:production");
|
|
185
|
+
expect(keys).not.toContain("prompt:my-prompt:staging");
|
|
131
186
|
});
|
|
132
187
|
});
|
|
@@ -4,30 +4,39 @@ import mongoose from "mongoose";
|
|
|
4
4
|
import passportLocalMongoose from "passport-local-mongoose";
|
|
5
5
|
import supertest from "supertest";
|
|
6
6
|
|
|
7
|
+
const scoreCreate = mock(() => {});
|
|
8
|
+
const promptCreate = mock(async (_params: Record<string, unknown>) => ({}));
|
|
9
|
+
const promptGet = mock(async (name: string) => ({
|
|
10
|
+
config: {temperature: 0.5},
|
|
11
|
+
labels: ["production"],
|
|
12
|
+
name,
|
|
13
|
+
prompt: "Hello {{name}}, welcome to {{place}}",
|
|
14
|
+
tags: ["tag"],
|
|
15
|
+
type: "text",
|
|
16
|
+
version: 1,
|
|
17
|
+
}));
|
|
18
|
+
const promptsList = mock(async () => ({
|
|
19
|
+
data: [{name: "p1", versions: [1]}],
|
|
20
|
+
meta: {limit: 20, page: 1, total: 1, totalPages: 1},
|
|
21
|
+
}));
|
|
22
|
+
const traceList = mock(async () => ({
|
|
23
|
+
data: [{id: "t1"}],
|
|
24
|
+
meta: {limit: 20, page: 1, total: 1, totalPages: 1},
|
|
25
|
+
}));
|
|
26
|
+
const traceGet = mock(async (traceId: string) => ({id: traceId, name: "Trace"}));
|
|
27
|
+
const flush = mock(async () => {});
|
|
28
|
+
|
|
29
|
+
const realLangfuseClient = await import("./langfuseClient");
|
|
7
30
|
mock.module("./langfuseClient", () => ({
|
|
31
|
+
...realLangfuseClient,
|
|
8
32
|
getLangfuseClient: () => ({
|
|
9
33
|
api: {
|
|
10
|
-
prompts: {
|
|
11
|
-
|
|
12
|
-
},
|
|
13
|
-
trace: {
|
|
14
|
-
get: async () => null,
|
|
15
|
-
list: async () => ({data: [], meta: {limit: 20, page: 1, total: 0, totalPages: 0}}),
|
|
16
|
-
},
|
|
34
|
+
prompts: {list: promptsList},
|
|
35
|
+
trace: {get: traceGet, list: traceList},
|
|
17
36
|
},
|
|
18
|
-
flush
|
|
19
|
-
prompt: {
|
|
20
|
-
|
|
21
|
-
config: {},
|
|
22
|
-
labels: [],
|
|
23
|
-
name: "test",
|
|
24
|
-
prompt: "",
|
|
25
|
-
tags: [],
|
|
26
|
-
type: "text",
|
|
27
|
-
version: 1,
|
|
28
|
-
}),
|
|
29
|
-
},
|
|
30
|
-
score: {create: () => {}},
|
|
37
|
+
flush,
|
|
38
|
+
prompt: {create: promptCreate, get: promptGet},
|
|
39
|
+
score: {create: scoreCreate},
|
|
31
40
|
}),
|
|
32
41
|
}));
|
|
33
42
|
|
|
@@ -99,10 +108,53 @@ describe("Langfuse routes", () => {
|
|
|
99
108
|
expect(res.status).toBe(403);
|
|
100
109
|
});
|
|
101
110
|
|
|
102
|
-
it("
|
|
111
|
+
it("lists prompts for admin users", async () => {
|
|
103
112
|
const agent = await authAsUser(app, "admin");
|
|
104
|
-
const res = await agent.get("/admin/langfuse/prompts");
|
|
105
|
-
expect(res.status).
|
|
113
|
+
const res = await agent.get("/admin/langfuse/prompts?page=2&limit=5");
|
|
114
|
+
expect(res.status).toBe(200);
|
|
115
|
+
expect(res.body.data).toBeDefined();
|
|
116
|
+
expect(promptsList).toHaveBeenCalled();
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
it("returns a prompt by name for admin users", async () => {
|
|
120
|
+
const agent = await authAsUser(app, "admin");
|
|
121
|
+
const res = await agent.get("/admin/langfuse/prompts/mine?version=2&label=staging");
|
|
122
|
+
expect(res.status).toBe(200);
|
|
123
|
+
expect(res.body.name).toBe("mine");
|
|
124
|
+
expect(res.body.version).toBe(1);
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
it("creates a text prompt", async () => {
|
|
128
|
+
const agent = await authAsUser(app, "admin");
|
|
129
|
+
const res = await agent
|
|
130
|
+
.post("/admin/langfuse/prompts")
|
|
131
|
+
.send({name: "new", prompt: "Hello", type: "text"});
|
|
132
|
+
expect(res.status).toBe(201);
|
|
133
|
+
expect(promptCreate).toHaveBeenCalled();
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
it("creates a chat prompt", async () => {
|
|
137
|
+
const agent = await authAsUser(app, "admin");
|
|
138
|
+
const res = await agent.post("/admin/langfuse/prompts").send({
|
|
139
|
+
name: "chat",
|
|
140
|
+
prompt: [{content: "hi", role: "user"}],
|
|
141
|
+
tags: ["x"],
|
|
142
|
+
type: "chat",
|
|
143
|
+
});
|
|
144
|
+
expect(res.status).toBe(201);
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
it("rejects prompt create when required fields are missing", async () => {
|
|
148
|
+
const agent = await authAsUser(app, "admin");
|
|
149
|
+
const res = await agent.post("/admin/langfuse/prompts").send({name: "x"});
|
|
150
|
+
expect(res.status).toBe(400);
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
it("invalidates prompt cache", async () => {
|
|
154
|
+
const agent = await authAsUser(app, "admin");
|
|
155
|
+
const res = await agent.delete("/admin/langfuse/prompts/mine/cache");
|
|
156
|
+
expect(res.status).toBe(200);
|
|
157
|
+
expect(res.body.invalidated).toBe(true);
|
|
106
158
|
});
|
|
107
159
|
});
|
|
108
160
|
|
|
@@ -113,10 +165,21 @@ describe("Langfuse routes", () => {
|
|
|
113
165
|
expect(res.status).toBe(403);
|
|
114
166
|
});
|
|
115
167
|
|
|
116
|
-
it("
|
|
168
|
+
it("lists traces for admin users with filters", async () => {
|
|
117
169
|
const agent = await authAsUser(app, "admin");
|
|
118
|
-
const res = await agent.get(
|
|
119
|
-
|
|
170
|
+
const res = await agent.get(
|
|
171
|
+
"/admin/langfuse/traces?page=3&limit=7&userId=u1&from=2024-01-01&to=2024-12-31"
|
|
172
|
+
);
|
|
173
|
+
expect(res.status).toBe(200);
|
|
174
|
+
expect(traceList).toHaveBeenCalled();
|
|
175
|
+
expect(res.body.data).toBeDefined();
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
it("returns a single trace by id", async () => {
|
|
179
|
+
const agent = await authAsUser(app, "admin");
|
|
180
|
+
const res = await agent.get("/admin/langfuse/traces/trace-1");
|
|
181
|
+
expect(res.status).toBe(200);
|
|
182
|
+
expect(res.body.id).toBe("trace-1");
|
|
120
183
|
});
|
|
121
184
|
});
|
|
122
185
|
|
|
@@ -127,10 +190,43 @@ describe("Langfuse routes", () => {
|
|
|
127
190
|
expect(res.status).toBe(403);
|
|
128
191
|
});
|
|
129
192
|
|
|
130
|
-
it("returns 400
|
|
193
|
+
it("returns 400 when promptName is missing", async () => {
|
|
131
194
|
const agent = await authAsUser(app, "admin");
|
|
132
195
|
const res = await agent.post("/admin/langfuse/playground").send({});
|
|
133
|
-
expect(res.status).
|
|
196
|
+
expect(res.status).toBe(400);
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
it("compiles a prompt for admin users", async () => {
|
|
200
|
+
const agent = await authAsUser(app, "admin");
|
|
201
|
+
const res = await agent
|
|
202
|
+
.post("/admin/langfuse/playground")
|
|
203
|
+
.send({promptName: "welcome", variables: {name: "Sam", place: "Earth"}});
|
|
204
|
+
expect(res.status).toBe(200);
|
|
205
|
+
expect(res.body.compiled).toBe("Hello Sam, welcome to Earth");
|
|
206
|
+
expect(res.body.variables.sort()).toEqual(["name", "place"]);
|
|
207
|
+
});
|
|
208
|
+
|
|
209
|
+
it("compiles a chat prompt and extracts variables from chat messages", async () => {
|
|
210
|
+
promptGet.mockImplementationOnce(async (name: string) => ({
|
|
211
|
+
config: {},
|
|
212
|
+
labels: ["production"],
|
|
213
|
+
name,
|
|
214
|
+
prompt: [
|
|
215
|
+
{content: "You are {{role}}.", role: "system"},
|
|
216
|
+
{content: "Greet {{user}}.", role: "user"},
|
|
217
|
+
],
|
|
218
|
+
tags: [],
|
|
219
|
+
type: "chat",
|
|
220
|
+
version: 2,
|
|
221
|
+
}));
|
|
222
|
+
const agent = await authAsUser(app, "admin");
|
|
223
|
+
const res = await agent
|
|
224
|
+
.post("/admin/langfuse/playground")
|
|
225
|
+
.send({promptName: "chatter", variables: {role: "bot", user: "Alex"}});
|
|
226
|
+
expect(res.status).toBe(200);
|
|
227
|
+
expect(res.body.type).toBe("chat");
|
|
228
|
+
expect(Array.isArray(res.body.compiled)).toBe(true);
|
|
229
|
+
expect(res.body.variables.sort()).toEqual(["role", "user"]);
|
|
134
230
|
});
|
|
135
231
|
});
|
|
136
232
|
|
|
@@ -141,6 +237,22 @@ describe("Langfuse routes", () => {
|
|
|
141
237
|
expect(res.status).toBe(403);
|
|
142
238
|
});
|
|
143
239
|
|
|
240
|
+
it("rejects evaluation submissions missing required fields", async () => {
|
|
241
|
+
const agent = await authAsUser(app, "admin");
|
|
242
|
+
const res = await agent.post("/admin/langfuse/evaluations").send({name: "x"});
|
|
243
|
+
expect(res.status).toBe(400);
|
|
244
|
+
});
|
|
245
|
+
|
|
246
|
+
it("creates an evaluation score", async () => {
|
|
247
|
+
const agent = await authAsUser(app, "admin");
|
|
248
|
+
const res = await agent
|
|
249
|
+
.post("/admin/langfuse/evaluations")
|
|
250
|
+
.send({dataType: "NUMERIC", name: "quality", traceId: "t1", value: 5});
|
|
251
|
+
expect(res.status).toBe(201);
|
|
252
|
+
expect(scoreCreate).toHaveBeenCalled();
|
|
253
|
+
expect(flush).toHaveBeenCalled();
|
|
254
|
+
});
|
|
255
|
+
|
|
144
256
|
it("returns non-403 for admin users on config", async () => {
|
|
145
257
|
const agent = await authAsUser(app, "admin");
|
|
146
258
|
const res = await agent.get("/admin/langfuse/evaluations/config");
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import {beforeEach, describe, expect, it, mock} from "bun:test";
|
|
2
|
+
|
|
3
|
+
interface FakeSdkOptions {
|
|
4
|
+
serviceName?: string;
|
|
5
|
+
spanProcessors?: unknown[];
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
const sdkStart = mock(() => {});
|
|
9
|
+
const sdkShutdown = mock(async () => {});
|
|
10
|
+
let lastSdkOptions: FakeSdkOptions | null = null;
|
|
11
|
+
|
|
12
|
+
class FakeNodeSDK {
|
|
13
|
+
constructor(public opts: FakeSdkOptions) {
|
|
14
|
+
lastSdkOptions = opts;
|
|
15
|
+
}
|
|
16
|
+
start() {
|
|
17
|
+
sdkStart();
|
|
18
|
+
}
|
|
19
|
+
async shutdown() {
|
|
20
|
+
await sdkShutdown();
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
class FakeLangfuseSpanProcessor {
|
|
25
|
+
constructor(public opts: Record<string, unknown>) {}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
mock.module("@opentelemetry/sdk-node", () => ({NodeSDK: FakeNodeSDK}));
|
|
29
|
+
mock.module("@langfuse/otel", () => ({LangfuseSpanProcessor: FakeLangfuseSpanProcessor}));
|
|
30
|
+
|
|
31
|
+
const {initTracing, shutdownTracing} = await import("./langfuseTracing");
|
|
32
|
+
|
|
33
|
+
describe("langfuseTracing", () => {
|
|
34
|
+
beforeEach(async () => {
|
|
35
|
+
await shutdownTracing();
|
|
36
|
+
sdkStart.mockClear();
|
|
37
|
+
sdkShutdown.mockClear();
|
|
38
|
+
lastSdkOptions = null;
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
it("initializes a NodeSDK with default service name", () => {
|
|
42
|
+
initTracing({publicKey: "pk", secretKey: "sk"});
|
|
43
|
+
expect(sdkStart).toHaveBeenCalled();
|
|
44
|
+
expect(lastSdkOptions?.serviceName).toBe("terreno-app");
|
|
45
|
+
expect(lastSdkOptions?.spanProcessors).toHaveLength(1);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
it("respects serviceName and baseUrl", () => {
|
|
49
|
+
initTracing({
|
|
50
|
+
baseUrl: "https://lf.example.com",
|
|
51
|
+
publicKey: "pk",
|
|
52
|
+
secretKey: "sk",
|
|
53
|
+
serviceName: "my-service",
|
|
54
|
+
});
|
|
55
|
+
expect(lastSdkOptions?.serviceName).toBe("my-service");
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
it("shuts down the SDK", async () => {
|
|
59
|
+
initTracing({publicKey: "pk", secretKey: "sk"});
|
|
60
|
+
await shutdownTracing();
|
|
61
|
+
expect(sdkShutdown).toHaveBeenCalled();
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
it("no-ops shutdown when not initialized", async () => {
|
|
65
|
+
await shutdownTracing();
|
|
66
|
+
expect(sdkShutdown).not.toHaveBeenCalled();
|
|
67
|
+
});
|
|
68
|
+
});
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import {beforeEach, describe, expect, it, mock} from "bun:test";
|
|
2
|
+
|
|
3
|
+
import {LangfuseCache} from "./langfuseCache";
|
|
4
|
+
|
|
5
|
+
const mockPromptGet = mock(async (_name: string, _options?: Record<string, unknown>) => ({
|
|
6
|
+
config: {temperature: 0.5},
|
|
7
|
+
labels: [],
|
|
8
|
+
name: "welcome",
|
|
9
|
+
prompt: "Hi {{name}}",
|
|
10
|
+
tags: [],
|
|
11
|
+
type: "text" as const,
|
|
12
|
+
version: 3,
|
|
13
|
+
}));
|
|
14
|
+
|
|
15
|
+
// Mock the langfuse SDK client so the real `getPrompt`/`compilePrompt` from
|
|
16
|
+
// `./langfusePrompts` can be exercised without network access. We intentionally
|
|
17
|
+
// do NOT mock `./langfusePrompts` directly because that would leak into other
|
|
18
|
+
// test suites that share the module cache in the same bun process.
|
|
19
|
+
const realLangfuseClient = await import("./langfuseClient");
|
|
20
|
+
mock.module("./langfuseClient", () => ({
|
|
21
|
+
...realLangfuseClient,
|
|
22
|
+
getLangfuseClient: () => ({prompt: {get: mockPromptGet}}),
|
|
23
|
+
}));
|
|
24
|
+
|
|
25
|
+
const {createTelemetryConfig, preparePromptForAI} = await import("./langfuseVercelAi");
|
|
26
|
+
|
|
27
|
+
describe("langfuseVercelAi", () => {
|
|
28
|
+
beforeEach(async () => {
|
|
29
|
+
mockPromptGet.mockClear();
|
|
30
|
+
await LangfuseCache.deleteMany({});
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
describe("preparePromptForAI", () => {
|
|
34
|
+
it("returns a compiled text prompt with telemetry", async () => {
|
|
35
|
+
const result = await preparePromptForAI({
|
|
36
|
+
promptName: "welcome",
|
|
37
|
+
userId: "user-1",
|
|
38
|
+
variables: {name: "Sam"},
|
|
39
|
+
});
|
|
40
|
+
expect(result.prompt).toBe("Hi Sam");
|
|
41
|
+
expect(result.telemetry.functionId).toBe("prompt:welcome");
|
|
42
|
+
expect(result.telemetry.metadata?.userId).toBe("user-1");
|
|
43
|
+
expect(result.telemetry.metadata?.langfusePromptVersion).toBe(3);
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
it("omits userId metadata when not provided", async () => {
|
|
47
|
+
const result = await preparePromptForAI({promptName: "welcome"});
|
|
48
|
+
expect(result.telemetry.metadata?.userId).toBeUndefined();
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
it("returns chat messages for chat prompts", async () => {
|
|
52
|
+
mockPromptGet.mockImplementationOnce(async (name: string) => ({
|
|
53
|
+
config: {},
|
|
54
|
+
labels: [],
|
|
55
|
+
name,
|
|
56
|
+
prompt: [{content: "Hello", role: "user" as const}],
|
|
57
|
+
tags: [],
|
|
58
|
+
type: "chat" as const,
|
|
59
|
+
version: 1,
|
|
60
|
+
}));
|
|
61
|
+
const result = await preparePromptForAI({promptName: "chat"});
|
|
62
|
+
expect(result.messages).toEqual([{content: "Hello", role: "user"}]);
|
|
63
|
+
});
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
describe("createTelemetryConfig", () => {
|
|
67
|
+
it("builds a telemetry config with userId and metadata", () => {
|
|
68
|
+
const t = createTelemetryConfig({
|
|
69
|
+
functionId: "fn-1",
|
|
70
|
+
metadata: {foo: "bar"},
|
|
71
|
+
userId: "user-1",
|
|
72
|
+
});
|
|
73
|
+
expect(t.functionId).toBe("fn-1");
|
|
74
|
+
expect(t.isEnabled).toBe(true);
|
|
75
|
+
expect(t.metadata).toEqual({foo: "bar", userId: "user-1"});
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
it("omits userId when not provided", () => {
|
|
79
|
+
const t = createTelemetryConfig({functionId: "fn-2"});
|
|
80
|
+
expect(t.metadata).toEqual({});
|
|
81
|
+
});
|
|
82
|
+
});
|
|
83
|
+
});
|
|
@@ -79,6 +79,87 @@ describe("AIRequest Model", () => {
|
|
|
79
79
|
});
|
|
80
80
|
});
|
|
81
81
|
|
|
82
|
+
describe("logMultiAgentRequest static", () => {
|
|
83
|
+
it("should create a parent request with sub-request references", async () => {
|
|
84
|
+
const sub1 = await AIRequest.create({
|
|
85
|
+
aiModel: "gpt-4",
|
|
86
|
+
prompt: "sub task 1",
|
|
87
|
+
requestType: "general",
|
|
88
|
+
response: "result 1",
|
|
89
|
+
responseTime: 100,
|
|
90
|
+
tokensUsed: 10,
|
|
91
|
+
});
|
|
92
|
+
const sub2 = await AIRequest.create({
|
|
93
|
+
aiModel: "gpt-4",
|
|
94
|
+
prompt: "sub task 2",
|
|
95
|
+
requestType: "general",
|
|
96
|
+
response: "result 2",
|
|
97
|
+
responseTime: 200,
|
|
98
|
+
tokensUsed: 20,
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
const parent = await AIRequest.logMultiAgentRequest({
|
|
102
|
+
aiModel: "gpt-4",
|
|
103
|
+
requestType: "multi-agent",
|
|
104
|
+
subRequestIds: [sub1._id, sub2._id],
|
|
105
|
+
totalResponseTime: 300,
|
|
106
|
+
totalTokensUsed: 30,
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
expect(parent._id).toBeDefined();
|
|
110
|
+
expect(parent.prompt).toBe("[multi-agent parent request]");
|
|
111
|
+
expect(parent.requestType).toBe("multi-agent");
|
|
112
|
+
expect(parent.totalResponseTime).toBe(300);
|
|
113
|
+
expect(parent.totalTokensUsed).toBe(30);
|
|
114
|
+
expect(parent.subRequestIds).toHaveLength(2);
|
|
115
|
+
|
|
116
|
+
// Verify sub-requests were updated with parentRequestId
|
|
117
|
+
const updatedSub1 = await AIRequest.findById(sub1._id);
|
|
118
|
+
const updatedSub2 = await AIRequest.findById(sub2._id);
|
|
119
|
+
expect(updatedSub1?.parentRequestId?.toString()).toBe(parent._id.toString());
|
|
120
|
+
expect(updatedSub2?.parentRequestId?.toString()).toBe(parent._id.toString());
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
it("should create a parent request with metadata", async () => {
|
|
124
|
+
const sub = await AIRequest.create({
|
|
125
|
+
aiModel: "gpt-4",
|
|
126
|
+
prompt: "sub",
|
|
127
|
+
requestType: "general",
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
const parent = await AIRequest.logMultiAgentRequest({
|
|
131
|
+
aiModel: "gpt-4",
|
|
132
|
+
metadata: {workflow: "test"},
|
|
133
|
+
requestType: "multi-agent",
|
|
134
|
+
subRequestIds: [sub._id],
|
|
135
|
+
totalResponseTime: 100,
|
|
136
|
+
totalTokensUsed: 10,
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
expect(parent.metadata).toEqual({workflow: "test"});
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
it("should create a parent request with userId", async () => {
|
|
143
|
+
const userId = new mongoose.Types.ObjectId();
|
|
144
|
+
const sub = await AIRequest.create({
|
|
145
|
+
aiModel: "gpt-4",
|
|
146
|
+
prompt: "sub",
|
|
147
|
+
requestType: "general",
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
const parent = await AIRequest.logMultiAgentRequest({
|
|
151
|
+
aiModel: "gpt-4",
|
|
152
|
+
requestType: "multi-agent",
|
|
153
|
+
subRequestIds: [sub._id],
|
|
154
|
+
totalResponseTime: 50,
|
|
155
|
+
totalTokensUsed: 5,
|
|
156
|
+
userId,
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
expect(parent.userId?.toString()).toBe(userId.toString());
|
|
160
|
+
});
|
|
161
|
+
});
|
|
162
|
+
|
|
82
163
|
describe("soft delete", () => {
|
|
83
164
|
it("should filter out deleted records by default", async () => {
|
|
84
165
|
await AIRequest.create({
|