@terreno/ai 57.20.0 → 57.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. package/CHANGELOG.md +81 -0
  2. package/README.md +3 -3
  3. package/biome.jsonc +3 -2
  4. package/dist/harness/agentLoop.js +1 -0
  5. package/dist/harness/commit.d.ts +4 -3
  6. package/dist/harness/commit.js +23 -4
  7. package/dist/index.d.ts +6 -9
  8. package/dist/index.js +14 -25
  9. package/dist/observability/adminScreens.js +5 -1
  10. package/dist/observability/evaluate.d.ts +8 -1
  11. package/dist/observability/evaluate.js +10 -2
  12. package/dist/observability/judgeAgreement.d.ts +53 -0
  13. package/dist/observability/judgeAgreement.js +123 -0
  14. package/dist/observability/local/datasetStore.d.ts +5 -0
  15. package/dist/observability/local/datasetStore.js +7 -2
  16. package/dist/observability/local/evaluatorStore.d.ts +17 -0
  17. package/dist/observability/local/evaluatorStore.js +73 -2
  18. package/dist/observability/local/experimentRunner.d.ts +42 -8
  19. package/dist/observability/local/experimentRunner.js +390 -11
  20. package/dist/observability/local/humanAnswers.d.ts +57 -0
  21. package/dist/observability/local/humanAnswers.js +174 -0
  22. package/dist/observability/local/judgeRun.d.ts +56 -0
  23. package/dist/observability/local/judgeRun.js +111 -0
  24. package/dist/observability/local/localPlugin.js +4 -0
  25. package/dist/observability/local/models/obsDatasetItem.js +4 -0
  26. package/dist/observability/local/models/obsEvaluator.js +4 -0
  27. package/dist/observability/local/models/obsExperiment.js +28 -0
  28. package/dist/observability/local/models/obsExperimentItem.js +4 -0
  29. package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
  30. package/dist/observability/local/models/obsJudgeRun.js +60 -0
  31. package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
  32. package/dist/observability/local/models/obsReviewBatch.js +48 -0
  33. package/dist/observability/local/models/obsReviewItem.js +10 -2
  34. package/dist/observability/local/models/obsTrace.js +4 -0
  35. package/dist/observability/local/promptReferences.d.ts +31 -0
  36. package/dist/observability/local/promptReferences.js +114 -0
  37. package/dist/observability/local/promptStore.d.ts +55 -19
  38. package/dist/observability/local/promptStore.js +171 -7
  39. package/dist/observability/local/reviewBatchStore.d.ts +106 -0
  40. package/dist/observability/local/reviewBatchStore.js +517 -0
  41. package/dist/observability/local/reviewStore.d.ts +24 -3
  42. package/dist/observability/local/reviewStore.js +98 -22
  43. package/dist/observability/local/traceStore.d.ts +14 -1
  44. package/dist/observability/local/traceStore.js +144 -11
  45. package/dist/observability/observabilityApp.js +10 -1
  46. package/dist/observability/questionFingerprint.d.ts +7 -0
  47. package/dist/observability/questionFingerprint.js +23 -0
  48. package/dist/observability/routes/experiments.js +12 -1
  49. package/dist/observability/routes/review.js +3 -1
  50. package/dist/observability/routes/reviewBatches.d.ts +11 -0
  51. package/dist/observability/routes/reviewBatches.js +78 -0
  52. package/dist/observability/routes/traces.d.ts +1 -1
  53. package/dist/observability/routes/traces.js +34 -2
  54. package/dist/observability/types.d.ts +10 -1
  55. package/dist/service/aiService.js +4 -1
  56. package/dist/service/chatTurn.js +38 -31
  57. package/dist/types/harness.d.ts +10 -4
  58. package/dist/types/index.d.ts +8 -3
  59. package/dist/types/observability.d.ts +74 -2
  60. package/e2e/smoke.ts +227 -0
  61. package/package.json +14 -28
  62. package/src/aiApp.test.ts +22 -17
  63. package/src/harness/agentLoop.test.ts +0 -3
  64. package/src/harness/agentLoop.ts +2 -1
  65. package/src/harness/agentRecovery.test.ts +30 -30
  66. package/src/harness/approvals.test.ts +11 -9
  67. package/src/harness/commit.ts +24 -4
  68. package/src/harness/events.test.ts +10 -6
  69. package/src/harness/extensions.test.ts +6 -8
  70. package/src/harness/ownership.test.ts +20 -9
  71. package/src/harness/retries.test.ts +6 -3
  72. package/src/harness/routes/conversations.test.ts +2 -2
  73. package/src/harness/routes/tasks.test.ts +2 -2
  74. package/src/harness/runners/jobsRunner.test.ts +1 -1
  75. package/src/harness/subagent.test.ts +29 -4
  76. package/src/harness/waits.test.ts +6 -4
  77. package/src/index.test.ts +3 -20
  78. package/src/index.ts +23 -26
  79. package/src/isolated/gptStream.isolated.ts +25 -28
  80. package/src/isolated/lazyProjectModel.isolated.ts +2 -2
  81. package/src/models/gptHistory.test.ts +10 -6
  82. package/src/observability/adminScreens.test.ts +7 -0
  83. package/src/observability/adminScreens.ts +6 -1
  84. package/src/observability/evaluate.test.ts +12 -10
  85. package/src/observability/evaluate.ts +12 -2
  86. package/src/observability/judgeAgreement.test.ts +111 -0
  87. package/src/observability/judgeAgreement.ts +184 -0
  88. package/src/observability/local/datasetStore.ts +30 -11
  89. package/src/observability/local/evaluatorStore.ts +96 -2
  90. package/src/observability/local/experimentRunner.test.ts +9 -12
  91. package/src/observability/local/experimentRunner.ts +497 -17
  92. package/src/observability/local/humanAnswers.ts +233 -0
  93. package/src/observability/local/judgePipeline.test.ts +444 -0
  94. package/src/observability/local/judgeRun.test.ts +173 -0
  95. package/src/observability/local/judgeRun.ts +184 -0
  96. package/src/observability/local/localPlugin.ts +4 -0
  97. package/src/observability/local/models/obsDatasetItem.ts +5 -0
  98. package/src/observability/local/models/obsEvaluator.ts +4 -0
  99. package/src/observability/local/models/obsExperiment.ts +29 -0
  100. package/src/observability/local/models/obsExperimentItem.ts +4 -0
  101. package/src/observability/local/models/obsJudgeRun.ts +61 -0
  102. package/src/observability/local/models/obsReviewBatch.ts +52 -0
  103. package/src/observability/local/models/obsReviewItem.ts +11 -2
  104. package/src/observability/local/models/obsTrace.ts +5 -0
  105. package/src/observability/local/obsModels.test.ts +13 -0
  106. package/src/observability/local/promptReferences.test.ts +128 -0
  107. package/src/observability/local/promptReferences.ts +149 -0
  108. package/src/observability/local/promptStore.test.ts +295 -3
  109. package/src/observability/local/promptStore.ts +243 -32
  110. package/src/observability/local/reviewBatchStore.test.ts +251 -0
  111. package/src/observability/local/reviewBatchStore.ts +656 -0
  112. package/src/observability/local/reviewStore.ts +122 -25
  113. package/src/observability/local/traceStore.test.ts +139 -1
  114. package/src/observability/local/traceStore.ts +162 -11
  115. package/src/observability/observabilityApp.test.ts +13 -10
  116. package/src/observability/observabilityApp.ts +10 -1
  117. package/src/observability/questionFingerprint.test.ts +33 -0
  118. package/src/observability/questionFingerprint.ts +24 -0
  119. package/src/observability/routes/datasets.test.ts +2 -2
  120. package/src/observability/routes/evaluators.test.ts +2 -2
  121. package/src/observability/routes/experiments.test.ts +39 -2
  122. package/src/observability/routes/experiments.ts +13 -1
  123. package/src/observability/routes/prompts.test.ts +3 -3
  124. package/src/observability/routes/review.test.ts +2 -2
  125. package/src/observability/routes/review.ts +3 -1
  126. package/src/observability/routes/reviewBatches.test.ts +209 -0
  127. package/src/observability/routes/reviewBatches.ts +138 -0
  128. package/src/observability/routes/status.test.ts +8 -8
  129. package/src/observability/routes/testMultiStage.test.ts +6 -6
  130. package/src/observability/routes/traces.test.ts +63 -2
  131. package/src/observability/routes/traces.ts +46 -4
  132. package/src/observability/types.ts +5 -2
  133. package/src/routes/files.test.ts +5 -3
  134. package/src/routes/gpt.test.ts +21 -14
  135. package/src/routes/gptActions.test.ts +7 -8
  136. package/src/routes/gptDatasets.test.ts +1 -1
  137. package/src/routes/gptFiles.test.ts +1 -1
  138. package/src/routes/gptHistories.test.ts +15 -9
  139. package/src/routes/mcp.test.ts +3 -3
  140. package/src/routes/projects.test.ts +5 -3
  141. package/src/routes/routes.test.ts +156 -115
  142. package/src/service/aiService.test.ts +22 -1
  143. package/src/service/aiService.ts +4 -1
  144. package/src/service/asks.test.ts +14 -6
  145. package/src/service/chatTurn.ts +42 -33
  146. package/src/service/fileStorage.test.ts +5 -2
  147. package/src/service/getMCPTools.test.ts +9 -5
  148. package/src/service/mcpService.test.ts +6 -3
  149. package/src/tests/bunSetup.ts +0 -54
  150. package/src/tests/chatHarness.ts +4 -4
  151. package/src/tests/fakeJudge.ts +8 -0
  152. package/src/tests/helpers.ts +26 -4
  153. package/src/types/harness.ts +11 -4
  154. package/src/types/index.ts +8 -3
  155. package/src/types/observability.ts +86 -2
  156. package/tsconfig.typecheck.json +18 -0
  157. package/dist/langfuseApp.d.ts +0 -9
  158. package/dist/langfuseApp.js +0 -53
  159. package/dist/langfuseCache.d.ts +0 -15
  160. package/dist/langfuseCache.js +0 -54
  161. package/dist/langfuseClient.d.ts +0 -6
  162. package/dist/langfuseClient.js +0 -34
  163. package/dist/langfusePrompts.d.ts +0 -12
  164. package/dist/langfusePrompts.js +0 -89
  165. package/dist/langfuseRoutesEvaluations.d.ts +0 -3
  166. package/dist/langfuseRoutesEvaluations.js +0 -32
  167. package/dist/langfuseRoutesMiddleware.d.ts +0 -2
  168. package/dist/langfuseRoutesMiddleware.js +0 -13
  169. package/dist/langfuseRoutesPlayground.d.ts +0 -2
  170. package/dist/langfuseRoutesPlayground.js +0 -36
  171. package/dist/langfuseRoutesPrompts.d.ts +0 -2
  172. package/dist/langfuseRoutesPrompts.js +0 -56
  173. package/dist/langfuseRoutesTraces.d.ts +0 -2
  174. package/dist/langfuseRoutesTraces.js +0 -30
  175. package/dist/langfuseTracing.d.ts +0 -4
  176. package/dist/langfuseTracing.js +0 -32
  177. package/dist/langfuseTypes.d.ts +0 -96
  178. package/dist/langfuseTypes.js +0 -2
  179. package/dist/langfuseVercelAi.d.ts +0 -12
  180. package/dist/langfuseVercelAi.js +0 -49
  181. package/src/isolated/langfuseClient.isolated.ts +0 -60
  182. package/src/langfuseApp.test.ts +0 -135
  183. package/src/langfuseApp.ts +0 -62
  184. package/src/langfuseCache.test.ts +0 -75
  185. package/src/langfuseCache.ts +0 -72
  186. package/src/langfuseClient.test.ts +0 -69
  187. package/src/langfuseClient.ts +0 -34
  188. package/src/langfusePrompts.test.ts +0 -188
  189. package/src/langfusePrompts.ts +0 -121
  190. package/src/langfuseRoutes.test.ts +0 -256
  191. package/src/langfuseRoutesEvaluations.ts +0 -48
  192. package/src/langfuseRoutesMiddleware.test.ts +0 -57
  193. package/src/langfuseRoutesMiddleware.ts +0 -12
  194. package/src/langfuseRoutesPlayground.ts +0 -50
  195. package/src/langfuseRoutesPrompts.ts +0 -87
  196. package/src/langfuseRoutesTraces.ts +0 -40
  197. package/src/langfuseTracing.test.ts +0 -68
  198. package/src/langfuseTracing.ts +0 -33
  199. package/src/langfuseTypes.ts +0 -103
  200. package/src/langfuseVercelAi.test.ts +0 -79
  201. package/src/langfuseVercelAi.ts +0 -72
@@ -1,34 +0,0 @@
1
- import {LangfuseClient} from "@langfuse/client";
2
- import {APIError} from "@terreno/api";
3
-
4
- import type {LangfuseAppOptions} from "./langfuseTypes";
5
-
6
- let langfuseInstance: LangfuseClient | null = null;
7
-
8
- export const initLangfuseClient = (options: LangfuseAppOptions): LangfuseClient => {
9
- langfuseInstance = new LangfuseClient({
10
- baseUrl: options.baseUrl ?? "https://cloud.langfuse.com",
11
- publicKey: options.publicKey,
12
- secretKey: options.secretKey,
13
- });
14
- return langfuseInstance;
15
- };
16
-
17
- export const getLangfuseClient = (): LangfuseClient => {
18
- if (!langfuseInstance) {
19
- throw new APIError({
20
- status: 500,
21
- title: "Langfuse client not initialized. Call initLangfuseClient first.",
22
- });
23
- }
24
- return langfuseInstance;
25
- };
26
-
27
- export const isLangfuseInitialized = (): boolean => langfuseInstance != null;
28
-
29
- export const shutdownLangfuseClient = async (): Promise<void> => {
30
- if (langfuseInstance) {
31
- await langfuseInstance.shutdown();
32
- langfuseInstance = null;
33
- }
34
- };
@@ -1,188 +0,0 @@
1
- import {beforeEach, describe, expect, it, mock} from "bun:test";
2
-
3
- import {LangfuseCache} from "./langfuseCache";
4
- import {getLangfuseClient, initLangfuseClient} from "./langfuseClient";
5
- import {compilePrompt, createPrompt, getPrompt, invalidatePromptCache} from "./langfusePrompts";
6
- import type {LangfuseCachedPrompt} from "./langfuseTypes";
7
-
8
- const mockPromptCreate = mock(async (_params: Record<string, unknown>) => {});
9
- const mockPromptGet = mock(async (_name: string, _options?: Record<string, unknown>) => ({
10
- config: {},
11
- labels: ["production"],
12
- name: "test-prompt",
13
- prompt: "Hello world",
14
- tags: [],
15
- type: "text" as const,
16
- version: 1,
17
- }));
18
-
19
- const textPrompt: LangfuseCachedPrompt = {
20
- config: {},
21
- labels: ["production"],
22
- name: "test-prompt",
23
- prompt: "Hello {{name}}, you are {{age}} years old.",
24
- tags: [],
25
- type: "text",
26
- version: 1,
27
- };
28
-
29
- const chatPrompt: LangfuseCachedPrompt = {
30
- config: {},
31
- labels: ["production"],
32
- name: "chat-prompt",
33
- prompt: [
34
- {content: "You are a helpful assistant for {{company}}.", role: "system"},
35
- {content: "My name is {{name}}.", role: "user"},
36
- ],
37
- tags: [],
38
- type: "chat",
39
- version: 1,
40
- };
41
-
42
- beforeEach(async () => {
43
- await LangfuseCache.deleteMany({});
44
- mockPromptCreate.mockClear();
45
- mockPromptGet.mockClear();
46
- mockPromptGet.mockImplementation(async () => ({
47
- config: {},
48
- labels: ["production"],
49
- name: "test-prompt",
50
- prompt: "Hello world",
51
- tags: [],
52
- type: "text" as const,
53
- version: 1,
54
- }));
55
- // Init the (fake) langfuse client and wire our local mocks onto its prompt
56
- // methods so the real `langfusePrompts` code exercises customizable
57
- // responses without hitting the network.
58
- initLangfuseClient({publicKey: "pk-test", secretKey: "sk-test"});
59
- const client = getLangfuseClient() as unknown as {
60
- prompt: {create: typeof mockPromptCreate; get: typeof mockPromptGet};
61
- };
62
- client.prompt.get = mockPromptGet;
63
- client.prompt.create = mockPromptCreate;
64
- });
65
-
66
- describe("compilePrompt", () => {
67
- it("replaces variables in text prompts", () => {
68
- const result = compilePrompt(textPrompt, {age: "30", name: "Alice"});
69
- expect(result).toBe("Hello Alice, you are 30 years old.");
70
- });
71
-
72
- it("keeps missing text variables unreplaced", () => {
73
- const result = compilePrompt(textPrompt, {name: "Alice"});
74
- expect(result).toBe("Hello Alice, you are {{age}} years old.");
75
- });
76
-
77
- it("replaces variables in chat prompts", () => {
78
- const result = compilePrompt(chatPrompt, {company: "Acme", name: "Bob"}) as Array<{
79
- content: string;
80
- role: string;
81
- }>;
82
-
83
- expect(result[0]).toEqual({content: "You are a helpful assistant for Acme.", role: "system"});
84
- expect(result[1]).toEqual({content: "My name is Bob.", role: "user"});
85
- });
86
- });
87
-
88
- describe("getPrompt", () => {
89
- it("returns cached prompts without calling Langfuse", async () => {
90
- await LangfuseCache.create({
91
- expiresAt: new Date(Date.now() + 60_000),
92
- key: "prompt:test-prompt:production",
93
- value: textPrompt,
94
- });
95
-
96
- const result = await getPrompt("test-prompt");
97
-
98
- expect(result.name).toBe(textPrompt.name);
99
- expect(result.prompt).toBe(textPrompt.prompt);
100
- expect(result.version).toBe(textPrompt.version);
101
- expect(mockPromptGet).toHaveBeenCalledTimes(0);
102
- });
103
-
104
- it("fetches and caches prompts with default ttl/label", async () => {
105
- const result = await getPrompt("test-prompt");
106
-
107
- expect(mockPromptGet).toHaveBeenCalledWith("test-prompt", {
108
- cacheTtlSeconds: 0,
109
- label: "production",
110
- });
111
- const cached = await LangfuseCache.findOne({key: "prompt:test-prompt:production"});
112
- expect(cached?.value).toEqual(result as unknown as LangfuseCachedPrompt);
113
- });
114
-
115
- it("fetches and caches prompts with custom ttl/label", async () => {
116
- await getPrompt("test-prompt", {label: "staging"}, {cache: {promptTtlSeconds: 120}});
117
-
118
- expect(mockPromptGet).toHaveBeenCalledWith("test-prompt", {
119
- cacheTtlSeconds: 0,
120
- label: "staging",
121
- });
122
- const cached = await LangfuseCache.findOne({key: "prompt:test-prompt:staging"});
123
- expect(cached).toBeTruthy();
124
- const expectedExpiryMs = Date.now() + 120_000;
125
- // Allow some slack for test execution time
126
- expect(cached?.expiresAt.getTime()).toBeGreaterThan(expectedExpiryMs - 5000);
127
- expect(cached?.expiresAt.getTime()).toBeLessThan(expectedExpiryMs + 5000);
128
- });
129
-
130
- it("throws when Langfuse fetch fails", async () => {
131
- mockPromptGet.mockImplementation(async () => {
132
- throw new Error("boom");
133
- });
134
- await expect(getPrompt("test-prompt")).rejects.toThrow("boom");
135
- });
136
- });
137
-
138
- describe("createPrompt", () => {
139
- it("creates text prompts with default labels/tags", async () => {
140
- await createPrompt({name: "new-prompt", prompt: "Hello", type: "text"});
141
- expect(mockPromptCreate).toHaveBeenCalled();
142
- const call = (mockPromptCreate.mock.calls[0] as Array<Record<string, unknown>>)[0];
143
- expect(call.name).toBe("new-prompt");
144
- expect(call.type).toBe("text");
145
- });
146
-
147
- it("creates chat prompts preserving role and content", async () => {
148
- await createPrompt({
149
- name: "chat-new",
150
- prompt: [{content: "hi", role: "user"}],
151
- tags: ["x"],
152
- type: "chat",
153
- });
154
- expect(mockPromptCreate).toHaveBeenCalled();
155
- const call = (mockPromptCreate.mock.calls[0] as Array<Record<string, unknown>>)[0];
156
- expect(call.type).toBe("chat");
157
- expect(Array.isArray(call.prompt)).toBe(true);
158
- expect(call.tags).toEqual(["x"]);
159
- });
160
- });
161
-
162
- describe("invalidatePromptCache", () => {
163
- it("invalidates cache entries for a prompt prefix", async () => {
164
- await LangfuseCache.create({
165
- expiresAt: new Date(Date.now() + 60_000),
166
- key: "prompt:my-prompt:production",
167
- value: textPrompt,
168
- });
169
- await LangfuseCache.create({
170
- expiresAt: new Date(Date.now() + 60_000),
171
- key: "prompt:my-prompt:staging",
172
- value: textPrompt,
173
- });
174
- await LangfuseCache.create({
175
- expiresAt: new Date(Date.now() + 60_000),
176
- key: "prompt:other-prompt:production",
177
- value: textPrompt,
178
- });
179
-
180
- await invalidatePromptCache("my-prompt");
181
-
182
- const remaining = await LangfuseCache.find({});
183
- const keys = remaining.map((r) => r.key);
184
- expect(keys).toContain("prompt:other-prompt:production");
185
- expect(keys).not.toContain("prompt:my-prompt:production");
186
- expect(keys).not.toContain("prompt:my-prompt:staging");
187
- });
188
- });
@@ -1,121 +0,0 @@
1
- import {logger} from "@terreno/api";
2
-
3
- import {getCached, invalidateCache, setCached} from "./langfuseCache";
4
- import {getLangfuseClient} from "./langfuseClient";
5
- import type {
6
- ChatMessage,
7
- GetPromptOptions,
8
- LangfuseAppOptions,
9
- LangfuseCachedPrompt,
10
- } from "./langfuseTypes";
11
-
12
- const DEFAULT_PROMPT_TTL = 60;
13
-
14
- const DEFAULT_LABEL = "production";
15
-
16
- const buildPromptCacheKey = (name: string, label?: string): string => {
17
- return `prompt:${name}:${label ?? DEFAULT_LABEL}`;
18
- };
19
-
20
- export const getPrompt = async (
21
- name: string,
22
- options: GetPromptOptions = {},
23
- appOptions?: Pick<LangfuseAppOptions, "cache">
24
- ): Promise<LangfuseCachedPrompt> => {
25
- const ttl = appOptions?.cache?.promptTtlSeconds ?? DEFAULT_PROMPT_TTL;
26
- const cacheKey = buildPromptCacheKey(name, options.label);
27
-
28
- const cached = await getCached(cacheKey);
29
- if (cached) {
30
- logger.debug(
31
- `Langfuse prompt cache hit: "${name}" v${cached.version} (label: ${options.label ?? DEFAULT_LABEL})`
32
- );
33
- return cached;
34
- }
35
-
36
- logger.info(
37
- `Langfuse prompt cache miss: fetching "${name}" from API (label: ${options.label ?? DEFAULT_LABEL})`
38
- );
39
- const client = getLangfuseClient();
40
- const prompt = await client.prompt.get(name, {
41
- cacheTtlSeconds: 0,
42
- label: options.label ?? DEFAULT_LABEL,
43
- });
44
-
45
- const cachedPrompt: LangfuseCachedPrompt = {
46
- config: (prompt.config ?? {}) as Record<string, unknown>,
47
- labels: prompt.labels ?? [],
48
- name: prompt.name,
49
- prompt: prompt.prompt as string | ChatMessage[],
50
- tags: prompt.tags ?? [],
51
- type: prompt.type as "text" | "chat",
52
- version: prompt.version,
53
- };
54
-
55
- logger.info(`Langfuse prompt fetched: "${name}" v${cachedPrompt.version} (ttl: ${ttl}s)`);
56
- await setCached(cacheKey, cachedPrompt, ttl);
57
- return cachedPrompt;
58
- };
59
-
60
- export const compilePrompt = (
61
- cached: LangfuseCachedPrompt,
62
- variables: Record<string, string> = {}
63
- ): string | ChatMessage[] => {
64
- const replace = (template: string): string => {
65
- return template.replace(/\{\{(\w+)\}\}/g, (_match, key: string) => {
66
- return variables[key] ?? `{{${key}}}`;
67
- });
68
- };
69
-
70
- if (cached.type === "text") {
71
- return replace(cached.prompt as string);
72
- }
73
-
74
- return (cached.prompt as ChatMessage[]).map((msg) => ({
75
- content: replace(msg.content),
76
- role: msg.role,
77
- }));
78
- };
79
-
80
- export const createPrompt = async (params: {
81
- name: string;
82
- type: "text" | "chat";
83
- prompt: string | ChatMessage[];
84
- labels?: string[];
85
- tags?: string[];
86
- config?: Record<string, unknown>;
87
- }): Promise<LangfuseCachedPrompt> => {
88
- const client = getLangfuseClient();
89
-
90
- if (params.type === "text") {
91
- await client.prompt.create({
92
- config: params.config,
93
- labels: params.labels ?? [DEFAULT_LABEL],
94
- name: params.name,
95
- prompt: params.prompt as string,
96
- tags: params.tags ?? [],
97
- type: "text",
98
- });
99
- } else {
100
- const chatMessages = (params.prompt as ChatMessage[]).map((msg) => ({
101
- content: msg.content,
102
- role: msg.role,
103
- }));
104
- await client.prompt.create({
105
- config: params.config,
106
- labels: params.labels ?? [DEFAULT_LABEL],
107
- name: params.name,
108
- prompt: chatMessages,
109
- tags: params.tags ?? [],
110
- type: "chat",
111
- });
112
- }
113
-
114
- await invalidatePromptCache(params.name);
115
- return getPrompt(params.name, {label: DEFAULT_LABEL});
116
- };
117
-
118
- export const invalidatePromptCache = async (name: string): Promise<void> => {
119
- await invalidateCache(`prompt:${name}:`);
120
- logger.info(`Langfuse prompt cache invalidated for: ${name}`);
121
- };
@@ -1,256 +0,0 @@
1
- import {beforeAll, beforeEach, describe, expect, it, mock} from "bun:test";
2
- import {TerrenoApp, type TerrenoAppOptions} from "@terreno/api";
3
- import type express from "express";
4
-
5
- import {getLangfuseClient, initLangfuseClient} from "./langfuseClient";
6
- import {addEvaluationRoutes} from "./langfuseRoutesEvaluations";
7
- import {addPlaygroundRoutes} from "./langfuseRoutesPlayground";
8
- import {addPromptRoutes} from "./langfuseRoutesPrompts";
9
- import {addTraceRoutes} from "./langfuseRoutesTraces";
10
- import {
11
- authAsUserWithCredentials,
12
- ensureTestUsers,
13
- type StandardAiTestUserRole,
14
- UserModel,
15
- } from "./tests/helpers";
16
-
17
- const LANGFUSE_TEST_USERS = [
18
- {admin: true, email: "lf-admin@example.com", name: "LF Admin", password: "adminPass123"},
19
- {admin: false, email: "lf-user@example.com", name: "LF User", password: "userPass123"},
20
- ] as const;
21
-
22
- const authAsUser = async (appInstance: express.Application, type: StandardAiTestUserRole) => {
23
- const user = LANGFUSE_TEST_USERS[type === "admin" ? 0 : 1];
24
- return authAsUserWithCredentials(appInstance, {email: user.email, password: user.password});
25
- };
26
-
27
- const scoreCreate = mock(() => {});
28
- const promptCreate = mock(async (_params: Record<string, unknown>) => ({}));
29
- const promptGet = mock(async (name: string) => ({
30
- config: {temperature: 0.5},
31
- labels: ["production"],
32
- name,
33
- prompt: "Hello {{name}}, welcome to {{place}}",
34
- tags: ["tag"],
35
- type: "text",
36
- version: 1,
37
- }));
38
- const promptsList = mock(async () => ({
39
- data: [{name: "p1", versions: [1]}],
40
- meta: {limit: 20, page: 1, total: 1, totalPages: 1},
41
- }));
42
- const traceList = mock(async () => ({
43
- data: [{id: "t1"}],
44
- meta: {limit: 20, page: 1, total: 1, totalPages: 1},
45
- }));
46
- const traceGet = mock(async (traceId: string) => ({id: traceId, name: "Trace"}));
47
- const flush = mock(async () => {});
48
-
49
- let app: express.Application;
50
-
51
- describe("Langfuse routes", () => {
52
- beforeAll(async () => {
53
- await ensureTestUsers([...LANGFUSE_TEST_USERS]);
54
- });
55
-
56
- beforeEach(() => {
57
- // Init the (fake) langfuse client and wire our local mocks onto its
58
- // methods so the route handlers exercise customizable responses without
59
- // making real SDK calls.
60
- initLangfuseClient({publicKey: "pk-test", secretKey: "sk-test"});
61
- const client = getLangfuseClient() as unknown as {
62
- api: {
63
- prompts: {list: typeof promptsList};
64
- trace: {get: typeof traceGet; list: typeof traceList};
65
- };
66
- flush: typeof flush;
67
- prompt: {create: typeof promptCreate; get: typeof promptGet};
68
- score: {create: typeof scoreCreate};
69
- };
70
- client.api.prompts.list = promptsList;
71
- client.api.trace.get = traceGet;
72
- client.api.trace.list = traceList;
73
- client.flush = flush;
74
- client.prompt.create = promptCreate;
75
- client.prompt.get = promptGet;
76
- client.score.create = scoreCreate;
77
-
78
- app = new TerrenoApp({
79
- configureApp: (router) => {
80
- addPromptRoutes(router, "/admin/langfuse");
81
- addTraceRoutes(router, "/admin/langfuse");
82
- addPlaygroundRoutes(router, "/admin/langfuse");
83
- addEvaluationRoutes(router, "/admin/langfuse", []);
84
- },
85
- skipListen: true,
86
- userModel: UserModel as unknown as TerrenoAppOptions["userModel"],
87
- }).build();
88
- });
89
-
90
- describe("prompt routes", () => {
91
- it("returns 403 for non-admin users", async () => {
92
- const agent = await authAsUser(app, "notAdmin");
93
- const res = await agent.get("/admin/langfuse/prompts");
94
- expect(res.status).toBe(403);
95
- });
96
-
97
- it("lists prompts for admin users", async () => {
98
- const agent = await authAsUser(app, "admin");
99
- const res = await agent.get("/admin/langfuse/prompts?page=2&limit=5");
100
- expect(res.status).toBe(200);
101
- expect(res.body.data).toBeDefined();
102
- expect(promptsList).toHaveBeenCalled();
103
- });
104
-
105
- it("returns a prompt by name for admin users", async () => {
106
- const agent = await authAsUser(app, "admin");
107
- const res = await agent.get("/admin/langfuse/prompts/mine?version=2&label=staging");
108
- expect(res.status).toBe(200);
109
- expect(res.body.name).toBe("mine");
110
- expect(res.body.version).toBe(1);
111
- });
112
-
113
- it("creates a text prompt", async () => {
114
- const agent = await authAsUser(app, "admin");
115
- const res = await agent
116
- .post("/admin/langfuse/prompts")
117
- .send({name: "new", prompt: "Hello", type: "text"});
118
- expect(res.status).toBe(201);
119
- expect(promptCreate).toHaveBeenCalled();
120
- });
121
-
122
- it("creates a chat prompt", async () => {
123
- const agent = await authAsUser(app, "admin");
124
- const res = await agent.post("/admin/langfuse/prompts").send({
125
- name: "chat",
126
- prompt: [{content: "hi", role: "user"}],
127
- tags: ["x"],
128
- type: "chat",
129
- });
130
- expect(res.status).toBe(201);
131
- });
132
-
133
- it("rejects prompt create when required fields are missing", async () => {
134
- const agent = await authAsUser(app, "admin");
135
- const res = await agent.post("/admin/langfuse/prompts").send({name: "x"});
136
- expect(res.status).toBe(400);
137
- });
138
-
139
- it("invalidates prompt cache", async () => {
140
- const agent = await authAsUser(app, "admin");
141
- const res = await agent.delete("/admin/langfuse/prompts/mine/cache");
142
- expect(res.status).toBe(200);
143
- expect(res.body.invalidated).toBe(true);
144
- });
145
- });
146
-
147
- describe("trace routes", () => {
148
- it("returns 403 for non-admin users", async () => {
149
- const agent = await authAsUser(app, "notAdmin");
150
- const res = await agent.get("/admin/langfuse/traces");
151
- expect(res.status).toBe(403);
152
- });
153
-
154
- it("lists traces for admin users with filters", async () => {
155
- const agent = await authAsUser(app, "admin");
156
- const res = await agent.get(
157
- "/admin/langfuse/traces?page=3&limit=7&userId=u1&from=2024-01-01&to=2024-12-31"
158
- );
159
- expect(res.status).toBe(200);
160
- expect(traceList).toHaveBeenCalled();
161
- expect(res.body.data).toBeDefined();
162
- });
163
-
164
- it("returns a single trace by id", async () => {
165
- const agent = await authAsUser(app, "admin");
166
- const res = await agent.get("/admin/langfuse/traces/trace-1");
167
- expect(res.status).toBe(200);
168
- expect(res.body.id).toBe("trace-1");
169
- });
170
- });
171
-
172
- describe("playground routes", () => {
173
- it("returns 403 for non-admin users", async () => {
174
- const agent = await authAsUser(app, "notAdmin");
175
- const res = await agent.post("/admin/langfuse/playground").send({promptName: "test"});
176
- expect(res.status).toBe(403);
177
- });
178
-
179
- it("returns 400 when promptName is missing", async () => {
180
- const agent = await authAsUser(app, "admin");
181
- const res = await agent.post("/admin/langfuse/playground").send({});
182
- expect(res.status).toBe(400);
183
- });
184
-
185
- it("compiles a prompt for admin users", async () => {
186
- const agent = await authAsUser(app, "admin");
187
- const res = await agent
188
- .post("/admin/langfuse/playground")
189
- .send({promptName: "welcome", variables: {name: "Sam", place: "Earth"}});
190
- expect(res.status).toBe(200);
191
- expect(res.body.compiled).toBe("Hello Sam, welcome to Earth");
192
- expect(res.body.variables.sort()).toEqual(["name", "place"]);
193
- });
194
-
195
- it("compiles a chat prompt and extracts variables from chat messages", async () => {
196
- promptGet.mockImplementationOnce(async (name: string) => ({
197
- config: {},
198
- labels: ["production"],
199
- name,
200
- prompt: [
201
- {content: "You are {{role}}.", role: "system"},
202
- {content: "Greet {{user}}.", role: "user"},
203
- ],
204
- tags: [],
205
- type: "chat",
206
- version: 2,
207
- }));
208
- const agent = await authAsUser(app, "admin");
209
- const res = await agent
210
- .post("/admin/langfuse/playground")
211
- .send({promptName: "chatter", variables: {role: "bot", user: "Alex"}});
212
- expect(res.status).toBe(200);
213
- expect(res.body.type).toBe("chat");
214
- expect(Array.isArray(res.body.compiled)).toBe(true);
215
- expect(res.body.variables.sort()).toEqual(["role", "user"]);
216
- });
217
- });
218
-
219
- describe("evaluation routes", () => {
220
- it("returns 403 for non-admin users on config", async () => {
221
- const agent = await authAsUser(app, "notAdmin");
222
- const res = await agent.get("/admin/langfuse/evaluations/config");
223
- expect(res.status).toBe(403);
224
- });
225
-
226
- it("rejects evaluation submissions missing required fields", async () => {
227
- const agent = await authAsUser(app, "admin");
228
- const res = await agent.post("/admin/langfuse/evaluations").send({name: "x"});
229
- expect(res.status).toBe(400);
230
- });
231
-
232
- it("creates an evaluation score", async () => {
233
- const agent = await authAsUser(app, "admin");
234
- const res = await agent
235
- .post("/admin/langfuse/evaluations")
236
- .send({dataType: "NUMERIC", name: "quality", traceId: "t1", value: 5});
237
- expect(res.status).toBe(201);
238
- expect(scoreCreate).toHaveBeenCalled();
239
- expect(flush).toHaveBeenCalled();
240
- });
241
-
242
- it("returns non-403 for admin users on config", async () => {
243
- const agent = await authAsUser(app, "admin");
244
- const res = await agent.get("/admin/langfuse/evaluations/config");
245
- expect(res.status).not.toBe(403);
246
- });
247
-
248
- it("returns 403 for non-admin users on submit", async () => {
249
- const agent = await authAsUser(app, "notAdmin");
250
- const res = await agent
251
- .post("/admin/langfuse/evaluations")
252
- .send({name: "score", traceId: "t1", value: 1});
253
- expect(res.status).toBe(403);
254
- });
255
- });
256
- });
@@ -1,48 +0,0 @@
1
- import {APIError, asyncHandler, authenticateMiddleware} from "@terreno/api";
2
- import type express from "express";
3
-
4
- import {getLangfuseClient} from "./langfuseClient";
5
- import {requireAdmin} from "./langfuseRoutesMiddleware";
6
- import type {ScoreSubmission, ScoringFunction} from "./langfuseTypes";
7
-
8
- export const addEvaluationRoutes = (
9
- router: express.Application,
10
- basePath: string,
11
- scoringFunctions: ScoringFunction[] = []
12
- ): void => {
13
- router.get(
14
- `${basePath}/evaluations/config`,
15
- [authenticateMiddleware(), requireAdmin],
16
- asyncHandler(async (_req, res) => {
17
- return res.json({scoringFunctions});
18
- })
19
- );
20
-
21
- router.post(
22
- `${basePath}/evaluations`,
23
- [authenticateMiddleware(), requireAdmin],
24
- asyncHandler(async (req, res) => {
25
- const {traceId, name, value, comment, dataType} = req.body as ScoreSubmission;
26
-
27
- if (!traceId || !name || value === undefined) {
28
- throw new APIError({
29
- detail: "traceId, name, and value are required",
30
- status: 400,
31
- title: "Invalid request",
32
- });
33
- }
34
-
35
- const client = getLangfuseClient();
36
- client.score.create({
37
- comment: comment ?? undefined,
38
- dataType: dataType as "NUMERIC" | "CATEGORICAL" | "BOOLEAN" | undefined,
39
- name,
40
- traceId,
41
- value,
42
- });
43
-
44
- await client.flush();
45
- return res.status(201).json({comment, dataType, name, traceId, value});
46
- })
47
- );
48
- };