@terreno/ai 58.0.0-beta.2 → 58.0.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +146 -0
- package/README.md +3 -3
- package/dist/harness/commit.d.ts +4 -3
- package/dist/harness/commit.js +23 -4
- package/dist/index.d.ts +6 -9
- package/dist/index.js +14 -25
- package/dist/observability/adminScreens.js +5 -1
- package/dist/observability/evaluate.d.ts +8 -1
- package/dist/observability/evaluate.js +10 -2
- package/dist/observability/judgeAgreement.d.ts +53 -0
- package/dist/observability/judgeAgreement.js +123 -0
- package/dist/observability/local/datasetStore.d.ts +5 -0
- package/dist/observability/local/datasetStore.js +7 -2
- package/dist/observability/local/evaluatorStore.d.ts +17 -0
- package/dist/observability/local/evaluatorStore.js +73 -2
- package/dist/observability/local/experimentRunner.d.ts +40 -3
- package/dist/observability/local/experimentRunner.js +390 -11
- package/dist/observability/local/humanAnswers.d.ts +57 -0
- package/dist/observability/local/humanAnswers.js +174 -0
- package/dist/observability/local/judgeRun.d.ts +56 -0
- package/dist/observability/local/judgeRun.js +111 -0
- package/dist/observability/local/localPlugin.js +4 -0
- package/dist/observability/local/models/obsDatasetItem.js +4 -0
- package/dist/observability/local/models/obsEvaluator.js +4 -0
- package/dist/observability/local/models/obsExperiment.js +28 -0
- package/dist/observability/local/models/obsExperimentItem.js +4 -0
- package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
- package/dist/observability/local/models/obsJudgeRun.js +60 -0
- package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
- package/dist/observability/local/models/obsReviewBatch.js +48 -0
- package/dist/observability/local/models/obsReviewItem.js +10 -2
- package/dist/observability/local/models/obsTrace.js +4 -0
- package/dist/observability/local/promptReferences.d.ts +31 -0
- package/dist/observability/local/promptReferences.js +114 -0
- package/dist/observability/local/promptStore.d.ts +55 -19
- package/dist/observability/local/promptStore.js +171 -7
- package/dist/observability/local/reviewBatchStore.d.ts +106 -0
- package/dist/observability/local/reviewBatchStore.js +517 -0
- package/dist/observability/local/reviewStore.d.ts +24 -3
- package/dist/observability/local/reviewStore.js +98 -22
- package/dist/observability/local/traceStore.d.ts +14 -1
- package/dist/observability/local/traceStore.js +144 -11
- package/dist/observability/observabilityApp.js +10 -1
- package/dist/observability/questionFingerprint.d.ts +7 -0
- package/dist/observability/questionFingerprint.js +23 -0
- package/dist/observability/routes/experiments.js +12 -1
- package/dist/observability/routes/review.js +3 -1
- package/dist/observability/routes/reviewBatches.d.ts +11 -0
- package/dist/observability/routes/reviewBatches.js +78 -0
- package/dist/observability/routes/traces.d.ts +1 -1
- package/dist/observability/routes/traces.js +34 -2
- package/dist/observability/types.d.ts +10 -1
- package/dist/service/aiService.js +4 -1
- package/dist/service/chatTurn.js +38 -31
- package/dist/types/harness.d.ts +5 -2
- package/dist/types/index.d.ts +8 -3
- package/dist/types/observability.d.ts +74 -2
- package/package.json +8 -24
- package/src/harness/agentLoop.test.ts +0 -3
- package/src/harness/agentRecovery.test.ts +0 -3
- package/src/harness/commit.ts +24 -4
- package/src/harness/extensions.test.ts +0 -3
- package/src/harness/subagent.test.ts +27 -3
- package/src/index.test.ts +0 -17
- package/src/index.ts +23 -26
- package/src/isolated/gptStream.isolated.ts +0 -8
- package/src/observability/adminScreens.test.ts +7 -0
- package/src/observability/adminScreens.ts +6 -1
- package/src/observability/evaluate.ts +12 -2
- package/src/observability/judgeAgreement.test.ts +111 -0
- package/src/observability/judgeAgreement.ts +184 -0
- package/src/observability/local/datasetStore.ts +30 -11
- package/src/observability/local/evaluatorStore.ts +96 -2
- package/src/observability/local/experimentRunner.ts +495 -15
- package/src/observability/local/humanAnswers.ts +233 -0
- package/src/observability/local/judgePipeline.test.ts +444 -0
- package/src/observability/local/judgeRun.test.ts +173 -0
- package/src/observability/local/judgeRun.ts +184 -0
- package/src/observability/local/localPlugin.ts +4 -0
- package/src/observability/local/models/obsDatasetItem.ts +5 -0
- package/src/observability/local/models/obsEvaluator.ts +4 -0
- package/src/observability/local/models/obsExperiment.ts +29 -0
- package/src/observability/local/models/obsExperimentItem.ts +4 -0
- package/src/observability/local/models/obsJudgeRun.ts +61 -0
- package/src/observability/local/models/obsReviewBatch.ts +52 -0
- package/src/observability/local/models/obsReviewItem.ts +11 -2
- package/src/observability/local/models/obsTrace.ts +5 -0
- package/src/observability/local/obsModels.test.ts +13 -0
- package/src/observability/local/promptReferences.test.ts +128 -0
- package/src/observability/local/promptReferences.ts +149 -0
- package/src/observability/local/promptStore.test.ts +291 -0
- package/src/observability/local/promptStore.ts +243 -32
- package/src/observability/local/reviewBatchStore.test.ts +251 -0
- package/src/observability/local/reviewBatchStore.ts +656 -0
- package/src/observability/local/reviewStore.ts +122 -25
- package/src/observability/local/traceStore.test.ts +139 -1
- package/src/observability/local/traceStore.ts +162 -11
- package/src/observability/observabilityApp.test.ts +13 -10
- package/src/observability/observabilityApp.ts +10 -1
- package/src/observability/questionFingerprint.test.ts +33 -0
- package/src/observability/questionFingerprint.ts +24 -0
- package/src/observability/routes/experiments.test.ts +37 -0
- package/src/observability/routes/experiments.ts +13 -1
- package/src/observability/routes/review.ts +3 -1
- package/src/observability/routes/reviewBatches.test.ts +209 -0
- package/src/observability/routes/reviewBatches.ts +138 -0
- package/src/observability/routes/status.test.ts +4 -4
- package/src/observability/routes/traces.test.ts +61 -0
- package/src/observability/routes/traces.ts +46 -4
- package/src/observability/types.ts +5 -2
- package/src/routes/routes.test.ts +101 -76
- package/src/service/aiService.test.ts +21 -0
- package/src/service/aiService.ts +4 -1
- package/src/service/chatTurn.ts +42 -33
- package/src/tests/bunSetup.ts +0 -54
- package/src/types/harness.ts +5 -2
- package/src/types/index.ts +8 -3
- package/src/types/observability.ts +86 -2
- package/dist/langfuseApp.d.ts +0 -9
- package/dist/langfuseApp.js +0 -53
- package/dist/langfuseCache.d.ts +0 -15
- package/dist/langfuseCache.js +0 -54
- package/dist/langfuseClient.d.ts +0 -6
- package/dist/langfuseClient.js +0 -34
- package/dist/langfusePrompts.d.ts +0 -12
- package/dist/langfusePrompts.js +0 -89
- package/dist/langfuseRoutesEvaluations.d.ts +0 -3
- package/dist/langfuseRoutesEvaluations.js +0 -32
- package/dist/langfuseRoutesMiddleware.d.ts +0 -2
- package/dist/langfuseRoutesMiddleware.js +0 -13
- package/dist/langfuseRoutesPlayground.d.ts +0 -2
- package/dist/langfuseRoutesPlayground.js +0 -36
- package/dist/langfuseRoutesPrompts.d.ts +0 -2
- package/dist/langfuseRoutesPrompts.js +0 -56
- package/dist/langfuseRoutesTraces.d.ts +0 -2
- package/dist/langfuseRoutesTraces.js +0 -30
- package/dist/langfuseTracing.d.ts +0 -4
- package/dist/langfuseTracing.js +0 -32
- package/dist/langfuseTypes.d.ts +0 -96
- package/dist/langfuseTypes.js +0 -2
- package/dist/langfuseVercelAi.d.ts +0 -12
- package/dist/langfuseVercelAi.js +0 -49
- package/src/isolated/langfuseClient.isolated.ts +0 -66
- package/src/langfuseApp.test.ts +0 -119
- package/src/langfuseApp.ts +0 -62
- package/src/langfuseCache.test.ts +0 -75
- package/src/langfuseCache.ts +0 -72
- package/src/langfuseClient.test.ts +0 -69
- package/src/langfuseClient.ts +0 -34
- package/src/langfusePrompts.test.ts +0 -188
- package/src/langfusePrompts.ts +0 -121
- package/src/langfuseRoutes.test.ts +0 -261
- package/src/langfuseRoutesEvaluations.ts +0 -48
- package/src/langfuseRoutesMiddleware.test.ts +0 -57
- package/src/langfuseRoutesMiddleware.ts +0 -12
- package/src/langfuseRoutesPlayground.ts +0 -50
- package/src/langfuseRoutesPrompts.ts +0 -87
- package/src/langfuseRoutesTraces.ts +0 -40
- package/src/langfuseTracing.test.ts +0 -72
- package/src/langfuseTracing.ts +0 -33
- package/src/langfuseTypes.ts +0 -103
- package/src/langfuseVercelAi.test.ts +0 -82
- package/src/langfuseVercelAi.ts +0 -72
- package/src/tests/fakeLangfusePrompt.ts +0 -10
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
import {describe, expect, it, mock} from "bun:test";
|
|
2
|
-
import type {NextFunction, Request, Response} from "express";
|
|
3
|
-
|
|
4
|
-
import {requireAdmin} from "./langfuseRoutesMiddleware";
|
|
5
|
-
|
|
6
|
-
const buildRequest = (user: unknown): Request => {
|
|
7
|
-
return {user} as Request;
|
|
8
|
-
};
|
|
9
|
-
|
|
10
|
-
const buildResponse = (): Response => {
|
|
11
|
-
return {} as Response;
|
|
12
|
-
};
|
|
13
|
-
|
|
14
|
-
describe("requireAdmin", () => {
|
|
15
|
-
it("calls next when user.isAdmin returns true", () => {
|
|
16
|
-
const req = buildRequest({
|
|
17
|
-
admin: false,
|
|
18
|
-
isAdmin: () => true,
|
|
19
|
-
});
|
|
20
|
-
const next = mock(() => {}) as NextFunction;
|
|
21
|
-
|
|
22
|
-
requireAdmin(req, buildResponse(), next);
|
|
23
|
-
|
|
24
|
-
expect(next).toHaveBeenCalledTimes(1);
|
|
25
|
-
});
|
|
26
|
-
|
|
27
|
-
it("uses Permissions.IsAdmin fallback when isAdmin is missing", () => {
|
|
28
|
-
const req = buildRequest({admin: true});
|
|
29
|
-
const next = mock(() => {}) as NextFunction;
|
|
30
|
-
|
|
31
|
-
requireAdmin(req, buildResponse(), next);
|
|
32
|
-
|
|
33
|
-
expect(next).toHaveBeenCalledTimes(1);
|
|
34
|
-
});
|
|
35
|
-
|
|
36
|
-
it("throws APIError with 403 when user is not admin", () => {
|
|
37
|
-
const req = buildRequest({admin: false});
|
|
38
|
-
const next = mock(() => {}) as NextFunction;
|
|
39
|
-
|
|
40
|
-
try {
|
|
41
|
-
requireAdmin(req, buildResponse(), next);
|
|
42
|
-
throw new Error("Expected requireAdmin to throw");
|
|
43
|
-
} catch (error) {
|
|
44
|
-
const apiError = error as {
|
|
45
|
-
disableExternalErrorTracking?: boolean;
|
|
46
|
-
name?: string;
|
|
47
|
-
status?: number;
|
|
48
|
-
title?: string;
|
|
49
|
-
};
|
|
50
|
-
expect(apiError.name).toBe("ForbiddenError");
|
|
51
|
-
expect(apiError.status).toBe(403);
|
|
52
|
-
expect(apiError.title).toBe("Forbidden");
|
|
53
|
-
expect(apiError.disableExternalErrorTracking).toBe(true);
|
|
54
|
-
expect(next).toHaveBeenCalledTimes(0);
|
|
55
|
-
}
|
|
56
|
-
});
|
|
57
|
-
});
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
import {APIError, Permissions} from "@terreno/api";
|
|
2
|
-
import type {NextFunction, Request, Response} from "express";
|
|
3
|
-
|
|
4
|
-
export const requireAdmin = (req: Request, _res: Response, next: NextFunction): void => {
|
|
5
|
-
const user = req.user as Parameters<typeof Permissions.IsAdmin>[1] & {isAdmin?: () => boolean};
|
|
6
|
-
const isAdmin =
|
|
7
|
-
typeof user?.isAdmin === "function" ? user.isAdmin() : Permissions.IsAdmin("read", user);
|
|
8
|
-
if (!isAdmin) {
|
|
9
|
-
throw new APIError({disableExternalErrorTracking: true, status: 403, title: "Forbidden"});
|
|
10
|
-
}
|
|
11
|
-
next();
|
|
12
|
-
};
|
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
import {APIError, asyncHandler, authenticateMiddleware} from "@terreno/api";
|
|
2
|
-
import type express from "express";
|
|
3
|
-
|
|
4
|
-
import {compilePrompt, getPrompt} from "./langfusePrompts";
|
|
5
|
-
import {requireAdmin} from "./langfuseRoutesMiddleware";
|
|
6
|
-
import type {ChatMessage} from "./langfuseTypes";
|
|
7
|
-
|
|
8
|
-
export const addPlaygroundRoutes = (router: express.Application, basePath: string): void => {
|
|
9
|
-
router.post(
|
|
10
|
-
`${basePath}/playground`,
|
|
11
|
-
[authenticateMiddleware(), requireAdmin],
|
|
12
|
-
asyncHandler(async (req, res) => {
|
|
13
|
-
const {promptName, variables, label} = req.body as {
|
|
14
|
-
promptName: string;
|
|
15
|
-
variables?: Record<string, string>;
|
|
16
|
-
label?: string;
|
|
17
|
-
};
|
|
18
|
-
|
|
19
|
-
if (!promptName) {
|
|
20
|
-
throw new APIError({
|
|
21
|
-
detail: "promptName is required",
|
|
22
|
-
status: 400,
|
|
23
|
-
title: "Invalid request",
|
|
24
|
-
});
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
const cached = await getPrompt(promptName, {label, variables});
|
|
28
|
-
const compiled = compilePrompt(cached, variables ?? {});
|
|
29
|
-
|
|
30
|
-
return res.json({
|
|
31
|
-
compiled,
|
|
32
|
-
config: cached.config,
|
|
33
|
-
labels: cached.labels,
|
|
34
|
-
name: cached.name,
|
|
35
|
-
type: cached.type,
|
|
36
|
-
variables: extractVariables(
|
|
37
|
-
cached.type === "text"
|
|
38
|
-
? (cached.prompt as string)
|
|
39
|
-
: (cached.prompt as ChatMessage[]).map((m) => m.content).join(" ")
|
|
40
|
-
),
|
|
41
|
-
version: cached.version,
|
|
42
|
-
});
|
|
43
|
-
})
|
|
44
|
-
);
|
|
45
|
-
};
|
|
46
|
-
|
|
47
|
-
const extractVariables = (template: string): string[] => {
|
|
48
|
-
const matches = template.match(/\{\{(\w+)\}\}/g) ?? [];
|
|
49
|
-
return [...new Set(matches.map((m) => m.slice(2, -2)))];
|
|
50
|
-
};
|
|
@@ -1,87 +0,0 @@
|
|
|
1
|
-
import {APIError, asyncHandler, authenticateMiddleware} from "@terreno/api";
|
|
2
|
-
import type express from "express";
|
|
3
|
-
|
|
4
|
-
import {getLangfuseClient} from "./langfuseClient";
|
|
5
|
-
import {createPrompt, invalidatePromptCache} from "./langfusePrompts";
|
|
6
|
-
import {requireAdmin} from "./langfuseRoutesMiddleware";
|
|
7
|
-
import type {ChatMessage} from "./langfuseTypes";
|
|
8
|
-
|
|
9
|
-
export const addPromptRoutes = (router: express.Application, basePath: string): void => {
|
|
10
|
-
router.get(
|
|
11
|
-
`${basePath}/prompts`,
|
|
12
|
-
[authenticateMiddleware(), requireAdmin],
|
|
13
|
-
asyncHandler(async (req, res) => {
|
|
14
|
-
const client = getLangfuseClient();
|
|
15
|
-
const page = Number(req.query.page) || 1;
|
|
16
|
-
const limit = Number(req.query.limit) || 20;
|
|
17
|
-
|
|
18
|
-
const result = await client.api.prompts.list({limit, page});
|
|
19
|
-
return res.json({
|
|
20
|
-
data: result.data,
|
|
21
|
-
meta: result.meta,
|
|
22
|
-
});
|
|
23
|
-
})
|
|
24
|
-
);
|
|
25
|
-
|
|
26
|
-
router.get(
|
|
27
|
-
`${basePath}/prompts/:name`,
|
|
28
|
-
[authenticateMiddleware(), requireAdmin],
|
|
29
|
-
asyncHandler(async (req, res) => {
|
|
30
|
-
const client = getLangfuseClient();
|
|
31
|
-
const {name} = req.params;
|
|
32
|
-
const version = req.query.version ? Number(req.query.version) : undefined;
|
|
33
|
-
const label = req.query.label as string | undefined;
|
|
34
|
-
|
|
35
|
-
const prompt = await client.prompt.get(name, {
|
|
36
|
-
cacheTtlSeconds: 0,
|
|
37
|
-
...(version !== undefined ? {version} : {}),
|
|
38
|
-
...(label ? {label} : {}),
|
|
39
|
-
});
|
|
40
|
-
|
|
41
|
-
return res.json({
|
|
42
|
-
config: prompt.config,
|
|
43
|
-
labels: prompt.labels,
|
|
44
|
-
name: prompt.name,
|
|
45
|
-
prompt: prompt.prompt,
|
|
46
|
-
tags: prompt.tags,
|
|
47
|
-
type: prompt.type,
|
|
48
|
-
version: prompt.version,
|
|
49
|
-
});
|
|
50
|
-
})
|
|
51
|
-
);
|
|
52
|
-
|
|
53
|
-
router.post(
|
|
54
|
-
`${basePath}/prompts`,
|
|
55
|
-
[authenticateMiddleware(), requireAdmin],
|
|
56
|
-
asyncHandler(async (req, res) => {
|
|
57
|
-
const {name, type, prompt, labels, tags, config} = req.body as {
|
|
58
|
-
name: string;
|
|
59
|
-
type: "text" | "chat";
|
|
60
|
-
prompt: string | ChatMessage[];
|
|
61
|
-
labels?: string[];
|
|
62
|
-
tags?: string[];
|
|
63
|
-
config?: Record<string, unknown>;
|
|
64
|
-
};
|
|
65
|
-
|
|
66
|
-
if (!name || !type || prompt === undefined) {
|
|
67
|
-
throw new APIError({
|
|
68
|
-
detail: "name, type, and prompt are required",
|
|
69
|
-
status: 400,
|
|
70
|
-
title: "Invalid request",
|
|
71
|
-
});
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
const result = await createPrompt({config, labels, name, prompt, tags, type});
|
|
75
|
-
return res.status(201).json(result);
|
|
76
|
-
})
|
|
77
|
-
);
|
|
78
|
-
|
|
79
|
-
router.delete(
|
|
80
|
-
`${basePath}/prompts/:name/cache`,
|
|
81
|
-
[authenticateMiddleware(), requireAdmin],
|
|
82
|
-
asyncHandler(async (req, res) => {
|
|
83
|
-
await invalidatePromptCache(req.params.name);
|
|
84
|
-
return res.json({invalidated: true, name: req.params.name});
|
|
85
|
-
})
|
|
86
|
-
);
|
|
87
|
-
};
|
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
import {asyncHandler, authenticateMiddleware} from "@terreno/api";
|
|
2
|
-
import type express from "express";
|
|
3
|
-
|
|
4
|
-
import {getLangfuseClient} from "./langfuseClient";
|
|
5
|
-
import {requireAdmin} from "./langfuseRoutesMiddleware";
|
|
6
|
-
|
|
7
|
-
export const addTraceRoutes = (router: express.Application, basePath: string): void => {
|
|
8
|
-
router.get(
|
|
9
|
-
`${basePath}/traces`,
|
|
10
|
-
[authenticateMiddleware(), requireAdmin],
|
|
11
|
-
asyncHandler(async (req, res) => {
|
|
12
|
-
const client = getLangfuseClient();
|
|
13
|
-
const page = Number(req.query.page) || 1;
|
|
14
|
-
const limit = Number(req.query.limit) || 20;
|
|
15
|
-
const userId = req.query.userId as string | undefined;
|
|
16
|
-
const fromTimestamp = req.query.from ? (req.query.from as string) : undefined;
|
|
17
|
-
const toTimestamp = req.query.to ? (req.query.to as string) : undefined;
|
|
18
|
-
|
|
19
|
-
const result = await client.api.trace.list({
|
|
20
|
-
fromTimestamp,
|
|
21
|
-
limit,
|
|
22
|
-
page,
|
|
23
|
-
toTimestamp,
|
|
24
|
-
userId,
|
|
25
|
-
});
|
|
26
|
-
|
|
27
|
-
return res.json({data: result.data, meta: result.meta});
|
|
28
|
-
})
|
|
29
|
-
);
|
|
30
|
-
|
|
31
|
-
router.get(
|
|
32
|
-
`${basePath}/traces/:traceId`,
|
|
33
|
-
[authenticateMiddleware(), requireAdmin],
|
|
34
|
-
asyncHandler(async (req, res) => {
|
|
35
|
-
const client = getLangfuseClient();
|
|
36
|
-
const result = await client.api.trace.get(req.params.traceId);
|
|
37
|
-
return res.json(result);
|
|
38
|
-
})
|
|
39
|
-
);
|
|
40
|
-
};
|
|
@@ -1,72 +0,0 @@
|
|
|
1
|
-
import {beforeAll, beforeEach, describe, expect, it, mock} from "bun:test";
|
|
2
|
-
|
|
3
|
-
interface FakeSdkOptions {
|
|
4
|
-
serviceName?: string;
|
|
5
|
-
spanProcessors?: unknown[];
|
|
6
|
-
}
|
|
7
|
-
|
|
8
|
-
const sdkStart = mock(() => {});
|
|
9
|
-
const sdkShutdown = mock(async () => {});
|
|
10
|
-
let lastSdkOptions: FakeSdkOptions | null = null;
|
|
11
|
-
|
|
12
|
-
class FakeNodeSDK {
|
|
13
|
-
constructor(public opts: FakeSdkOptions) {
|
|
14
|
-
lastSdkOptions = opts;
|
|
15
|
-
}
|
|
16
|
-
start() {
|
|
17
|
-
sdkStart();
|
|
18
|
-
}
|
|
19
|
-
async shutdown() {
|
|
20
|
-
await sdkShutdown();
|
|
21
|
-
}
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
class FakeLangfuseSpanProcessor {
|
|
25
|
-
constructor(public opts: Record<string, unknown>) {}
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
mock.module("@opentelemetry/sdk-node", () => ({NodeSDK: FakeNodeSDK}));
|
|
29
|
-
mock.module("@langfuse/otel", () => ({LangfuseSpanProcessor: FakeLangfuseSpanProcessor}));
|
|
30
|
-
|
|
31
|
-
let initTracing: typeof import("./langfuseTracing.js").initTracing;
|
|
32
|
-
let shutdownTracing: typeof import("./langfuseTracing.js").shutdownTracing;
|
|
33
|
-
beforeAll(async () => {
|
|
34
|
-
({initTracing, shutdownTracing} = await import("./langfuseTracing.js"));
|
|
35
|
-
});
|
|
36
|
-
|
|
37
|
-
describe("langfuseTracing", () => {
|
|
38
|
-
beforeEach(async () => {
|
|
39
|
-
await shutdownTracing();
|
|
40
|
-
sdkStart.mockClear();
|
|
41
|
-
sdkShutdown.mockClear();
|
|
42
|
-
lastSdkOptions = null;
|
|
43
|
-
});
|
|
44
|
-
|
|
45
|
-
it("initializes a NodeSDK with default service name", () => {
|
|
46
|
-
initTracing({publicKey: "pk", secretKey: "sk"});
|
|
47
|
-
expect(sdkStart).toHaveBeenCalled();
|
|
48
|
-
expect(lastSdkOptions?.serviceName).toBe("terreno-app");
|
|
49
|
-
expect(lastSdkOptions?.spanProcessors).toHaveLength(1);
|
|
50
|
-
});
|
|
51
|
-
|
|
52
|
-
it("respects serviceName and baseUrl", () => {
|
|
53
|
-
initTracing({
|
|
54
|
-
baseUrl: "https://lf.example.com",
|
|
55
|
-
publicKey: "pk",
|
|
56
|
-
secretKey: "sk",
|
|
57
|
-
serviceName: "my-service",
|
|
58
|
-
});
|
|
59
|
-
expect(lastSdkOptions?.serviceName).toBe("my-service");
|
|
60
|
-
});
|
|
61
|
-
|
|
62
|
-
it("shuts down the SDK", async () => {
|
|
63
|
-
initTracing({publicKey: "pk", secretKey: "sk"});
|
|
64
|
-
await shutdownTracing();
|
|
65
|
-
expect(sdkShutdown).toHaveBeenCalled();
|
|
66
|
-
});
|
|
67
|
-
|
|
68
|
-
it("no-ops shutdown when not initialized", async () => {
|
|
69
|
-
await shutdownTracing();
|
|
70
|
-
expect(sdkShutdown).not.toHaveBeenCalled();
|
|
71
|
-
});
|
|
72
|
-
});
|
package/src/langfuseTracing.ts
DELETED
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
import {LangfuseSpanProcessor} from "@langfuse/otel";
|
|
2
|
-
import {NodeSDK} from "@opentelemetry/sdk-node";
|
|
3
|
-
import {logger} from "@terreno/api";
|
|
4
|
-
|
|
5
|
-
import type {LangfuseAppOptions} from "./langfuseTypes";
|
|
6
|
-
|
|
7
|
-
let sdkInstance: NodeSDK | null = null;
|
|
8
|
-
|
|
9
|
-
export const initTracing = (options: LangfuseAppOptions): NodeSDK => {
|
|
10
|
-
const sdk = new NodeSDK({
|
|
11
|
-
serviceName: options.serviceName ?? "terreno-app",
|
|
12
|
-
spanProcessors: [
|
|
13
|
-
new LangfuseSpanProcessor({
|
|
14
|
-
baseUrl: options.baseUrl ?? "https://cloud.langfuse.com",
|
|
15
|
-
publicKey: options.publicKey,
|
|
16
|
-
secretKey: options.secretKey,
|
|
17
|
-
}),
|
|
18
|
-
],
|
|
19
|
-
});
|
|
20
|
-
|
|
21
|
-
sdk.start();
|
|
22
|
-
sdkInstance = sdk;
|
|
23
|
-
logger.info("Langfuse OpenTelemetry tracing initialized");
|
|
24
|
-
return sdk;
|
|
25
|
-
};
|
|
26
|
-
|
|
27
|
-
export const shutdownTracing = async (): Promise<void> => {
|
|
28
|
-
if (sdkInstance) {
|
|
29
|
-
await sdkInstance.shutdown();
|
|
30
|
-
sdkInstance = null;
|
|
31
|
-
logger.info("Langfuse OpenTelemetry tracing shut down");
|
|
32
|
-
}
|
|
33
|
-
};
|
package/src/langfuseTypes.ts
DELETED
|
@@ -1,103 +0,0 @@
|
|
|
1
|
-
export interface ScoringFunction {
|
|
2
|
-
categories?: string[];
|
|
3
|
-
description: string;
|
|
4
|
-
name: string;
|
|
5
|
-
range?: {max: number; min: number};
|
|
6
|
-
scoreType: "numeric" | "categorical" | "boolean";
|
|
7
|
-
}
|
|
8
|
-
|
|
9
|
-
export interface LangfuseAppOptions {
|
|
10
|
-
adminPath?: string;
|
|
11
|
-
baseUrl?: string;
|
|
12
|
-
cache?: {
|
|
13
|
-
promptTtlSeconds?: number;
|
|
14
|
-
traceTtlSeconds?: number;
|
|
15
|
-
};
|
|
16
|
-
enableAdminUI?: boolean;
|
|
17
|
-
enableTracing?: boolean;
|
|
18
|
-
evaluation?: {
|
|
19
|
-
enabled?: boolean;
|
|
20
|
-
scoringFunctions?: ScoringFunction[];
|
|
21
|
-
};
|
|
22
|
-
/** Organization slug in Langfuse (default: "flourish-health") */
|
|
23
|
-
organization?: string;
|
|
24
|
-
/** Project name/slug in Langfuse (default: "terreno") */
|
|
25
|
-
project?: string;
|
|
26
|
-
/** Langfuse project ID (UUID) for trace URLs; optional, keys usually imply project */
|
|
27
|
-
projectId?: string;
|
|
28
|
-
publicKey: string;
|
|
29
|
-
secretKey: string;
|
|
30
|
-
serviceName?: string;
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
export interface ChatMessage {
|
|
34
|
-
content: string;
|
|
35
|
-
role: string;
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
export interface LangfuseCachedPrompt {
|
|
39
|
-
config: Record<string, unknown>;
|
|
40
|
-
labels: string[];
|
|
41
|
-
name: string;
|
|
42
|
-
prompt: string | ChatMessage[];
|
|
43
|
-
tags: string[];
|
|
44
|
-
type: "text" | "chat";
|
|
45
|
-
version: number;
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
export interface TelemetrySettings {
|
|
49
|
-
functionId?: string;
|
|
50
|
-
isEnabled?: boolean;
|
|
51
|
-
metadata?: Record<string, string | number | boolean>;
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
export interface PreparePromptResult {
|
|
55
|
-
config: Record<string, unknown>;
|
|
56
|
-
messages?: ChatMessage[];
|
|
57
|
-
prompt?: string;
|
|
58
|
-
telemetry: TelemetrySettings;
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
export interface GetPromptOptions {
|
|
62
|
-
label?: string;
|
|
63
|
-
userId?: string;
|
|
64
|
-
variables?: Record<string, string>;
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
export interface PromptListItem {
|
|
68
|
-
createdAt: string;
|
|
69
|
-
labels: string[];
|
|
70
|
-
name: string;
|
|
71
|
-
tags: string[];
|
|
72
|
-
type: "text" | "chat";
|
|
73
|
-
version: number;
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
export interface TraceListItem {
|
|
77
|
-
id: string;
|
|
78
|
-
input?: unknown;
|
|
79
|
-
metadata?: Record<string, unknown>;
|
|
80
|
-
name: string;
|
|
81
|
-
output?: unknown;
|
|
82
|
-
sessionId?: string;
|
|
83
|
-
timestamp: string;
|
|
84
|
-
userId?: string;
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
export interface PaginatedResult<T> {
|
|
88
|
-
data: T[];
|
|
89
|
-
meta: {
|
|
90
|
-
limit: number;
|
|
91
|
-
page: number;
|
|
92
|
-
total: number;
|
|
93
|
-
totalPages: number;
|
|
94
|
-
};
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
export interface ScoreSubmission {
|
|
98
|
-
comment?: string;
|
|
99
|
-
dataType?: "NUMERIC" | "CATEGORICAL" | "BOOLEAN";
|
|
100
|
-
name: string;
|
|
101
|
-
traceId: string;
|
|
102
|
-
value: number | string;
|
|
103
|
-
}
|
|
@@ -1,82 +0,0 @@
|
|
|
1
|
-
import {beforeEach, describe, expect, it, mock} from "bun:test";
|
|
2
|
-
|
|
3
|
-
import {LangfuseCache} from "./langfuseCache";
|
|
4
|
-
import {getLangfuseClient, initLangfuseClient} from "./langfuseClient";
|
|
5
|
-
import {createTelemetryConfig, preparePromptForAI} from "./langfuseVercelAi";
|
|
6
|
-
import type {FakeLangfusePrompt} from "./tests/fakeLangfusePrompt";
|
|
7
|
-
|
|
8
|
-
const mockPromptGet = mock(
|
|
9
|
-
async (_name: string, _options?: Record<string, unknown>): Promise<FakeLangfusePrompt> => ({
|
|
10
|
-
config: {temperature: 0.5},
|
|
11
|
-
labels: [],
|
|
12
|
-
name: "welcome",
|
|
13
|
-
prompt: "Hi {{name}}",
|
|
14
|
-
tags: [],
|
|
15
|
-
type: "text" as const,
|
|
16
|
-
version: 3,
|
|
17
|
-
})
|
|
18
|
-
);
|
|
19
|
-
|
|
20
|
-
describe("langfuseVercelAi", () => {
|
|
21
|
-
beforeEach(async () => {
|
|
22
|
-
mockPromptGet.mockClear();
|
|
23
|
-
await LangfuseCache.deleteMany({});
|
|
24
|
-
// Re-init the (fake) langfuse client and wire the prompt.get mock so the
|
|
25
|
-
// real `getPrompt`/`compilePrompt` from `./langfusePrompts` runs against
|
|
26
|
-
// a customizable response without network access.
|
|
27
|
-
initLangfuseClient({publicKey: "pk-test", secretKey: "sk-test"});
|
|
28
|
-
(getLangfuseClient() as unknown as {prompt: {get: typeof mockPromptGet}}).prompt.get =
|
|
29
|
-
mockPromptGet;
|
|
30
|
-
});
|
|
31
|
-
|
|
32
|
-
describe("preparePromptForAI", () => {
|
|
33
|
-
it("returns a compiled text prompt with telemetry", async () => {
|
|
34
|
-
const result = await preparePromptForAI({
|
|
35
|
-
promptName: "welcome",
|
|
36
|
-
userId: "user-1",
|
|
37
|
-
variables: {name: "Sam"},
|
|
38
|
-
});
|
|
39
|
-
expect(result.prompt).toBe("Hi Sam");
|
|
40
|
-
expect(result.telemetry.functionId).toBe("prompt:welcome");
|
|
41
|
-
expect(result.telemetry.metadata?.userId).toBe("user-1");
|
|
42
|
-
expect(result.telemetry.metadata?.langfusePromptVersion).toBe(3);
|
|
43
|
-
});
|
|
44
|
-
|
|
45
|
-
it("omits userId metadata when not provided", async () => {
|
|
46
|
-
const result = await preparePromptForAI({promptName: "welcome"});
|
|
47
|
-
expect(result.telemetry.metadata?.userId).toBeUndefined();
|
|
48
|
-
});
|
|
49
|
-
|
|
50
|
-
it("returns chat messages for chat prompts", async () => {
|
|
51
|
-
mockPromptGet.mockImplementationOnce(async (name: string) => ({
|
|
52
|
-
config: {},
|
|
53
|
-
labels: [],
|
|
54
|
-
name,
|
|
55
|
-
prompt: [{content: "Hello", role: "user" as const}],
|
|
56
|
-
tags: [],
|
|
57
|
-
type: "chat" as const,
|
|
58
|
-
version: 1,
|
|
59
|
-
}));
|
|
60
|
-
const result = await preparePromptForAI({promptName: "chat"});
|
|
61
|
-
expect(result.messages).toEqual([{content: "Hello", role: "user"}]);
|
|
62
|
-
});
|
|
63
|
-
});
|
|
64
|
-
|
|
65
|
-
describe("createTelemetryConfig", () => {
|
|
66
|
-
it("builds a telemetry config with userId and metadata", () => {
|
|
67
|
-
const t = createTelemetryConfig({
|
|
68
|
-
functionId: "fn-1",
|
|
69
|
-
metadata: {foo: "bar"},
|
|
70
|
-
userId: "user-1",
|
|
71
|
-
});
|
|
72
|
-
expect(t.functionId).toBe("fn-1");
|
|
73
|
-
expect(t.isEnabled).toBe(true);
|
|
74
|
-
expect(t.metadata).toEqual({foo: "bar", userId: "user-1"});
|
|
75
|
-
});
|
|
76
|
-
|
|
77
|
-
it("omits userId when not provided", () => {
|
|
78
|
-
const t = createTelemetryConfig({functionId: "fn-2"});
|
|
79
|
-
expect(t.metadata).toEqual({});
|
|
80
|
-
});
|
|
81
|
-
});
|
|
82
|
-
});
|
package/src/langfuseVercelAi.ts
DELETED
|
@@ -1,72 +0,0 @@
|
|
|
1
|
-
import {logger} from "@terreno/api";
|
|
2
|
-
|
|
3
|
-
import {compilePrompt, getPrompt} from "./langfusePrompts";
|
|
4
|
-
import type {
|
|
5
|
-
ChatMessage,
|
|
6
|
-
GetPromptOptions,
|
|
7
|
-
LangfuseAppOptions,
|
|
8
|
-
PreparePromptResult,
|
|
9
|
-
} from "./langfuseTypes";
|
|
10
|
-
|
|
11
|
-
export const preparePromptForAI = async (
|
|
12
|
-
params: {
|
|
13
|
-
promptName: string;
|
|
14
|
-
variables?: Record<string, string>;
|
|
15
|
-
userId?: string;
|
|
16
|
-
label?: string;
|
|
17
|
-
},
|
|
18
|
-
appOptions?: Pick<LangfuseAppOptions, "cache">
|
|
19
|
-
): Promise<PreparePromptResult> => {
|
|
20
|
-
const options: GetPromptOptions = {
|
|
21
|
-
label: params.label,
|
|
22
|
-
userId: params.userId,
|
|
23
|
-
variables: params.variables,
|
|
24
|
-
};
|
|
25
|
-
|
|
26
|
-
const cached = await getPrompt(params.promptName, options, appOptions);
|
|
27
|
-
const compiled = compilePrompt(cached, params.variables ?? {});
|
|
28
|
-
|
|
29
|
-
const varKeys = Object.keys(params.variables ?? {});
|
|
30
|
-
logger.debug(
|
|
31
|
-
`Langfuse prompt used: "${params.promptName}" v${cached.version}${varKeys.length ? ` (vars: ${varKeys.join(", ")})` : ""}${params.userId ? ` (user: ${params.userId})` : ""}`
|
|
32
|
-
);
|
|
33
|
-
|
|
34
|
-
const telemetry: PreparePromptResult["telemetry"] = {
|
|
35
|
-
functionId: `prompt:${params.promptName}`,
|
|
36
|
-
isEnabled: true,
|
|
37
|
-
metadata: {
|
|
38
|
-
langfusePromptName: params.promptName,
|
|
39
|
-
langfusePromptVersion: cached.version,
|
|
40
|
-
...(params.userId ? {userId: params.userId} : {}),
|
|
41
|
-
},
|
|
42
|
-
};
|
|
43
|
-
|
|
44
|
-
if (cached.type === "text") {
|
|
45
|
-
return {
|
|
46
|
-
config: cached.config,
|
|
47
|
-
prompt: compiled as string,
|
|
48
|
-
telemetry,
|
|
49
|
-
};
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
return {
|
|
53
|
-
config: cached.config,
|
|
54
|
-
messages: compiled as ChatMessage[],
|
|
55
|
-
telemetry,
|
|
56
|
-
};
|
|
57
|
-
};
|
|
58
|
-
|
|
59
|
-
export const createTelemetryConfig = (params: {
|
|
60
|
-
functionId: string;
|
|
61
|
-
userId?: string;
|
|
62
|
-
metadata?: Record<string, string | number | boolean>;
|
|
63
|
-
}): PreparePromptResult["telemetry"] => {
|
|
64
|
-
return {
|
|
65
|
-
functionId: params.functionId,
|
|
66
|
-
isEnabled: true,
|
|
67
|
-
metadata: {
|
|
68
|
-
...params.metadata,
|
|
69
|
-
...(params.userId ? {userId: params.userId} : {}),
|
|
70
|
-
},
|
|
71
|
-
};
|
|
72
|
-
};
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
/** The fields of a Langfuse prompt that tests stub `client.prompt.get` with. */
|
|
2
|
-
export interface FakeLangfusePrompt {
|
|
3
|
-
config: Record<string, unknown>;
|
|
4
|
-
labels: string[];
|
|
5
|
-
name: string;
|
|
6
|
-
prompt: string | Array<{content: string; role: string}>;
|
|
7
|
-
tags: string[];
|
|
8
|
-
type: "chat" | "text";
|
|
9
|
-
version: number;
|
|
10
|
-
}
|