@terreno/ai 58.0.0-beta.3 → 58.0.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +99 -0
- package/README.md +3 -3
- package/dist/harness/commit.d.ts +4 -3
- package/dist/harness/commit.js +23 -4
- package/dist/index.d.ts +6 -9
- package/dist/index.js +14 -25
- package/dist/observability/adminScreens.js +5 -1
- package/dist/observability/evaluate.d.ts +8 -1
- package/dist/observability/evaluate.js +10 -2
- package/dist/observability/judgeAgreement.d.ts +53 -0
- package/dist/observability/judgeAgreement.js +123 -0
- package/dist/observability/local/datasetStore.d.ts +5 -0
- package/dist/observability/local/datasetStore.js +7 -2
- package/dist/observability/local/evaluatorStore.d.ts +17 -0
- package/dist/observability/local/evaluatorStore.js +73 -2
- package/dist/observability/local/experimentRunner.d.ts +40 -3
- package/dist/observability/local/experimentRunner.js +390 -11
- package/dist/observability/local/humanAnswers.d.ts +57 -0
- package/dist/observability/local/humanAnswers.js +174 -0
- package/dist/observability/local/judgeRun.d.ts +56 -0
- package/dist/observability/local/judgeRun.js +111 -0
- package/dist/observability/local/localPlugin.js +4 -0
- package/dist/observability/local/models/obsDatasetItem.js +4 -0
- package/dist/observability/local/models/obsEvaluator.js +4 -0
- package/dist/observability/local/models/obsExperiment.js +28 -0
- package/dist/observability/local/models/obsExperimentItem.js +4 -0
- package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
- package/dist/observability/local/models/obsJudgeRun.js +60 -0
- package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
- package/dist/observability/local/models/obsReviewBatch.js +48 -0
- package/dist/observability/local/models/obsReviewItem.js +10 -2
- package/dist/observability/local/models/obsTrace.js +4 -0
- package/dist/observability/local/promptReferences.d.ts +31 -0
- package/dist/observability/local/promptReferences.js +114 -0
- package/dist/observability/local/promptStore.d.ts +55 -19
- package/dist/observability/local/promptStore.js +171 -7
- package/dist/observability/local/reviewBatchStore.d.ts +106 -0
- package/dist/observability/local/reviewBatchStore.js +517 -0
- package/dist/observability/local/reviewStore.d.ts +24 -3
- package/dist/observability/local/reviewStore.js +98 -22
- package/dist/observability/local/traceStore.d.ts +14 -1
- package/dist/observability/local/traceStore.js +144 -11
- package/dist/observability/observabilityApp.js +10 -1
- package/dist/observability/questionFingerprint.d.ts +7 -0
- package/dist/observability/questionFingerprint.js +23 -0
- package/dist/observability/routes/experiments.js +12 -1
- package/dist/observability/routes/review.js +3 -1
- package/dist/observability/routes/reviewBatches.d.ts +11 -0
- package/dist/observability/routes/reviewBatches.js +78 -0
- package/dist/observability/routes/traces.d.ts +1 -1
- package/dist/observability/routes/traces.js +34 -2
- package/dist/observability/types.d.ts +10 -1
- package/dist/service/aiService.js +4 -1
- package/dist/service/chatTurn.js +38 -31
- package/dist/types/harness.d.ts +5 -2
- package/dist/types/index.d.ts +8 -3
- package/dist/types/observability.d.ts +74 -2
- package/package.json +8 -24
- package/src/harness/agentLoop.test.ts +0 -3
- package/src/harness/agentRecovery.test.ts +0 -3
- package/src/harness/commit.ts +24 -4
- package/src/harness/extensions.test.ts +0 -3
- package/src/harness/subagent.test.ts +27 -3
- package/src/index.test.ts +0 -17
- package/src/index.ts +23 -26
- package/src/isolated/gptStream.isolated.ts +0 -8
- package/src/observability/adminScreens.test.ts +7 -0
- package/src/observability/adminScreens.ts +6 -1
- package/src/observability/evaluate.ts +12 -2
- package/src/observability/judgeAgreement.test.ts +111 -0
- package/src/observability/judgeAgreement.ts +184 -0
- package/src/observability/local/datasetStore.ts +30 -11
- package/src/observability/local/evaluatorStore.ts +96 -2
- package/src/observability/local/experimentRunner.ts +495 -15
- package/src/observability/local/humanAnswers.ts +233 -0
- package/src/observability/local/judgePipeline.test.ts +444 -0
- package/src/observability/local/judgeRun.test.ts +173 -0
- package/src/observability/local/judgeRun.ts +184 -0
- package/src/observability/local/localPlugin.ts +4 -0
- package/src/observability/local/models/obsDatasetItem.ts +5 -0
- package/src/observability/local/models/obsEvaluator.ts +4 -0
- package/src/observability/local/models/obsExperiment.ts +29 -0
- package/src/observability/local/models/obsExperimentItem.ts +4 -0
- package/src/observability/local/models/obsJudgeRun.ts +61 -0
- package/src/observability/local/models/obsReviewBatch.ts +52 -0
- package/src/observability/local/models/obsReviewItem.ts +11 -2
- package/src/observability/local/models/obsTrace.ts +5 -0
- package/src/observability/local/obsModels.test.ts +13 -0
- package/src/observability/local/promptReferences.test.ts +128 -0
- package/src/observability/local/promptReferences.ts +149 -0
- package/src/observability/local/promptStore.test.ts +291 -0
- package/src/observability/local/promptStore.ts +243 -32
- package/src/observability/local/reviewBatchStore.test.ts +251 -0
- package/src/observability/local/reviewBatchStore.ts +656 -0
- package/src/observability/local/reviewStore.ts +122 -25
- package/src/observability/local/traceStore.test.ts +139 -1
- package/src/observability/local/traceStore.ts +162 -11
- package/src/observability/observabilityApp.test.ts +13 -10
- package/src/observability/observabilityApp.ts +10 -1
- package/src/observability/questionFingerprint.test.ts +33 -0
- package/src/observability/questionFingerprint.ts +24 -0
- package/src/observability/routes/experiments.test.ts +37 -0
- package/src/observability/routes/experiments.ts +13 -1
- package/src/observability/routes/review.ts +3 -1
- package/src/observability/routes/reviewBatches.test.ts +209 -0
- package/src/observability/routes/reviewBatches.ts +138 -0
- package/src/observability/routes/status.test.ts +4 -4
- package/src/observability/routes/traces.test.ts +61 -0
- package/src/observability/routes/traces.ts +46 -4
- package/src/observability/types.ts +5 -2
- package/src/routes/routes.test.ts +101 -76
- package/src/service/aiService.test.ts +21 -0
- package/src/service/aiService.ts +4 -1
- package/src/service/chatTurn.ts +42 -33
- package/src/tests/bunSetup.ts +0 -54
- package/src/types/harness.ts +5 -2
- package/src/types/index.ts +8 -3
- package/src/types/observability.ts +86 -2
- package/dist/langfuseApp.d.ts +0 -9
- package/dist/langfuseApp.js +0 -53
- package/dist/langfuseCache.d.ts +0 -15
- package/dist/langfuseCache.js +0 -54
- package/dist/langfuseClient.d.ts +0 -6
- package/dist/langfuseClient.js +0 -34
- package/dist/langfusePrompts.d.ts +0 -12
- package/dist/langfusePrompts.js +0 -89
- package/dist/langfuseRoutesEvaluations.d.ts +0 -3
- package/dist/langfuseRoutesEvaluations.js +0 -32
- package/dist/langfuseRoutesMiddleware.d.ts +0 -2
- package/dist/langfuseRoutesMiddleware.js +0 -13
- package/dist/langfuseRoutesPlayground.d.ts +0 -2
- package/dist/langfuseRoutesPlayground.js +0 -36
- package/dist/langfuseRoutesPrompts.d.ts +0 -2
- package/dist/langfuseRoutesPrompts.js +0 -56
- package/dist/langfuseRoutesTraces.d.ts +0 -2
- package/dist/langfuseRoutesTraces.js +0 -30
- package/dist/langfuseTracing.d.ts +0 -4
- package/dist/langfuseTracing.js +0 -32
- package/dist/langfuseTypes.d.ts +0 -96
- package/dist/langfuseTypes.js +0 -2
- package/dist/langfuseVercelAi.d.ts +0 -12
- package/dist/langfuseVercelAi.js +0 -49
- package/src/isolated/langfuseClient.isolated.ts +0 -66
- package/src/langfuseApp.test.ts +0 -119
- package/src/langfuseApp.ts +0 -62
- package/src/langfuseCache.test.ts +0 -75
- package/src/langfuseCache.ts +0 -72
- package/src/langfuseClient.test.ts +0 -69
- package/src/langfuseClient.ts +0 -34
- package/src/langfusePrompts.test.ts +0 -188
- package/src/langfusePrompts.ts +0 -121
- package/src/langfuseRoutes.test.ts +0 -261
- package/src/langfuseRoutesEvaluations.ts +0 -48
- package/src/langfuseRoutesMiddleware.test.ts +0 -57
- package/src/langfuseRoutesMiddleware.ts +0 -12
- package/src/langfuseRoutesPlayground.ts +0 -50
- package/src/langfuseRoutesPrompts.ts +0 -87
- package/src/langfuseRoutesTraces.ts +0 -40
- package/src/langfuseTracing.test.ts +0 -72
- package/src/langfuseTracing.ts +0 -33
- package/src/langfuseTypes.ts +0 -103
- package/src/langfuseVercelAi.test.ts +0 -82
- package/src/langfuseVercelAi.ts +0 -72
- package/src/tests/fakeLangfusePrompt.ts +0 -10
package/dist/langfuseTracing.js
DELETED
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.shutdownTracing = exports.initTracing = void 0;
|
|
4
|
-
const otel_1 = require("@langfuse/otel");
|
|
5
|
-
const sdk_node_1 = require("@opentelemetry/sdk-node");
|
|
6
|
-
const api_1 = require("@terreno/api");
|
|
7
|
-
let sdkInstance = null;
|
|
8
|
-
const initTracing = (options) => {
|
|
9
|
-
const sdk = new sdk_node_1.NodeSDK({
|
|
10
|
-
serviceName: options.serviceName ?? "terreno-app",
|
|
11
|
-
spanProcessors: [
|
|
12
|
-
new otel_1.LangfuseSpanProcessor({
|
|
13
|
-
baseUrl: options.baseUrl ?? "https://cloud.langfuse.com",
|
|
14
|
-
publicKey: options.publicKey,
|
|
15
|
-
secretKey: options.secretKey,
|
|
16
|
-
}),
|
|
17
|
-
],
|
|
18
|
-
});
|
|
19
|
-
sdk.start();
|
|
20
|
-
sdkInstance = sdk;
|
|
21
|
-
api_1.logger.info("Langfuse OpenTelemetry tracing initialized");
|
|
22
|
-
return sdk;
|
|
23
|
-
};
|
|
24
|
-
exports.initTracing = initTracing;
|
|
25
|
-
const shutdownTracing = async () => {
|
|
26
|
-
if (sdkInstance) {
|
|
27
|
-
await sdkInstance.shutdown();
|
|
28
|
-
sdkInstance = null;
|
|
29
|
-
api_1.logger.info("Langfuse OpenTelemetry tracing shut down");
|
|
30
|
-
}
|
|
31
|
-
};
|
|
32
|
-
exports.shutdownTracing = shutdownTracing;
|
package/dist/langfuseTypes.d.ts
DELETED
|
@@ -1,96 +0,0 @@
|
|
|
1
|
-
export interface ScoringFunction {
|
|
2
|
-
categories?: string[];
|
|
3
|
-
description: string;
|
|
4
|
-
name: string;
|
|
5
|
-
range?: {
|
|
6
|
-
max: number;
|
|
7
|
-
min: number;
|
|
8
|
-
};
|
|
9
|
-
scoreType: "numeric" | "categorical" | "boolean";
|
|
10
|
-
}
|
|
11
|
-
export interface LangfuseAppOptions {
|
|
12
|
-
adminPath?: string;
|
|
13
|
-
baseUrl?: string;
|
|
14
|
-
cache?: {
|
|
15
|
-
promptTtlSeconds?: number;
|
|
16
|
-
traceTtlSeconds?: number;
|
|
17
|
-
};
|
|
18
|
-
enableAdminUI?: boolean;
|
|
19
|
-
enableTracing?: boolean;
|
|
20
|
-
evaluation?: {
|
|
21
|
-
enabled?: boolean;
|
|
22
|
-
scoringFunctions?: ScoringFunction[];
|
|
23
|
-
};
|
|
24
|
-
/** Organization slug in Langfuse (default: "flourish-health") */
|
|
25
|
-
organization?: string;
|
|
26
|
-
/** Project name/slug in Langfuse (default: "terreno") */
|
|
27
|
-
project?: string;
|
|
28
|
-
/** Langfuse project ID (UUID) for trace URLs; optional, keys usually imply project */
|
|
29
|
-
projectId?: string;
|
|
30
|
-
publicKey: string;
|
|
31
|
-
secretKey: string;
|
|
32
|
-
serviceName?: string;
|
|
33
|
-
}
|
|
34
|
-
export interface ChatMessage {
|
|
35
|
-
content: string;
|
|
36
|
-
role: string;
|
|
37
|
-
}
|
|
38
|
-
export interface LangfuseCachedPrompt {
|
|
39
|
-
config: Record<string, unknown>;
|
|
40
|
-
labels: string[];
|
|
41
|
-
name: string;
|
|
42
|
-
prompt: string | ChatMessage[];
|
|
43
|
-
tags: string[];
|
|
44
|
-
type: "text" | "chat";
|
|
45
|
-
version: number;
|
|
46
|
-
}
|
|
47
|
-
export interface TelemetrySettings {
|
|
48
|
-
functionId?: string;
|
|
49
|
-
isEnabled?: boolean;
|
|
50
|
-
metadata?: Record<string, string | number | boolean>;
|
|
51
|
-
}
|
|
52
|
-
export interface PreparePromptResult {
|
|
53
|
-
config: Record<string, unknown>;
|
|
54
|
-
messages?: ChatMessage[];
|
|
55
|
-
prompt?: string;
|
|
56
|
-
telemetry: TelemetrySettings;
|
|
57
|
-
}
|
|
58
|
-
export interface GetPromptOptions {
|
|
59
|
-
label?: string;
|
|
60
|
-
userId?: string;
|
|
61
|
-
variables?: Record<string, string>;
|
|
62
|
-
}
|
|
63
|
-
export interface PromptListItem {
|
|
64
|
-
createdAt: string;
|
|
65
|
-
labels: string[];
|
|
66
|
-
name: string;
|
|
67
|
-
tags: string[];
|
|
68
|
-
type: "text" | "chat";
|
|
69
|
-
version: number;
|
|
70
|
-
}
|
|
71
|
-
export interface TraceListItem {
|
|
72
|
-
id: string;
|
|
73
|
-
input?: unknown;
|
|
74
|
-
metadata?: Record<string, unknown>;
|
|
75
|
-
name: string;
|
|
76
|
-
output?: unknown;
|
|
77
|
-
sessionId?: string;
|
|
78
|
-
timestamp: string;
|
|
79
|
-
userId?: string;
|
|
80
|
-
}
|
|
81
|
-
export interface PaginatedResult<T> {
|
|
82
|
-
data: T[];
|
|
83
|
-
meta: {
|
|
84
|
-
limit: number;
|
|
85
|
-
page: number;
|
|
86
|
-
total: number;
|
|
87
|
-
totalPages: number;
|
|
88
|
-
};
|
|
89
|
-
}
|
|
90
|
-
export interface ScoreSubmission {
|
|
91
|
-
comment?: string;
|
|
92
|
-
dataType?: "NUMERIC" | "CATEGORICAL" | "BOOLEAN";
|
|
93
|
-
name: string;
|
|
94
|
-
traceId: string;
|
|
95
|
-
value: number | string;
|
|
96
|
-
}
|
package/dist/langfuseTypes.js
DELETED
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
import type { LangfuseAppOptions, PreparePromptResult } from "./langfuseTypes";
|
|
2
|
-
export declare const preparePromptForAI: (params: {
|
|
3
|
-
promptName: string;
|
|
4
|
-
variables?: Record<string, string>;
|
|
5
|
-
userId?: string;
|
|
6
|
-
label?: string;
|
|
7
|
-
}, appOptions?: Pick<LangfuseAppOptions, "cache">) => Promise<PreparePromptResult>;
|
|
8
|
-
export declare const createTelemetryConfig: (params: {
|
|
9
|
-
functionId: string;
|
|
10
|
-
userId?: string;
|
|
11
|
-
metadata?: Record<string, string | number | boolean>;
|
|
12
|
-
}) => PreparePromptResult["telemetry"];
|
package/dist/langfuseVercelAi.js
DELETED
|
@@ -1,49 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.createTelemetryConfig = exports.preparePromptForAI = void 0;
|
|
4
|
-
const api_1 = require("@terreno/api");
|
|
5
|
-
const langfusePrompts_1 = require("./langfusePrompts");
|
|
6
|
-
const preparePromptForAI = async (params, appOptions) => {
|
|
7
|
-
const options = {
|
|
8
|
-
label: params.label,
|
|
9
|
-
userId: params.userId,
|
|
10
|
-
variables: params.variables,
|
|
11
|
-
};
|
|
12
|
-
const cached = await (0, langfusePrompts_1.getPrompt)(params.promptName, options, appOptions);
|
|
13
|
-
const compiled = (0, langfusePrompts_1.compilePrompt)(cached, params.variables ?? {});
|
|
14
|
-
const varKeys = Object.keys(params.variables ?? {});
|
|
15
|
-
api_1.logger.debug(`Langfuse prompt used: "${params.promptName}" v${cached.version}${varKeys.length ? ` (vars: ${varKeys.join(", ")})` : ""}${params.userId ? ` (user: ${params.userId})` : ""}`);
|
|
16
|
-
const telemetry = {
|
|
17
|
-
functionId: `prompt:${params.promptName}`,
|
|
18
|
-
isEnabled: true,
|
|
19
|
-
metadata: {
|
|
20
|
-
langfusePromptName: params.promptName,
|
|
21
|
-
langfusePromptVersion: cached.version,
|
|
22
|
-
...(params.userId ? { userId: params.userId } : {}),
|
|
23
|
-
},
|
|
24
|
-
};
|
|
25
|
-
if (cached.type === "text") {
|
|
26
|
-
return {
|
|
27
|
-
config: cached.config,
|
|
28
|
-
prompt: compiled,
|
|
29
|
-
telemetry,
|
|
30
|
-
};
|
|
31
|
-
}
|
|
32
|
-
return {
|
|
33
|
-
config: cached.config,
|
|
34
|
-
messages: compiled,
|
|
35
|
-
telemetry,
|
|
36
|
-
};
|
|
37
|
-
};
|
|
38
|
-
exports.preparePromptForAI = preparePromptForAI;
|
|
39
|
-
const createTelemetryConfig = (params) => {
|
|
40
|
-
return {
|
|
41
|
-
functionId: params.functionId,
|
|
42
|
-
isEnabled: true,
|
|
43
|
-
metadata: {
|
|
44
|
-
...params.metadata,
|
|
45
|
-
...(params.userId ? { userId: params.userId } : {}),
|
|
46
|
-
},
|
|
47
|
-
};
|
|
48
|
-
};
|
|
49
|
-
exports.createTelemetryConfig = createTelemetryConfig;
|
|
@@ -1,66 +0,0 @@
|
|
|
1
|
-
import {beforeAll, beforeEach, describe, expect, it, mock} from "bun:test";
|
|
2
|
-
|
|
3
|
-
const shutdownMock = mock(async () => {});
|
|
4
|
-
class FakeLangfuseClient {
|
|
5
|
-
constructor(public opts: {baseUrl?: string; publicKey: string; secretKey: string}) {}
|
|
6
|
-
async shutdown(): Promise<void> {
|
|
7
|
-
return shutdownMock();
|
|
8
|
-
}
|
|
9
|
-
}
|
|
10
|
-
|
|
11
|
-
mock.module("@langfuse/client", () => ({
|
|
12
|
-
LangfuseClient: FakeLangfuseClient,
|
|
13
|
-
}));
|
|
14
|
-
|
|
15
|
-
let getLangfuseClient: typeof import("../langfuseClient.js").getLangfuseClient;
|
|
16
|
-
let initLangfuseClient: typeof import("../langfuseClient.js").initLangfuseClient;
|
|
17
|
-
let isLangfuseInitialized: typeof import("../langfuseClient.js").isLangfuseInitialized;
|
|
18
|
-
let shutdownLangfuseClient: typeof import("../langfuseClient.js").shutdownLangfuseClient;
|
|
19
|
-
beforeAll(async () => {
|
|
20
|
-
({getLangfuseClient, initLangfuseClient, isLangfuseInitialized, shutdownLangfuseClient} =
|
|
21
|
-
await import("../langfuseClient.js"));
|
|
22
|
-
});
|
|
23
|
-
|
|
24
|
-
describe("langfuseClient", () => {
|
|
25
|
-
beforeEach(async () => {
|
|
26
|
-
await shutdownLangfuseClient();
|
|
27
|
-
});
|
|
28
|
-
|
|
29
|
-
it("throws when accessing client before initialization", () => {
|
|
30
|
-
expect(() => getLangfuseClient()).toThrow(/not initialized/);
|
|
31
|
-
expect(isLangfuseInitialized()).toBe(false);
|
|
32
|
-
});
|
|
33
|
-
|
|
34
|
-
it("initializes a client with defaults", () => {
|
|
35
|
-
const client = initLangfuseClient({
|
|
36
|
-
publicKey: "pk",
|
|
37
|
-
secretKey: "sk",
|
|
38
|
-
});
|
|
39
|
-
expect((client as unknown as FakeLangfuseClient).opts.baseUrl).toBe(
|
|
40
|
-
"https://cloud.langfuse.com"
|
|
41
|
-
);
|
|
42
|
-
expect(isLangfuseInitialized()).toBe(true);
|
|
43
|
-
expect(getLangfuseClient()).toBe(client);
|
|
44
|
-
});
|
|
45
|
-
|
|
46
|
-
it("respects a custom baseUrl", () => {
|
|
47
|
-
const client = initLangfuseClient({
|
|
48
|
-
baseUrl: "https://lf.example.com",
|
|
49
|
-
publicKey: "pk",
|
|
50
|
-
secretKey: "sk",
|
|
51
|
-
});
|
|
52
|
-
expect((client as unknown as FakeLangfuseClient).opts.baseUrl).toBe("https://lf.example.com");
|
|
53
|
-
});
|
|
54
|
-
|
|
55
|
-
it("shuts down the client", async () => {
|
|
56
|
-
initLangfuseClient({publicKey: "pk", secretKey: "sk"});
|
|
57
|
-
await shutdownLangfuseClient();
|
|
58
|
-
expect(shutdownMock).toHaveBeenCalled();
|
|
59
|
-
expect(isLangfuseInitialized()).toBe(false);
|
|
60
|
-
});
|
|
61
|
-
|
|
62
|
-
it("no-ops shutdown when no client is initialized", async () => {
|
|
63
|
-
await shutdownLangfuseClient();
|
|
64
|
-
expect(isLangfuseInitialized()).toBe(false);
|
|
65
|
-
});
|
|
66
|
-
});
|
package/src/langfuseApp.test.ts
DELETED
|
@@ -1,119 +0,0 @@
|
|
|
1
|
-
import {beforeAll, beforeEach, describe, expect, it} from "bun:test";
|
|
2
|
-
import {TerrenoApp} from "@terreno/api";
|
|
3
|
-
import type express from "express";
|
|
4
|
-
import supertest from "supertest";
|
|
5
|
-
|
|
6
|
-
import {LangfuseApp} from "./langfuseApp";
|
|
7
|
-
import {terrenoUserModel, UserModel} from "./tests/helpers";
|
|
8
|
-
|
|
9
|
-
type PasswordedUser = {setPassword: (password: string) => Promise<void>};
|
|
10
|
-
|
|
11
|
-
const buildApp = (plugin: LangfuseApp) =>
|
|
12
|
-
new TerrenoApp({
|
|
13
|
-
configureApp: (router) => plugin.register(router as unknown as express.Application),
|
|
14
|
-
skipListen: true,
|
|
15
|
-
userModel: terrenoUserModel,
|
|
16
|
-
}).build();
|
|
17
|
-
|
|
18
|
-
describe("LangfuseApp", () => {
|
|
19
|
-
beforeAll(async () => {
|
|
20
|
-
await UserModel.deleteMany({});
|
|
21
|
-
const u = await UserModel.create({email: "lf@example.com", name: "User"});
|
|
22
|
-
await (u as unknown as PasswordedUser).setPassword("password");
|
|
23
|
-
await u.save();
|
|
24
|
-
});
|
|
25
|
-
|
|
26
|
-
beforeEach(async () => {
|
|
27
|
-
const {shutdownLangfuseClient} = await import("./langfuseClient.js");
|
|
28
|
-
await shutdownLangfuseClient();
|
|
29
|
-
});
|
|
30
|
-
|
|
31
|
-
it("mounts admin routes at the default path", async () => {
|
|
32
|
-
const plugin = new LangfuseApp({
|
|
33
|
-
enableTracing: false,
|
|
34
|
-
publicKey: "pk",
|
|
35
|
-
secretKey: "sk",
|
|
36
|
-
});
|
|
37
|
-
const app = buildApp(plugin);
|
|
38
|
-
const res = await supertest(app).get("/admin/langfuse/prompts");
|
|
39
|
-
expect(res.status).not.toBe(404);
|
|
40
|
-
});
|
|
41
|
-
|
|
42
|
-
it("mounts admin routes at a custom path", async () => {
|
|
43
|
-
const plugin = new LangfuseApp({
|
|
44
|
-
adminPath: "/lf-admin",
|
|
45
|
-
enableTracing: false,
|
|
46
|
-
publicKey: "pk",
|
|
47
|
-
secretKey: "sk",
|
|
48
|
-
});
|
|
49
|
-
const app = buildApp(plugin);
|
|
50
|
-
const custom = await supertest(app).get("/lf-admin/prompts");
|
|
51
|
-
expect(custom.status).not.toBe(404);
|
|
52
|
-
const defaultPath = await supertest(app).get("/admin/langfuse/prompts");
|
|
53
|
-
expect(defaultPath.status).toBe(404);
|
|
54
|
-
});
|
|
55
|
-
|
|
56
|
-
it("skips admin routes when enableAdminUI is false", async () => {
|
|
57
|
-
const plugin = new LangfuseApp({
|
|
58
|
-
enableAdminUI: false,
|
|
59
|
-
enableTracing: false,
|
|
60
|
-
publicKey: "pk",
|
|
61
|
-
secretKey: "sk",
|
|
62
|
-
});
|
|
63
|
-
const app = buildApp(plugin);
|
|
64
|
-
const res = await supertest(app).get("/admin/langfuse/prompts");
|
|
65
|
-
expect(res.status).toBe(404);
|
|
66
|
-
});
|
|
67
|
-
|
|
68
|
-
it("registers evaluation routes when evaluation.enabled is true", async () => {
|
|
69
|
-
const plugin = new LangfuseApp({
|
|
70
|
-
enableTracing: false,
|
|
71
|
-
evaluation: {enabled: true, scoringFunctions: []},
|
|
72
|
-
publicKey: "pk",
|
|
73
|
-
secretKey: "sk",
|
|
74
|
-
});
|
|
75
|
-
const app = buildApp(plugin);
|
|
76
|
-
const res = await supertest(app).get("/admin/langfuse/evaluations/config");
|
|
77
|
-
expect(res.status).not.toBe(404);
|
|
78
|
-
});
|
|
79
|
-
|
|
80
|
-
it("does not register evaluation routes when evaluation.enabled is false", async () => {
|
|
81
|
-
const plugin = new LangfuseApp({
|
|
82
|
-
enableTracing: false,
|
|
83
|
-
evaluation: {enabled: false, scoringFunctions: []},
|
|
84
|
-
publicKey: "pk",
|
|
85
|
-
secretKey: "sk",
|
|
86
|
-
});
|
|
87
|
-
const app = buildApp(plugin);
|
|
88
|
-
const res = await supertest(app).get("/admin/langfuse/evaluations/config");
|
|
89
|
-
expect(res.status).toBe(404);
|
|
90
|
-
});
|
|
91
|
-
|
|
92
|
-
it("shuts down cleanly", async () => {
|
|
93
|
-
const plugin = new LangfuseApp({
|
|
94
|
-
enableTracing: false,
|
|
95
|
-
publicKey: "pk",
|
|
96
|
-
secretKey: "sk",
|
|
97
|
-
});
|
|
98
|
-
const app = buildApp(plugin);
|
|
99
|
-
expect(app).toBeDefined();
|
|
100
|
-
await plugin.shutdown();
|
|
101
|
-
});
|
|
102
|
-
|
|
103
|
-
it("invokes shutdown via the SIGTERM handler registered on register", async () => {
|
|
104
|
-
const plugin = new LangfuseApp({
|
|
105
|
-
enableTracing: false,
|
|
106
|
-
publicKey: "pk",
|
|
107
|
-
secretKey: "sk",
|
|
108
|
-
});
|
|
109
|
-
buildApp(plugin);
|
|
110
|
-
// Find the SIGTERM listener that was added last by register() and invoke it directly.
|
|
111
|
-
const listeners = process.listeners("SIGTERM");
|
|
112
|
-
const last = listeners[listeners.length - 1] as (sig: NodeJS.Signals) => void;
|
|
113
|
-
expect(typeof last).toBe("function");
|
|
114
|
-
last("SIGTERM");
|
|
115
|
-
process.removeListener("SIGTERM", last);
|
|
116
|
-
// Give the fire-and-forget shutdown a tick to settle.
|
|
117
|
-
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
118
|
-
});
|
|
119
|
-
});
|
package/src/langfuseApp.ts
DELETED
|
@@ -1,62 +0,0 @@
|
|
|
1
|
-
import type {TerrenoPlugin} from "@terreno/api";
|
|
2
|
-
import {logger} from "@terreno/api";
|
|
3
|
-
import type express from "express";
|
|
4
|
-
|
|
5
|
-
import {initLangfuseClient, shutdownLangfuseClient} from "./langfuseClient";
|
|
6
|
-
import {addEvaluationRoutes} from "./langfuseRoutesEvaluations";
|
|
7
|
-
import {addPlaygroundRoutes} from "./langfuseRoutesPlayground";
|
|
8
|
-
import {addPromptRoutes} from "./langfuseRoutesPrompts";
|
|
9
|
-
import {addTraceRoutes} from "./langfuseRoutesTraces";
|
|
10
|
-
import {initTracing, shutdownTracing} from "./langfuseTracing";
|
|
11
|
-
import type {LangfuseAppOptions} from "./langfuseTypes";
|
|
12
|
-
|
|
13
|
-
export class LangfuseApp implements TerrenoPlugin {
|
|
14
|
-
private options: LangfuseAppOptions;
|
|
15
|
-
|
|
16
|
-
constructor(options: LangfuseAppOptions) {
|
|
17
|
-
this.options = options;
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
register(app: express.Application): void {
|
|
21
|
-
const organization = this.options.organization ?? "flourish-health";
|
|
22
|
-
const project = this.options.project ?? "terreno";
|
|
23
|
-
try {
|
|
24
|
-
initLangfuseClient(this.options);
|
|
25
|
-
} catch (err) {
|
|
26
|
-
logger.warn(`Langfuse client initialization failed; continuing without Langfuse: ${err}`);
|
|
27
|
-
return;
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
if (this.options.enableTracing !== false) {
|
|
31
|
-
try {
|
|
32
|
-
initTracing(this.options);
|
|
33
|
-
} catch (err) {
|
|
34
|
-
logger.warn(`Langfuse tracing initialization failed: ${err}`);
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
if (this.options.enableAdminUI !== false) {
|
|
39
|
-
const adminPath = this.options.adminPath ?? "/admin/langfuse";
|
|
40
|
-
addPromptRoutes(app, adminPath);
|
|
41
|
-
addTraceRoutes(app, adminPath);
|
|
42
|
-
addPlaygroundRoutes(app, adminPath);
|
|
43
|
-
|
|
44
|
-
if (this.options.evaluation?.enabled) {
|
|
45
|
-
addEvaluationRoutes(app, adminPath, this.options.evaluation.scoringFunctions ?? []);
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
logger.info(
|
|
49
|
-
`Langfuse admin routes mounted at ${adminPath} (org: ${organization}, project: ${project})`
|
|
50
|
-
);
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
process.once("SIGTERM", () => {
|
|
54
|
-
void this.shutdown();
|
|
55
|
-
});
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
async shutdown(): Promise<void> {
|
|
59
|
-
await shutdownLangfuseClient();
|
|
60
|
-
await shutdownTracing();
|
|
61
|
-
}
|
|
62
|
-
}
|
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
import {beforeEach, describe, expect, it} from "bun:test";
|
|
2
|
-
|
|
3
|
-
import {getCached, invalidateCache, LangfuseCache, setCached} from "./langfuseCache";
|
|
4
|
-
|
|
5
|
-
describe("langfuseCache", () => {
|
|
6
|
-
beforeEach(async () => {
|
|
7
|
-
await LangfuseCache.deleteMany({});
|
|
8
|
-
});
|
|
9
|
-
|
|
10
|
-
it("returns null for missing keys", async () => {
|
|
11
|
-
expect(await getCached("missing")).toBeNull();
|
|
12
|
-
});
|
|
13
|
-
|
|
14
|
-
it("stores and retrieves a cached prompt", async () => {
|
|
15
|
-
await setCached(
|
|
16
|
-
"prompt:hello:production",
|
|
17
|
-
{
|
|
18
|
-
config: {},
|
|
19
|
-
labels: [],
|
|
20
|
-
name: "hello",
|
|
21
|
-
prompt: "Hi",
|
|
22
|
-
tags: [],
|
|
23
|
-
type: "text",
|
|
24
|
-
version: 1,
|
|
25
|
-
},
|
|
26
|
-
60
|
|
27
|
-
);
|
|
28
|
-
const got = await getCached("prompt:hello:production");
|
|
29
|
-
expect(got?.name).toBe("hello");
|
|
30
|
-
});
|
|
31
|
-
|
|
32
|
-
it("upserts existing entries via setCached", async () => {
|
|
33
|
-
await setCached(
|
|
34
|
-
"prompt:x:production",
|
|
35
|
-
{config: {}, labels: [], name: "x", prompt: "old", tags: [], type: "text", version: 1},
|
|
36
|
-
60
|
|
37
|
-
);
|
|
38
|
-
await setCached(
|
|
39
|
-
"prompt:x:production",
|
|
40
|
-
{config: {}, labels: [], name: "x", prompt: "new", tags: [], type: "text", version: 2},
|
|
41
|
-
60
|
|
42
|
-
);
|
|
43
|
-
const got = await getCached("prompt:x:production");
|
|
44
|
-
expect(got?.version).toBe(2);
|
|
45
|
-
});
|
|
46
|
-
|
|
47
|
-
it("returns null for expired entries", async () => {
|
|
48
|
-
await setCached(
|
|
49
|
-
"prompt:expired:production",
|
|
50
|
-
{config: {}, labels: [], name: "expired", prompt: "", tags: [], type: "text", version: 1},
|
|
51
|
-
60
|
|
52
|
-
);
|
|
53
|
-
await LangfuseCache.updateOne(
|
|
54
|
-
{key: "prompt:expired:production"},
|
|
55
|
-
{expiresAt: new Date(Date.now() - 1000)}
|
|
56
|
-
);
|
|
57
|
-
expect(await getCached("prompt:expired:production")).toBeNull();
|
|
58
|
-
});
|
|
59
|
-
|
|
60
|
-
it("invalidates entries matching a pattern", async () => {
|
|
61
|
-
await setCached(
|
|
62
|
-
"prompt:keep:production",
|
|
63
|
-
{config: {}, labels: [], name: "keep", prompt: "", tags: [], type: "text", version: 1},
|
|
64
|
-
60
|
|
65
|
-
);
|
|
66
|
-
await setCached(
|
|
67
|
-
"prompt:drop:production",
|
|
68
|
-
{config: {}, labels: [], name: "drop", prompt: "", tags: [], type: "text", version: 1},
|
|
69
|
-
60
|
|
70
|
-
);
|
|
71
|
-
await invalidateCache("prompt:drop");
|
|
72
|
-
expect(await getCached("prompt:drop:production")).toBeNull();
|
|
73
|
-
expect(await getCached("prompt:keep:production")).not.toBeNull();
|
|
74
|
-
});
|
|
75
|
-
});
|
package/src/langfuseCache.ts
DELETED
|
@@ -1,72 +0,0 @@
|
|
|
1
|
-
import {createdUpdatedPlugin, findExactlyOne, findOneOrNone, isDeletedPlugin} from "@terreno/api";
|
|
2
|
-
import {DateTime} from "luxon";
|
|
3
|
-
import mongoose, {type Document} from "mongoose";
|
|
4
|
-
|
|
5
|
-
import type {LangfuseCachedPrompt} from "./langfuseTypes";
|
|
6
|
-
|
|
7
|
-
interface LangfuseCacheDocument extends mongoose.Document {
|
|
8
|
-
key: string;
|
|
9
|
-
value: LangfuseCachedPrompt;
|
|
10
|
-
expiresAt: Date;
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
interface LangfuseCacheModel extends mongoose.Model<LangfuseCacheDocument> {
|
|
14
|
-
findOneOrNone(query: Record<string, unknown>): Promise<(Document & LangfuseCacheDocument) | null>;
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
const langfuseCacheSchema = new mongoose.Schema<LangfuseCacheDocument>(
|
|
18
|
-
{
|
|
19
|
-
expiresAt: {
|
|
20
|
-
description: "When this cache entry expires",
|
|
21
|
-
required: true,
|
|
22
|
-
type: Date,
|
|
23
|
-
},
|
|
24
|
-
key: {
|
|
25
|
-
description: "Cache key (e.g. prompt:chat-assistant:production)",
|
|
26
|
-
required: true,
|
|
27
|
-
type: String,
|
|
28
|
-
unique: true,
|
|
29
|
-
},
|
|
30
|
-
value: {
|
|
31
|
-
description: "Cached Langfuse object",
|
|
32
|
-
required: true,
|
|
33
|
-
type: mongoose.Schema.Types.Mixed,
|
|
34
|
-
},
|
|
35
|
-
},
|
|
36
|
-
{strict: "throw", toJSON: {virtuals: true}, toObject: {virtuals: true}}
|
|
37
|
-
);
|
|
38
|
-
|
|
39
|
-
langfuseCacheSchema.index({expiresAt: 1}, {expireAfterSeconds: 0});
|
|
40
|
-
|
|
41
|
-
langfuseCacheSchema.plugin(createdUpdatedPlugin);
|
|
42
|
-
langfuseCacheSchema.plugin(isDeletedPlugin);
|
|
43
|
-
langfuseCacheSchema.plugin(findOneOrNone);
|
|
44
|
-
langfuseCacheSchema.plugin(findExactlyOne);
|
|
45
|
-
|
|
46
|
-
export const LangfuseCache =
|
|
47
|
-
(mongoose.models.LangfuseCache as LangfuseCacheModel) ||
|
|
48
|
-
mongoose.model<LangfuseCacheDocument, LangfuseCacheModel>("LangfuseCache", langfuseCacheSchema);
|
|
49
|
-
|
|
50
|
-
export const getCached = async (key: string): Promise<LangfuseCachedPrompt | null> => {
|
|
51
|
-
const entry = await LangfuseCache.findOneOrNone({
|
|
52
|
-
expiresAt: {$gt: DateTime.now().toJSDate()},
|
|
53
|
-
key,
|
|
54
|
-
});
|
|
55
|
-
if (!entry) {
|
|
56
|
-
return null;
|
|
57
|
-
}
|
|
58
|
-
return entry.value as LangfuseCachedPrompt;
|
|
59
|
-
};
|
|
60
|
-
|
|
61
|
-
export const setCached = async (
|
|
62
|
-
key: string,
|
|
63
|
-
value: LangfuseCachedPrompt,
|
|
64
|
-
ttlSeconds: number
|
|
65
|
-
): Promise<void> => {
|
|
66
|
-
const expiresAt = DateTime.now().plus({seconds: ttlSeconds}).toJSDate();
|
|
67
|
-
await LangfuseCache.findOneAndUpdate({key}, {expiresAt, key, value}, {upsert: true});
|
|
68
|
-
};
|
|
69
|
-
|
|
70
|
-
export const invalidateCache = async (keyPattern: string): Promise<void> => {
|
|
71
|
-
await LangfuseCache.deleteMany({key: {$regex: keyPattern}});
|
|
72
|
-
};
|
|
@@ -1,69 +0,0 @@
|
|
|
1
|
-
import {afterEach, describe, expect, it, spyOn} from "bun:test";
|
|
2
|
-
import {LangfuseClient} from "@langfuse/client";
|
|
3
|
-
|
|
4
|
-
import {
|
|
5
|
-
getLangfuseClient,
|
|
6
|
-
initLangfuseClient,
|
|
7
|
-
isLangfuseInitialized,
|
|
8
|
-
shutdownLangfuseClient,
|
|
9
|
-
} from "./langfuseClient";
|
|
10
|
-
|
|
11
|
-
describe("langfuseClient", () => {
|
|
12
|
-
afterEach(async () => {
|
|
13
|
-
await shutdownLangfuseClient();
|
|
14
|
-
});
|
|
15
|
-
|
|
16
|
-
it("initLangfuseClient creates a client with the given options", () => {
|
|
17
|
-
const client = initLangfuseClient({publicKey: "pk-test", secretKey: "sk-test"});
|
|
18
|
-
expect(client).toBeInstanceOf(LangfuseClient);
|
|
19
|
-
});
|
|
20
|
-
|
|
21
|
-
it("initLangfuseClient uses default baseUrl when not provided", () => {
|
|
22
|
-
const client = initLangfuseClient({publicKey: "pk", secretKey: "sk"});
|
|
23
|
-
expect(client).toBeDefined();
|
|
24
|
-
});
|
|
25
|
-
|
|
26
|
-
it("initLangfuseClient respects a custom baseUrl", () => {
|
|
27
|
-
const client = initLangfuseClient({
|
|
28
|
-
baseUrl: "https://custom.langfuse.dev",
|
|
29
|
-
publicKey: "pk",
|
|
30
|
-
secretKey: "sk",
|
|
31
|
-
});
|
|
32
|
-
expect(client).toBeDefined();
|
|
33
|
-
});
|
|
34
|
-
|
|
35
|
-
it("getLangfuseClient throws when not initialized", () => {
|
|
36
|
-
expect(() => getLangfuseClient()).toThrow(
|
|
37
|
-
"Langfuse client not initialized. Call initLangfuseClient first."
|
|
38
|
-
);
|
|
39
|
-
});
|
|
40
|
-
|
|
41
|
-
it("getLangfuseClient returns the initialized client", () => {
|
|
42
|
-
const created = initLangfuseClient({publicKey: "pk", secretKey: "sk"});
|
|
43
|
-
const retrieved = getLangfuseClient();
|
|
44
|
-
expect(retrieved).toBe(created);
|
|
45
|
-
});
|
|
46
|
-
|
|
47
|
-
it("isLangfuseInitialized returns false before init and true after", () => {
|
|
48
|
-
expect(isLangfuseInitialized()).toBe(false);
|
|
49
|
-
initLangfuseClient({publicKey: "pk", secretKey: "sk"});
|
|
50
|
-
expect(isLangfuseInitialized()).toBe(true);
|
|
51
|
-
});
|
|
52
|
-
|
|
53
|
-
it("shutdownLangfuseClient calls shutdown and resets state", async () => {
|
|
54
|
-
const client = initLangfuseClient({publicKey: "pk", secretKey: "sk"});
|
|
55
|
-
const shutdownSpy = spyOn(client, "shutdown").mockResolvedValue(undefined);
|
|
56
|
-
expect(isLangfuseInitialized()).toBe(true);
|
|
57
|
-
|
|
58
|
-
await shutdownLangfuseClient();
|
|
59
|
-
expect(isLangfuseInitialized()).toBe(false);
|
|
60
|
-
expect(shutdownSpy).toHaveBeenCalledTimes(1);
|
|
61
|
-
shutdownSpy.mockRestore();
|
|
62
|
-
});
|
|
63
|
-
|
|
64
|
-
it("shutdownLangfuseClient is a no-op when not initialized", async () => {
|
|
65
|
-
expect(isLangfuseInitialized()).toBe(false);
|
|
66
|
-
await shutdownLangfuseClient();
|
|
67
|
-
expect(isLangfuseInitialized()).toBe(false);
|
|
68
|
-
});
|
|
69
|
-
});
|