@terreno/ai 58.0.0-beta.3 → 58.0.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +99 -0
- package/README.md +3 -3
- package/dist/harness/commit.d.ts +4 -3
- package/dist/harness/commit.js +23 -4
- package/dist/index.d.ts +6 -9
- package/dist/index.js +14 -25
- package/dist/observability/adminScreens.js +5 -1
- package/dist/observability/evaluate.d.ts +8 -1
- package/dist/observability/evaluate.js +10 -2
- package/dist/observability/judgeAgreement.d.ts +53 -0
- package/dist/observability/judgeAgreement.js +123 -0
- package/dist/observability/local/datasetStore.d.ts +5 -0
- package/dist/observability/local/datasetStore.js +7 -2
- package/dist/observability/local/evaluatorStore.d.ts +17 -0
- package/dist/observability/local/evaluatorStore.js +73 -2
- package/dist/observability/local/experimentRunner.d.ts +40 -3
- package/dist/observability/local/experimentRunner.js +390 -11
- package/dist/observability/local/humanAnswers.d.ts +57 -0
- package/dist/observability/local/humanAnswers.js +174 -0
- package/dist/observability/local/judgeRun.d.ts +56 -0
- package/dist/observability/local/judgeRun.js +111 -0
- package/dist/observability/local/localPlugin.js +4 -0
- package/dist/observability/local/models/obsDatasetItem.js +4 -0
- package/dist/observability/local/models/obsEvaluator.js +4 -0
- package/dist/observability/local/models/obsExperiment.js +28 -0
- package/dist/observability/local/models/obsExperimentItem.js +4 -0
- package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
- package/dist/observability/local/models/obsJudgeRun.js +60 -0
- package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
- package/dist/observability/local/models/obsReviewBatch.js +48 -0
- package/dist/observability/local/models/obsReviewItem.js +10 -2
- package/dist/observability/local/models/obsTrace.js +4 -0
- package/dist/observability/local/promptReferences.d.ts +31 -0
- package/dist/observability/local/promptReferences.js +114 -0
- package/dist/observability/local/promptStore.d.ts +55 -19
- package/dist/observability/local/promptStore.js +171 -7
- package/dist/observability/local/reviewBatchStore.d.ts +106 -0
- package/dist/observability/local/reviewBatchStore.js +517 -0
- package/dist/observability/local/reviewStore.d.ts +24 -3
- package/dist/observability/local/reviewStore.js +98 -22
- package/dist/observability/local/traceStore.d.ts +14 -1
- package/dist/observability/local/traceStore.js +144 -11
- package/dist/observability/observabilityApp.js +10 -1
- package/dist/observability/questionFingerprint.d.ts +7 -0
- package/dist/observability/questionFingerprint.js +23 -0
- package/dist/observability/routes/experiments.js +12 -1
- package/dist/observability/routes/review.js +3 -1
- package/dist/observability/routes/reviewBatches.d.ts +11 -0
- package/dist/observability/routes/reviewBatches.js +78 -0
- package/dist/observability/routes/traces.d.ts +1 -1
- package/dist/observability/routes/traces.js +34 -2
- package/dist/observability/types.d.ts +10 -1
- package/dist/service/aiService.js +4 -1
- package/dist/service/chatTurn.js +38 -31
- package/dist/types/harness.d.ts +5 -2
- package/dist/types/index.d.ts +8 -3
- package/dist/types/observability.d.ts +74 -2
- package/package.json +8 -24
- package/src/harness/agentLoop.test.ts +0 -3
- package/src/harness/agentRecovery.test.ts +0 -3
- package/src/harness/commit.ts +24 -4
- package/src/harness/extensions.test.ts +0 -3
- package/src/harness/subagent.test.ts +27 -3
- package/src/index.test.ts +0 -17
- package/src/index.ts +23 -26
- package/src/isolated/gptStream.isolated.ts +0 -8
- package/src/observability/adminScreens.test.ts +7 -0
- package/src/observability/adminScreens.ts +6 -1
- package/src/observability/evaluate.ts +12 -2
- package/src/observability/judgeAgreement.test.ts +111 -0
- package/src/observability/judgeAgreement.ts +184 -0
- package/src/observability/local/datasetStore.ts +30 -11
- package/src/observability/local/evaluatorStore.ts +96 -2
- package/src/observability/local/experimentRunner.ts +495 -15
- package/src/observability/local/humanAnswers.ts +233 -0
- package/src/observability/local/judgePipeline.test.ts +444 -0
- package/src/observability/local/judgeRun.test.ts +173 -0
- package/src/observability/local/judgeRun.ts +184 -0
- package/src/observability/local/localPlugin.ts +4 -0
- package/src/observability/local/models/obsDatasetItem.ts +5 -0
- package/src/observability/local/models/obsEvaluator.ts +4 -0
- package/src/observability/local/models/obsExperiment.ts +29 -0
- package/src/observability/local/models/obsExperimentItem.ts +4 -0
- package/src/observability/local/models/obsJudgeRun.ts +61 -0
- package/src/observability/local/models/obsReviewBatch.ts +52 -0
- package/src/observability/local/models/obsReviewItem.ts +11 -2
- package/src/observability/local/models/obsTrace.ts +5 -0
- package/src/observability/local/obsModels.test.ts +13 -0
- package/src/observability/local/promptReferences.test.ts +128 -0
- package/src/observability/local/promptReferences.ts +149 -0
- package/src/observability/local/promptStore.test.ts +291 -0
- package/src/observability/local/promptStore.ts +243 -32
- package/src/observability/local/reviewBatchStore.test.ts +251 -0
- package/src/observability/local/reviewBatchStore.ts +656 -0
- package/src/observability/local/reviewStore.ts +122 -25
- package/src/observability/local/traceStore.test.ts +139 -1
- package/src/observability/local/traceStore.ts +162 -11
- package/src/observability/observabilityApp.test.ts +13 -10
- package/src/observability/observabilityApp.ts +10 -1
- package/src/observability/questionFingerprint.test.ts +33 -0
- package/src/observability/questionFingerprint.ts +24 -0
- package/src/observability/routes/experiments.test.ts +37 -0
- package/src/observability/routes/experiments.ts +13 -1
- package/src/observability/routes/review.ts +3 -1
- package/src/observability/routes/reviewBatches.test.ts +209 -0
- package/src/observability/routes/reviewBatches.ts +138 -0
- package/src/observability/routes/status.test.ts +4 -4
- package/src/observability/routes/traces.test.ts +61 -0
- package/src/observability/routes/traces.ts +46 -4
- package/src/observability/types.ts +5 -2
- package/src/routes/routes.test.ts +101 -76
- package/src/service/aiService.test.ts +21 -0
- package/src/service/aiService.ts +4 -1
- package/src/service/chatTurn.ts +42 -33
- package/src/tests/bunSetup.ts +0 -54
- package/src/types/harness.ts +5 -2
- package/src/types/index.ts +8 -3
- package/src/types/observability.ts +86 -2
- package/dist/langfuseApp.d.ts +0 -9
- package/dist/langfuseApp.js +0 -53
- package/dist/langfuseCache.d.ts +0 -15
- package/dist/langfuseCache.js +0 -54
- package/dist/langfuseClient.d.ts +0 -6
- package/dist/langfuseClient.js +0 -34
- package/dist/langfusePrompts.d.ts +0 -12
- package/dist/langfusePrompts.js +0 -89
- package/dist/langfuseRoutesEvaluations.d.ts +0 -3
- package/dist/langfuseRoutesEvaluations.js +0 -32
- package/dist/langfuseRoutesMiddleware.d.ts +0 -2
- package/dist/langfuseRoutesMiddleware.js +0 -13
- package/dist/langfuseRoutesPlayground.d.ts +0 -2
- package/dist/langfuseRoutesPlayground.js +0 -36
- package/dist/langfuseRoutesPrompts.d.ts +0 -2
- package/dist/langfuseRoutesPrompts.js +0 -56
- package/dist/langfuseRoutesTraces.d.ts +0 -2
- package/dist/langfuseRoutesTraces.js +0 -30
- package/dist/langfuseTracing.d.ts +0 -4
- package/dist/langfuseTracing.js +0 -32
- package/dist/langfuseTypes.d.ts +0 -96
- package/dist/langfuseTypes.js +0 -2
- package/dist/langfuseVercelAi.d.ts +0 -12
- package/dist/langfuseVercelAi.js +0 -49
- package/src/isolated/langfuseClient.isolated.ts +0 -66
- package/src/langfuseApp.test.ts +0 -119
- package/src/langfuseApp.ts +0 -62
- package/src/langfuseCache.test.ts +0 -75
- package/src/langfuseCache.ts +0 -72
- package/src/langfuseClient.test.ts +0 -69
- package/src/langfuseClient.ts +0 -34
- package/src/langfusePrompts.test.ts +0 -188
- package/src/langfusePrompts.ts +0 -121
- package/src/langfuseRoutes.test.ts +0 -261
- package/src/langfuseRoutesEvaluations.ts +0 -48
- package/src/langfuseRoutesMiddleware.test.ts +0 -57
- package/src/langfuseRoutesMiddleware.ts +0 -12
- package/src/langfuseRoutesPlayground.ts +0 -50
- package/src/langfuseRoutesPrompts.ts +0 -87
- package/src/langfuseRoutesTraces.ts +0 -40
- package/src/langfuseTracing.test.ts +0 -72
- package/src/langfuseTracing.ts +0 -33
- package/src/langfuseTypes.ts +0 -103
- package/src/langfuseVercelAi.test.ts +0 -82
- package/src/langfuseVercelAi.ts +0 -72
- package/src/tests/fakeLangfusePrompt.ts +0 -10
|
@@ -109,6 +109,67 @@ describe("observability trace routes", () => {
|
|
|
109
109
|
).toEqual([[inScope.id, "ws-1"]]);
|
|
110
110
|
});
|
|
111
111
|
|
|
112
|
+
it("sorts the trace list across pages and rejects unknown sort keys", async () => {
|
|
113
|
+
const base = DateTime.utc().minus({hours: 1});
|
|
114
|
+
const exportAt = async (
|
|
115
|
+
name: string,
|
|
116
|
+
offsetMs: number,
|
|
117
|
+
latencyMs: number,
|
|
118
|
+
costUsd: number,
|
|
119
|
+
inputTokens: number
|
|
120
|
+
): Promise<string> => {
|
|
121
|
+
const exported = await store.exportTrace({
|
|
122
|
+
endedAt: base.plus({milliseconds: offsetMs + latencyMs}).toISO() ?? "",
|
|
123
|
+
id: name,
|
|
124
|
+
name,
|
|
125
|
+
prompts: [{name, version: 1}],
|
|
126
|
+
sensitive: false,
|
|
127
|
+
spans: [],
|
|
128
|
+
startedAt: base.plus({milliseconds: offsetMs}).toISO() ?? "",
|
|
129
|
+
status: "ok",
|
|
130
|
+
usage: {costUsd, inputTokens, outputTokens: 0},
|
|
131
|
+
});
|
|
132
|
+
return exported.id;
|
|
133
|
+
};
|
|
134
|
+
const cheapSlow = await exportAt("cheap-slow", 0, 900, 0.0001, 500);
|
|
135
|
+
const priceyFast = await exportAt("pricey-fast", 1000, 50, 0.05, 20);
|
|
136
|
+
const midMid = await exportAt("mid-mid", 2000, 300, 0.001, 12_345);
|
|
137
|
+
const admin = await authAsUser(app, "admin");
|
|
138
|
+
|
|
139
|
+
const pageIds = async (query: string): Promise<string[]> => {
|
|
140
|
+
const first = await admin.get(`/ai/observability/traces?${query}&limit=2&page=1`);
|
|
141
|
+
const second = await admin.get(`/ai/observability/traces?${query}&limit=2&page=2`);
|
|
142
|
+
expect(first.status).toBe(200);
|
|
143
|
+
expect(first.body.more).toBe(true);
|
|
144
|
+
expect(second.body.more).toBe(false);
|
|
145
|
+
return [...first.body.data, ...second.body.data].map((row: {id: string}) => row.id);
|
|
146
|
+
};
|
|
147
|
+
|
|
148
|
+
expect(await pageIds("sort=-cost")).toEqual([priceyFast, midMid, cheapSlow]);
|
|
149
|
+
expect(await pageIds("sort=cost")).toEqual([cheapSlow, midMid, priceyFast]);
|
|
150
|
+
expect(await pageIds("sort=-latency")).toEqual([cheapSlow, midMid, priceyFast]);
|
|
151
|
+
expect(await pageIds("sort=latency")).toEqual([priceyFast, midMid, cheapSlow]);
|
|
152
|
+
expect(await pageIds("sort=-tokens")).toEqual([midMid, cheapSlow, priceyFast]);
|
|
153
|
+
expect(await pageIds("sort=tokens")).toEqual([priceyFast, cheapSlow, midMid]);
|
|
154
|
+
expect(await pageIds("sort=startedAt")).toEqual([cheapSlow, priceyFast, midMid]);
|
|
155
|
+
expect(await pageIds("sort=-name")).toEqual([priceyFast, midMid, cheapSlow]);
|
|
156
|
+
// Default stays newest first.
|
|
157
|
+
expect(await pageIds("status=ok")).toEqual([midMid, priceyFast, cheapSlow]);
|
|
158
|
+
|
|
159
|
+
const multiPrompt = await admin.get(
|
|
160
|
+
"/ai/observability/traces?prompt[0]=cheap-slow&prompt[1]=mid-mid&sort=name"
|
|
161
|
+
);
|
|
162
|
+
expect(multiPrompt.status).toBe(200);
|
|
163
|
+
expect(multiPrompt.body.data.map((row: {id: string}) => row.id)).toEqual([cheapSlow, midMid]);
|
|
164
|
+
|
|
165
|
+
const unknown = await admin.get("/ai/observability/traces?sort=created");
|
|
166
|
+
expect(unknown.status).toBe(400);
|
|
167
|
+
const listSort = await admin.get("/ai/observability/traces?sort[0]=cost");
|
|
168
|
+
expect(listSort.status).toBe(400);
|
|
169
|
+
const objectPrompt = await admin.get("/ai/observability/traces?prompt[name]=x");
|
|
170
|
+
expect(objectPrompt.status).toBe(400);
|
|
171
|
+
});
|
|
172
|
+
|
|
112
173
|
it("forbids non-admins from listing traces", async () => {
|
|
113
174
|
const agent = await authAsUser(app, "notAdmin");
|
|
114
175
|
const res = await agent.get("/ai/observability/traces");
|
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
import {APIError, asyncHandler, createOpenApiBuilder} from "@terreno/api";
|
|
2
2
|
import type express from "express";
|
|
3
|
-
import
|
|
3
|
+
import {
|
|
4
|
+
DEFAULT_TRACE_LIST_SORT,
|
|
5
|
+
type LocalTraceStore,
|
|
6
|
+
parseTraceListSort,
|
|
7
|
+
TRACE_LIST_SORT_KEYS,
|
|
8
|
+
} from "../local/traceStore";
|
|
4
9
|
import {getObservabilityApp} from "../observabilityAppRegistry";
|
|
5
10
|
import type {ScoreRecord, ScoreSink} from "../types";
|
|
6
11
|
import {
|
|
@@ -27,7 +32,7 @@ const parseBoolean = (value: unknown): boolean | undefined => {
|
|
|
27
32
|
throw new APIError({status: 400, title: "boolean query must be true or false"});
|
|
28
33
|
};
|
|
29
34
|
|
|
30
|
-
const parsePromptVersion = (value: unknown, prompt?: string): number | undefined => {
|
|
35
|
+
const parsePromptVersion = (value: unknown, prompt?: string | string[]): number | undefined => {
|
|
31
36
|
if (value === undefined) {
|
|
32
37
|
return undefined;
|
|
33
38
|
}
|
|
@@ -35,12 +40,37 @@ const parsePromptVersion = (value: unknown, prompt?: string): number | undefined
|
|
|
35
40
|
if (!Number.isInteger(parsed) || parsed < 1) {
|
|
36
41
|
throw new APIError({status: 400, title: "promptVersion must be a positive integer"});
|
|
37
42
|
}
|
|
38
|
-
if (!prompt) {
|
|
43
|
+
if (!prompt || prompt.length === 0) {
|
|
39
44
|
throw new APIError({status: 400, title: "prompt is required when promptVersion is set"});
|
|
40
45
|
}
|
|
41
46
|
return parsed;
|
|
42
47
|
};
|
|
43
48
|
|
|
49
|
+
const parsePrompt = (value: unknown): string | string[] | undefined => {
|
|
50
|
+
if (typeof value === "string") {
|
|
51
|
+
return value;
|
|
52
|
+
}
|
|
53
|
+
if (Array.isArray(value) && value.every((entry) => typeof entry === "string")) {
|
|
54
|
+
return value as string[];
|
|
55
|
+
}
|
|
56
|
+
if (value === undefined) {
|
|
57
|
+
return undefined;
|
|
58
|
+
}
|
|
59
|
+
throw new APIError({status: 400, title: "prompt must be a string or a list of strings"});
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
const parseSort = (value: unknown): string | undefined => {
|
|
63
|
+
if (value === undefined) {
|
|
64
|
+
return undefined;
|
|
65
|
+
}
|
|
66
|
+
if (typeof value !== "string") {
|
|
67
|
+
throw new APIError({status: 400, title: "Unknown trace sort key"});
|
|
68
|
+
}
|
|
69
|
+
// Validate here so a bad key fails before the store runs any query.
|
|
70
|
+
parseTraceListSort(value);
|
|
71
|
+
return value;
|
|
72
|
+
};
|
|
73
|
+
|
|
44
74
|
const parsePositiveInt = (value: unknown, fallback: number): number => {
|
|
45
75
|
if (value === undefined) {
|
|
46
76
|
return fallback;
|
|
@@ -72,11 +102,22 @@ export const addObservabilityTraceRoutes = (
|
|
|
72
102
|
.withQueryParameter("prompt", {type: "string"}, {required: false})
|
|
73
103
|
.withQueryParameter("promptVersion", {type: "number"}, {required: false})
|
|
74
104
|
.withQueryParameter("scope", {type: "string"}, {required: false})
|
|
105
|
+
.withQueryParameter(
|
|
106
|
+
"sort",
|
|
107
|
+
{
|
|
108
|
+
enum: TRACE_LIST_SORT_KEYS.flatMap((key) => [key, `-${key}`]),
|
|
109
|
+
type: "string",
|
|
110
|
+
},
|
|
111
|
+
{
|
|
112
|
+
description: `Sort column, \`-\` prefix for descending. Defaults to \`${DEFAULT_TRACE_LIST_SORT}\`.`,
|
|
113
|
+
required: false,
|
|
114
|
+
}
|
|
115
|
+
)
|
|
75
116
|
.withResponse(200, {data: {type: "array"}})
|
|
76
117
|
.build()
|
|
77
118
|
),
|
|
78
119
|
asyncHandler(async (req, res) => {
|
|
79
|
-
const prompt =
|
|
120
|
+
const prompt = parsePrompt(req.query.prompt);
|
|
80
121
|
const listed = await options.store.list({
|
|
81
122
|
flaggedForDataset: parseBoolean(req.query.flaggedForDataset),
|
|
82
123
|
from: typeof req.query.from === "string" ? req.query.from : undefined,
|
|
@@ -88,6 +129,7 @@ export const addObservabilityTraceRoutes = (
|
|
|
88
129
|
scope: typeof req.query.scope === "string" ? req.query.scope : undefined,
|
|
89
130
|
sensitive: parseBoolean(req.query.sensitive),
|
|
90
131
|
sessionId: typeof req.query.sessionId === "string" ? req.query.sessionId : undefined,
|
|
132
|
+
sort: parseSort(req.query.sort),
|
|
91
133
|
status:
|
|
92
134
|
req.query.status === "ok" || req.query.status === "error" ? req.query.status : undefined,
|
|
93
135
|
to: typeof req.query.to === "string" ? req.query.to : undefined,
|
|
@@ -8,7 +8,7 @@ export type ObservabilityCapability =
|
|
|
8
8
|
| "scores"
|
|
9
9
|
| "traces";
|
|
10
10
|
|
|
11
|
-
export type ControlPrimary = "
|
|
11
|
+
export type ControlPrimary = "local";
|
|
12
12
|
|
|
13
13
|
export interface ObservabilityControlConfig {
|
|
14
14
|
datasets: ControlPrimary;
|
|
@@ -50,7 +50,7 @@ export interface TraceRecord {
|
|
|
50
50
|
metadata?: Record<string, unknown>;
|
|
51
51
|
name: string;
|
|
52
52
|
output?: unknown;
|
|
53
|
-
prompts: {label?: string; name: string; version: number}[];
|
|
53
|
+
prompts: {includedBy?: string; label?: string; name: string; version: number}[];
|
|
54
54
|
sensitive: boolean;
|
|
55
55
|
sessionId?: string;
|
|
56
56
|
spans: SpanRecord[];
|
|
@@ -78,7 +78,10 @@ export interface ScoreRecord {
|
|
|
78
78
|
}
|
|
79
79
|
|
|
80
80
|
export interface PromptVersionRef {
|
|
81
|
+
/** `body` with every prompt reference resolved. */
|
|
81
82
|
body: string;
|
|
83
|
+
/** Prompt versions pulled in by references, recorded on the trace after this prompt. */
|
|
84
|
+
includes?: {includedBy?: string; label?: string; name: string; version: number}[];
|
|
82
85
|
label?: string;
|
|
83
86
|
name: string;
|
|
84
87
|
sensitive?: boolean;
|
|
@@ -30,16 +30,7 @@ import {addGptHistoryRoutes} from "./gptHistories";
|
|
|
30
30
|
|
|
31
31
|
type SseResponse = supertest.Response & {body: string};
|
|
32
32
|
|
|
33
|
-
// Mock langfuseVercelAi to avoid transitive langfuse SDK import in tests.
|
|
34
|
-
// Spread the real module so later-loaded test files still see the real exports.
|
|
35
33
|
// The package compiles as CommonJS, so modules load with require instead of top-level await.
|
|
36
|
-
const realLangfuseVercelAi: typeof import("../langfuseVercelAi.js") = require("../langfuseVercelAi");
|
|
37
|
-
mock.module("../langfuseVercelAi", () => ({
|
|
38
|
-
...realLangfuseVercelAi,
|
|
39
|
-
createTelemetryConfig: () => ({functionId: "test", isEnabled: false}),
|
|
40
|
-
preparePromptForAI: async () => ({config: {}, prompt: "test", telemetry: {isEnabled: false}}),
|
|
41
|
-
}));
|
|
42
|
-
|
|
43
34
|
const realAi: typeof import("ai") = require("ai");
|
|
44
35
|
type StreamTextOptions = Parameters<typeof realAi.streamText>[0];
|
|
45
36
|
type StreamTextResult = ReturnType<typeof realAi.streamText>;
|
|
@@ -364,6 +355,107 @@ describe("AI Routes", () => {
|
|
|
364
355
|
assert.isTrue(sink.traces[0]?.sensitive);
|
|
365
356
|
});
|
|
366
357
|
|
|
358
|
+
it("prepends the route registry system prompt and records its version", async () => {
|
|
359
|
+
const sink = new MemoryTraceSink();
|
|
360
|
+
const lookups: Array<{label?: string; name: string}> = [];
|
|
361
|
+
const plugin: ObservabilityPlugin = {
|
|
362
|
+
capabilities: new Set([
|
|
363
|
+
"datasets",
|
|
364
|
+
"experiments",
|
|
365
|
+
"prompts",
|
|
366
|
+
"reviewQueue",
|
|
367
|
+
"scores",
|
|
368
|
+
"traces",
|
|
369
|
+
]),
|
|
370
|
+
datasetStore: {},
|
|
371
|
+
experimentRunner: {},
|
|
372
|
+
id: "local",
|
|
373
|
+
promptRegistry: {
|
|
374
|
+
get: async (args) => {
|
|
375
|
+
lookups.push(args);
|
|
376
|
+
return {
|
|
377
|
+
body: "Registry assistant prompt",
|
|
378
|
+
label: "staging",
|
|
379
|
+
name: "chat-assistant",
|
|
380
|
+
version: 2,
|
|
381
|
+
};
|
|
382
|
+
},
|
|
383
|
+
},
|
|
384
|
+
reviewQueue: {},
|
|
385
|
+
traceSink: sink,
|
|
386
|
+
};
|
|
387
|
+
new ObservabilityApp({plugins: [plugin]});
|
|
388
|
+
const model = createMockModel();
|
|
389
|
+
const registryApp = new TerrenoApp({
|
|
390
|
+
configureApp: (router, options) => {
|
|
391
|
+
addGptRoutes(router, {
|
|
392
|
+
aiService: new AIService({model: model as unknown as LanguageModel}),
|
|
393
|
+
openApiOptions: options,
|
|
394
|
+
systemPromptLabel: "staging",
|
|
395
|
+
systemPromptName: "chat-assistant",
|
|
396
|
+
});
|
|
397
|
+
},
|
|
398
|
+
skipListen: true,
|
|
399
|
+
userModel: terrenoUserModel,
|
|
400
|
+
}).build();
|
|
401
|
+
|
|
402
|
+
const agent = await authAsUser(registryApp, "notAdmin");
|
|
403
|
+
const res = await agent
|
|
404
|
+
.post("/gpt/prompt")
|
|
405
|
+
.send({prompt: "Hi", systemPrompt: "Caller context"})
|
|
406
|
+
.buffer(true)
|
|
407
|
+
.parse(sseCollect);
|
|
408
|
+
|
|
409
|
+
assert.equal(res.status, 200);
|
|
410
|
+
assert.deepEqual(lookups, [{label: "staging", name: "chat-assistant"}]);
|
|
411
|
+
const callOptions = (model.doStream.mock.calls[0] as unknown[] | undefined)?.[0] as {
|
|
412
|
+
prompt: Array<{content: unknown; role: string}>;
|
|
413
|
+
};
|
|
414
|
+
const system = callOptions.prompt.find((message) => message.role === "system");
|
|
415
|
+
assert.match(String(system?.content), /^Registry assistant prompt\n\nCaller context/);
|
|
416
|
+
assert.deepEqual(sink.traces[0]?.prompts, [
|
|
417
|
+
{label: "staging", name: "chat-assistant", version: 2},
|
|
418
|
+
]);
|
|
419
|
+
});
|
|
420
|
+
|
|
421
|
+
it("keeps chatting without a prompt when the route registry prompt is missing", async () => {
|
|
422
|
+
const sink = new MemoryTraceSink();
|
|
423
|
+
const plugin: ObservabilityPlugin = {
|
|
424
|
+
capabilities: new Set([
|
|
425
|
+
"datasets",
|
|
426
|
+
"experiments",
|
|
427
|
+
"prompts",
|
|
428
|
+
"reviewQueue",
|
|
429
|
+
"scores",
|
|
430
|
+
"traces",
|
|
431
|
+
]),
|
|
432
|
+
datasetStore: {},
|
|
433
|
+
experimentRunner: {},
|
|
434
|
+
id: "local",
|
|
435
|
+
promptRegistry: {get: async () => undefined},
|
|
436
|
+
reviewQueue: {},
|
|
437
|
+
traceSink: sink,
|
|
438
|
+
};
|
|
439
|
+
new ObservabilityApp({plugins: [plugin]});
|
|
440
|
+
const registryApp = new TerrenoApp({
|
|
441
|
+
configureApp: (router, options) => {
|
|
442
|
+
addGptRoutes(router, {aiService, openApiOptions: options, systemPromptName: "absent"});
|
|
443
|
+
},
|
|
444
|
+
skipListen: true,
|
|
445
|
+
userModel: terrenoUserModel,
|
|
446
|
+
}).build();
|
|
447
|
+
|
|
448
|
+
const agent = await authAsUser(registryApp, "notAdmin");
|
|
449
|
+
const res = await agent
|
|
450
|
+
.post("/gpt/prompt")
|
|
451
|
+
.send({prompt: "Hi"})
|
|
452
|
+
.buffer(true)
|
|
453
|
+
.parse(sseCollect);
|
|
454
|
+
|
|
455
|
+
assert.equal(res.status, 200);
|
|
456
|
+
assert.deepEqual(sink.traces[0]?.prompts, []);
|
|
457
|
+
});
|
|
458
|
+
|
|
367
459
|
it("emits a memory-sink trace with userId and sessionId", async () => {
|
|
368
460
|
const sink = new MemoryTraceSink();
|
|
369
461
|
const plugin: ObservabilityPlugin = {
|
|
@@ -809,27 +901,6 @@ describe("AI Routes", () => {
|
|
|
809
901
|
expect(res.status).toBe(200);
|
|
810
902
|
});
|
|
811
903
|
|
|
812
|
-
it("uses the langfuse system prompt when configured", async () => {
|
|
813
|
-
const customApp = new TerrenoApp({
|
|
814
|
-
configureApp: (router, options) => {
|
|
815
|
-
addGptRoutes(router, {
|
|
816
|
-
aiService,
|
|
817
|
-
langfuseSystemPromptName: "missing-prompt-name",
|
|
818
|
-
openApiOptions: options,
|
|
819
|
-
});
|
|
820
|
-
},
|
|
821
|
-
skipListen: true,
|
|
822
|
-
userModel: terrenoUserModel,
|
|
823
|
-
}).build();
|
|
824
|
-
const agent = await authAsUser(customApp, "notAdmin");
|
|
825
|
-
const res = await agent
|
|
826
|
-
.post("/gpt/prompt")
|
|
827
|
-
.send({prompt: "Hi"})
|
|
828
|
-
.buffer(true)
|
|
829
|
-
.parse(sseCollect);
|
|
830
|
-
expect(res.status).toBe(200);
|
|
831
|
-
});
|
|
832
|
-
|
|
833
904
|
it("forwards model tool-call and tool-result stream events via SSE", async () => {
|
|
834
905
|
const sink = new MemoryTraceSink();
|
|
835
906
|
const plugin: ObservabilityPlugin = {
|
|
@@ -2147,52 +2218,6 @@ describe("AI Routes", () => {
|
|
|
2147
2218
|
}
|
|
2148
2219
|
});
|
|
2149
2220
|
|
|
2150
|
-
it("swallows preparePromptForAI failure when langfuse prompt loading throws", async () => {
|
|
2151
|
-
// Re-mock preparePromptForAI to throw, exercising the catch branch that
|
|
2152
|
-
// logs `Langfuse system prompt skipped: ...` and continues without it.
|
|
2153
|
-
mock.module("../langfuseVercelAi", () => ({
|
|
2154
|
-
...realLangfuseVercelAi,
|
|
2155
|
-
createTelemetryConfig: () => ({functionId: "test", isEnabled: false}),
|
|
2156
|
-
preparePromptForAI: async () => {
|
|
2157
|
-
throw new Error("langfuse offline");
|
|
2158
|
-
},
|
|
2159
|
-
}));
|
|
2160
|
-
try {
|
|
2161
|
-
const customApp = new TerrenoApp({
|
|
2162
|
-
configureApp: (router, options) => {
|
|
2163
|
-
addGptRoutes(router, {
|
|
2164
|
-
aiService,
|
|
2165
|
-
langfuseSystemPromptName: "broken-prompt",
|
|
2166
|
-
openApiOptions: options,
|
|
2167
|
-
});
|
|
2168
|
-
},
|
|
2169
|
-
skipListen: true,
|
|
2170
|
-
userModel: terrenoUserModel,
|
|
2171
|
-
}).build();
|
|
2172
|
-
const agent = await authAsUser(customApp, "notAdmin");
|
|
2173
|
-
const res = await agent
|
|
2174
|
-
.post("/gpt/prompt")
|
|
2175
|
-
.send({prompt: "Hi"})
|
|
2176
|
-
.buffer(true)
|
|
2177
|
-
.parse(sseCollect);
|
|
2178
|
-
// Failure is silently caught; request still completes successfully.
|
|
2179
|
-
expect(res.status).toBe(200);
|
|
2180
|
-
const body = (res as SseResponse).body;
|
|
2181
|
-
expect(body).toContain("done");
|
|
2182
|
-
} finally {
|
|
2183
|
-
// Restore the original happy-path mock for subsequent tests.
|
|
2184
|
-
mock.module("../langfuseVercelAi", () => ({
|
|
2185
|
-
...realLangfuseVercelAi,
|
|
2186
|
-
createTelemetryConfig: () => ({functionId: "test", isEnabled: false}),
|
|
2187
|
-
preparePromptForAI: async () => ({
|
|
2188
|
-
config: {},
|
|
2189
|
-
prompt: "test",
|
|
2190
|
-
telemetry: {isEnabled: false},
|
|
2191
|
-
}),
|
|
2192
|
-
}));
|
|
2193
|
-
}
|
|
2194
|
-
});
|
|
2195
|
-
|
|
2196
2221
|
it("logs warning when AIRequest.logRequest throws inside the stream error catch", async () => {
|
|
2197
2222
|
// Combine a stream that errors mid-iteration with a logRequest that also throws,
|
|
2198
2223
|
// exercising the catch-within-catch that logs `Failed to log AIRequest error`.
|
|
@@ -1201,6 +1201,27 @@ describe("AIService", () => {
|
|
|
1201
1201
|
expect(sink.traces[0].prompts).toEqual([{label: "production", name: "greeter", version: 2}]);
|
|
1202
1202
|
});
|
|
1203
1203
|
|
|
1204
|
+
it("records a prompt's shared includes on the trace after the prompt itself", async () => {
|
|
1205
|
+
const sink = new MemoryTraceSink();
|
|
1206
|
+
registerLocalApp({
|
|
1207
|
+
promptVersion: {
|
|
1208
|
+
...productionVersion,
|
|
1209
|
+
includes: [{includedBy: "greeter", label: "production", name: "base-rules", version: 4}],
|
|
1210
|
+
},
|
|
1211
|
+
traceSink: sink,
|
|
1212
|
+
});
|
|
1213
|
+
const model = createMockModel("Hello world");
|
|
1214
|
+
const service = new AIService({model: model as unknown as LanguageModel});
|
|
1215
|
+
|
|
1216
|
+
await service.generateText({prompt: "User says hi", promptName: "greeter"});
|
|
1217
|
+
|
|
1218
|
+
expect(sink.traces[0].name).toBe("greeter");
|
|
1219
|
+
expect(sink.traces[0].prompts).toEqual([
|
|
1220
|
+
{label: "production", name: "greeter", version: 2},
|
|
1221
|
+
{includedBy: "greeter", label: "production", name: "base-rules", version: 4},
|
|
1222
|
+
]);
|
|
1223
|
+
});
|
|
1224
|
+
|
|
1204
1225
|
it("still writes AIRequest for a chat stream when a sink is registered", async () => {
|
|
1205
1226
|
const sink = new MemoryTraceSink();
|
|
1206
1227
|
registerLocalApp({traceSink: sink});
|
package/src/service/aiService.ts
CHANGED
|
@@ -397,7 +397,10 @@ export class AIService {
|
|
|
397
397
|
name: spanName,
|
|
398
398
|
output: params.response,
|
|
399
399
|
prompts: promptRef
|
|
400
|
-
? [
|
|
400
|
+
? [
|
|
401
|
+
{label: promptRef.label, name: promptRef.name, version: promptRef.version},
|
|
402
|
+
...(promptRef.includes ?? []),
|
|
403
|
+
]
|
|
401
404
|
: [],
|
|
402
405
|
sensitive: params.observability.sensitive,
|
|
403
406
|
sessionId: params.observability.sessionId,
|
package/src/service/chatTurn.ts
CHANGED
|
@@ -24,12 +24,12 @@ import type express from "express";
|
|
|
24
24
|
import {DateTime} from "luxon";
|
|
25
25
|
import type mongoose from "mongoose";
|
|
26
26
|
|
|
27
|
-
import {createTelemetryConfig, preparePromptForAI} from "../langfuseVercelAi";
|
|
28
27
|
import {AIRequest} from "../models/aiRequest";
|
|
29
28
|
import {FileAttachment} from "../models/fileAttachment";
|
|
30
29
|
import {GptHistory} from "../models/gptHistory";
|
|
31
30
|
import {getProjectModel} from "../models/project";
|
|
32
|
-
import
|
|
31
|
+
import {getObservabilityApp} from "../observability/observabilityAppRegistry";
|
|
32
|
+
import type {PromptVersionRef, SpanRecord} from "../observability/types";
|
|
33
33
|
import type {
|
|
34
34
|
AskOrigin,
|
|
35
35
|
GptHistoryAskStatus,
|
|
@@ -1145,17 +1145,39 @@ const startTurn = async ({
|
|
|
1145
1145
|
const lastUserText = (prompts: GptHistoryPrompt[]): string =>
|
|
1146
1146
|
prompts.findLast((prompt) => prompt.type === "user")?.text ?? "";
|
|
1147
1147
|
|
|
1148
|
-
/** The
|
|
1148
|
+
/** The route's default registry system prompt, or undefined when it cannot be loaded. */
|
|
1149
|
+
const loadRegistrySystemPrompt = async (
|
|
1150
|
+
options: GptRouteOptions
|
|
1151
|
+
): Promise<PromptVersionRef | undefined> => {
|
|
1152
|
+
const registry = getObservabilityApp()?.promptRegistry;
|
|
1153
|
+
if (!options.systemPromptName || !registry) {
|
|
1154
|
+
return undefined;
|
|
1155
|
+
}
|
|
1156
|
+
try {
|
|
1157
|
+
return await registry.get({
|
|
1158
|
+
label: options.systemPromptLabel ?? "production",
|
|
1159
|
+
name: options.systemPromptName,
|
|
1160
|
+
});
|
|
1161
|
+
} catch (error) {
|
|
1162
|
+
logger.warn("Registry system prompt skipped", {
|
|
1163
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1164
|
+
promptName: options.systemPromptName,
|
|
1165
|
+
});
|
|
1166
|
+
return undefined;
|
|
1167
|
+
}
|
|
1168
|
+
};
|
|
1169
|
+
|
|
1170
|
+
/** The request's system prompt with project context and the registry prompt prepended. */
|
|
1149
1171
|
const buildSystemPrompt = async ({
|
|
1150
1172
|
history,
|
|
1151
|
-
options,
|
|
1152
1173
|
projectId,
|
|
1174
|
+
registryPrompt,
|
|
1153
1175
|
systemPrompt,
|
|
1154
1176
|
userId,
|
|
1155
1177
|
}: {
|
|
1156
1178
|
history: GptHistoryDocument;
|
|
1157
|
-
options: GptRouteOptions;
|
|
1158
1179
|
projectId: unknown;
|
|
1180
|
+
registryPrompt?: string;
|
|
1159
1181
|
systemPrompt: unknown;
|
|
1160
1182
|
userId: mongoose.Types.ObjectId | undefined;
|
|
1161
1183
|
}): Promise<string | undefined> => {
|
|
@@ -1186,22 +1208,10 @@ const buildSystemPrompt = async ({
|
|
|
1186
1208
|
}
|
|
1187
1209
|
}
|
|
1188
1210
|
|
|
1189
|
-
if (
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
userId: userId?.toString(),
|
|
1194
|
-
});
|
|
1195
|
-
const langfusePrompt =
|
|
1196
|
-
typeof langfuseResult.prompt === "string" ? langfuseResult.prompt : undefined;
|
|
1197
|
-
if (langfusePrompt) {
|
|
1198
|
-
effectiveSystemPrompt = effectiveSystemPrompt
|
|
1199
|
-
? `${langfusePrompt}\n\n${effectiveSystemPrompt}`
|
|
1200
|
-
: langfusePrompt;
|
|
1201
|
-
}
|
|
1202
|
-
} catch (err) {
|
|
1203
|
-
logger.debug(`Langfuse system prompt skipped: ${(err as Error).message}`);
|
|
1204
|
-
}
|
|
1211
|
+
if (registryPrompt) {
|
|
1212
|
+
effectiveSystemPrompt = effectiveSystemPrompt
|
|
1213
|
+
? `${registryPrompt}\n\n${effectiveSystemPrompt}`
|
|
1214
|
+
: registryPrompt;
|
|
1205
1215
|
}
|
|
1206
1216
|
return effectiveSystemPrompt;
|
|
1207
1217
|
};
|
|
@@ -1783,20 +1793,29 @@ export const runChatTurn = async ({
|
|
|
1783
1793
|
const {answer, isNewHistory, logPrompt, messages, replayedMessages, resolvedAsk, titlePrompt} =
|
|
1784
1794
|
turn;
|
|
1785
1795
|
let {history} = turn;
|
|
1796
|
+
// An admin-selected promptName replaces the system prompt, so the route default is skipped.
|
|
1797
|
+
const registryPrompt = promptName ? undefined : await loadRegistrySystemPrompt(options);
|
|
1786
1798
|
let effectiveSystemPrompt = await buildSystemPrompt({
|
|
1787
1799
|
history,
|
|
1788
|
-
options,
|
|
1789
1800
|
projectId,
|
|
1801
|
+
registryPrompt: registryPrompt?.body,
|
|
1790
1802
|
systemPrompt,
|
|
1791
1803
|
userId,
|
|
1792
1804
|
});
|
|
1793
|
-
const
|
|
1805
|
+
const resolvedObservability = await aiService.resolveGenerateObservability({
|
|
1794
1806
|
promptLabel,
|
|
1795
1807
|
promptName,
|
|
1796
1808
|
sensitive: body.sensitive === true ? true : undefined,
|
|
1797
1809
|
sessionId,
|
|
1798
1810
|
systemPrompt: effectiveSystemPrompt,
|
|
1799
1811
|
});
|
|
1812
|
+
const observability = registryPrompt
|
|
1813
|
+
? {
|
|
1814
|
+
...resolvedObservability,
|
|
1815
|
+
promptRef: registryPrompt,
|
|
1816
|
+
sensitive: resolvedObservability.sensitive || Boolean(registryPrompt.sensitive),
|
|
1817
|
+
}
|
|
1818
|
+
: resolvedObservability;
|
|
1800
1819
|
effectiveSystemPrompt = observability.systemPrompt;
|
|
1801
1820
|
|
|
1802
1821
|
// Some models (e.g. gemini-2.5-flash-image) don't support tool calling
|
|
@@ -2042,17 +2061,7 @@ export const runChatTurn = async ({
|
|
|
2042
2061
|
sink.emit({askResolved: {action: resolvedAsk.action, toolCallId: resolvedAsk.toolCallId}});
|
|
2043
2062
|
}
|
|
2044
2063
|
logger.debug("Starting streamText", {model: modelId, supportsTools});
|
|
2045
|
-
const telemetry = createTelemetryConfig({
|
|
2046
|
-
functionId: "gpt-prompt",
|
|
2047
|
-
metadata: {
|
|
2048
|
-
...(options.langfuseSystemPromptName
|
|
2049
|
-
? {langfusePromptName: options.langfuseSystemPromptName}
|
|
2050
|
-
: {}),
|
|
2051
|
-
},
|
|
2052
|
-
userId: userId?.toString(),
|
|
2053
|
-
});
|
|
2054
2064
|
const result = streamText({
|
|
2055
|
-
experimental_telemetry: telemetry,
|
|
2056
2065
|
messages,
|
|
2057
2066
|
model: aiService.model,
|
|
2058
2067
|
providerOptions: !supportsTools
|
package/src/tests/bunSetup.ts
CHANGED
|
@@ -6,61 +6,7 @@ import {registerBackendPreload} from "@terreno/test";
|
|
|
6
6
|
registerBackendPreload({
|
|
7
7
|
connectMongoInBeforeAll: true,
|
|
8
8
|
mongo: {baseDatabaseName: "terreno-ai-test", useReplSet: true},
|
|
9
|
-
onBeforeEach: async () => {
|
|
10
|
-
const {shutdownLangfuseClient} = await import("../langfuseClient.js");
|
|
11
|
-
await shutdownLangfuseClient();
|
|
12
|
-
},
|
|
13
9
|
testEnv: {
|
|
14
10
|
tokenIssuer: "terreno-ai.test",
|
|
15
11
|
},
|
|
16
12
|
});
|
|
17
|
-
|
|
18
|
-
// Mock @langfuse/client globally so the real `./langfuseClient` module runs
|
|
19
|
-
// without making network calls.
|
|
20
|
-
import {mock} from "bun:test";
|
|
21
|
-
|
|
22
|
-
mock.module("@langfuse/client", () => {
|
|
23
|
-
return {
|
|
24
|
-
LangfuseClient: class FakeLangfuseClient {
|
|
25
|
-
baseUrl: string;
|
|
26
|
-
publicKey: string;
|
|
27
|
-
secretKey: string;
|
|
28
|
-
api = {
|
|
29
|
-
prompts: {
|
|
30
|
-
list: mock(async () => ({
|
|
31
|
-
data: [],
|
|
32
|
-
meta: {limit: 20, page: 1, total: 0, totalPages: 0},
|
|
33
|
-
})),
|
|
34
|
-
},
|
|
35
|
-
trace: {
|
|
36
|
-
get: mock(async (id: string) => ({id, name: "Trace"})),
|
|
37
|
-
list: mock(async () => ({
|
|
38
|
-
data: [],
|
|
39
|
-
meta: {limit: 20, page: 1, total: 0, totalPages: 0},
|
|
40
|
-
})),
|
|
41
|
-
},
|
|
42
|
-
};
|
|
43
|
-
prompt = {
|
|
44
|
-
create: mock(async (_params: Record<string, unknown>) => undefined),
|
|
45
|
-
get: mock(async (name: string) => ({
|
|
46
|
-
config: {},
|
|
47
|
-
labels: [],
|
|
48
|
-
name,
|
|
49
|
-
prompt: "",
|
|
50
|
-
tags: [],
|
|
51
|
-
type: "text" as const,
|
|
52
|
-
version: 1,
|
|
53
|
-
})),
|
|
54
|
-
};
|
|
55
|
-
score = {create: mock(() => {})};
|
|
56
|
-
flush = mock(async () => {});
|
|
57
|
-
shutdown = mock(async () => {});
|
|
58
|
-
|
|
59
|
-
constructor(opts: {baseUrl: string; publicKey: string; secretKey: string}) {
|
|
60
|
-
this.baseUrl = opts.baseUrl;
|
|
61
|
-
this.publicKey = opts.publicKey;
|
|
62
|
-
this.secretKey = opts.secretKey;
|
|
63
|
-
}
|
|
64
|
-
},
|
|
65
|
-
};
|
|
66
|
-
});
|
package/src/types/harness.ts
CHANGED
|
@@ -8,9 +8,12 @@ import type {ObsPromptRef} from "./observability";
|
|
|
8
8
|
|
|
9
9
|
/**
|
|
10
10
|
* A prompt version a task used, recorded on its `ObsTrace.prompts`. A `PromptVersionRef`
|
|
11
|
-
* from `promptRegistry.get` fits as-is; its `body` is ignored
|
|
11
|
+
* from `promptRegistry.get` fits as-is; its `body` is ignored and its `includes` (shared
|
|
12
|
+
* prompts it referenced) are recorded right after it.
|
|
12
13
|
*/
|
|
13
|
-
export
|
|
14
|
+
export interface HarnessPromptRef extends ObsPromptRef {
|
|
15
|
+
includes?: ObsPromptRef[];
|
|
16
|
+
}
|
|
14
17
|
|
|
15
18
|
/** A registry prompt passed as instructions: its `body` is used and its version recorded. */
|
|
16
19
|
export interface HarnessPromptInstructions extends HarnessPromptRef {
|
package/src/types/index.ts
CHANGED
|
@@ -452,9 +452,14 @@ export interface GptRouteOptions {
|
|
|
452
452
|
* `POST /gpt/actions`. `{repair: true}` runs one repair call when that check fails.
|
|
453
453
|
*/
|
|
454
454
|
uiBlocks?: boolean | UiBlocksOptions;
|
|
455
|
-
/**
|
|
456
|
-
*
|
|
457
|
-
|
|
455
|
+
/**
|
|
456
|
+
* Observability prompt registry name used as the default system prompt (prepended to the
|
|
457
|
+
* request's system prompt). The trace records the version. Skipped when no registry is registered, the
|
|
458
|
+
* label has no version, or an admin request selects `promptName`.
|
|
459
|
+
*/
|
|
460
|
+
systemPromptName?: string;
|
|
461
|
+
/** Registry label read for `systemPromptName`. Defaults to `production`. */
|
|
462
|
+
systemPromptLabel?: string;
|
|
458
463
|
/**
|
|
459
464
|
* When `false` or the function returns `false`, prompts that include attachments are rejected.
|
|
460
465
|
* Omit or pass `true` to leave uploads enabled.
|