@terreno/ai 57.21.0 → 57.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. package/CHANGELOG.md +31 -0
  2. package/README.md +3 -3
  3. package/biome.jsonc +2 -2
  4. package/dist/harness/commit.d.ts +4 -3
  5. package/dist/harness/commit.js +23 -4
  6. package/dist/index.d.ts +6 -9
  7. package/dist/index.js +14 -25
  8. package/dist/observability/adminScreens.js +5 -1
  9. package/dist/observability/evaluate.d.ts +8 -1
  10. package/dist/observability/evaluate.js +10 -2
  11. package/dist/observability/judgeAgreement.d.ts +53 -0
  12. package/dist/observability/judgeAgreement.js +123 -0
  13. package/dist/observability/local/datasetStore.d.ts +5 -0
  14. package/dist/observability/local/datasetStore.js +7 -2
  15. package/dist/observability/local/evaluatorStore.d.ts +17 -0
  16. package/dist/observability/local/evaluatorStore.js +73 -2
  17. package/dist/observability/local/experimentRunner.d.ts +40 -3
  18. package/dist/observability/local/experimentRunner.js +390 -11
  19. package/dist/observability/local/humanAnswers.d.ts +57 -0
  20. package/dist/observability/local/humanAnswers.js +174 -0
  21. package/dist/observability/local/judgeRun.d.ts +56 -0
  22. package/dist/observability/local/judgeRun.js +111 -0
  23. package/dist/observability/local/localPlugin.js +4 -0
  24. package/dist/observability/local/models/obsDatasetItem.js +4 -0
  25. package/dist/observability/local/models/obsEvaluator.js +4 -0
  26. package/dist/observability/local/models/obsExperiment.js +28 -0
  27. package/dist/observability/local/models/obsExperimentItem.js +4 -0
  28. package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
  29. package/dist/observability/local/models/obsJudgeRun.js +60 -0
  30. package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
  31. package/dist/observability/local/models/obsReviewBatch.js +48 -0
  32. package/dist/observability/local/models/obsReviewItem.js +10 -2
  33. package/dist/observability/local/models/obsTrace.js +4 -0
  34. package/dist/observability/local/promptReferences.d.ts +31 -0
  35. package/dist/observability/local/promptReferences.js +114 -0
  36. package/dist/observability/local/promptStore.d.ts +55 -19
  37. package/dist/observability/local/promptStore.js +171 -7
  38. package/dist/observability/local/reviewBatchStore.d.ts +106 -0
  39. package/dist/observability/local/reviewBatchStore.js +517 -0
  40. package/dist/observability/local/reviewStore.d.ts +24 -3
  41. package/dist/observability/local/reviewStore.js +98 -22
  42. package/dist/observability/local/traceStore.d.ts +14 -1
  43. package/dist/observability/local/traceStore.js +144 -11
  44. package/dist/observability/observabilityApp.js +10 -1
  45. package/dist/observability/questionFingerprint.d.ts +7 -0
  46. package/dist/observability/questionFingerprint.js +23 -0
  47. package/dist/observability/routes/experiments.js +12 -1
  48. package/dist/observability/routes/review.js +3 -1
  49. package/dist/observability/routes/reviewBatches.d.ts +11 -0
  50. package/dist/observability/routes/reviewBatches.js +78 -0
  51. package/dist/observability/routes/traces.d.ts +1 -1
  52. package/dist/observability/routes/traces.js +34 -2
  53. package/dist/observability/types.d.ts +10 -1
  54. package/dist/service/aiService.js +4 -1
  55. package/dist/service/chatTurn.js +38 -31
  56. package/dist/types/harness.d.ts +5 -2
  57. package/dist/types/index.d.ts +8 -3
  58. package/dist/types/observability.d.ts +74 -2
  59. package/package.json +8 -24
  60. package/src/harness/agentLoop.test.ts +0 -3
  61. package/src/harness/agentRecovery.test.ts +0 -3
  62. package/src/harness/commit.ts +24 -4
  63. package/src/harness/extensions.test.ts +0 -3
  64. package/src/harness/subagent.test.ts +27 -3
  65. package/src/index.test.ts +0 -17
  66. package/src/index.ts +23 -26
  67. package/src/isolated/gptStream.isolated.ts +0 -8
  68. package/src/observability/adminScreens.test.ts +7 -0
  69. package/src/observability/adminScreens.ts +6 -1
  70. package/src/observability/evaluate.ts +12 -2
  71. package/src/observability/judgeAgreement.test.ts +111 -0
  72. package/src/observability/judgeAgreement.ts +184 -0
  73. package/src/observability/local/datasetStore.ts +30 -11
  74. package/src/observability/local/evaluatorStore.ts +96 -2
  75. package/src/observability/local/experimentRunner.ts +495 -15
  76. package/src/observability/local/humanAnswers.ts +233 -0
  77. package/src/observability/local/judgePipeline.test.ts +444 -0
  78. package/src/observability/local/judgeRun.test.ts +173 -0
  79. package/src/observability/local/judgeRun.ts +184 -0
  80. package/src/observability/local/localPlugin.ts +4 -0
  81. package/src/observability/local/models/obsDatasetItem.ts +5 -0
  82. package/src/observability/local/models/obsEvaluator.ts +4 -0
  83. package/src/observability/local/models/obsExperiment.ts +29 -0
  84. package/src/observability/local/models/obsExperimentItem.ts +4 -0
  85. package/src/observability/local/models/obsJudgeRun.ts +61 -0
  86. package/src/observability/local/models/obsReviewBatch.ts +52 -0
  87. package/src/observability/local/models/obsReviewItem.ts +11 -2
  88. package/src/observability/local/models/obsTrace.ts +5 -0
  89. package/src/observability/local/obsModels.test.ts +13 -0
  90. package/src/observability/local/promptReferences.test.ts +128 -0
  91. package/src/observability/local/promptReferences.ts +149 -0
  92. package/src/observability/local/promptStore.test.ts +291 -0
  93. package/src/observability/local/promptStore.ts +243 -32
  94. package/src/observability/local/reviewBatchStore.test.ts +251 -0
  95. package/src/observability/local/reviewBatchStore.ts +656 -0
  96. package/src/observability/local/reviewStore.ts +122 -25
  97. package/src/observability/local/traceStore.test.ts +139 -1
  98. package/src/observability/local/traceStore.ts +162 -11
  99. package/src/observability/observabilityApp.test.ts +13 -10
  100. package/src/observability/observabilityApp.ts +10 -1
  101. package/src/observability/questionFingerprint.test.ts +33 -0
  102. package/src/observability/questionFingerprint.ts +24 -0
  103. package/src/observability/routes/experiments.test.ts +37 -0
  104. package/src/observability/routes/experiments.ts +13 -1
  105. package/src/observability/routes/review.ts +3 -1
  106. package/src/observability/routes/reviewBatches.test.ts +209 -0
  107. package/src/observability/routes/reviewBatches.ts +138 -0
  108. package/src/observability/routes/status.test.ts +4 -4
  109. package/src/observability/routes/traces.test.ts +61 -0
  110. package/src/observability/routes/traces.ts +46 -4
  111. package/src/observability/types.ts +5 -2
  112. package/src/routes/routes.test.ts +101 -76
  113. package/src/service/aiService.test.ts +21 -0
  114. package/src/service/aiService.ts +4 -1
  115. package/src/service/chatTurn.ts +42 -33
  116. package/src/tests/bunSetup.ts +0 -54
  117. package/src/types/harness.ts +5 -2
  118. package/src/types/index.ts +8 -3
  119. package/src/types/observability.ts +86 -2
  120. package/dist/langfuseApp.d.ts +0 -9
  121. package/dist/langfuseApp.js +0 -53
  122. package/dist/langfuseCache.d.ts +0 -15
  123. package/dist/langfuseCache.js +0 -54
  124. package/dist/langfuseClient.d.ts +0 -6
  125. package/dist/langfuseClient.js +0 -34
  126. package/dist/langfusePrompts.d.ts +0 -12
  127. package/dist/langfusePrompts.js +0 -89
  128. package/dist/langfuseRoutesEvaluations.d.ts +0 -3
  129. package/dist/langfuseRoutesEvaluations.js +0 -32
  130. package/dist/langfuseRoutesMiddleware.d.ts +0 -2
  131. package/dist/langfuseRoutesMiddleware.js +0 -13
  132. package/dist/langfuseRoutesPlayground.d.ts +0 -2
  133. package/dist/langfuseRoutesPlayground.js +0 -36
  134. package/dist/langfuseRoutesPrompts.d.ts +0 -2
  135. package/dist/langfuseRoutesPrompts.js +0 -56
  136. package/dist/langfuseRoutesTraces.d.ts +0 -2
  137. package/dist/langfuseRoutesTraces.js +0 -30
  138. package/dist/langfuseTracing.d.ts +0 -4
  139. package/dist/langfuseTracing.js +0 -32
  140. package/dist/langfuseTypes.d.ts +0 -96
  141. package/dist/langfuseTypes.js +0 -2
  142. package/dist/langfuseVercelAi.d.ts +0 -12
  143. package/dist/langfuseVercelAi.js +0 -49
  144. package/src/isolated/langfuseClient.isolated.ts +0 -66
  145. package/src/langfuseApp.test.ts +0 -119
  146. package/src/langfuseApp.ts +0 -62
  147. package/src/langfuseCache.test.ts +0 -75
  148. package/src/langfuseCache.ts +0 -72
  149. package/src/langfuseClient.test.ts +0 -69
  150. package/src/langfuseClient.ts +0 -34
  151. package/src/langfusePrompts.test.ts +0 -188
  152. package/src/langfusePrompts.ts +0 -121
  153. package/src/langfuseRoutes.test.ts +0 -261
  154. package/src/langfuseRoutesEvaluations.ts +0 -48
  155. package/src/langfuseRoutesMiddleware.test.ts +0 -57
  156. package/src/langfuseRoutesMiddleware.ts +0 -12
  157. package/src/langfuseRoutesPlayground.ts +0 -50
  158. package/src/langfuseRoutesPrompts.ts +0 -87
  159. package/src/langfuseRoutesTraces.ts +0 -40
  160. package/src/langfuseTracing.test.ts +0 -72
  161. package/src/langfuseTracing.ts +0 -33
  162. package/src/langfuseTypes.ts +0 -103
  163. package/src/langfuseVercelAi.test.ts +0 -82
  164. package/src/langfuseVercelAi.ts +0 -72
  165. package/src/tests/fakeLangfusePrompt.ts +0 -10
@@ -109,6 +109,67 @@ describe("observability trace routes", () => {
109
109
  ).toEqual([[inScope.id, "ws-1"]]);
110
110
  });
111
111
 
112
+ it("sorts the trace list across pages and rejects unknown sort keys", async () => {
113
+ const base = DateTime.utc().minus({hours: 1});
114
+ const exportAt = async (
115
+ name: string,
116
+ offsetMs: number,
117
+ latencyMs: number,
118
+ costUsd: number,
119
+ inputTokens: number
120
+ ): Promise<string> => {
121
+ const exported = await store.exportTrace({
122
+ endedAt: base.plus({milliseconds: offsetMs + latencyMs}).toISO() ?? "",
123
+ id: name,
124
+ name,
125
+ prompts: [{name, version: 1}],
126
+ sensitive: false,
127
+ spans: [],
128
+ startedAt: base.plus({milliseconds: offsetMs}).toISO() ?? "",
129
+ status: "ok",
130
+ usage: {costUsd, inputTokens, outputTokens: 0},
131
+ });
132
+ return exported.id;
133
+ };
134
+ const cheapSlow = await exportAt("cheap-slow", 0, 900, 0.0001, 500);
135
+ const priceyFast = await exportAt("pricey-fast", 1000, 50, 0.05, 20);
136
+ const midMid = await exportAt("mid-mid", 2000, 300, 0.001, 12_345);
137
+ const admin = await authAsUser(app, "admin");
138
+
139
+ const pageIds = async (query: string): Promise<string[]> => {
140
+ const first = await admin.get(`/ai/observability/traces?${query}&limit=2&page=1`);
141
+ const second = await admin.get(`/ai/observability/traces?${query}&limit=2&page=2`);
142
+ expect(first.status).toBe(200);
143
+ expect(first.body.more).toBe(true);
144
+ expect(second.body.more).toBe(false);
145
+ return [...first.body.data, ...second.body.data].map((row: {id: string}) => row.id);
146
+ };
147
+
148
+ expect(await pageIds("sort=-cost")).toEqual([priceyFast, midMid, cheapSlow]);
149
+ expect(await pageIds("sort=cost")).toEqual([cheapSlow, midMid, priceyFast]);
150
+ expect(await pageIds("sort=-latency")).toEqual([cheapSlow, midMid, priceyFast]);
151
+ expect(await pageIds("sort=latency")).toEqual([priceyFast, midMid, cheapSlow]);
152
+ expect(await pageIds("sort=-tokens")).toEqual([midMid, cheapSlow, priceyFast]);
153
+ expect(await pageIds("sort=tokens")).toEqual([priceyFast, cheapSlow, midMid]);
154
+ expect(await pageIds("sort=startedAt")).toEqual([cheapSlow, priceyFast, midMid]);
155
+ expect(await pageIds("sort=-name")).toEqual([priceyFast, midMid, cheapSlow]);
156
+ // Default stays newest first.
157
+ expect(await pageIds("status=ok")).toEqual([midMid, priceyFast, cheapSlow]);
158
+
159
+ const multiPrompt = await admin.get(
160
+ "/ai/observability/traces?prompt[0]=cheap-slow&prompt[1]=mid-mid&sort=name"
161
+ );
162
+ expect(multiPrompt.status).toBe(200);
163
+ expect(multiPrompt.body.data.map((row: {id: string}) => row.id)).toEqual([cheapSlow, midMid]);
164
+
165
+ const unknown = await admin.get("/ai/observability/traces?sort=created");
166
+ expect(unknown.status).toBe(400);
167
+ const listSort = await admin.get("/ai/observability/traces?sort[0]=cost");
168
+ expect(listSort.status).toBe(400);
169
+ const objectPrompt = await admin.get("/ai/observability/traces?prompt[name]=x");
170
+ expect(objectPrompt.status).toBe(400);
171
+ });
172
+
112
173
  it("forbids non-admins from listing traces", async () => {
113
174
  const agent = await authAsUser(app, "notAdmin");
114
175
  const res = await agent.get("/ai/observability/traces");
@@ -1,6 +1,11 @@
1
1
  import {APIError, asyncHandler, createOpenApiBuilder} from "@terreno/api";
2
2
  import type express from "express";
3
- import type {LocalTraceStore} from "../local/traceStore";
3
+ import {
4
+ DEFAULT_TRACE_LIST_SORT,
5
+ type LocalTraceStore,
6
+ parseTraceListSort,
7
+ TRACE_LIST_SORT_KEYS,
8
+ } from "../local/traceStore";
4
9
  import {getObservabilityApp} from "../observabilityAppRegistry";
5
10
  import type {ScoreRecord, ScoreSink} from "../types";
6
11
  import {
@@ -27,7 +32,7 @@ const parseBoolean = (value: unknown): boolean | undefined => {
27
32
  throw new APIError({status: 400, title: "boolean query must be true or false"});
28
33
  };
29
34
 
30
- const parsePromptVersion = (value: unknown, prompt?: string): number | undefined => {
35
+ const parsePromptVersion = (value: unknown, prompt?: string | string[]): number | undefined => {
31
36
  if (value === undefined) {
32
37
  return undefined;
33
38
  }
@@ -35,12 +40,37 @@ const parsePromptVersion = (value: unknown, prompt?: string): number | undefined
35
40
  if (!Number.isInteger(parsed) || parsed < 1) {
36
41
  throw new APIError({status: 400, title: "promptVersion must be a positive integer"});
37
42
  }
38
- if (!prompt) {
43
+ if (!prompt || prompt.length === 0) {
39
44
  throw new APIError({status: 400, title: "prompt is required when promptVersion is set"});
40
45
  }
41
46
  return parsed;
42
47
  };
43
48
 
49
+ const parsePrompt = (value: unknown): string | string[] | undefined => {
50
+ if (typeof value === "string") {
51
+ return value;
52
+ }
53
+ if (Array.isArray(value) && value.every((entry) => typeof entry === "string")) {
54
+ return value as string[];
55
+ }
56
+ if (value === undefined) {
57
+ return undefined;
58
+ }
59
+ throw new APIError({status: 400, title: "prompt must be a string or a list of strings"});
60
+ };
61
+
62
+ const parseSort = (value: unknown): string | undefined => {
63
+ if (value === undefined) {
64
+ return undefined;
65
+ }
66
+ if (typeof value !== "string") {
67
+ throw new APIError({status: 400, title: "Unknown trace sort key"});
68
+ }
69
+ // Validate here so a bad key fails before the store runs any query.
70
+ parseTraceListSort(value);
71
+ return value;
72
+ };
73
+
44
74
  const parsePositiveInt = (value: unknown, fallback: number): number => {
45
75
  if (value === undefined) {
46
76
  return fallback;
@@ -72,11 +102,22 @@ export const addObservabilityTraceRoutes = (
72
102
  .withQueryParameter("prompt", {type: "string"}, {required: false})
73
103
  .withQueryParameter("promptVersion", {type: "number"}, {required: false})
74
104
  .withQueryParameter("scope", {type: "string"}, {required: false})
105
+ .withQueryParameter(
106
+ "sort",
107
+ {
108
+ enum: TRACE_LIST_SORT_KEYS.flatMap((key) => [key, `-${key}`]),
109
+ type: "string",
110
+ },
111
+ {
112
+ description: `Sort column, \`-\` prefix for descending. Defaults to \`${DEFAULT_TRACE_LIST_SORT}\`.`,
113
+ required: false,
114
+ }
115
+ )
75
116
  .withResponse(200, {data: {type: "array"}})
76
117
  .build()
77
118
  ),
78
119
  asyncHandler(async (req, res) => {
79
- const prompt = typeof req.query.prompt === "string" ? req.query.prompt : undefined;
120
+ const prompt = parsePrompt(req.query.prompt);
80
121
  const listed = await options.store.list({
81
122
  flaggedForDataset: parseBoolean(req.query.flaggedForDataset),
82
123
  from: typeof req.query.from === "string" ? req.query.from : undefined,
@@ -88,6 +129,7 @@ export const addObservabilityTraceRoutes = (
88
129
  scope: typeof req.query.scope === "string" ? req.query.scope : undefined,
89
130
  sensitive: parseBoolean(req.query.sensitive),
90
131
  sessionId: typeof req.query.sessionId === "string" ? req.query.sessionId : undefined,
132
+ sort: parseSort(req.query.sort),
91
133
  status:
92
134
  req.query.status === "ok" || req.query.status === "error" ? req.query.status : undefined,
93
135
  to: typeof req.query.to === "string" ? req.query.to : undefined,
@@ -8,7 +8,7 @@ export type ObservabilityCapability =
8
8
  | "scores"
9
9
  | "traces";
10
10
 
11
- export type ControlPrimary = "langfuse" | "local";
11
+ export type ControlPrimary = "local";
12
12
 
13
13
  export interface ObservabilityControlConfig {
14
14
  datasets: ControlPrimary;
@@ -50,7 +50,7 @@ export interface TraceRecord {
50
50
  metadata?: Record<string, unknown>;
51
51
  name: string;
52
52
  output?: unknown;
53
- prompts: {label?: string; name: string; version: number}[];
53
+ prompts: {includedBy?: string; label?: string; name: string; version: number}[];
54
54
  sensitive: boolean;
55
55
  sessionId?: string;
56
56
  spans: SpanRecord[];
@@ -78,7 +78,10 @@ export interface ScoreRecord {
78
78
  }
79
79
 
80
80
  export interface PromptVersionRef {
81
+ /** `body` with every prompt reference resolved. */
81
82
  body: string;
83
+ /** Prompt versions pulled in by references, recorded on the trace after this prompt. */
84
+ includes?: {includedBy?: string; label?: string; name: string; version: number}[];
82
85
  label?: string;
83
86
  name: string;
84
87
  sensitive?: boolean;
@@ -30,16 +30,7 @@ import {addGptHistoryRoutes} from "./gptHistories";
30
30
 
31
31
  type SseResponse = supertest.Response & {body: string};
32
32
 
33
- // Mock langfuseVercelAi to avoid transitive langfuse SDK import in tests.
34
- // Spread the real module so later-loaded test files still see the real exports.
35
33
  // The package compiles as CommonJS, so modules load with require instead of top-level await.
36
- const realLangfuseVercelAi: typeof import("../langfuseVercelAi.js") = require("../langfuseVercelAi");
37
- mock.module("../langfuseVercelAi", () => ({
38
- ...realLangfuseVercelAi,
39
- createTelemetryConfig: () => ({functionId: "test", isEnabled: false}),
40
- preparePromptForAI: async () => ({config: {}, prompt: "test", telemetry: {isEnabled: false}}),
41
- }));
42
-
43
34
  const realAi: typeof import("ai") = require("ai");
44
35
  type StreamTextOptions = Parameters<typeof realAi.streamText>[0];
45
36
  type StreamTextResult = ReturnType<typeof realAi.streamText>;
@@ -364,6 +355,107 @@ describe("AI Routes", () => {
364
355
  assert.isTrue(sink.traces[0]?.sensitive);
365
356
  });
366
357
 
358
+ it("prepends the route registry system prompt and records its version", async () => {
359
+ const sink = new MemoryTraceSink();
360
+ const lookups: Array<{label?: string; name: string}> = [];
361
+ const plugin: ObservabilityPlugin = {
362
+ capabilities: new Set([
363
+ "datasets",
364
+ "experiments",
365
+ "prompts",
366
+ "reviewQueue",
367
+ "scores",
368
+ "traces",
369
+ ]),
370
+ datasetStore: {},
371
+ experimentRunner: {},
372
+ id: "local",
373
+ promptRegistry: {
374
+ get: async (args) => {
375
+ lookups.push(args);
376
+ return {
377
+ body: "Registry assistant prompt",
378
+ label: "staging",
379
+ name: "chat-assistant",
380
+ version: 2,
381
+ };
382
+ },
383
+ },
384
+ reviewQueue: {},
385
+ traceSink: sink,
386
+ };
387
+ new ObservabilityApp({plugins: [plugin]});
388
+ const model = createMockModel();
389
+ const registryApp = new TerrenoApp({
390
+ configureApp: (router, options) => {
391
+ addGptRoutes(router, {
392
+ aiService: new AIService({model: model as unknown as LanguageModel}),
393
+ openApiOptions: options,
394
+ systemPromptLabel: "staging",
395
+ systemPromptName: "chat-assistant",
396
+ });
397
+ },
398
+ skipListen: true,
399
+ userModel: terrenoUserModel,
400
+ }).build();
401
+
402
+ const agent = await authAsUser(registryApp, "notAdmin");
403
+ const res = await agent
404
+ .post("/gpt/prompt")
405
+ .send({prompt: "Hi", systemPrompt: "Caller context"})
406
+ .buffer(true)
407
+ .parse(sseCollect);
408
+
409
+ assert.equal(res.status, 200);
410
+ assert.deepEqual(lookups, [{label: "staging", name: "chat-assistant"}]);
411
+ const callOptions = (model.doStream.mock.calls[0] as unknown[] | undefined)?.[0] as {
412
+ prompt: Array<{content: unknown; role: string}>;
413
+ };
414
+ const system = callOptions.prompt.find((message) => message.role === "system");
415
+ assert.match(String(system?.content), /^Registry assistant prompt\n\nCaller context/);
416
+ assert.deepEqual(sink.traces[0]?.prompts, [
417
+ {label: "staging", name: "chat-assistant", version: 2},
418
+ ]);
419
+ });
420
+
421
+ it("keeps chatting without a prompt when the route registry prompt is missing", async () => {
422
+ const sink = new MemoryTraceSink();
423
+ const plugin: ObservabilityPlugin = {
424
+ capabilities: new Set([
425
+ "datasets",
426
+ "experiments",
427
+ "prompts",
428
+ "reviewQueue",
429
+ "scores",
430
+ "traces",
431
+ ]),
432
+ datasetStore: {},
433
+ experimentRunner: {},
434
+ id: "local",
435
+ promptRegistry: {get: async () => undefined},
436
+ reviewQueue: {},
437
+ traceSink: sink,
438
+ };
439
+ new ObservabilityApp({plugins: [plugin]});
440
+ const registryApp = new TerrenoApp({
441
+ configureApp: (router, options) => {
442
+ addGptRoutes(router, {aiService, openApiOptions: options, systemPromptName: "absent"});
443
+ },
444
+ skipListen: true,
445
+ userModel: terrenoUserModel,
446
+ }).build();
447
+
448
+ const agent = await authAsUser(registryApp, "notAdmin");
449
+ const res = await agent
450
+ .post("/gpt/prompt")
451
+ .send({prompt: "Hi"})
452
+ .buffer(true)
453
+ .parse(sseCollect);
454
+
455
+ assert.equal(res.status, 200);
456
+ assert.deepEqual(sink.traces[0]?.prompts, []);
457
+ });
458
+
367
459
  it("emits a memory-sink trace with userId and sessionId", async () => {
368
460
  const sink = new MemoryTraceSink();
369
461
  const plugin: ObservabilityPlugin = {
@@ -809,27 +901,6 @@ describe("AI Routes", () => {
809
901
  expect(res.status).toBe(200);
810
902
  });
811
903
 
812
- it("uses the langfuse system prompt when configured", async () => {
813
- const customApp = new TerrenoApp({
814
- configureApp: (router, options) => {
815
- addGptRoutes(router, {
816
- aiService,
817
- langfuseSystemPromptName: "missing-prompt-name",
818
- openApiOptions: options,
819
- });
820
- },
821
- skipListen: true,
822
- userModel: terrenoUserModel,
823
- }).build();
824
- const agent = await authAsUser(customApp, "notAdmin");
825
- const res = await agent
826
- .post("/gpt/prompt")
827
- .send({prompt: "Hi"})
828
- .buffer(true)
829
- .parse(sseCollect);
830
- expect(res.status).toBe(200);
831
- });
832
-
833
904
  it("forwards model tool-call and tool-result stream events via SSE", async () => {
834
905
  const sink = new MemoryTraceSink();
835
906
  const plugin: ObservabilityPlugin = {
@@ -2146,52 +2217,6 @@ describe("AI Routes", () => {
2146
2217
  }
2147
2218
  });
2148
2219
 
2149
- it("swallows preparePromptForAI failure when langfuse prompt loading throws", async () => {
2150
- // Re-mock preparePromptForAI to throw, exercising the catch branch that
2151
- // logs `Langfuse system prompt skipped: ...` and continues without it.
2152
- mock.module("../langfuseVercelAi", () => ({
2153
- ...realLangfuseVercelAi,
2154
- createTelemetryConfig: () => ({functionId: "test", isEnabled: false}),
2155
- preparePromptForAI: async () => {
2156
- throw new Error("langfuse offline");
2157
- },
2158
- }));
2159
- try {
2160
- const customApp = new TerrenoApp({
2161
- configureApp: (router, options) => {
2162
- addGptRoutes(router, {
2163
- aiService,
2164
- langfuseSystemPromptName: "broken-prompt",
2165
- openApiOptions: options,
2166
- });
2167
- },
2168
- skipListen: true,
2169
- userModel: terrenoUserModel,
2170
- }).build();
2171
- const agent = await authAsUser(customApp, "notAdmin");
2172
- const res = await agent
2173
- .post("/gpt/prompt")
2174
- .send({prompt: "Hi"})
2175
- .buffer(true)
2176
- .parse(sseCollect);
2177
- // Failure is silently caught; request still completes successfully.
2178
- expect(res.status).toBe(200);
2179
- const body = (res as SseResponse).body;
2180
- expect(body).toContain("done");
2181
- } finally {
2182
- // Restore the original happy-path mock for subsequent tests.
2183
- mock.module("../langfuseVercelAi", () => ({
2184
- ...realLangfuseVercelAi,
2185
- createTelemetryConfig: () => ({functionId: "test", isEnabled: false}),
2186
- preparePromptForAI: async () => ({
2187
- config: {},
2188
- prompt: "test",
2189
- telemetry: {isEnabled: false},
2190
- }),
2191
- }));
2192
- }
2193
- });
2194
-
2195
2220
  it("logs warning when AIRequest.logRequest throws inside the stream error catch", async () => {
2196
2221
  // Combine a stream that errors mid-iteration with a logRequest that also throws,
2197
2222
  // exercising the catch-within-catch that logs `Failed to log AIRequest error`.
@@ -1201,6 +1201,27 @@ describe("AIService", () => {
1201
1201
  expect(sink.traces[0].prompts).toEqual([{label: "production", name: "greeter", version: 2}]);
1202
1202
  });
1203
1203
 
1204
+ it("records a prompt's shared includes on the trace after the prompt itself", async () => {
1205
+ const sink = new MemoryTraceSink();
1206
+ registerLocalApp({
1207
+ promptVersion: {
1208
+ ...productionVersion,
1209
+ includes: [{includedBy: "greeter", label: "production", name: "base-rules", version: 4}],
1210
+ },
1211
+ traceSink: sink,
1212
+ });
1213
+ const model = createMockModel("Hello world");
1214
+ const service = new AIService({model: model as unknown as LanguageModel});
1215
+
1216
+ await service.generateText({prompt: "User says hi", promptName: "greeter"});
1217
+
1218
+ expect(sink.traces[0].name).toBe("greeter");
1219
+ expect(sink.traces[0].prompts).toEqual([
1220
+ {label: "production", name: "greeter", version: 2},
1221
+ {includedBy: "greeter", label: "production", name: "base-rules", version: 4},
1222
+ ]);
1223
+ });
1224
+
1204
1225
  it("still writes AIRequest for a chat stream when a sink is registered", async () => {
1205
1226
  const sink = new MemoryTraceSink();
1206
1227
  registerLocalApp({traceSink: sink});
@@ -397,7 +397,10 @@ export class AIService {
397
397
  name: spanName,
398
398
  output: params.response,
399
399
  prompts: promptRef
400
- ? [{label: promptRef.label, name: promptRef.name, version: promptRef.version}]
400
+ ? [
401
+ {label: promptRef.label, name: promptRef.name, version: promptRef.version},
402
+ ...(promptRef.includes ?? []),
403
+ ]
401
404
  : [],
402
405
  sensitive: params.observability.sensitive,
403
406
  sessionId: params.observability.sessionId,
@@ -24,12 +24,12 @@ import type express from "express";
24
24
  import {DateTime} from "luxon";
25
25
  import type mongoose from "mongoose";
26
26
 
27
- import {createTelemetryConfig, preparePromptForAI} from "../langfuseVercelAi";
28
27
  import {AIRequest} from "../models/aiRequest";
29
28
  import {FileAttachment} from "../models/fileAttachment";
30
29
  import {GptHistory} from "../models/gptHistory";
31
30
  import {getProjectModel} from "../models/project";
32
- import type {SpanRecord} from "../observability/types";
31
+ import {getObservabilityApp} from "../observability/observabilityAppRegistry";
32
+ import type {PromptVersionRef, SpanRecord} from "../observability/types";
33
33
  import type {
34
34
  AskOrigin,
35
35
  GptHistoryAskStatus,
@@ -1140,17 +1140,39 @@ const startTurn = async ({
1140
1140
  const lastUserText = (prompts: GptHistoryPrompt[]): string =>
1141
1141
  prompts.findLast((prompt) => prompt.type === "user")?.text ?? "";
1142
1142
 
1143
- /** The request's system prompt with project context and the Langfuse prompt prepended. */
1143
+ /** The route's default registry system prompt, or undefined when it cannot be loaded. */
1144
+ const loadRegistrySystemPrompt = async (
1145
+ options: GptRouteOptions
1146
+ ): Promise<PromptVersionRef | undefined> => {
1147
+ const registry = getObservabilityApp()?.promptRegistry;
1148
+ if (!options.systemPromptName || !registry) {
1149
+ return undefined;
1150
+ }
1151
+ try {
1152
+ return await registry.get({
1153
+ label: options.systemPromptLabel ?? "production",
1154
+ name: options.systemPromptName,
1155
+ });
1156
+ } catch (error) {
1157
+ logger.warn("Registry system prompt skipped", {
1158
+ error: error instanceof Error ? error.message : String(error),
1159
+ promptName: options.systemPromptName,
1160
+ });
1161
+ return undefined;
1162
+ }
1163
+ };
1164
+
1165
+ /** The request's system prompt with project context and the registry prompt prepended. */
1144
1166
  const buildSystemPrompt = async ({
1145
1167
  history,
1146
- options,
1147
1168
  projectId,
1169
+ registryPrompt,
1148
1170
  systemPrompt,
1149
1171
  userId,
1150
1172
  }: {
1151
1173
  history: GptHistoryDocument;
1152
- options: GptRouteOptions;
1153
1174
  projectId: unknown;
1175
+ registryPrompt?: string;
1154
1176
  systemPrompt: unknown;
1155
1177
  userId: mongoose.Types.ObjectId | undefined;
1156
1178
  }): Promise<string | undefined> => {
@@ -1181,22 +1203,10 @@ const buildSystemPrompt = async ({
1181
1203
  }
1182
1204
  }
1183
1205
 
1184
- if (options.langfuseSystemPromptName) {
1185
- try {
1186
- const langfuseResult = await preparePromptForAI({
1187
- promptName: options.langfuseSystemPromptName,
1188
- userId: userId?.toString(),
1189
- });
1190
- const langfusePrompt =
1191
- typeof langfuseResult.prompt === "string" ? langfuseResult.prompt : undefined;
1192
- if (langfusePrompt) {
1193
- effectiveSystemPrompt = effectiveSystemPrompt
1194
- ? `${langfusePrompt}\n\n${effectiveSystemPrompt}`
1195
- : langfusePrompt;
1196
- }
1197
- } catch (err) {
1198
- logger.debug(`Langfuse system prompt skipped: ${(err as Error).message}`);
1199
- }
1206
+ if (registryPrompt) {
1207
+ effectiveSystemPrompt = effectiveSystemPrompt
1208
+ ? `${registryPrompt}\n\n${effectiveSystemPrompt}`
1209
+ : registryPrompt;
1200
1210
  }
1201
1211
  return effectiveSystemPrompt;
1202
1212
  };
@@ -1778,20 +1788,29 @@ export const runChatTurn = async ({
1778
1788
  const {answer, isNewHistory, logPrompt, messages, replayedMessages, resolvedAsk, titlePrompt} =
1779
1789
  turn;
1780
1790
  let {history} = turn;
1791
+ // An admin-selected promptName replaces the system prompt, so the route default is skipped.
1792
+ const registryPrompt = promptName ? undefined : await loadRegistrySystemPrompt(options);
1781
1793
  let effectiveSystemPrompt = await buildSystemPrompt({
1782
1794
  history,
1783
- options,
1784
1795
  projectId,
1796
+ registryPrompt: registryPrompt?.body,
1785
1797
  systemPrompt,
1786
1798
  userId,
1787
1799
  });
1788
- const observability = await aiService.resolveGenerateObservability({
1800
+ const resolvedObservability = await aiService.resolveGenerateObservability({
1789
1801
  promptLabel,
1790
1802
  promptName,
1791
1803
  sensitive: body.sensitive === true ? true : undefined,
1792
1804
  sessionId,
1793
1805
  systemPrompt: effectiveSystemPrompt,
1794
1806
  });
1807
+ const observability = registryPrompt
1808
+ ? {
1809
+ ...resolvedObservability,
1810
+ promptRef: registryPrompt,
1811
+ sensitive: resolvedObservability.sensitive || Boolean(registryPrompt.sensitive),
1812
+ }
1813
+ : resolvedObservability;
1795
1814
  effectiveSystemPrompt = observability.systemPrompt;
1796
1815
 
1797
1816
  // Some models (e.g. gemini-2.5-flash-image) don't support tool calling
@@ -2037,17 +2056,7 @@ export const runChatTurn = async ({
2037
2056
  sink.emit({askResolved: {action: resolvedAsk.action, toolCallId: resolvedAsk.toolCallId}});
2038
2057
  }
2039
2058
  logger.debug("Starting streamText", {model: modelId, supportsTools});
2040
- const telemetry = createTelemetryConfig({
2041
- functionId: "gpt-prompt",
2042
- metadata: {
2043
- ...(options.langfuseSystemPromptName
2044
- ? {langfusePromptName: options.langfuseSystemPromptName}
2045
- : {}),
2046
- },
2047
- userId: userId?.toString(),
2048
- });
2049
2059
  const result = streamText({
2050
- experimental_telemetry: telemetry,
2051
2060
  messages,
2052
2061
  model: aiService.model,
2053
2062
  providerOptions: !supportsTools
@@ -6,61 +6,7 @@ import {registerBackendPreload} from "@terreno/test";
6
6
  registerBackendPreload({
7
7
  connectMongoInBeforeAll: true,
8
8
  mongo: {baseDatabaseName: "terreno-ai-test", useReplSet: true},
9
- onBeforeEach: async () => {
10
- const {shutdownLangfuseClient} = await import("../langfuseClient.js");
11
- await shutdownLangfuseClient();
12
- },
13
9
  testEnv: {
14
10
  tokenIssuer: "terreno-ai.test",
15
11
  },
16
12
  });
17
-
18
- // Mock @langfuse/client globally so the real `./langfuseClient` module runs
19
- // without making network calls.
20
- import {mock} from "bun:test";
21
-
22
- mock.module("@langfuse/client", () => {
23
- return {
24
- LangfuseClient: class FakeLangfuseClient {
25
- baseUrl: string;
26
- publicKey: string;
27
- secretKey: string;
28
- api = {
29
- prompts: {
30
- list: mock(async () => ({
31
- data: [],
32
- meta: {limit: 20, page: 1, total: 0, totalPages: 0},
33
- })),
34
- },
35
- trace: {
36
- get: mock(async (id: string) => ({id, name: "Trace"})),
37
- list: mock(async () => ({
38
- data: [],
39
- meta: {limit: 20, page: 1, total: 0, totalPages: 0},
40
- })),
41
- },
42
- };
43
- prompt = {
44
- create: mock(async (_params: Record<string, unknown>) => undefined),
45
- get: mock(async (name: string) => ({
46
- config: {},
47
- labels: [],
48
- name,
49
- prompt: "",
50
- tags: [],
51
- type: "text" as const,
52
- version: 1,
53
- })),
54
- };
55
- score = {create: mock(() => {})};
56
- flush = mock(async () => {});
57
- shutdown = mock(async () => {});
58
-
59
- constructor(opts: {baseUrl: string; publicKey: string; secretKey: string}) {
60
- this.baseUrl = opts.baseUrl;
61
- this.publicKey = opts.publicKey;
62
- this.secretKey = opts.secretKey;
63
- }
64
- },
65
- };
66
- });
@@ -8,9 +8,12 @@ import type {ObsPromptRef} from "./observability";
8
8
 
9
9
  /**
10
10
  * A prompt version a task used, recorded on its `ObsTrace.prompts`. A `PromptVersionRef`
11
- * from `promptRegistry.get` fits as-is; its `body` is ignored.
11
+ * from `promptRegistry.get` fits as-is; its `body` is ignored and its `includes` (shared
12
+ * prompts it referenced) are recorded right after it.
12
13
  */
13
- export type HarnessPromptRef = ObsPromptRef;
14
+ export interface HarnessPromptRef extends ObsPromptRef {
15
+ includes?: ObsPromptRef[];
16
+ }
14
17
 
15
18
  /** A registry prompt passed as instructions: its `body` is used and its version recorded. */
16
19
  export interface HarnessPromptInstructions extends HarnessPromptRef {
@@ -448,9 +448,14 @@ export interface GptRouteOptions {
448
448
  * `POST /gpt/actions`. `{repair: true}` runs one repair call when that check fails.
449
449
  */
450
450
  uiBlocks?: boolean | UiBlocksOptions;
451
- /** Langfuse prompt name to load and use as the system prompt. Compiled with no variables.
452
- * Falls back gracefully if Langfuse is not configured or the prompt is not found. */
453
- langfuseSystemPromptName?: string;
451
+ /**
452
+ * Observability prompt registry name used as the default system prompt (prepended to the
453
+ * request's system prompt). The trace records the version. Skipped when no registry is registered, the
454
+ * label has no version, or an admin request selects `promptName`.
455
+ */
456
+ systemPromptName?: string;
457
+ /** Registry label read for `systemPromptName`. Defaults to `production`. */
458
+ systemPromptLabel?: string;
454
459
  /**
455
460
  * When `false` or the function returns `false`, prompts that include attachments are rejected.
456
461
  * Omit or pass `true` to leave uploads enabled.