@terreno/ai 58.0.0-beta.3 → 58.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. package/CHANGELOG.md +166 -0
  2. package/README.md +3 -3
  3. package/dist/harness/commit.d.ts +4 -3
  4. package/dist/harness/commit.js +23 -4
  5. package/dist/index.d.ts +6 -9
  6. package/dist/index.js +14 -25
  7. package/dist/observability/adminScreens.js +5 -1
  8. package/dist/observability/evaluate.d.ts +8 -1
  9. package/dist/observability/evaluate.js +10 -2
  10. package/dist/observability/judgeAgreement.d.ts +53 -0
  11. package/dist/observability/judgeAgreement.js +123 -0
  12. package/dist/observability/local/datasetStore.d.ts +5 -0
  13. package/dist/observability/local/datasetStore.js +7 -2
  14. package/dist/observability/local/evaluatorStore.d.ts +17 -0
  15. package/dist/observability/local/evaluatorStore.js +73 -2
  16. package/dist/observability/local/experimentRunner.d.ts +40 -3
  17. package/dist/observability/local/experimentRunner.js +390 -11
  18. package/dist/observability/local/humanAnswers.d.ts +57 -0
  19. package/dist/observability/local/humanAnswers.js +174 -0
  20. package/dist/observability/local/judgeRun.d.ts +56 -0
  21. package/dist/observability/local/judgeRun.js +111 -0
  22. package/dist/observability/local/localPlugin.js +4 -0
  23. package/dist/observability/local/models/obsDatasetItem.js +4 -0
  24. package/dist/observability/local/models/obsEvaluator.js +4 -0
  25. package/dist/observability/local/models/obsExperiment.js +28 -0
  26. package/dist/observability/local/models/obsExperimentItem.js +4 -0
  27. package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
  28. package/dist/observability/local/models/obsJudgeRun.js +60 -0
  29. package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
  30. package/dist/observability/local/models/obsReviewBatch.js +48 -0
  31. package/dist/observability/local/models/obsReviewItem.js +10 -2
  32. package/dist/observability/local/models/obsTrace.js +4 -0
  33. package/dist/observability/local/promptReferences.d.ts +31 -0
  34. package/dist/observability/local/promptReferences.js +114 -0
  35. package/dist/observability/local/promptStore.d.ts +55 -19
  36. package/dist/observability/local/promptStore.js +171 -7
  37. package/dist/observability/local/reviewBatchStore.d.ts +106 -0
  38. package/dist/observability/local/reviewBatchStore.js +517 -0
  39. package/dist/observability/local/reviewStore.d.ts +24 -3
  40. package/dist/observability/local/reviewStore.js +98 -22
  41. package/dist/observability/local/traceStore.d.ts +14 -1
  42. package/dist/observability/local/traceStore.js +144 -11
  43. package/dist/observability/observabilityApp.js +10 -1
  44. package/dist/observability/questionFingerprint.d.ts +7 -0
  45. package/dist/observability/questionFingerprint.js +23 -0
  46. package/dist/observability/routes/experiments.js +12 -1
  47. package/dist/observability/routes/review.js +3 -1
  48. package/dist/observability/routes/reviewBatches.d.ts +11 -0
  49. package/dist/observability/routes/reviewBatches.js +78 -0
  50. package/dist/observability/routes/traces.d.ts +1 -1
  51. package/dist/observability/routes/traces.js +34 -2
  52. package/dist/observability/types.d.ts +10 -1
  53. package/dist/service/aiService.js +4 -1
  54. package/dist/service/chatTurn.js +38 -31
  55. package/dist/types/harness.d.ts +5 -2
  56. package/dist/types/index.d.ts +8 -3
  57. package/dist/types/observability.d.ts +74 -2
  58. package/package.json +8 -24
  59. package/src/harness/agentLoop.test.ts +0 -3
  60. package/src/harness/agentRecovery.test.ts +0 -3
  61. package/src/harness/commit.ts +24 -4
  62. package/src/harness/extensions.test.ts +0 -3
  63. package/src/harness/subagent.test.ts +27 -3
  64. package/src/index.test.ts +0 -17
  65. package/src/index.ts +23 -26
  66. package/src/isolated/gptStream.isolated.ts +0 -8
  67. package/src/observability/adminScreens.test.ts +7 -0
  68. package/src/observability/adminScreens.ts +6 -1
  69. package/src/observability/evaluate.ts +12 -2
  70. package/src/observability/judgeAgreement.test.ts +111 -0
  71. package/src/observability/judgeAgreement.ts +184 -0
  72. package/src/observability/local/datasetStore.ts +30 -11
  73. package/src/observability/local/evaluatorStore.ts +96 -2
  74. package/src/observability/local/experimentRunner.ts +495 -15
  75. package/src/observability/local/humanAnswers.ts +233 -0
  76. package/src/observability/local/judgePipeline.test.ts +444 -0
  77. package/src/observability/local/judgeRun.test.ts +173 -0
  78. package/src/observability/local/judgeRun.ts +184 -0
  79. package/src/observability/local/localPlugin.ts +4 -0
  80. package/src/observability/local/models/obsDatasetItem.ts +5 -0
  81. package/src/observability/local/models/obsEvaluator.ts +4 -0
  82. package/src/observability/local/models/obsExperiment.ts +29 -0
  83. package/src/observability/local/models/obsExperimentItem.ts +4 -0
  84. package/src/observability/local/models/obsJudgeRun.ts +61 -0
  85. package/src/observability/local/models/obsReviewBatch.ts +52 -0
  86. package/src/observability/local/models/obsReviewItem.ts +11 -2
  87. package/src/observability/local/models/obsTrace.ts +5 -0
  88. package/src/observability/local/obsModels.test.ts +13 -0
  89. package/src/observability/local/promptReferences.test.ts +128 -0
  90. package/src/observability/local/promptReferences.ts +149 -0
  91. package/src/observability/local/promptStore.test.ts +291 -0
  92. package/src/observability/local/promptStore.ts +243 -32
  93. package/src/observability/local/reviewBatchStore.test.ts +251 -0
  94. package/src/observability/local/reviewBatchStore.ts +656 -0
  95. package/src/observability/local/reviewStore.ts +122 -25
  96. package/src/observability/local/traceStore.test.ts +139 -1
  97. package/src/observability/local/traceStore.ts +162 -11
  98. package/src/observability/observabilityApp.test.ts +13 -10
  99. package/src/observability/observabilityApp.ts +10 -1
  100. package/src/observability/questionFingerprint.test.ts +33 -0
  101. package/src/observability/questionFingerprint.ts +24 -0
  102. package/src/observability/routes/experiments.test.ts +37 -0
  103. package/src/observability/routes/experiments.ts +13 -1
  104. package/src/observability/routes/review.ts +3 -1
  105. package/src/observability/routes/reviewBatches.test.ts +209 -0
  106. package/src/observability/routes/reviewBatches.ts +138 -0
  107. package/src/observability/routes/status.test.ts +4 -4
  108. package/src/observability/routes/traces.test.ts +61 -0
  109. package/src/observability/routes/traces.ts +46 -4
  110. package/src/observability/types.ts +5 -2
  111. package/src/routes/routes.test.ts +101 -76
  112. package/src/service/aiService.test.ts +21 -0
  113. package/src/service/aiService.ts +4 -1
  114. package/src/service/chatTurn.ts +42 -33
  115. package/src/tests/bunSetup.ts +0 -54
  116. package/src/types/harness.ts +5 -2
  117. package/src/types/index.ts +8 -3
  118. package/src/types/observability.ts +86 -2
  119. package/dist/langfuseApp.d.ts +0 -9
  120. package/dist/langfuseApp.js +0 -53
  121. package/dist/langfuseCache.d.ts +0 -15
  122. package/dist/langfuseCache.js +0 -54
  123. package/dist/langfuseClient.d.ts +0 -6
  124. package/dist/langfuseClient.js +0 -34
  125. package/dist/langfusePrompts.d.ts +0 -12
  126. package/dist/langfusePrompts.js +0 -89
  127. package/dist/langfuseRoutesEvaluations.d.ts +0 -3
  128. package/dist/langfuseRoutesEvaluations.js +0 -32
  129. package/dist/langfuseRoutesMiddleware.d.ts +0 -2
  130. package/dist/langfuseRoutesMiddleware.js +0 -13
  131. package/dist/langfuseRoutesPlayground.d.ts +0 -2
  132. package/dist/langfuseRoutesPlayground.js +0 -36
  133. package/dist/langfuseRoutesPrompts.d.ts +0 -2
  134. package/dist/langfuseRoutesPrompts.js +0 -56
  135. package/dist/langfuseRoutesTraces.d.ts +0 -2
  136. package/dist/langfuseRoutesTraces.js +0 -30
  137. package/dist/langfuseTracing.d.ts +0 -4
  138. package/dist/langfuseTracing.js +0 -32
  139. package/dist/langfuseTypes.d.ts +0 -96
  140. package/dist/langfuseTypes.js +0 -2
  141. package/dist/langfuseVercelAi.d.ts +0 -12
  142. package/dist/langfuseVercelAi.js +0 -49
  143. package/src/isolated/langfuseClient.isolated.ts +0 -66
  144. package/src/langfuseApp.test.ts +0 -119
  145. package/src/langfuseApp.ts +0 -62
  146. package/src/langfuseCache.test.ts +0 -75
  147. package/src/langfuseCache.ts +0 -72
  148. package/src/langfuseClient.test.ts +0 -69
  149. package/src/langfuseClient.ts +0 -34
  150. package/src/langfusePrompts.test.ts +0 -188
  151. package/src/langfusePrompts.ts +0 -121
  152. package/src/langfuseRoutes.test.ts +0 -261
  153. package/src/langfuseRoutesEvaluations.ts +0 -48
  154. package/src/langfuseRoutesMiddleware.test.ts +0 -57
  155. package/src/langfuseRoutesMiddleware.ts +0 -12
  156. package/src/langfuseRoutesPlayground.ts +0 -50
  157. package/src/langfuseRoutesPrompts.ts +0 -87
  158. package/src/langfuseRoutesTraces.ts +0 -40
  159. package/src/langfuseTracing.test.ts +0 -72
  160. package/src/langfuseTracing.ts +0 -33
  161. package/src/langfuseTypes.ts +0 -103
  162. package/src/langfuseVercelAi.test.ts +0 -82
  163. package/src/langfuseVercelAi.ts +0 -72
  164. package/src/tests/fakeLangfusePrompt.ts +0 -10
@@ -109,6 +109,67 @@ describe("observability trace routes", () => {
109
109
  ).toEqual([[inScope.id, "ws-1"]]);
110
110
  });
111
111
 
112
+ it("sorts the trace list across pages and rejects unknown sort keys", async () => {
113
+ const base = DateTime.utc().minus({hours: 1});
114
+ const exportAt = async (
115
+ name: string,
116
+ offsetMs: number,
117
+ latencyMs: number,
118
+ costUsd: number,
119
+ inputTokens: number
120
+ ): Promise<string> => {
121
+ const exported = await store.exportTrace({
122
+ endedAt: base.plus({milliseconds: offsetMs + latencyMs}).toISO() ?? "",
123
+ id: name,
124
+ name,
125
+ prompts: [{name, version: 1}],
126
+ sensitive: false,
127
+ spans: [],
128
+ startedAt: base.plus({milliseconds: offsetMs}).toISO() ?? "",
129
+ status: "ok",
130
+ usage: {costUsd, inputTokens, outputTokens: 0},
131
+ });
132
+ return exported.id;
133
+ };
134
+ const cheapSlow = await exportAt("cheap-slow", 0, 900, 0.0001, 500);
135
+ const priceyFast = await exportAt("pricey-fast", 1000, 50, 0.05, 20);
136
+ const midMid = await exportAt("mid-mid", 2000, 300, 0.001, 12_345);
137
+ const admin = await authAsUser(app, "admin");
138
+
139
+ const pageIds = async (query: string): Promise<string[]> => {
140
+ const first = await admin.get(`/ai/observability/traces?${query}&limit=2&page=1`);
141
+ const second = await admin.get(`/ai/observability/traces?${query}&limit=2&page=2`);
142
+ expect(first.status).toBe(200);
143
+ expect(first.body.more).toBe(true);
144
+ expect(second.body.more).toBe(false);
145
+ return [...first.body.data, ...second.body.data].map((row: {id: string}) => row.id);
146
+ };
147
+
148
+ expect(await pageIds("sort=-cost")).toEqual([priceyFast, midMid, cheapSlow]);
149
+ expect(await pageIds("sort=cost")).toEqual([cheapSlow, midMid, priceyFast]);
150
+ expect(await pageIds("sort=-latency")).toEqual([cheapSlow, midMid, priceyFast]);
151
+ expect(await pageIds("sort=latency")).toEqual([priceyFast, midMid, cheapSlow]);
152
+ expect(await pageIds("sort=-tokens")).toEqual([midMid, cheapSlow, priceyFast]);
153
+ expect(await pageIds("sort=tokens")).toEqual([priceyFast, cheapSlow, midMid]);
154
+ expect(await pageIds("sort=startedAt")).toEqual([cheapSlow, priceyFast, midMid]);
155
+ expect(await pageIds("sort=-name")).toEqual([priceyFast, midMid, cheapSlow]);
156
+ // Default stays newest first.
157
+ expect(await pageIds("status=ok")).toEqual([midMid, priceyFast, cheapSlow]);
158
+
159
+ const multiPrompt = await admin.get(
160
+ "/ai/observability/traces?prompt[0]=cheap-slow&prompt[1]=mid-mid&sort=name"
161
+ );
162
+ expect(multiPrompt.status).toBe(200);
163
+ expect(multiPrompt.body.data.map((row: {id: string}) => row.id)).toEqual([cheapSlow, midMid]);
164
+
165
+ const unknown = await admin.get("/ai/observability/traces?sort=created");
166
+ expect(unknown.status).toBe(400);
167
+ const listSort = await admin.get("/ai/observability/traces?sort[0]=cost");
168
+ expect(listSort.status).toBe(400);
169
+ const objectPrompt = await admin.get("/ai/observability/traces?prompt[name]=x");
170
+ expect(objectPrompt.status).toBe(400);
171
+ });
172
+
112
173
  it("forbids non-admins from listing traces", async () => {
113
174
  const agent = await authAsUser(app, "notAdmin");
114
175
  const res = await agent.get("/ai/observability/traces");
@@ -1,6 +1,11 @@
1
1
  import {APIError, asyncHandler, createOpenApiBuilder} from "@terreno/api";
2
2
  import type express from "express";
3
- import type {LocalTraceStore} from "../local/traceStore";
3
+ import {
4
+ DEFAULT_TRACE_LIST_SORT,
5
+ type LocalTraceStore,
6
+ parseTraceListSort,
7
+ TRACE_LIST_SORT_KEYS,
8
+ } from "../local/traceStore";
4
9
  import {getObservabilityApp} from "../observabilityAppRegistry";
5
10
  import type {ScoreRecord, ScoreSink} from "../types";
6
11
  import {
@@ -27,7 +32,7 @@ const parseBoolean = (value: unknown): boolean | undefined => {
27
32
  throw new APIError({status: 400, title: "boolean query must be true or false"});
28
33
  };
29
34
 
30
- const parsePromptVersion = (value: unknown, prompt?: string): number | undefined => {
35
+ const parsePromptVersion = (value: unknown, prompt?: string | string[]): number | undefined => {
31
36
  if (value === undefined) {
32
37
  return undefined;
33
38
  }
@@ -35,12 +40,37 @@ const parsePromptVersion = (value: unknown, prompt?: string): number | undefined
35
40
  if (!Number.isInteger(parsed) || parsed < 1) {
36
41
  throw new APIError({status: 400, title: "promptVersion must be a positive integer"});
37
42
  }
38
- if (!prompt) {
43
+ if (!prompt || prompt.length === 0) {
39
44
  throw new APIError({status: 400, title: "prompt is required when promptVersion is set"});
40
45
  }
41
46
  return parsed;
42
47
  };
43
48
 
49
+ const parsePrompt = (value: unknown): string | string[] | undefined => {
50
+ if (typeof value === "string") {
51
+ return value;
52
+ }
53
+ if (Array.isArray(value) && value.every((entry) => typeof entry === "string")) {
54
+ return value as string[];
55
+ }
56
+ if (value === undefined) {
57
+ return undefined;
58
+ }
59
+ throw new APIError({status: 400, title: "prompt must be a string or a list of strings"});
60
+ };
61
+
62
+ const parseSort = (value: unknown): string | undefined => {
63
+ if (value === undefined) {
64
+ return undefined;
65
+ }
66
+ if (typeof value !== "string") {
67
+ throw new APIError({status: 400, title: "Unknown trace sort key"});
68
+ }
69
+ // Validate here so a bad key fails before the store runs any query.
70
+ parseTraceListSort(value);
71
+ return value;
72
+ };
73
+
44
74
  const parsePositiveInt = (value: unknown, fallback: number): number => {
45
75
  if (value === undefined) {
46
76
  return fallback;
@@ -72,11 +102,22 @@ export const addObservabilityTraceRoutes = (
72
102
  .withQueryParameter("prompt", {type: "string"}, {required: false})
73
103
  .withQueryParameter("promptVersion", {type: "number"}, {required: false})
74
104
  .withQueryParameter("scope", {type: "string"}, {required: false})
105
+ .withQueryParameter(
106
+ "sort",
107
+ {
108
+ enum: TRACE_LIST_SORT_KEYS.flatMap((key) => [key, `-${key}`]),
109
+ type: "string",
110
+ },
111
+ {
112
+ description: `Sort column, \`-\` prefix for descending. Defaults to \`${DEFAULT_TRACE_LIST_SORT}\`.`,
113
+ required: false,
114
+ }
115
+ )
75
116
  .withResponse(200, {data: {type: "array"}})
76
117
  .build()
77
118
  ),
78
119
  asyncHandler(async (req, res) => {
79
- const prompt = typeof req.query.prompt === "string" ? req.query.prompt : undefined;
120
+ const prompt = parsePrompt(req.query.prompt);
80
121
  const listed = await options.store.list({
81
122
  flaggedForDataset: parseBoolean(req.query.flaggedForDataset),
82
123
  from: typeof req.query.from === "string" ? req.query.from : undefined,
@@ -88,6 +129,7 @@ export const addObservabilityTraceRoutes = (
88
129
  scope: typeof req.query.scope === "string" ? req.query.scope : undefined,
89
130
  sensitive: parseBoolean(req.query.sensitive),
90
131
  sessionId: typeof req.query.sessionId === "string" ? req.query.sessionId : undefined,
132
+ sort: parseSort(req.query.sort),
91
133
  status:
92
134
  req.query.status === "ok" || req.query.status === "error" ? req.query.status : undefined,
93
135
  to: typeof req.query.to === "string" ? req.query.to : undefined,
@@ -8,7 +8,7 @@ export type ObservabilityCapability =
8
8
  | "scores"
9
9
  | "traces";
10
10
 
11
- export type ControlPrimary = "langfuse" | "local";
11
+ export type ControlPrimary = "local";
12
12
 
13
13
  export interface ObservabilityControlConfig {
14
14
  datasets: ControlPrimary;
@@ -50,7 +50,7 @@ export interface TraceRecord {
50
50
  metadata?: Record<string, unknown>;
51
51
  name: string;
52
52
  output?: unknown;
53
- prompts: {label?: string; name: string; version: number}[];
53
+ prompts: {includedBy?: string; label?: string; name: string; version: number}[];
54
54
  sensitive: boolean;
55
55
  sessionId?: string;
56
56
  spans: SpanRecord[];
@@ -78,7 +78,10 @@ export interface ScoreRecord {
78
78
  }
79
79
 
80
80
  export interface PromptVersionRef {
81
+ /** `body` with every prompt reference resolved. */
81
82
  body: string;
83
+ /** Prompt versions pulled in by references, recorded on the trace after this prompt. */
84
+ includes?: {includedBy?: string; label?: string; name: string; version: number}[];
82
85
  label?: string;
83
86
  name: string;
84
87
  sensitive?: boolean;
@@ -30,16 +30,7 @@ import {addGptHistoryRoutes} from "./gptHistories";
30
30
 
31
31
  type SseResponse = supertest.Response & {body: string};
32
32
 
33
- // Mock langfuseVercelAi to avoid transitive langfuse SDK import in tests.
34
- // Spread the real module so later-loaded test files still see the real exports.
35
33
  // The package compiles as CommonJS, so modules load with require instead of top-level await.
36
- const realLangfuseVercelAi: typeof import("../langfuseVercelAi.js") = require("../langfuseVercelAi");
37
- mock.module("../langfuseVercelAi", () => ({
38
- ...realLangfuseVercelAi,
39
- createTelemetryConfig: () => ({functionId: "test", isEnabled: false}),
40
- preparePromptForAI: async () => ({config: {}, prompt: "test", telemetry: {isEnabled: false}}),
41
- }));
42
-
43
34
  const realAi: typeof import("ai") = require("ai");
44
35
  type StreamTextOptions = Parameters<typeof realAi.streamText>[0];
45
36
  type StreamTextResult = ReturnType<typeof realAi.streamText>;
@@ -364,6 +355,107 @@ describe("AI Routes", () => {
364
355
  assert.isTrue(sink.traces[0]?.sensitive);
365
356
  });
366
357
 
358
+ it("prepends the route registry system prompt and records its version", async () => {
359
+ const sink = new MemoryTraceSink();
360
+ const lookups: Array<{label?: string; name: string}> = [];
361
+ const plugin: ObservabilityPlugin = {
362
+ capabilities: new Set([
363
+ "datasets",
364
+ "experiments",
365
+ "prompts",
366
+ "reviewQueue",
367
+ "scores",
368
+ "traces",
369
+ ]),
370
+ datasetStore: {},
371
+ experimentRunner: {},
372
+ id: "local",
373
+ promptRegistry: {
374
+ get: async (args) => {
375
+ lookups.push(args);
376
+ return {
377
+ body: "Registry assistant prompt",
378
+ label: "staging",
379
+ name: "chat-assistant",
380
+ version: 2,
381
+ };
382
+ },
383
+ },
384
+ reviewQueue: {},
385
+ traceSink: sink,
386
+ };
387
+ new ObservabilityApp({plugins: [plugin]});
388
+ const model = createMockModel();
389
+ const registryApp = new TerrenoApp({
390
+ configureApp: (router, options) => {
391
+ addGptRoutes(router, {
392
+ aiService: new AIService({model: model as unknown as LanguageModel}),
393
+ openApiOptions: options,
394
+ systemPromptLabel: "staging",
395
+ systemPromptName: "chat-assistant",
396
+ });
397
+ },
398
+ skipListen: true,
399
+ userModel: terrenoUserModel,
400
+ }).build();
401
+
402
+ const agent = await authAsUser(registryApp, "notAdmin");
403
+ const res = await agent
404
+ .post("/gpt/prompt")
405
+ .send({prompt: "Hi", systemPrompt: "Caller context"})
406
+ .buffer(true)
407
+ .parse(sseCollect);
408
+
409
+ assert.equal(res.status, 200);
410
+ assert.deepEqual(lookups, [{label: "staging", name: "chat-assistant"}]);
411
+ const callOptions = (model.doStream.mock.calls[0] as unknown[] | undefined)?.[0] as {
412
+ prompt: Array<{content: unknown; role: string}>;
413
+ };
414
+ const system = callOptions.prompt.find((message) => message.role === "system");
415
+ assert.match(String(system?.content), /^Registry assistant prompt\n\nCaller context/);
416
+ assert.deepEqual(sink.traces[0]?.prompts, [
417
+ {label: "staging", name: "chat-assistant", version: 2},
418
+ ]);
419
+ });
420
+
421
+ it("keeps chatting without a prompt when the route registry prompt is missing", async () => {
422
+ const sink = new MemoryTraceSink();
423
+ const plugin: ObservabilityPlugin = {
424
+ capabilities: new Set([
425
+ "datasets",
426
+ "experiments",
427
+ "prompts",
428
+ "reviewQueue",
429
+ "scores",
430
+ "traces",
431
+ ]),
432
+ datasetStore: {},
433
+ experimentRunner: {},
434
+ id: "local",
435
+ promptRegistry: {get: async () => undefined},
436
+ reviewQueue: {},
437
+ traceSink: sink,
438
+ };
439
+ new ObservabilityApp({plugins: [plugin]});
440
+ const registryApp = new TerrenoApp({
441
+ configureApp: (router, options) => {
442
+ addGptRoutes(router, {aiService, openApiOptions: options, systemPromptName: "absent"});
443
+ },
444
+ skipListen: true,
445
+ userModel: terrenoUserModel,
446
+ }).build();
447
+
448
+ const agent = await authAsUser(registryApp, "notAdmin");
449
+ const res = await agent
450
+ .post("/gpt/prompt")
451
+ .send({prompt: "Hi"})
452
+ .buffer(true)
453
+ .parse(sseCollect);
454
+
455
+ assert.equal(res.status, 200);
456
+ assert.deepEqual(sink.traces[0]?.prompts, []);
457
+ });
458
+
367
459
  it("emits a memory-sink trace with userId and sessionId", async () => {
368
460
  const sink = new MemoryTraceSink();
369
461
  const plugin: ObservabilityPlugin = {
@@ -809,27 +901,6 @@ describe("AI Routes", () => {
809
901
  expect(res.status).toBe(200);
810
902
  });
811
903
 
812
- it("uses the langfuse system prompt when configured", async () => {
813
- const customApp = new TerrenoApp({
814
- configureApp: (router, options) => {
815
- addGptRoutes(router, {
816
- aiService,
817
- langfuseSystemPromptName: "missing-prompt-name",
818
- openApiOptions: options,
819
- });
820
- },
821
- skipListen: true,
822
- userModel: terrenoUserModel,
823
- }).build();
824
- const agent = await authAsUser(customApp, "notAdmin");
825
- const res = await agent
826
- .post("/gpt/prompt")
827
- .send({prompt: "Hi"})
828
- .buffer(true)
829
- .parse(sseCollect);
830
- expect(res.status).toBe(200);
831
- });
832
-
833
904
  it("forwards model tool-call and tool-result stream events via SSE", async () => {
834
905
  const sink = new MemoryTraceSink();
835
906
  const plugin: ObservabilityPlugin = {
@@ -2147,52 +2218,6 @@ describe("AI Routes", () => {
2147
2218
  }
2148
2219
  });
2149
2220
 
2150
- it("swallows preparePromptForAI failure when langfuse prompt loading throws", async () => {
2151
- // Re-mock preparePromptForAI to throw, exercising the catch branch that
2152
- // logs `Langfuse system prompt skipped: ...` and continues without it.
2153
- mock.module("../langfuseVercelAi", () => ({
2154
- ...realLangfuseVercelAi,
2155
- createTelemetryConfig: () => ({functionId: "test", isEnabled: false}),
2156
- preparePromptForAI: async () => {
2157
- throw new Error("langfuse offline");
2158
- },
2159
- }));
2160
- try {
2161
- const customApp = new TerrenoApp({
2162
- configureApp: (router, options) => {
2163
- addGptRoutes(router, {
2164
- aiService,
2165
- langfuseSystemPromptName: "broken-prompt",
2166
- openApiOptions: options,
2167
- });
2168
- },
2169
- skipListen: true,
2170
- userModel: terrenoUserModel,
2171
- }).build();
2172
- const agent = await authAsUser(customApp, "notAdmin");
2173
- const res = await agent
2174
- .post("/gpt/prompt")
2175
- .send({prompt: "Hi"})
2176
- .buffer(true)
2177
- .parse(sseCollect);
2178
- // Failure is silently caught; request still completes successfully.
2179
- expect(res.status).toBe(200);
2180
- const body = (res as SseResponse).body;
2181
- expect(body).toContain("done");
2182
- } finally {
2183
- // Restore the original happy-path mock for subsequent tests.
2184
- mock.module("../langfuseVercelAi", () => ({
2185
- ...realLangfuseVercelAi,
2186
- createTelemetryConfig: () => ({functionId: "test", isEnabled: false}),
2187
- preparePromptForAI: async () => ({
2188
- config: {},
2189
- prompt: "test",
2190
- telemetry: {isEnabled: false},
2191
- }),
2192
- }));
2193
- }
2194
- });
2195
-
2196
2221
  it("logs warning when AIRequest.logRequest throws inside the stream error catch", async () => {
2197
2222
  // Combine a stream that errors mid-iteration with a logRequest that also throws,
2198
2223
  // exercising the catch-within-catch that logs `Failed to log AIRequest error`.
@@ -1201,6 +1201,27 @@ describe("AIService", () => {
1201
1201
  expect(sink.traces[0].prompts).toEqual([{label: "production", name: "greeter", version: 2}]);
1202
1202
  });
1203
1203
 
1204
+ it("records a prompt's shared includes on the trace after the prompt itself", async () => {
1205
+ const sink = new MemoryTraceSink();
1206
+ registerLocalApp({
1207
+ promptVersion: {
1208
+ ...productionVersion,
1209
+ includes: [{includedBy: "greeter", label: "production", name: "base-rules", version: 4}],
1210
+ },
1211
+ traceSink: sink,
1212
+ });
1213
+ const model = createMockModel("Hello world");
1214
+ const service = new AIService({model: model as unknown as LanguageModel});
1215
+
1216
+ await service.generateText({prompt: "User says hi", promptName: "greeter"});
1217
+
1218
+ expect(sink.traces[0].name).toBe("greeter");
1219
+ expect(sink.traces[0].prompts).toEqual([
1220
+ {label: "production", name: "greeter", version: 2},
1221
+ {includedBy: "greeter", label: "production", name: "base-rules", version: 4},
1222
+ ]);
1223
+ });
1224
+
1204
1225
  it("still writes AIRequest for a chat stream when a sink is registered", async () => {
1205
1226
  const sink = new MemoryTraceSink();
1206
1227
  registerLocalApp({traceSink: sink});
@@ -397,7 +397,10 @@ export class AIService {
397
397
  name: spanName,
398
398
  output: params.response,
399
399
  prompts: promptRef
400
- ? [{label: promptRef.label, name: promptRef.name, version: promptRef.version}]
400
+ ? [
401
+ {label: promptRef.label, name: promptRef.name, version: promptRef.version},
402
+ ...(promptRef.includes ?? []),
403
+ ]
401
404
  : [],
402
405
  sensitive: params.observability.sensitive,
403
406
  sessionId: params.observability.sessionId,
@@ -24,12 +24,12 @@ import type express from "express";
24
24
  import {DateTime} from "luxon";
25
25
  import type mongoose from "mongoose";
26
26
 
27
- import {createTelemetryConfig, preparePromptForAI} from "../langfuseVercelAi";
28
27
  import {AIRequest} from "../models/aiRequest";
29
28
  import {FileAttachment} from "../models/fileAttachment";
30
29
  import {GptHistory} from "../models/gptHistory";
31
30
  import {getProjectModel} from "../models/project";
32
- import type {SpanRecord} from "../observability/types";
31
+ import {getObservabilityApp} from "../observability/observabilityAppRegistry";
32
+ import type {PromptVersionRef, SpanRecord} from "../observability/types";
33
33
  import type {
34
34
  AskOrigin,
35
35
  GptHistoryAskStatus,
@@ -1145,17 +1145,39 @@ const startTurn = async ({
1145
1145
  const lastUserText = (prompts: GptHistoryPrompt[]): string =>
1146
1146
  prompts.findLast((prompt) => prompt.type === "user")?.text ?? "";
1147
1147
 
1148
- /** The request's system prompt with project context and the Langfuse prompt prepended. */
1148
+ /** The route's default registry system prompt, or undefined when it cannot be loaded. */
1149
+ const loadRegistrySystemPrompt = async (
1150
+ options: GptRouteOptions
1151
+ ): Promise<PromptVersionRef | undefined> => {
1152
+ const registry = getObservabilityApp()?.promptRegistry;
1153
+ if (!options.systemPromptName || !registry) {
1154
+ return undefined;
1155
+ }
1156
+ try {
1157
+ return await registry.get({
1158
+ label: options.systemPromptLabel ?? "production",
1159
+ name: options.systemPromptName,
1160
+ });
1161
+ } catch (error) {
1162
+ logger.warn("Registry system prompt skipped", {
1163
+ error: error instanceof Error ? error.message : String(error),
1164
+ promptName: options.systemPromptName,
1165
+ });
1166
+ return undefined;
1167
+ }
1168
+ };
1169
+
1170
+ /** The request's system prompt with project context and the registry prompt prepended. */
1149
1171
  const buildSystemPrompt = async ({
1150
1172
  history,
1151
- options,
1152
1173
  projectId,
1174
+ registryPrompt,
1153
1175
  systemPrompt,
1154
1176
  userId,
1155
1177
  }: {
1156
1178
  history: GptHistoryDocument;
1157
- options: GptRouteOptions;
1158
1179
  projectId: unknown;
1180
+ registryPrompt?: string;
1159
1181
  systemPrompt: unknown;
1160
1182
  userId: mongoose.Types.ObjectId | undefined;
1161
1183
  }): Promise<string | undefined> => {
@@ -1186,22 +1208,10 @@ const buildSystemPrompt = async ({
1186
1208
  }
1187
1209
  }
1188
1210
 
1189
- if (options.langfuseSystemPromptName) {
1190
- try {
1191
- const langfuseResult = await preparePromptForAI({
1192
- promptName: options.langfuseSystemPromptName,
1193
- userId: userId?.toString(),
1194
- });
1195
- const langfusePrompt =
1196
- typeof langfuseResult.prompt === "string" ? langfuseResult.prompt : undefined;
1197
- if (langfusePrompt) {
1198
- effectiveSystemPrompt = effectiveSystemPrompt
1199
- ? `${langfusePrompt}\n\n${effectiveSystemPrompt}`
1200
- : langfusePrompt;
1201
- }
1202
- } catch (err) {
1203
- logger.debug(`Langfuse system prompt skipped: ${(err as Error).message}`);
1204
- }
1211
+ if (registryPrompt) {
1212
+ effectiveSystemPrompt = effectiveSystemPrompt
1213
+ ? `${registryPrompt}\n\n${effectiveSystemPrompt}`
1214
+ : registryPrompt;
1205
1215
  }
1206
1216
  return effectiveSystemPrompt;
1207
1217
  };
@@ -1783,20 +1793,29 @@ export const runChatTurn = async ({
1783
1793
  const {answer, isNewHistory, logPrompt, messages, replayedMessages, resolvedAsk, titlePrompt} =
1784
1794
  turn;
1785
1795
  let {history} = turn;
1796
+ // An admin-selected promptName replaces the system prompt, so the route default is skipped.
1797
+ const registryPrompt = promptName ? undefined : await loadRegistrySystemPrompt(options);
1786
1798
  let effectiveSystemPrompt = await buildSystemPrompt({
1787
1799
  history,
1788
- options,
1789
1800
  projectId,
1801
+ registryPrompt: registryPrompt?.body,
1790
1802
  systemPrompt,
1791
1803
  userId,
1792
1804
  });
1793
- const observability = await aiService.resolveGenerateObservability({
1805
+ const resolvedObservability = await aiService.resolveGenerateObservability({
1794
1806
  promptLabel,
1795
1807
  promptName,
1796
1808
  sensitive: body.sensitive === true ? true : undefined,
1797
1809
  sessionId,
1798
1810
  systemPrompt: effectiveSystemPrompt,
1799
1811
  });
1812
+ const observability = registryPrompt
1813
+ ? {
1814
+ ...resolvedObservability,
1815
+ promptRef: registryPrompt,
1816
+ sensitive: resolvedObservability.sensitive || Boolean(registryPrompt.sensitive),
1817
+ }
1818
+ : resolvedObservability;
1800
1819
  effectiveSystemPrompt = observability.systemPrompt;
1801
1820
 
1802
1821
  // Some models (e.g. gemini-2.5-flash-image) don't support tool calling
@@ -2042,17 +2061,7 @@ export const runChatTurn = async ({
2042
2061
  sink.emit({askResolved: {action: resolvedAsk.action, toolCallId: resolvedAsk.toolCallId}});
2043
2062
  }
2044
2063
  logger.debug("Starting streamText", {model: modelId, supportsTools});
2045
- const telemetry = createTelemetryConfig({
2046
- functionId: "gpt-prompt",
2047
- metadata: {
2048
- ...(options.langfuseSystemPromptName
2049
- ? {langfusePromptName: options.langfuseSystemPromptName}
2050
- : {}),
2051
- },
2052
- userId: userId?.toString(),
2053
- });
2054
2064
  const result = streamText({
2055
- experimental_telemetry: telemetry,
2056
2065
  messages,
2057
2066
  model: aiService.model,
2058
2067
  providerOptions: !supportsTools
@@ -6,61 +6,7 @@ import {registerBackendPreload} from "@terreno/test";
6
6
  registerBackendPreload({
7
7
  connectMongoInBeforeAll: true,
8
8
  mongo: {baseDatabaseName: "terreno-ai-test", useReplSet: true},
9
- onBeforeEach: async () => {
10
- const {shutdownLangfuseClient} = await import("../langfuseClient.js");
11
- await shutdownLangfuseClient();
12
- },
13
9
  testEnv: {
14
10
  tokenIssuer: "terreno-ai.test",
15
11
  },
16
12
  });
17
-
18
- // Mock @langfuse/client globally so the real `./langfuseClient` module runs
19
- // without making network calls.
20
- import {mock} from "bun:test";
21
-
22
- mock.module("@langfuse/client", () => {
23
- return {
24
- LangfuseClient: class FakeLangfuseClient {
25
- baseUrl: string;
26
- publicKey: string;
27
- secretKey: string;
28
- api = {
29
- prompts: {
30
- list: mock(async () => ({
31
- data: [],
32
- meta: {limit: 20, page: 1, total: 0, totalPages: 0},
33
- })),
34
- },
35
- trace: {
36
- get: mock(async (id: string) => ({id, name: "Trace"})),
37
- list: mock(async () => ({
38
- data: [],
39
- meta: {limit: 20, page: 1, total: 0, totalPages: 0},
40
- })),
41
- },
42
- };
43
- prompt = {
44
- create: mock(async (_params: Record<string, unknown>) => undefined),
45
- get: mock(async (name: string) => ({
46
- config: {},
47
- labels: [],
48
- name,
49
- prompt: "",
50
- tags: [],
51
- type: "text" as const,
52
- version: 1,
53
- })),
54
- };
55
- score = {create: mock(() => {})};
56
- flush = mock(async () => {});
57
- shutdown = mock(async () => {});
58
-
59
- constructor(opts: {baseUrl: string; publicKey: string; secretKey: string}) {
60
- this.baseUrl = opts.baseUrl;
61
- this.publicKey = opts.publicKey;
62
- this.secretKey = opts.secretKey;
63
- }
64
- },
65
- };
66
- });
@@ -8,9 +8,12 @@ import type {ObsPromptRef} from "./observability";
8
8
 
9
9
  /**
10
10
  * A prompt version a task used, recorded on its `ObsTrace.prompts`. A `PromptVersionRef`
11
- * from `promptRegistry.get` fits as-is; its `body` is ignored.
11
+ * from `promptRegistry.get` fits as-is; its `body` is ignored and its `includes` (shared
12
+ * prompts it referenced) are recorded right after it.
12
13
  */
13
- export type HarnessPromptRef = ObsPromptRef;
14
+ export interface HarnessPromptRef extends ObsPromptRef {
15
+ includes?: ObsPromptRef[];
16
+ }
14
17
 
15
18
  /** A registry prompt passed as instructions: its `body` is used and its version recorded. */
16
19
  export interface HarnessPromptInstructions extends HarnessPromptRef {
@@ -452,9 +452,14 @@ export interface GptRouteOptions {
452
452
  * `POST /gpt/actions`. `{repair: true}` runs one repair call when that check fails.
453
453
  */
454
454
  uiBlocks?: boolean | UiBlocksOptions;
455
- /** Langfuse prompt name to load and use as the system prompt. Compiled with no variables.
456
- * Falls back gracefully if Langfuse is not configured or the prompt is not found. */
457
- langfuseSystemPromptName?: string;
455
+ /**
456
+ * Observability prompt registry name used as the default system prompt (prepended to the
457
+ * request's system prompt). The trace records the version. Skipped when no registry is registered, the
458
+ * label has no version, or an admin request selects `promptName`.
459
+ */
460
+ systemPromptName?: string;
461
+ /** Registry label read for `systemPromptName`. Defaults to `production`. */
462
+ systemPromptLabel?: string;
458
463
  /**
459
464
  * When `false` or the function returns `false`, prompts that include attachments are rejected.
460
465
  * Omit or pass `true` to leave uploads enabled.