@terreno/ai 57.20.0 → 57.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. package/CHANGELOG.md +81 -0
  2. package/README.md +3 -3
  3. package/biome.jsonc +3 -2
  4. package/dist/harness/agentLoop.js +1 -0
  5. package/dist/harness/commit.d.ts +4 -3
  6. package/dist/harness/commit.js +23 -4
  7. package/dist/index.d.ts +6 -9
  8. package/dist/index.js +14 -25
  9. package/dist/observability/adminScreens.js +5 -1
  10. package/dist/observability/evaluate.d.ts +8 -1
  11. package/dist/observability/evaluate.js +10 -2
  12. package/dist/observability/judgeAgreement.d.ts +53 -0
  13. package/dist/observability/judgeAgreement.js +123 -0
  14. package/dist/observability/local/datasetStore.d.ts +5 -0
  15. package/dist/observability/local/datasetStore.js +7 -2
  16. package/dist/observability/local/evaluatorStore.d.ts +17 -0
  17. package/dist/observability/local/evaluatorStore.js +73 -2
  18. package/dist/observability/local/experimentRunner.d.ts +42 -8
  19. package/dist/observability/local/experimentRunner.js +390 -11
  20. package/dist/observability/local/humanAnswers.d.ts +57 -0
  21. package/dist/observability/local/humanAnswers.js +174 -0
  22. package/dist/observability/local/judgeRun.d.ts +56 -0
  23. package/dist/observability/local/judgeRun.js +111 -0
  24. package/dist/observability/local/localPlugin.js +4 -0
  25. package/dist/observability/local/models/obsDatasetItem.js +4 -0
  26. package/dist/observability/local/models/obsEvaluator.js +4 -0
  27. package/dist/observability/local/models/obsExperiment.js +28 -0
  28. package/dist/observability/local/models/obsExperimentItem.js +4 -0
  29. package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
  30. package/dist/observability/local/models/obsJudgeRun.js +60 -0
  31. package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
  32. package/dist/observability/local/models/obsReviewBatch.js +48 -0
  33. package/dist/observability/local/models/obsReviewItem.js +10 -2
  34. package/dist/observability/local/models/obsTrace.js +4 -0
  35. package/dist/observability/local/promptReferences.d.ts +31 -0
  36. package/dist/observability/local/promptReferences.js +114 -0
  37. package/dist/observability/local/promptStore.d.ts +55 -19
  38. package/dist/observability/local/promptStore.js +171 -7
  39. package/dist/observability/local/reviewBatchStore.d.ts +106 -0
  40. package/dist/observability/local/reviewBatchStore.js +517 -0
  41. package/dist/observability/local/reviewStore.d.ts +24 -3
  42. package/dist/observability/local/reviewStore.js +98 -22
  43. package/dist/observability/local/traceStore.d.ts +14 -1
  44. package/dist/observability/local/traceStore.js +144 -11
  45. package/dist/observability/observabilityApp.js +10 -1
  46. package/dist/observability/questionFingerprint.d.ts +7 -0
  47. package/dist/observability/questionFingerprint.js +23 -0
  48. package/dist/observability/routes/experiments.js +12 -1
  49. package/dist/observability/routes/review.js +3 -1
  50. package/dist/observability/routes/reviewBatches.d.ts +11 -0
  51. package/dist/observability/routes/reviewBatches.js +78 -0
  52. package/dist/observability/routes/traces.d.ts +1 -1
  53. package/dist/observability/routes/traces.js +34 -2
  54. package/dist/observability/types.d.ts +10 -1
  55. package/dist/service/aiService.js +4 -1
  56. package/dist/service/chatTurn.js +38 -31
  57. package/dist/types/harness.d.ts +10 -4
  58. package/dist/types/index.d.ts +8 -3
  59. package/dist/types/observability.d.ts +74 -2
  60. package/e2e/smoke.ts +227 -0
  61. package/package.json +14 -28
  62. package/src/aiApp.test.ts +22 -17
  63. package/src/harness/agentLoop.test.ts +0 -3
  64. package/src/harness/agentLoop.ts +2 -1
  65. package/src/harness/agentRecovery.test.ts +30 -30
  66. package/src/harness/approvals.test.ts +11 -9
  67. package/src/harness/commit.ts +24 -4
  68. package/src/harness/events.test.ts +10 -6
  69. package/src/harness/extensions.test.ts +6 -8
  70. package/src/harness/ownership.test.ts +20 -9
  71. package/src/harness/retries.test.ts +6 -3
  72. package/src/harness/routes/conversations.test.ts +2 -2
  73. package/src/harness/routes/tasks.test.ts +2 -2
  74. package/src/harness/runners/jobsRunner.test.ts +1 -1
  75. package/src/harness/subagent.test.ts +29 -4
  76. package/src/harness/waits.test.ts +6 -4
  77. package/src/index.test.ts +3 -20
  78. package/src/index.ts +23 -26
  79. package/src/isolated/gptStream.isolated.ts +25 -28
  80. package/src/isolated/lazyProjectModel.isolated.ts +2 -2
  81. package/src/models/gptHistory.test.ts +10 -6
  82. package/src/observability/adminScreens.test.ts +7 -0
  83. package/src/observability/adminScreens.ts +6 -1
  84. package/src/observability/evaluate.test.ts +12 -10
  85. package/src/observability/evaluate.ts +12 -2
  86. package/src/observability/judgeAgreement.test.ts +111 -0
  87. package/src/observability/judgeAgreement.ts +184 -0
  88. package/src/observability/local/datasetStore.ts +30 -11
  89. package/src/observability/local/evaluatorStore.ts +96 -2
  90. package/src/observability/local/experimentRunner.test.ts +9 -12
  91. package/src/observability/local/experimentRunner.ts +497 -17
  92. package/src/observability/local/humanAnswers.ts +233 -0
  93. package/src/observability/local/judgePipeline.test.ts +444 -0
  94. package/src/observability/local/judgeRun.test.ts +173 -0
  95. package/src/observability/local/judgeRun.ts +184 -0
  96. package/src/observability/local/localPlugin.ts +4 -0
  97. package/src/observability/local/models/obsDatasetItem.ts +5 -0
  98. package/src/observability/local/models/obsEvaluator.ts +4 -0
  99. package/src/observability/local/models/obsExperiment.ts +29 -0
  100. package/src/observability/local/models/obsExperimentItem.ts +4 -0
  101. package/src/observability/local/models/obsJudgeRun.ts +61 -0
  102. package/src/observability/local/models/obsReviewBatch.ts +52 -0
  103. package/src/observability/local/models/obsReviewItem.ts +11 -2
  104. package/src/observability/local/models/obsTrace.ts +5 -0
  105. package/src/observability/local/obsModels.test.ts +13 -0
  106. package/src/observability/local/promptReferences.test.ts +128 -0
  107. package/src/observability/local/promptReferences.ts +149 -0
  108. package/src/observability/local/promptStore.test.ts +295 -3
  109. package/src/observability/local/promptStore.ts +243 -32
  110. package/src/observability/local/reviewBatchStore.test.ts +251 -0
  111. package/src/observability/local/reviewBatchStore.ts +656 -0
  112. package/src/observability/local/reviewStore.ts +122 -25
  113. package/src/observability/local/traceStore.test.ts +139 -1
  114. package/src/observability/local/traceStore.ts +162 -11
  115. package/src/observability/observabilityApp.test.ts +13 -10
  116. package/src/observability/observabilityApp.ts +10 -1
  117. package/src/observability/questionFingerprint.test.ts +33 -0
  118. package/src/observability/questionFingerprint.ts +24 -0
  119. package/src/observability/routes/datasets.test.ts +2 -2
  120. package/src/observability/routes/evaluators.test.ts +2 -2
  121. package/src/observability/routes/experiments.test.ts +39 -2
  122. package/src/observability/routes/experiments.ts +13 -1
  123. package/src/observability/routes/prompts.test.ts +3 -3
  124. package/src/observability/routes/review.test.ts +2 -2
  125. package/src/observability/routes/review.ts +3 -1
  126. package/src/observability/routes/reviewBatches.test.ts +209 -0
  127. package/src/observability/routes/reviewBatches.ts +138 -0
  128. package/src/observability/routes/status.test.ts +8 -8
  129. package/src/observability/routes/testMultiStage.test.ts +6 -6
  130. package/src/observability/routes/traces.test.ts +63 -2
  131. package/src/observability/routes/traces.ts +46 -4
  132. package/src/observability/types.ts +5 -2
  133. package/src/routes/files.test.ts +5 -3
  134. package/src/routes/gpt.test.ts +21 -14
  135. package/src/routes/gptActions.test.ts +7 -8
  136. package/src/routes/gptDatasets.test.ts +1 -1
  137. package/src/routes/gptFiles.test.ts +1 -1
  138. package/src/routes/gptHistories.test.ts +15 -9
  139. package/src/routes/mcp.test.ts +3 -3
  140. package/src/routes/projects.test.ts +5 -3
  141. package/src/routes/routes.test.ts +156 -115
  142. package/src/service/aiService.test.ts +22 -1
  143. package/src/service/aiService.ts +4 -1
  144. package/src/service/asks.test.ts +14 -6
  145. package/src/service/chatTurn.ts +42 -33
  146. package/src/service/fileStorage.test.ts +5 -2
  147. package/src/service/getMCPTools.test.ts +9 -5
  148. package/src/service/mcpService.test.ts +6 -3
  149. package/src/tests/bunSetup.ts +0 -54
  150. package/src/tests/chatHarness.ts +4 -4
  151. package/src/tests/fakeJudge.ts +8 -0
  152. package/src/tests/helpers.ts +26 -4
  153. package/src/types/harness.ts +11 -4
  154. package/src/types/index.ts +8 -3
  155. package/src/types/observability.ts +86 -2
  156. package/tsconfig.typecheck.json +18 -0
  157. package/dist/langfuseApp.d.ts +0 -9
  158. package/dist/langfuseApp.js +0 -53
  159. package/dist/langfuseCache.d.ts +0 -15
  160. package/dist/langfuseCache.js +0 -54
  161. package/dist/langfuseClient.d.ts +0 -6
  162. package/dist/langfuseClient.js +0 -34
  163. package/dist/langfusePrompts.d.ts +0 -12
  164. package/dist/langfusePrompts.js +0 -89
  165. package/dist/langfuseRoutesEvaluations.d.ts +0 -3
  166. package/dist/langfuseRoutesEvaluations.js +0 -32
  167. package/dist/langfuseRoutesMiddleware.d.ts +0 -2
  168. package/dist/langfuseRoutesMiddleware.js +0 -13
  169. package/dist/langfuseRoutesPlayground.d.ts +0 -2
  170. package/dist/langfuseRoutesPlayground.js +0 -36
  171. package/dist/langfuseRoutesPrompts.d.ts +0 -2
  172. package/dist/langfuseRoutesPrompts.js +0 -56
  173. package/dist/langfuseRoutesTraces.d.ts +0 -2
  174. package/dist/langfuseRoutesTraces.js +0 -30
  175. package/dist/langfuseTracing.d.ts +0 -4
  176. package/dist/langfuseTracing.js +0 -32
  177. package/dist/langfuseTypes.d.ts +0 -96
  178. package/dist/langfuseTypes.js +0 -2
  179. package/dist/langfuseVercelAi.d.ts +0 -12
  180. package/dist/langfuseVercelAi.js +0 -49
  181. package/src/isolated/langfuseClient.isolated.ts +0 -60
  182. package/src/langfuseApp.test.ts +0 -135
  183. package/src/langfuseApp.ts +0 -62
  184. package/src/langfuseCache.test.ts +0 -75
  185. package/src/langfuseCache.ts +0 -72
  186. package/src/langfuseClient.test.ts +0 -69
  187. package/src/langfuseClient.ts +0 -34
  188. package/src/langfusePrompts.test.ts +0 -188
  189. package/src/langfusePrompts.ts +0 -121
  190. package/src/langfuseRoutes.test.ts +0 -256
  191. package/src/langfuseRoutesEvaluations.ts +0 -48
  192. package/src/langfuseRoutesMiddleware.test.ts +0 -57
  193. package/src/langfuseRoutesMiddleware.ts +0 -12
  194. package/src/langfuseRoutesPlayground.ts +0 -50
  195. package/src/langfuseRoutesPrompts.ts +0 -87
  196. package/src/langfuseRoutesTraces.ts +0 -40
  197. package/src/langfuseTracing.test.ts +0 -68
  198. package/src/langfuseTracing.ts +0 -33
  199. package/src/langfuseTypes.ts +0 -103
  200. package/src/langfuseVercelAi.test.ts +0 -79
  201. package/src/langfuseVercelAi.ts +0 -72
@@ -13,7 +13,7 @@ import {AIService, TemperaturePresets} from "./aiService";
13
13
  // Create a mock LanguageModelV2
14
14
  const createMockModel = (responseText = "Mock response") => {
15
15
  return {
16
- doGenerate: mock(async () => ({
16
+ doGenerate: mock(async (_options: unknown) => ({
17
17
  content: [{text: responseText, type: "text" as const}],
18
18
  finishReason: "stop" as const,
19
19
  usage: {inputTokens: 5, outputTokens: 10},
@@ -1201,6 +1201,27 @@ describe("AIService", () => {
1201
1201
  expect(sink.traces[0].prompts).toEqual([{label: "production", name: "greeter", version: 2}]);
1202
1202
  });
1203
1203
 
1204
+ it("records a prompt's shared includes on the trace after the prompt itself", async () => {
1205
+ const sink = new MemoryTraceSink();
1206
+ registerLocalApp({
1207
+ promptVersion: {
1208
+ ...productionVersion,
1209
+ includes: [{includedBy: "greeter", label: "production", name: "base-rules", version: 4}],
1210
+ },
1211
+ traceSink: sink,
1212
+ });
1213
+ const model = createMockModel("Hello world");
1214
+ const service = new AIService({model: model as unknown as LanguageModel});
1215
+
1216
+ await service.generateText({prompt: "User says hi", promptName: "greeter"});
1217
+
1218
+ expect(sink.traces[0].name).toBe("greeter");
1219
+ expect(sink.traces[0].prompts).toEqual([
1220
+ {label: "production", name: "greeter", version: 2},
1221
+ {includedBy: "greeter", label: "production", name: "base-rules", version: 4},
1222
+ ]);
1223
+ });
1224
+
1204
1225
  it("still writes AIRequest for a chat stream when a sink is registered", async () => {
1205
1226
  const sink = new MemoryTraceSink();
1206
1227
  registerLocalApp({traceSink: sink});
@@ -397,7 +397,10 @@ export class AIService {
397
397
  name: spanName,
398
398
  output: params.response,
399
399
  prompts: promptRef
400
- ? [{label: promptRef.label, name: promptRef.name, version: promptRef.version}]
400
+ ? [
401
+ {label: promptRef.label, name: promptRef.name, version: promptRef.version},
402
+ ...(promptRef.includes ?? []),
403
+ ]
401
404
  : [],
402
405
  sensitive: params.observability.sensitive,
403
406
  sessionId: params.observability.sessionId,
@@ -1,6 +1,14 @@
1
1
  import {describe, expect, it} from "bun:test";
2
2
  import {ASK_LIMITS, type AskKind, type AskResponse, askPromptSection} from "@terreno/blocks";
3
- import {asSchema, jsonSchema, type ModelMessage, type Tool, tool} from "ai";
3
+ import {
4
+ asSchema,
5
+ type JSONValue,
6
+ jsonSchema,
7
+ type ModelMessage,
8
+ type Tool,
9
+ type ToolResultPart,
10
+ tool,
11
+ } from "ai";
4
12
 
5
13
  import {
6
14
  approvalAskInput,
@@ -24,7 +32,7 @@ const PLAN_ASK_INPUT = {
24
32
  {id: "team", label: "Team"},
25
33
  ],
26
34
  prompt: "Which plan should I set up?",
27
- select: "one",
35
+ select: "one" as const,
28
36
  };
29
37
 
30
38
  const TEAM_ANSWER = {action: "accept" as const, content: {selected: ["team"]}};
@@ -57,8 +65,8 @@ const lookupResult = {
57
65
  type: "tool-result" as const,
58
66
  };
59
67
 
60
- const answerResult = (toolCallId: string, value: unknown) => ({
61
- output: {type: "json", value},
68
+ const answerResult = (toolCallId: string, value: unknown): ToolResultPart => ({
69
+ output: {type: "json", value: value as JSONValue},
62
70
  toolCallId,
63
71
  toolName: "ask_choice",
64
72
  type: "tool-result",
@@ -76,14 +84,14 @@ describe("asks", () => {
76
84
  expected: ["confirm"],
77
85
  label: '{kinds: ["confirm"]}',
78
86
  },
79
- {asks: {kinds: []}, expected: [], label: "{kinds: []}"},
87
+ {asks: {kinds: [] as AskKind[]}, expected: [], label: "{kinds: []}"},
80
88
  {
81
89
  asks: {kinds: ["choice", "choice"] as AskKind[]},
82
90
  expected: ["choice"],
83
91
  label: '{kinds: ["choice", "choice"]}',
84
92
  },
85
93
  ])("resolves $label", ({asks, expected}) => {
86
- expect(resolveAskKinds(asks)).toEqual(expected);
94
+ expect(resolveAskKinds(asks)).toEqual([...expected]);
87
95
  });
88
96
 
89
97
  it("throws on an unknown kind and names the known kinds", () => {
@@ -24,12 +24,12 @@ import type express from "express";
24
24
  import {DateTime} from "luxon";
25
25
  import type mongoose from "mongoose";
26
26
 
27
- import {createTelemetryConfig, preparePromptForAI} from "../langfuseVercelAi";
28
27
  import {AIRequest} from "../models/aiRequest";
29
28
  import {FileAttachment} from "../models/fileAttachment";
30
29
  import {GptHistory} from "../models/gptHistory";
31
30
  import {getProjectModel} from "../models/project";
32
- import type {SpanRecord} from "../observability/types";
31
+ import {getObservabilityApp} from "../observability/observabilityAppRegistry";
32
+ import type {PromptVersionRef, SpanRecord} from "../observability/types";
33
33
  import type {
34
34
  AskOrigin,
35
35
  GptHistoryAskStatus,
@@ -1140,17 +1140,39 @@ const startTurn = async ({
1140
1140
  const lastUserText = (prompts: GptHistoryPrompt[]): string =>
1141
1141
  prompts.findLast((prompt) => prompt.type === "user")?.text ?? "";
1142
1142
 
1143
- /** The request's system prompt with project context and the Langfuse prompt prepended. */
1143
+ /** The route's default registry system prompt, or undefined when it cannot be loaded. */
1144
+ const loadRegistrySystemPrompt = async (
1145
+ options: GptRouteOptions
1146
+ ): Promise<PromptVersionRef | undefined> => {
1147
+ const registry = getObservabilityApp()?.promptRegistry;
1148
+ if (!options.systemPromptName || !registry) {
1149
+ return undefined;
1150
+ }
1151
+ try {
1152
+ return await registry.get({
1153
+ label: options.systemPromptLabel ?? "production",
1154
+ name: options.systemPromptName,
1155
+ });
1156
+ } catch (error) {
1157
+ logger.warn("Registry system prompt skipped", {
1158
+ error: error instanceof Error ? error.message : String(error),
1159
+ promptName: options.systemPromptName,
1160
+ });
1161
+ return undefined;
1162
+ }
1163
+ };
1164
+
1165
+ /** The request's system prompt with project context and the registry prompt prepended. */
1144
1166
  const buildSystemPrompt = async ({
1145
1167
  history,
1146
- options,
1147
1168
  projectId,
1169
+ registryPrompt,
1148
1170
  systemPrompt,
1149
1171
  userId,
1150
1172
  }: {
1151
1173
  history: GptHistoryDocument;
1152
- options: GptRouteOptions;
1153
1174
  projectId: unknown;
1175
+ registryPrompt?: string;
1154
1176
  systemPrompt: unknown;
1155
1177
  userId: mongoose.Types.ObjectId | undefined;
1156
1178
  }): Promise<string | undefined> => {
@@ -1181,22 +1203,10 @@ const buildSystemPrompt = async ({
1181
1203
  }
1182
1204
  }
1183
1205
 
1184
- if (options.langfuseSystemPromptName) {
1185
- try {
1186
- const langfuseResult = await preparePromptForAI({
1187
- promptName: options.langfuseSystemPromptName,
1188
- userId: userId?.toString(),
1189
- });
1190
- const langfusePrompt =
1191
- typeof langfuseResult.prompt === "string" ? langfuseResult.prompt : undefined;
1192
- if (langfusePrompt) {
1193
- effectiveSystemPrompt = effectiveSystemPrompt
1194
- ? `${langfusePrompt}\n\n${effectiveSystemPrompt}`
1195
- : langfusePrompt;
1196
- }
1197
- } catch (err) {
1198
- logger.debug(`Langfuse system prompt skipped: ${(err as Error).message}`);
1199
- }
1206
+ if (registryPrompt) {
1207
+ effectiveSystemPrompt = effectiveSystemPrompt
1208
+ ? `${registryPrompt}\n\n${effectiveSystemPrompt}`
1209
+ : registryPrompt;
1200
1210
  }
1201
1211
  return effectiveSystemPrompt;
1202
1212
  };
@@ -1778,20 +1788,29 @@ export const runChatTurn = async ({
1778
1788
  const {answer, isNewHistory, logPrompt, messages, replayedMessages, resolvedAsk, titlePrompt} =
1779
1789
  turn;
1780
1790
  let {history} = turn;
1791
+ // An admin-selected promptName replaces the system prompt, so the route default is skipped.
1792
+ const registryPrompt = promptName ? undefined : await loadRegistrySystemPrompt(options);
1781
1793
  let effectiveSystemPrompt = await buildSystemPrompt({
1782
1794
  history,
1783
- options,
1784
1795
  projectId,
1796
+ registryPrompt: registryPrompt?.body,
1785
1797
  systemPrompt,
1786
1798
  userId,
1787
1799
  });
1788
- const observability = await aiService.resolveGenerateObservability({
1800
+ const resolvedObservability = await aiService.resolveGenerateObservability({
1789
1801
  promptLabel,
1790
1802
  promptName,
1791
1803
  sensitive: body.sensitive === true ? true : undefined,
1792
1804
  sessionId,
1793
1805
  systemPrompt: effectiveSystemPrompt,
1794
1806
  });
1807
+ const observability = registryPrompt
1808
+ ? {
1809
+ ...resolvedObservability,
1810
+ promptRef: registryPrompt,
1811
+ sensitive: resolvedObservability.sensitive || Boolean(registryPrompt.sensitive),
1812
+ }
1813
+ : resolvedObservability;
1795
1814
  effectiveSystemPrompt = observability.systemPrompt;
1796
1815
 
1797
1816
  // Some models (e.g. gemini-2.5-flash-image) don't support tool calling
@@ -2037,17 +2056,7 @@ export const runChatTurn = async ({
2037
2056
  sink.emit({askResolved: {action: resolvedAsk.action, toolCallId: resolvedAsk.toolCallId}});
2038
2057
  }
2039
2058
  logger.debug("Starting streamText", {model: modelId, supportsTools});
2040
- const telemetry = createTelemetryConfig({
2041
- functionId: "gpt-prompt",
2042
- metadata: {
2043
- ...(options.langfuseSystemPromptName
2044
- ? {langfusePromptName: options.langfuseSystemPromptName}
2045
- : {}),
2046
- },
2047
- userId: userId?.toString(),
2048
- });
2049
2059
  const result = streamText({
2050
- experimental_telemetry: telemetry,
2051
2060
  messages,
2052
2061
  model: aiService.model,
2053
2062
  providerOptions: !supportsTools
@@ -1,4 +1,4 @@
1
- import {afterEach, beforeEach, describe, expect, it} from "bun:test";
1
+ import {afterEach, beforeAll, beforeEach, describe, expect, it} from "bun:test";
2
2
  import {registerGcsMock} from "@terreno/test";
3
3
  import mongoose from "mongoose";
4
4
 
@@ -7,7 +7,10 @@ import {FileAttachment} from "../models/fileAttachment";
7
7
  // In-memory @google-cloud/storage so the service doesn't try to authenticate
8
8
  const gcs = registerGcsMock();
9
9
 
10
- const {FileStorageService} = await import("./fileStorage");
10
+ let FileStorageService: typeof import("./fileStorage.js").FileStorageService;
11
+ beforeAll(async () => {
12
+ ({FileStorageService} = await import("./fileStorage.js"));
13
+ });
11
14
 
12
15
  describe("FileStorageService", () => {
13
16
  beforeEach(async () => {
@@ -7,12 +7,16 @@ import {
7
7
  registerMCPTool,
8
8
  type User,
9
9
  } from "@terreno/api";
10
+ import type {ToolExecutionOptions} from "ai";
10
11
  import {assert} from "chai";
11
12
  import mongoose, {Schema} from "mongoose";
12
13
  import {z} from "zod";
13
14
 
14
15
  import {getMCPTools} from "./getMCPTools";
15
16
 
17
+ /** `execute` options as the AI SDK passes them; getMCPTools ignores them. */
18
+ const callOptions: ToolExecutionOptions = {messages: [], toolCallId: "call-1"};
19
+
16
20
  interface NoteFields {
17
21
  _id: mongoose.Types.ObjectId;
18
22
  title: string;
@@ -67,7 +71,7 @@ describe("getMCPTools", () => {
67
71
 
68
72
  expect(Object.keys(tools).sort()).toEqual(["notes_create", "notes_list"]);
69
73
 
70
- const created = await tools.notes_create.execute?.({title: "From AI tools"});
74
+ const created = await tools.notes_create.execute?.({title: "From AI tools"}, callOptions);
71
75
  expect(created).toEqual(
72
76
  expect.objectContaining({
73
77
  data: expect.objectContaining({title: "From AI tools"}),
@@ -88,7 +92,7 @@ describe("getMCPTools", () => {
88
92
  const tools = getMCPTools();
89
93
  assert.deepEqual(Object.keys(tools), ["echo_json"]);
90
94
  assert.equal(tools.echo_json.description, "Echoes structured json back to the caller");
91
- assert.deepEqual(await tools.echo_json.execute?.({value: "hi"}), {echoed: "hi"});
95
+ assert.deepEqual(await tools.echo_json.execute?.({value: "hi"}, callOptions), {echoed: "hi"});
92
96
  });
93
97
 
94
98
  it("returns joined plain text when the tool response is not json", async () => {
@@ -105,7 +109,7 @@ describe("getMCPTools", () => {
105
109
  });
106
110
 
107
111
  const tools = getMCPTools();
108
- assert.equal(await tools.prose.execute?.({}), "first line\nsecond line");
112
+ assert.equal(await tools.prose.execute?.({}, callOptions), "first line\nsecond line");
109
113
  });
110
114
 
111
115
  it("passes the authenticated user through to custom tool handlers", async () => {
@@ -121,8 +125,8 @@ describe("getMCPTools", () => {
121
125
  });
122
126
 
123
127
  const user = asUser();
124
- await getMCPTools(user).whoami.execute?.({});
125
- await getMCPTools().whoami.execute?.({});
128
+ await getMCPTools(user).whoami.execute?.({}, callOptions);
129
+ await getMCPTools().whoami.execute?.({}, callOptions);
126
130
  assert.deepEqual(seenUserIds, [user.id, undefined]);
127
131
  });
128
132
  });
@@ -1,15 +1,18 @@
1
- import {beforeEach, describe, expect, it, mock} from "bun:test";
1
+ import {beforeAll, beforeEach, describe, expect, it, mock} from "bun:test";
2
2
 
3
3
  const clientImpl = {
4
4
  close: mock(async () => {}),
5
5
  tools: mock(async () => ({search: {description: "Search"}})),
6
6
  };
7
- const createMCPClientMock = mock(async () => clientImpl);
7
+ const createMCPClientMock = mock(async (_opts: Record<string, unknown>) => clientImpl);
8
8
  mock.module("@ai-sdk/mcp", () => ({
9
9
  createMCPClient: (opts: Record<string, unknown>) => createMCPClientMock(opts),
10
10
  }));
11
11
 
12
- const {MCPService} = await import("./mcpService");
12
+ let MCPService: typeof import("./mcpService.js").MCPService;
13
+ beforeAll(async () => {
14
+ ({MCPService} = await import("./mcpService.js"));
15
+ });
13
16
 
14
17
  const makeConfig = (overrides?: Partial<Record<string, unknown>>) => ({
15
18
  name: "test-server",
@@ -6,61 +6,7 @@ import {registerBackendPreload} from "@terreno/test";
6
6
  registerBackendPreload({
7
7
  connectMongoInBeforeAll: true,
8
8
  mongo: {baseDatabaseName: "terreno-ai-test", useReplSet: true},
9
- onBeforeEach: async () => {
10
- const {shutdownLangfuseClient} = await import("../langfuseClient");
11
- await shutdownLangfuseClient();
12
- },
13
9
  testEnv: {
14
10
  tokenIssuer: "terreno-ai.test",
15
11
  },
16
12
  });
17
-
18
- // Mock @langfuse/client globally so the real `./langfuseClient` module runs
19
- // without making network calls.
20
- import {mock} from "bun:test";
21
-
22
- mock.module("@langfuse/client", () => {
23
- return {
24
- LangfuseClient: class FakeLangfuseClient {
25
- baseUrl: string;
26
- publicKey: string;
27
- secretKey: string;
28
- api = {
29
- prompts: {
30
- list: mock(async () => ({
31
- data: [],
32
- meta: {limit: 20, page: 1, total: 0, totalPages: 0},
33
- })),
34
- },
35
- trace: {
36
- get: mock(async (id: string) => ({id, name: "Trace"})),
37
- list: mock(async () => ({
38
- data: [],
39
- meta: {limit: 20, page: 1, total: 0, totalPages: 0},
40
- })),
41
- },
42
- };
43
- prompt = {
44
- create: mock(async (_params: Record<string, unknown>) => undefined),
45
- get: mock(async (name: string) => ({
46
- config: {},
47
- labels: [],
48
- name,
49
- prompt: "",
50
- tags: [],
51
- type: "text" as const,
52
- version: 1,
53
- })),
54
- };
55
- score = {create: mock(() => {})};
56
- flush = mock(async () => {});
57
- shutdown = mock(async () => {});
58
-
59
- constructor(opts: {baseUrl: string; publicKey: string; secretKey: string}) {
60
- this.baseUrl = opts.baseUrl;
61
- this.publicKey = opts.publicKey;
62
- this.secretKey = opts.secretKey;
63
- }
64
- },
65
- };
66
- });
@@ -11,7 +11,7 @@ import {addGptRoutes} from "../routes/gpt";
11
11
  import {addGptHistoryRoutes} from "../routes/gptHistories";
12
12
  import {AIService} from "../service/aiService";
13
13
  import type {GptHistoryDocument, GptRouteOptions} from "../types";
14
- import {type authAsUser, UserModel} from "./helpers";
14
+ import {type authAsUser, historyRouterOptions, terrenoUserModel} from "./helpers";
15
15
 
16
16
  type ModelStreamPart = {type: string; [key: string]: unknown};
17
17
 
@@ -232,7 +232,7 @@ export const createScriptedModel = ({
232
232
  const remaining = [...steps];
233
233
  let callCount = 0;
234
234
  return {
235
- doGenerate: mock(async () => ({
235
+ doGenerate: mock(async (_options: ModelCallOptions) => ({
236
236
  content: [{text: "Workspace setup", type: "text" as const}],
237
237
  finishReason: "stop" as const,
238
238
  usage: USAGE,
@@ -348,11 +348,11 @@ export const buildApp = ({
348
348
  openApiOptions: options,
349
349
  ...routeOptions,
350
350
  };
351
- addGptHistoryRoutes(router, {...options, chat});
351
+ addGptHistoryRoutes(router, {...historyRouterOptions(options), chat});
352
352
  addGptRoutes(router, chat);
353
353
  },
354
354
  skipListen: true,
355
- userModel: UserModel,
355
+ userModel: terrenoUserModel,
356
356
  }).build();
357
357
 
358
358
  export interface ServedApp {
@@ -0,0 +1,8 @@
1
+ /**
2
+ * A stub for `AIService.generateJsonObject` that answers every call with `answer()`, whatever
3
+ * schema the caller asks for. The real method is generic over the parsed object's type.
4
+ */
5
+ export const judgeAnswering =
6
+ (answer: () => Record<string, unknown>) =>
7
+ async <OBJECT>(): Promise<OBJECT> =>
8
+ answer() as OBJECT;
@@ -1,10 +1,16 @@
1
- import {createdUpdatedPlugin} from "@terreno/api";
1
+ import {
2
+ type UserModel as ApiUserModel,
3
+ createdUpdatedPlugin,
4
+ type ModelRouterOptions,
5
+ } from "@terreno/api";
2
6
  import {authAsUser as authAsUserWithCredentials} from "@terreno/test";
3
7
  import type express from "express";
4
- import mongoose, {type Model} from "mongoose";
8
+ import mongoose, {type HydratedDocument, type Model} from "mongoose";
5
9
  import passportLocalMongoose from "passport-local-mongoose";
6
10
  import type TestAgent from "supertest/lib/agent";
7
11
 
12
+ import type {GptHistoryDocument} from "../types/index";
13
+
8
14
  type PasswordedUser = {setPassword: (password: string) => Promise<void>};
9
15
 
10
16
  export interface AiTestUser {
@@ -43,6 +49,22 @@ const createTestUserModel = (): Model<AiTestUserDoc> => {
43
49
 
44
50
  export const UserModel = createTestUserModel();
45
51
 
52
+ /**
53
+ * `UserModel` as `TerrenoApp({userModel})` expects it. The passport-local-mongoose plugin adds
54
+ * the auth statics at runtime, and `@terreno/api`'s `User._id` uses mongoose's schema `ObjectId`
55
+ * type, which a hydrated document's `Types.ObjectId` does not satisfy.
56
+ */
57
+ export const terrenoUserModel = UserModel as unknown as ApiUserModel;
58
+
59
+ /**
60
+ * `TerrenoApp`'s `configureApp` options, typed as the GptHistory model router options tests
61
+ * spread them into. `configureApp` types them for no particular model.
62
+ */
63
+ export const historyRouterOptions = (
64
+ options: Partial<ModelRouterOptions<unknown>> | undefined
65
+ ): Partial<ModelRouterOptions<GptHistoryDocument>> =>
66
+ options as Partial<ModelRouterOptions<GptHistoryDocument>>;
67
+
46
68
  const STANDARD_AI_TEST_USERS = {
47
69
  admin: {admin: true, email: "admin@example.com", name: "Admin", password: "securePassword"},
48
70
  notAdmin: {admin: false, email: "notAdmin@example.com", name: "User", password: "password"},
@@ -52,10 +74,10 @@ export type StandardAiTestUserRole = keyof typeof STANDARD_AI_TEST_USERS;
52
74
 
53
75
  export const ensureTestUsers = async (
54
76
  users: AiTestUser[] = [STANDARD_AI_TEST_USERS.admin, STANDARD_AI_TEST_USERS.notAdmin]
55
- ): Promise<AiTestUserDoc[]> => {
77
+ ): Promise<HydratedDocument<AiTestUserDoc>[]> => {
56
78
  await UserModel.deleteMany({email: {$in: users.map((user) => user.email)}});
57
79
 
58
- const createdUsers: AiTestUserDoc[] = [];
80
+ const createdUsers: HydratedDocument<AiTestUserDoc>[] = [];
59
81
  for (const user of users) {
60
82
  const doc = await UserModel.create({
61
83
  admin: user.admin ?? false,
@@ -8,9 +8,12 @@ import type {ObsPromptRef} from "./observability";
8
8
 
9
9
  /**
10
10
  * A prompt version a task used, recorded on its `ObsTrace.prompts`. A `PromptVersionRef`
11
- * from `promptRegistry.get` fits as-is; its `body` is ignored.
11
+ * from `promptRegistry.get` fits as-is; its `body` is ignored and its `includes` (shared
12
+ * prompts it referenced) are recorded right after it.
12
13
  */
13
- export type HarnessPromptRef = ObsPromptRef;
14
+ export interface HarnessPromptRef extends ObsPromptRef {
15
+ includes?: ObsPromptRef[];
16
+ }
14
17
 
15
18
  /** A registry prompt passed as instructions: its `body` is used and its version recorded. */
16
19
  export interface HarnessPromptInstructions extends HarnessPromptRef {
@@ -625,9 +628,13 @@ export interface HarnessToolDefinition<Args = unknown, Result = unknown>
625
628
 
626
629
  /**
627
630
  * Any tool, whatever its argument and result types. `execute` arguments are contravariant, so
628
- * `never` accepts every tool in heterogeneous lists without an explicit `any`.
631
+ * `never` accepts every tool in heterogeneous lists without an explicit `any`. A Zod schema's
632
+ * output is covariant, so `parameters` widens to `unknown` instead.
629
633
  */
630
- export type AnyHarnessToolDefinition = HarnessToolDefinition<never, unknown>;
634
+ export interface AnyHarnessToolDefinition
635
+ extends Omit<HarnessToolDefinition<never, unknown>, "parameters"> {
636
+ parameters: z.ZodType<unknown>;
637
+ }
631
638
 
632
639
  export interface HarnessAgentDefinitionInput {
633
640
  /** Tried in order once the primary model's retryable failures exhaust `modelRetry`. */
@@ -448,9 +448,14 @@ export interface GptRouteOptions {
448
448
  * `POST /gpt/actions`. `{repair: true}` runs one repair call when that check fails.
449
449
  */
450
450
  uiBlocks?: boolean | UiBlocksOptions;
451
- /** Langfuse prompt name to load and use as the system prompt. Compiled with no variables.
452
- * Falls back gracefully if Langfuse is not configured or the prompt is not found. */
453
- langfuseSystemPromptName?: string;
451
+ /**
452
+ * Observability prompt registry name used as the default system prompt (prepended to the
453
+ * request's system prompt). The trace records the version. Skipped when no registry is registered, the
454
+ * label has no version, or an admin request selects `promptName`.
455
+ */
456
+ systemPromptName?: string;
457
+ /** Registry label read for `systemPromptName`. Defaults to `production`. */
458
+ systemPromptLabel?: string;
454
459
  /**
455
460
  * When `false` or the function returns `false`, prompts that include attachments are rejected.
456
461
  * Omit or pass `true` to leave uploads enabled.