@terreno/ai 57.20.0 → 57.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. package/CHANGELOG.md +81 -0
  2. package/README.md +3 -3
  3. package/biome.jsonc +3 -2
  4. package/dist/harness/agentLoop.js +1 -0
  5. package/dist/harness/commit.d.ts +4 -3
  6. package/dist/harness/commit.js +23 -4
  7. package/dist/index.d.ts +6 -9
  8. package/dist/index.js +14 -25
  9. package/dist/observability/adminScreens.js +5 -1
  10. package/dist/observability/evaluate.d.ts +8 -1
  11. package/dist/observability/evaluate.js +10 -2
  12. package/dist/observability/judgeAgreement.d.ts +53 -0
  13. package/dist/observability/judgeAgreement.js +123 -0
  14. package/dist/observability/local/datasetStore.d.ts +5 -0
  15. package/dist/observability/local/datasetStore.js +7 -2
  16. package/dist/observability/local/evaluatorStore.d.ts +17 -0
  17. package/dist/observability/local/evaluatorStore.js +73 -2
  18. package/dist/observability/local/experimentRunner.d.ts +42 -8
  19. package/dist/observability/local/experimentRunner.js +390 -11
  20. package/dist/observability/local/humanAnswers.d.ts +57 -0
  21. package/dist/observability/local/humanAnswers.js +174 -0
  22. package/dist/observability/local/judgeRun.d.ts +56 -0
  23. package/dist/observability/local/judgeRun.js +111 -0
  24. package/dist/observability/local/localPlugin.js +4 -0
  25. package/dist/observability/local/models/obsDatasetItem.js +4 -0
  26. package/dist/observability/local/models/obsEvaluator.js +4 -0
  27. package/dist/observability/local/models/obsExperiment.js +28 -0
  28. package/dist/observability/local/models/obsExperimentItem.js +4 -0
  29. package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
  30. package/dist/observability/local/models/obsJudgeRun.js +60 -0
  31. package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
  32. package/dist/observability/local/models/obsReviewBatch.js +48 -0
  33. package/dist/observability/local/models/obsReviewItem.js +10 -2
  34. package/dist/observability/local/models/obsTrace.js +4 -0
  35. package/dist/observability/local/promptReferences.d.ts +31 -0
  36. package/dist/observability/local/promptReferences.js +114 -0
  37. package/dist/observability/local/promptStore.d.ts +55 -19
  38. package/dist/observability/local/promptStore.js +171 -7
  39. package/dist/observability/local/reviewBatchStore.d.ts +106 -0
  40. package/dist/observability/local/reviewBatchStore.js +517 -0
  41. package/dist/observability/local/reviewStore.d.ts +24 -3
  42. package/dist/observability/local/reviewStore.js +98 -22
  43. package/dist/observability/local/traceStore.d.ts +14 -1
  44. package/dist/observability/local/traceStore.js +144 -11
  45. package/dist/observability/observabilityApp.js +10 -1
  46. package/dist/observability/questionFingerprint.d.ts +7 -0
  47. package/dist/observability/questionFingerprint.js +23 -0
  48. package/dist/observability/routes/experiments.js +12 -1
  49. package/dist/observability/routes/review.js +3 -1
  50. package/dist/observability/routes/reviewBatches.d.ts +11 -0
  51. package/dist/observability/routes/reviewBatches.js +78 -0
  52. package/dist/observability/routes/traces.d.ts +1 -1
  53. package/dist/observability/routes/traces.js +34 -2
  54. package/dist/observability/types.d.ts +10 -1
  55. package/dist/service/aiService.js +4 -1
  56. package/dist/service/chatTurn.js +38 -31
  57. package/dist/types/harness.d.ts +10 -4
  58. package/dist/types/index.d.ts +8 -3
  59. package/dist/types/observability.d.ts +74 -2
  60. package/e2e/smoke.ts +227 -0
  61. package/package.json +14 -28
  62. package/src/aiApp.test.ts +22 -17
  63. package/src/harness/agentLoop.test.ts +0 -3
  64. package/src/harness/agentLoop.ts +2 -1
  65. package/src/harness/agentRecovery.test.ts +30 -30
  66. package/src/harness/approvals.test.ts +11 -9
  67. package/src/harness/commit.ts +24 -4
  68. package/src/harness/events.test.ts +10 -6
  69. package/src/harness/extensions.test.ts +6 -8
  70. package/src/harness/ownership.test.ts +20 -9
  71. package/src/harness/retries.test.ts +6 -3
  72. package/src/harness/routes/conversations.test.ts +2 -2
  73. package/src/harness/routes/tasks.test.ts +2 -2
  74. package/src/harness/runners/jobsRunner.test.ts +1 -1
  75. package/src/harness/subagent.test.ts +29 -4
  76. package/src/harness/waits.test.ts +6 -4
  77. package/src/index.test.ts +3 -20
  78. package/src/index.ts +23 -26
  79. package/src/isolated/gptStream.isolated.ts +25 -28
  80. package/src/isolated/lazyProjectModel.isolated.ts +2 -2
  81. package/src/models/gptHistory.test.ts +10 -6
  82. package/src/observability/adminScreens.test.ts +7 -0
  83. package/src/observability/adminScreens.ts +6 -1
  84. package/src/observability/evaluate.test.ts +12 -10
  85. package/src/observability/evaluate.ts +12 -2
  86. package/src/observability/judgeAgreement.test.ts +111 -0
  87. package/src/observability/judgeAgreement.ts +184 -0
  88. package/src/observability/local/datasetStore.ts +30 -11
  89. package/src/observability/local/evaluatorStore.ts +96 -2
  90. package/src/observability/local/experimentRunner.test.ts +9 -12
  91. package/src/observability/local/experimentRunner.ts +497 -17
  92. package/src/observability/local/humanAnswers.ts +233 -0
  93. package/src/observability/local/judgePipeline.test.ts +444 -0
  94. package/src/observability/local/judgeRun.test.ts +173 -0
  95. package/src/observability/local/judgeRun.ts +184 -0
  96. package/src/observability/local/localPlugin.ts +4 -0
  97. package/src/observability/local/models/obsDatasetItem.ts +5 -0
  98. package/src/observability/local/models/obsEvaluator.ts +4 -0
  99. package/src/observability/local/models/obsExperiment.ts +29 -0
  100. package/src/observability/local/models/obsExperimentItem.ts +4 -0
  101. package/src/observability/local/models/obsJudgeRun.ts +61 -0
  102. package/src/observability/local/models/obsReviewBatch.ts +52 -0
  103. package/src/observability/local/models/obsReviewItem.ts +11 -2
  104. package/src/observability/local/models/obsTrace.ts +5 -0
  105. package/src/observability/local/obsModels.test.ts +13 -0
  106. package/src/observability/local/promptReferences.test.ts +128 -0
  107. package/src/observability/local/promptReferences.ts +149 -0
  108. package/src/observability/local/promptStore.test.ts +295 -3
  109. package/src/observability/local/promptStore.ts +243 -32
  110. package/src/observability/local/reviewBatchStore.test.ts +251 -0
  111. package/src/observability/local/reviewBatchStore.ts +656 -0
  112. package/src/observability/local/reviewStore.ts +122 -25
  113. package/src/observability/local/traceStore.test.ts +139 -1
  114. package/src/observability/local/traceStore.ts +162 -11
  115. package/src/observability/observabilityApp.test.ts +13 -10
  116. package/src/observability/observabilityApp.ts +10 -1
  117. package/src/observability/questionFingerprint.test.ts +33 -0
  118. package/src/observability/questionFingerprint.ts +24 -0
  119. package/src/observability/routes/datasets.test.ts +2 -2
  120. package/src/observability/routes/evaluators.test.ts +2 -2
  121. package/src/observability/routes/experiments.test.ts +39 -2
  122. package/src/observability/routes/experiments.ts +13 -1
  123. package/src/observability/routes/prompts.test.ts +3 -3
  124. package/src/observability/routes/review.test.ts +2 -2
  125. package/src/observability/routes/review.ts +3 -1
  126. package/src/observability/routes/reviewBatches.test.ts +209 -0
  127. package/src/observability/routes/reviewBatches.ts +138 -0
  128. package/src/observability/routes/status.test.ts +8 -8
  129. package/src/observability/routes/testMultiStage.test.ts +6 -6
  130. package/src/observability/routes/traces.test.ts +63 -2
  131. package/src/observability/routes/traces.ts +46 -4
  132. package/src/observability/types.ts +5 -2
  133. package/src/routes/files.test.ts +5 -3
  134. package/src/routes/gpt.test.ts +21 -14
  135. package/src/routes/gptActions.test.ts +7 -8
  136. package/src/routes/gptDatasets.test.ts +1 -1
  137. package/src/routes/gptFiles.test.ts +1 -1
  138. package/src/routes/gptHistories.test.ts +15 -9
  139. package/src/routes/mcp.test.ts +3 -3
  140. package/src/routes/projects.test.ts +5 -3
  141. package/src/routes/routes.test.ts +156 -115
  142. package/src/service/aiService.test.ts +22 -1
  143. package/src/service/aiService.ts +4 -1
  144. package/src/service/asks.test.ts +14 -6
  145. package/src/service/chatTurn.ts +42 -33
  146. package/src/service/fileStorage.test.ts +5 -2
  147. package/src/service/getMCPTools.test.ts +9 -5
  148. package/src/service/mcpService.test.ts +6 -3
  149. package/src/tests/bunSetup.ts +0 -54
  150. package/src/tests/chatHarness.ts +4 -4
  151. package/src/tests/fakeJudge.ts +8 -0
  152. package/src/tests/helpers.ts +26 -4
  153. package/src/types/harness.ts +11 -4
  154. package/src/types/index.ts +8 -3
  155. package/src/types/observability.ts +86 -2
  156. package/tsconfig.typecheck.json +18 -0
  157. package/dist/langfuseApp.d.ts +0 -9
  158. package/dist/langfuseApp.js +0 -53
  159. package/dist/langfuseCache.d.ts +0 -15
  160. package/dist/langfuseCache.js +0 -54
  161. package/dist/langfuseClient.d.ts +0 -6
  162. package/dist/langfuseClient.js +0 -34
  163. package/dist/langfusePrompts.d.ts +0 -12
  164. package/dist/langfusePrompts.js +0 -89
  165. package/dist/langfuseRoutesEvaluations.d.ts +0 -3
  166. package/dist/langfuseRoutesEvaluations.js +0 -32
  167. package/dist/langfuseRoutesMiddleware.d.ts +0 -2
  168. package/dist/langfuseRoutesMiddleware.js +0 -13
  169. package/dist/langfuseRoutesPlayground.d.ts +0 -2
  170. package/dist/langfuseRoutesPlayground.js +0 -36
  171. package/dist/langfuseRoutesPrompts.d.ts +0 -2
  172. package/dist/langfuseRoutesPrompts.js +0 -56
  173. package/dist/langfuseRoutesTraces.d.ts +0 -2
  174. package/dist/langfuseRoutesTraces.js +0 -30
  175. package/dist/langfuseTracing.d.ts +0 -4
  176. package/dist/langfuseTracing.js +0 -32
  177. package/dist/langfuseTypes.d.ts +0 -96
  178. package/dist/langfuseTypes.js +0 -2
  179. package/dist/langfuseVercelAi.d.ts +0 -12
  180. package/dist/langfuseVercelAi.js +0 -49
  181. package/src/isolated/langfuseClient.isolated.ts +0 -60
  182. package/src/langfuseApp.test.ts +0 -135
  183. package/src/langfuseApp.ts +0 -62
  184. package/src/langfuseCache.test.ts +0 -75
  185. package/src/langfuseCache.ts +0 -72
  186. package/src/langfuseClient.test.ts +0 -69
  187. package/src/langfuseClient.ts +0 -34
  188. package/src/langfusePrompts.test.ts +0 -188
  189. package/src/langfusePrompts.ts +0 -121
  190. package/src/langfuseRoutes.test.ts +0 -256
  191. package/src/langfuseRoutesEvaluations.ts +0 -48
  192. package/src/langfuseRoutesMiddleware.test.ts +0 -57
  193. package/src/langfuseRoutesMiddleware.ts +0 -12
  194. package/src/langfuseRoutesPlayground.ts +0 -50
  195. package/src/langfuseRoutesPrompts.ts +0 -87
  196. package/src/langfuseRoutesTraces.ts +0 -40
  197. package/src/langfuseTracing.test.ts +0 -68
  198. package/src/langfuseTracing.ts +0 -33
  199. package/src/langfuseTypes.ts +0 -103
  200. package/src/langfuseVercelAi.test.ts +0 -79
  201. package/src/langfuseVercelAi.ts +0 -72
@@ -2,6 +2,7 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.addObservabilityTraceRoutes = void 0;
4
4
  const api_1 = require("@terreno/api");
5
+ const traceStore_1 = require("../local/traceStore");
5
6
  const observabilityAppRegistry_1 = require("../observabilityAppRegistry");
6
7
  const observabilityRouteAccess_1 = require("./observabilityRouteAccess");
7
8
  const BASE_PATH = "/ai/observability";
@@ -25,11 +26,34 @@ const parsePromptVersion = (value, prompt) => {
25
26
  if (!Number.isInteger(parsed) || parsed < 1) {
26
27
  throw new api_1.APIError({ status: 400, title: "promptVersion must be a positive integer" });
27
28
  }
28
- if (!prompt) {
29
+ if (!prompt || prompt.length === 0) {
29
30
  throw new api_1.APIError({ status: 400, title: "prompt is required when promptVersion is set" });
30
31
  }
31
32
  return parsed;
32
33
  };
34
+ const parsePrompt = (value) => {
35
+ if (typeof value === "string") {
36
+ return value;
37
+ }
38
+ if (Array.isArray(value) && value.every((entry) => typeof entry === "string")) {
39
+ return value;
40
+ }
41
+ if (value === undefined) {
42
+ return undefined;
43
+ }
44
+ throw new api_1.APIError({ status: 400, title: "prompt must be a string or a list of strings" });
45
+ };
46
+ const parseSort = (value) => {
47
+ if (value === undefined) {
48
+ return undefined;
49
+ }
50
+ if (typeof value !== "string") {
51
+ throw new api_1.APIError({ status: 400, title: "Unknown trace sort key" });
52
+ }
53
+ // Validate here so a bad key fails before the store runs any query.
54
+ (0, traceStore_1.parseTraceListSort)(value);
55
+ return value;
56
+ };
33
57
  const parsePositiveInt = (value, fallback) => {
34
58
  if (value === undefined) {
35
59
  return fallback;
@@ -51,9 +75,16 @@ const addObservabilityTraceRoutes = (router, options) => {
51
75
  .withQueryParameter("prompt", { type: "string" }, { required: false })
52
76
  .withQueryParameter("promptVersion", { type: "number" }, { required: false })
53
77
  .withQueryParameter("scope", { type: "string" }, { required: false })
78
+ .withQueryParameter("sort", {
79
+ enum: traceStore_1.TRACE_LIST_SORT_KEYS.flatMap((key) => [key, `-${key}`]),
80
+ type: "string",
81
+ }, {
82
+ description: `Sort column, \`-\` prefix for descending. Defaults to \`${traceStore_1.DEFAULT_TRACE_LIST_SORT}\`.`,
83
+ required: false,
84
+ })
54
85
  .withResponse(200, { data: { type: "array" } })
55
86
  .build()), (0, api_1.asyncHandler)(async (req, res) => {
56
- const prompt = typeof req.query.prompt === "string" ? req.query.prompt : undefined;
87
+ const prompt = parsePrompt(req.query.prompt);
57
88
  const listed = await options.store.list({
58
89
  flaggedForDataset: parseBoolean(req.query.flaggedForDataset),
59
90
  from: typeof req.query.from === "string" ? req.query.from : undefined,
@@ -65,6 +96,7 @@ const addObservabilityTraceRoutes = (router, options) => {
65
96
  scope: typeof req.query.scope === "string" ? req.query.scope : undefined,
66
97
  sensitive: parseBoolean(req.query.sensitive),
67
98
  sessionId: typeof req.query.sessionId === "string" ? req.query.sessionId : undefined,
99
+ sort: parseSort(req.query.sort),
68
100
  status: req.query.status === "ok" || req.query.status === "error" ? req.query.status : undefined,
69
101
  to: typeof req.query.to === "string" ? req.query.to : undefined,
70
102
  userId: typeof req.query.userId === "string" ? req.query.userId : undefined,
@@ -1,6 +1,6 @@
1
1
  import { type AnyTerrenoAccess } from "@terreno/api";
2
2
  export type ObservabilityCapability = "datasets" | "experiments" | "prompts" | "reviewQueue" | "scores" | "traces";
3
- export type ControlPrimary = "langfuse" | "local";
3
+ export type ControlPrimary = "local";
4
4
  export interface ObservabilityControlConfig {
5
5
  datasets: ControlPrimary;
6
6
  experiments: ControlPrimary;
@@ -39,6 +39,7 @@ export interface TraceRecord {
39
39
  name: string;
40
40
  output?: unknown;
41
41
  prompts: {
42
+ includedBy?: string;
42
43
  label?: string;
43
44
  name: string;
44
45
  version: number;
@@ -68,7 +69,15 @@ export interface ScoreRecord {
68
69
  value: boolean | number | string;
69
70
  }
70
71
  export interface PromptVersionRef {
72
+ /** `body` with every prompt reference resolved. */
71
73
  body: string;
74
+ /** Prompt versions pulled in by references, recorded on the trace after this prompt. */
75
+ includes?: {
76
+ includedBy?: string;
77
+ label?: string;
78
+ name: string;
79
+ version: number;
80
+ }[];
72
81
  label?: string;
73
82
  name: string;
74
83
  sensitive?: boolean;
@@ -267,7 +267,10 @@ class AIService {
267
267
  name: spanName,
268
268
  output: params.response,
269
269
  prompts: promptRef
270
- ? [{ label: promptRef.label, name: promptRef.name, version: promptRef.version }]
270
+ ? [
271
+ { label: promptRef.label, name: promptRef.name, version: promptRef.version },
272
+ ...(promptRef.includes ?? []),
273
+ ]
271
274
  : [],
272
275
  sensitive: params.observability.sensitive,
273
276
  sessionId: params.observability.sessionId,
@@ -6,11 +6,11 @@ const api_1 = require("@terreno/api");
6
6
  const blocks_1 = require("@terreno/blocks");
7
7
  const ai_1 = require("ai");
8
8
  const luxon_1 = require("luxon");
9
- const langfuseVercelAi_1 = require("../langfuseVercelAi");
10
9
  const aiRequest_1 = require("../models/aiRequest");
11
10
  const fileAttachment_1 = require("../models/fileAttachment");
12
11
  const gptHistory_1 = require("../models/gptHistory");
13
12
  const project_1 = require("../models/project");
13
+ const observabilityAppRegistry_1 = require("../observability/observabilityAppRegistry");
14
14
  const aiService_1 = require("./aiService");
15
15
  const askFiles_1 = require("./askFiles");
16
16
  const asks_1 = require("./asks");
@@ -691,8 +691,28 @@ const startTurn = async ({ aiService, askAnswer, body, options, req, userId, })
691
691
  };
692
692
  };
693
693
  const lastUserText = (prompts) => prompts.findLast((prompt) => prompt.type === "user")?.text ?? "";
694
- /** The request's system prompt with project context and the Langfuse prompt prepended. */
695
- const buildSystemPrompt = async ({ history, options, projectId, systemPrompt, userId, }) => {
694
+ /** The route's default registry system prompt, or undefined when it cannot be loaded. */
695
+ const loadRegistrySystemPrompt = async (options) => {
696
+ const registry = (0, observabilityAppRegistry_1.getObservabilityApp)()?.promptRegistry;
697
+ if (!options.systemPromptName || !registry) {
698
+ return undefined;
699
+ }
700
+ try {
701
+ return await registry.get({
702
+ label: options.systemPromptLabel ?? "production",
703
+ name: options.systemPromptName,
704
+ });
705
+ }
706
+ catch (error) {
707
+ api_1.logger.warn("Registry system prompt skipped", {
708
+ error: error instanceof Error ? error.message : String(error),
709
+ promptName: options.systemPromptName,
710
+ });
711
+ return undefined;
712
+ }
713
+ };
714
+ /** The request's system prompt with project context and the registry prompt prepended. */
715
+ const buildSystemPrompt = async ({ history, projectId, registryPrompt, systemPrompt, userId, }) => {
696
716
  const effectiveProjectId = projectId ?? history.projectId;
697
717
  let effectiveSystemPrompt = systemPrompt;
698
718
  if (effectiveProjectId) {
@@ -718,22 +738,10 @@ const buildSystemPrompt = async ({ history, options, projectId, systemPrompt, us
718
738
  // Project loading failure should not block the request
719
739
  }
720
740
  }
721
- if (options.langfuseSystemPromptName) {
722
- try {
723
- const langfuseResult = await (0, langfuseVercelAi_1.preparePromptForAI)({
724
- promptName: options.langfuseSystemPromptName,
725
- userId: userId?.toString(),
726
- });
727
- const langfusePrompt = typeof langfuseResult.prompt === "string" ? langfuseResult.prompt : undefined;
728
- if (langfusePrompt) {
729
- effectiveSystemPrompt = effectiveSystemPrompt
730
- ? `${langfusePrompt}\n\n${effectiveSystemPrompt}`
731
- : langfusePrompt;
732
- }
733
- }
734
- catch (err) {
735
- api_1.logger.debug(`Langfuse system prompt skipped: ${err.message}`);
736
- }
741
+ if (registryPrompt) {
742
+ effectiveSystemPrompt = effectiveSystemPrompt
743
+ ? `${registryPrompt}\n\n${effectiveSystemPrompt}`
744
+ : registryPrompt;
737
745
  }
738
746
  return effectiveSystemPrompt;
739
747
  };
@@ -1192,20 +1200,29 @@ const runChatTurn = async ({ body, options, req, sink, }) => {
1192
1200
  const turn = await startTurn({ aiService, askAnswer, body, options, req, userId });
1193
1201
  const { answer, isNewHistory, logPrompt, messages, replayedMessages, resolvedAsk, titlePrompt } = turn;
1194
1202
  let { history } = turn;
1203
+ // An admin-selected promptName replaces the system prompt, so the route default is skipped.
1204
+ const registryPrompt = promptName ? undefined : await loadRegistrySystemPrompt(options);
1195
1205
  let effectiveSystemPrompt = await buildSystemPrompt({
1196
1206
  history,
1197
- options,
1198
1207
  projectId,
1208
+ registryPrompt: registryPrompt?.body,
1199
1209
  systemPrompt,
1200
1210
  userId,
1201
1211
  });
1202
- const observability = await aiService.resolveGenerateObservability({
1212
+ const resolvedObservability = await aiService.resolveGenerateObservability({
1203
1213
  promptLabel,
1204
1214
  promptName,
1205
1215
  sensitive: body.sensitive === true ? true : undefined,
1206
1216
  sessionId,
1207
1217
  systemPrompt: effectiveSystemPrompt,
1208
1218
  });
1219
+ const observability = registryPrompt
1220
+ ? {
1221
+ ...resolvedObservability,
1222
+ promptRef: registryPrompt,
1223
+ sensitive: resolvedObservability.sensitive || Boolean(registryPrompt.sensitive),
1224
+ }
1225
+ : resolvedObservability;
1209
1226
  effectiveSystemPrompt = observability.systemPrompt;
1210
1227
  // Some models (e.g. gemini-2.5-flash-image) don't support tool calling
1211
1228
  const modelId = aiService.modelId;
@@ -1435,17 +1452,7 @@ const runChatTurn = async ({ body, options, req, sink, }) => {
1435
1452
  sink.emit({ askResolved: { action: resolvedAsk.action, toolCallId: resolvedAsk.toolCallId } });
1436
1453
  }
1437
1454
  api_1.logger.debug("Starting streamText", { model: modelId, supportsTools });
1438
- const telemetry = (0, langfuseVercelAi_1.createTelemetryConfig)({
1439
- functionId: "gpt-prompt",
1440
- metadata: {
1441
- ...(options.langfuseSystemPromptName
1442
- ? { langfusePromptName: options.langfuseSystemPromptName }
1443
- : {}),
1444
- },
1445
- userId: userId?.toString(),
1446
- });
1447
1455
  const result = (0, ai_1.streamText)({
1448
- experimental_telemetry: telemetry,
1449
1456
  messages,
1450
1457
  model: aiService.model,
1451
1458
  providerOptions: !supportsTools
@@ -6,9 +6,12 @@ import type { ExecutionEnv } from "../harness/executionEnv";
6
6
  import type { ObsPromptRef } from "./observability";
7
7
  /**
8
8
  * A prompt version a task used, recorded on its `ObsTrace.prompts`. A `PromptVersionRef`
9
- * from `promptRegistry.get` fits as-is; its `body` is ignored.
9
+ * from `promptRegistry.get` fits as-is; its `body` is ignored and its `includes` (shared
10
+ * prompts it referenced) are recorded right after it.
10
11
  */
11
- export type HarnessPromptRef = ObsPromptRef;
12
+ export interface HarnessPromptRef extends ObsPromptRef {
13
+ includes?: ObsPromptRef[];
14
+ }
12
15
  /** A registry prompt passed as instructions: its `body` is used and its version recorded. */
13
16
  export interface HarnessPromptInstructions extends HarnessPromptRef {
14
17
  body: string;
@@ -536,9 +539,12 @@ export interface HarnessToolDefinition<Args = unknown, Result = unknown> extends
536
539
  }
537
540
  /**
538
541
  * Any tool, whatever its argument and result types. `execute` arguments are contravariant, so
539
- * `never` accepts every tool in heterogeneous lists without an explicit `any`.
542
+ * `never` accepts every tool in heterogeneous lists without an explicit `any`. A Zod schema's
543
+ * output is covariant, so `parameters` widens to `unknown` instead.
540
544
  */
541
- export type AnyHarnessToolDefinition = HarnessToolDefinition<never, unknown>;
545
+ export interface AnyHarnessToolDefinition extends Omit<HarnessToolDefinition<never, unknown>, "parameters"> {
546
+ parameters: z.ZodType<unknown>;
547
+ }
542
548
  export interface HarnessAgentDefinitionInput {
543
549
  /** Tried in order once the primary model's retryable failures exhaust `modelRetry`. */
544
550
  fallbackModels?: HarnessModelRef[];
@@ -384,9 +384,14 @@ export interface GptRouteOptions {
384
384
  * `POST /gpt/actions`. `{repair: true}` runs one repair call when that check fails.
385
385
  */
386
386
  uiBlocks?: boolean | UiBlocksOptions;
387
- /** Langfuse prompt name to load and use as the system prompt. Compiled with no variables.
388
- * Falls back gracefully if Langfuse is not configured or the prompt is not found. */
389
- langfuseSystemPromptName?: string;
387
+ /**
388
+ * Observability prompt registry name used as the default system prompt (prepended to the
389
+ * request's system prompt). The trace records the version. Skipped when no registry is registered, the
390
+ * label has no version, or an admin request selects `promptName`.
391
+ */
392
+ systemPromptName?: string;
393
+ /** Registry label read for `systemPromptName`. Defaults to `production`. */
394
+ systemPromptLabel?: string;
390
395
  /**
391
396
  * When `false` or the function returns `false`, prompts that include attachments are rejected.
392
397
  * Omit or pass `true` to leave uploads enabled.
@@ -1,5 +1,6 @@
1
1
  import type { FindExactlyOnePlugin, FindOneOrNonePlugin } from "@terreno/api";
2
2
  import type mongoose from "mongoose";
3
+ import type { JudgeAgreement } from "../observability/judgeAgreement";
3
4
  export interface ObsPromptVariable {
4
5
  key: string;
5
6
  label?: string;
@@ -13,6 +14,8 @@ export interface ObsPromptUsage {
13
14
  outputTokens?: number;
14
15
  }
15
16
  export interface ObsPromptRef {
17
+ /** Set when this version was pulled in by another prompt's reference, to that prompt's name. */
18
+ includedBy?: string;
16
19
  label?: string;
17
20
  name: string;
18
21
  version: number;
@@ -131,6 +134,8 @@ export interface ObsScoreModel extends mongoose.Model<ObsScoreDocument>, ObsScor
131
134
  export interface EvaluatorDimension {
132
135
  dataType: "boolean" | "categorical" | "numeric";
133
136
  key: string;
137
+ /** Question shown to human reviewers, e.g. "Does the summary mention every todo?" */
138
+ question?: string;
134
139
  range?: string;
135
140
  required: boolean;
136
141
  }
@@ -163,6 +168,7 @@ export interface ObsEvaluatorModel extends mongoose.Model<ObsEvaluatorDocument>,
163
168
  }
164
169
  export interface ObsReviewItemDocument extends mongoose.Document<mongoose.Types.ObjectId> {
165
170
  assigneeId?: mongoose.Types.ObjectId;
171
+ batchId?: mongoose.Types.ObjectId;
166
172
  comment?: string;
167
173
  created: Date;
168
174
  datasetItemId?: mongoose.Types.ObjectId;
@@ -173,13 +179,53 @@ export interface ObsReviewItemDocument extends mongoose.Document<mongoose.Types.
173
179
  scores?: Record<string, boolean | number | string>;
174
180
  spanId?: mongoose.Types.ObjectId;
175
181
  status: "done" | "in_progress" | "pending" | "skipped";
176
- traceId: mongoose.Types.ObjectId;
182
+ /** Trace under review. Unset when reviewers judge a dataset item that has no source trace. */
183
+ traceId?: mongoose.Types.ObjectId;
177
184
  updated: Date;
178
185
  }
179
186
  export interface ObsReviewItemStatics extends FindExactlyOnePlugin<ObsReviewItemDocument>, FindOneOrNonePlugin<ObsReviewItemDocument> {
180
187
  }
181
188
  export interface ObsReviewItemModel extends mongoose.Model<ObsReviewItemDocument>, ObsReviewItemStatics {
182
189
  }
190
+ export interface ObsReviewBatchDocument extends mongoose.Document<mongoose.Types.ObjectId> {
191
+ created: Date;
192
+ /** Dataset the batch was built from, when reviewers judge dataset items. */
193
+ datasetId?: mongoose.Types.ObjectId;
194
+ deleted: boolean;
195
+ description?: string;
196
+ evaluatorId: mongoose.Types.ObjectId;
197
+ name: string;
198
+ promptName?: string;
199
+ /** Question-set fingerprint when the batch was created (see `questionSetFingerprint`). */
200
+ questionsFingerprint?: string;
201
+ traceIds: mongoose.Types.ObjectId[];
202
+ updated: Date;
203
+ }
204
+ export interface ObsReviewBatchStatics extends FindExactlyOnePlugin<ObsReviewBatchDocument>, FindOneOrNonePlugin<ObsReviewBatchDocument> {
205
+ }
206
+ export interface ObsReviewBatchModel extends mongoose.Model<ObsReviewBatchDocument>, ObsReviewBatchStatics {
207
+ }
208
+ export interface ObsJudgeRunResult {
209
+ confidence?: number;
210
+ datasetItemId?: mongoose.Types.ObjectId;
211
+ error?: string;
212
+ scores?: Record<string, boolean | number | string>;
213
+ traceId?: mongoose.Types.ObjectId;
214
+ }
215
+ export interface ObsJudgeRunDocument extends mongoose.Document<mongoose.Types.ObjectId> {
216
+ batchId: mongoose.Types.ObjectId;
217
+ completedAt: Date;
218
+ created: Date;
219
+ deleted: boolean;
220
+ evaluatorId: mongoose.Types.ObjectId;
221
+ modelId?: string;
222
+ results: ObsJudgeRunResult[];
223
+ updated: Date;
224
+ }
225
+ export interface ObsJudgeRunStatics extends FindExactlyOnePlugin<ObsJudgeRunDocument>, FindOneOrNonePlugin<ObsJudgeRunDocument> {
226
+ }
227
+ export interface ObsJudgeRunModel extends mongoose.Model<ObsJudgeRunDocument>, ObsJudgeRunStatics {
228
+ }
183
229
  export interface ObsDatasetAnnotatedBy {
184
230
  label: string;
185
231
  reviewItemId?: string;
@@ -199,12 +245,23 @@ export interface ObsDatasetStatics extends FindExactlyOnePlugin<ObsDatasetDocume
199
245
  }
200
246
  export interface ObsDatasetModel extends mongoose.Model<ObsDatasetDocument>, ObsDatasetStatics {
201
247
  }
248
+ /** One question set's human answers for a dataset item. */
249
+ export interface ObsDatasetHumanAnswer {
250
+ answeredAt: Date;
251
+ /** `questionSetFingerprint` of the question set when the reviewer answered. */
252
+ fingerprint: string;
253
+ reviewItemId: string;
254
+ scores: Record<string, boolean | number | string>;
255
+ userId?: string;
256
+ }
202
257
  export interface ObsDatasetItemDocument extends mongoose.Document<mongoose.Types.ObjectId> {
203
258
  annotatedBy?: ObsDatasetAnnotatedBy;
204
259
  created: Date;
205
260
  datasetId: mongoose.Types.ObjectId;
206
261
  deleted: boolean;
207
262
  expectedOutput?: unknown;
263
+ /** Human answers keyed by question-set (human evaluator) id. */
264
+ humanAnswers?: Record<string, ObsDatasetHumanAnswer>;
208
265
  input: unknown;
209
266
  metadata?: Record<string, unknown>;
210
267
  origin: "manual" | "synthetic" | "trace";
@@ -219,7 +276,8 @@ export interface ObsDatasetItemStatics extends FindExactlyOnePlugin<ObsDatasetIt
219
276
  export interface ObsDatasetItemModel extends mongoose.Model<ObsDatasetItemDocument>, ObsDatasetItemStatics {
220
277
  }
221
278
  export interface ScoreThreshold {
222
- aggregate: "mean" | "trueRate";
279
+ /** `agreement` gates compare a judge prompt version's answers with human answers. */
280
+ aggregate: "agreement" | "mean" | "trueRate";
223
281
  dimension: string;
224
282
  evaluatorName: string;
225
283
  op: "eq" | "gte" | "lte";
@@ -235,7 +293,10 @@ export interface ExperimentEstimate {
235
293
  generations: number;
236
294
  wallClockSeconds: number;
237
295
  }
296
+ export type ExperimentKind = "judge" | "prompt";
238
297
  export interface ExperimentAggregates {
298
+ /** Judge experiments: agreement with the human answers per judge prompt version. */
299
+ agreement?: Record<string, JudgeAgreement>;
239
300
  gates: ExperimentGateResult[];
240
301
  lowConfidenceItemIds: string[];
241
302
  outlierItemIds: string[];
@@ -253,11 +314,20 @@ export interface ObsExperimentDocument extends mongoose.Document<mongoose.Types.
253
314
  estimate?: ExperimentEstimate;
254
315
  evaluatorIds: mongoose.Types.ObjectId[];
255
316
  includeUnproofread: boolean;
317
+ /** Judge experiments: the llm-judge evaluator whose prompt versions are compared. */
318
+ judgeEvaluatorId?: mongoose.Types.ObjectId;
319
+ kind: ExperimentKind;
256
320
  modelOverride?: string;
257
321
  name: string;
258
322
  promptName: string;
323
+ /** Judge experiments: question-set fingerprint of the human answers compared. */
324
+ questionsFingerprint?: string;
325
+ /** Judge experiments: the human question set whose answers are the reference. */
326
+ questionSetId?: mongoose.Types.ObjectId;
259
327
  results?: ExperimentAggregates;
260
328
  status: "completed" | "failed" | "pending" | "running";
329
+ /** Judge experiments: agreement rate (0–1) a version must reach to pass. */
330
+ targetAgreement?: number;
261
331
  thresholds: ScoreThreshold[];
262
332
  updated: Date;
263
333
  versions: number[];
@@ -282,6 +352,8 @@ export interface ObsExperimentItemDocument extends mongoose.Document<mongoose.Ty
282
352
  deleted: boolean;
283
353
  experimentId: mongoose.Types.ObjectId;
284
354
  failed: boolean;
355
+ /** Judge experiments: the human answers the judge versions are compared with. */
356
+ humanScores?: Record<string, boolean | number | string>;
285
357
  updated: Date;
286
358
  versionResults: Record<string, ExperimentVersionResult>;
287
359
  }
package/e2e/smoke.ts ADDED
@@ -0,0 +1,227 @@
1
+ /**
2
+ * Real-process smoke for @terreno/ai. Run `bun e2e/smoke.ts` after `bun run compile`.
3
+ *
4
+ * The smoke re-runs this file with `serve` as a child process: a minimal TerrenoApp from the
5
+ * compiled `@terreno/api` with the compiled `AiApp` registered, on a free port, against
6
+ * `smokeMongoUri()`. The child's `AIService` wraps a deterministic fake provider
7
+ * (`MockLanguageModelV3` from `ai/test`) and runs with every provider key unset. The parent then
8
+ * checks over real HTTP that a signed-in user's chat request streams back the fake provider's
9
+ * text. Exits non-zero on the first failed check.
10
+ */
11
+ import {
12
+ freePort,
13
+ type SpawnedProcess,
14
+ smokeMongoUri,
15
+ spawnUntilReady,
16
+ stopSmokeMongo,
17
+ } from "@terreno/test";
18
+ import type express from "express";
19
+ import {DateTime} from "luxon";
20
+ import type {Schema} from "mongoose";
21
+
22
+ const SERVE_ARG = "serve";
23
+ const FAKE_REPLY = "Hello from the ai smoke's fake provider.";
24
+ const FAKE_TITLE = "Smoke chat";
25
+
26
+ /** Provider credentials the smoke unsets in the child, so no real provider can answer. */
27
+ const PROVIDER_ENV_KEYS = [
28
+ "ANTHROPIC_API_KEY",
29
+ "GEMINI_API_KEY",
30
+ "GOOGLE_APPLICATION_CREDENTIALS",
31
+ "GOOGLE_GENERATIVE_AI_API_KEY",
32
+ "GOOGLE_VERTEX_PROJECT",
33
+ "LANGFUSE_PUBLIC_KEY",
34
+ "LANGFUSE_SECRET_KEY",
35
+ "OPENAI_API_KEY",
36
+ ];
37
+
38
+ interface ChatEvent {
39
+ done?: boolean;
40
+ error?: string;
41
+ historyId?: string;
42
+ text?: string;
43
+ title?: string;
44
+ }
45
+
46
+ const ZERO_USAGE = {
47
+ inputTokens: {cacheRead: 0, cacheWrite: 0, noCache: 0, total: 0},
48
+ outputTokens: {reasoning: 0, text: 0, total: 0},
49
+ };
50
+
51
+ /**
52
+ * Child: the compiled API with auth, the compiled AiApp on a fake model, and a health plugin.
53
+ * The packages are imported here, not at the top, so the parent process never loads them.
54
+ */
55
+ const serve = async (): Promise<void> => {
56
+ const {createdUpdatedPlugin, logger, TerrenoApp} = await import("@terreno/api");
57
+ const {AIService, AiApp} = await import("@terreno/ai");
58
+ const {MockLanguageModelV3} = await import("ai/test");
59
+ const {default: mongoose} = await import("mongoose");
60
+ const {default: passportLocalMongoose} = await import("passport-local-mongoose");
61
+
62
+ const userSchema = new mongoose.Schema({
63
+ admin: {default: false, description: "Whether the user is an admin", type: Boolean},
64
+ email: {description: "Sign-in email", type: String},
65
+ name: {description: "Display name", type: String},
66
+ });
67
+ userSchema.plugin(passportLocalMongoose as unknown as (schema: Schema, options: object) => void, {
68
+ usernameCaseInsensitive: true,
69
+ usernameField: "email",
70
+ });
71
+ userSchema.plugin(createdUpdatedPlugin);
72
+ const User = mongoose.model("User", userSchema);
73
+
74
+ // Streams the chat reply and answers the title request, the two calls one chat turn makes.
75
+ const model = new MockLanguageModelV3({
76
+ doGenerate: async () => ({
77
+ content: [{text: FAKE_TITLE, type: "text"}],
78
+ finishReason: {raw: "stop", unified: "stop"},
79
+ usage: ZERO_USAGE,
80
+ warnings: [],
81
+ }),
82
+ doStream: async () => ({
83
+ stream: new ReadableStream({
84
+ start(controller): void {
85
+ controller.enqueue({type: "stream-start", warnings: []});
86
+ controller.enqueue({id: "text-1", type: "text-start"});
87
+ controller.enqueue({delta: FAKE_REPLY, id: "text-1", type: "text-delta"});
88
+ controller.enqueue({id: "text-1", type: "text-end"});
89
+ controller.enqueue({
90
+ finishReason: {raw: "stop", unified: "stop"},
91
+ type: "finish",
92
+ usage: ZERO_USAGE,
93
+ });
94
+ controller.close();
95
+ },
96
+ }),
97
+ }),
98
+ modelId: "ai-smoke-fake-model",
99
+ provider: "ai-smoke-fake-provider",
100
+ });
101
+
102
+ const healthPlugin = {
103
+ register: (app: express.Application): void => {
104
+ app.get("/health", (_req, res) => {
105
+ res.json({healthy: mongoose.connection.readyState === 1});
106
+ });
107
+ },
108
+ };
109
+
110
+ await mongoose.connect(process.env.MONGO_URI as string);
111
+ const app = new TerrenoApp({
112
+ loggingOptions: {logRequests: false},
113
+ userModel: User as unknown as ConstructorParameters<typeof TerrenoApp>[0]["userModel"],
114
+ })
115
+ .register(new AiApp({aiService: new AIService({model})}))
116
+ .register(healthPlugin);
117
+ app.start();
118
+ await app.whenReady();
119
+ logger.info(`[aiSmoke] serving on port ${process.env.PORT}`);
120
+ };
121
+
122
+ const expectStatus = async (res: Response, status: number, step: string): Promise<string> => {
123
+ const body = await res.text();
124
+ if (res.status !== status) {
125
+ throw new Error(`${step}: expected ${status}, got ${res.status}: ${body}`);
126
+ }
127
+ return body;
128
+ };
129
+
130
+ /** Splits a finished `text/event-stream` body into its JSON `data:` events. */
131
+ const parseEvents = (body: string): ChatEvent[] =>
132
+ body
133
+ .split("\n\n")
134
+ .map((chunk) => chunk.trim())
135
+ .filter((chunk) => chunk.startsWith("data: "))
136
+ .map((chunk) => JSON.parse(chunk.slice("data: ".length)) as ChatEvent);
137
+
138
+ /** Parent: boots the child and sends one chat request over HTTP. */
139
+ const run = async (): Promise<void> => {
140
+ const startedAt = DateTime.now();
141
+ const port = await freePort();
142
+ const baseUrl = `http://127.0.0.1:${port}`;
143
+ let server: SpawnedProcess | undefined;
144
+
145
+ try {
146
+ server = await spawnUntilReady({
147
+ cmd: [process.execPath, __filename, SERVE_ARG],
148
+ env: {
149
+ ...Object.fromEntries(PROVIDER_ENV_KEYS.map((key) => [key, undefined])),
150
+ MONGO_URI: await smokeMongoUri({databaseName: "aiSmoke"}),
151
+ PORT: String(port),
152
+ REFRESH_TOKEN_SECRET: "ai-smoke-refresh-secret",
153
+ SESSION_SECRET: "ai-smoke-session-secret",
154
+ TOKEN_ISSUER: "ai-smoke",
155
+ TOKEN_SECRET: "ai-smoke-token-secret",
156
+ },
157
+ timeoutMs: 30000,
158
+ url: `${baseUrl}/health`,
159
+ });
160
+
161
+ const signup = JSON.parse(
162
+ await expectStatus(
163
+ await fetch(`${baseUrl}/auth/signup`, {
164
+ body: JSON.stringify({
165
+ email: `ai-smoke-${startedAt.toMillis()}@example.com`,
166
+ password: "ai-smoke-password-1",
167
+ }),
168
+ headers: {"Content-Type": "application/json"},
169
+ method: "POST",
170
+ }),
171
+ 200,
172
+ "POST /auth/signup"
173
+ )
174
+ ) as {data: {token: string}};
175
+ console.info("✅ POST /auth/signup signs in a user");
176
+
177
+ const chat = await fetch(`${baseUrl}/gpt/prompt`, {
178
+ body: JSON.stringify({prompt: "Say hello."}),
179
+ headers: {
180
+ Authorization: `Bearer ${signup.data.token}`,
181
+ "Content-Type": "application/json",
182
+ },
183
+ method: "POST",
184
+ });
185
+ const events = parseEvents(await expectStatus(chat, 200, "POST /gpt/prompt"));
186
+ const failed = events.find((event) => event.error);
187
+ if (failed) {
188
+ throw new Error(`POST /gpt/prompt: the stream reported an error: ${failed.error}`);
189
+ }
190
+ const reply = events.map((event) => event.text ?? "").join("");
191
+ if (reply !== FAKE_REPLY) {
192
+ throw new Error(
193
+ `POST /gpt/prompt: expected the fake provider's text, got ${JSON.stringify(events)}`
194
+ );
195
+ }
196
+ const done = events.find((event) => event.done);
197
+ if (!done?.historyId || done.title !== FAKE_TITLE) {
198
+ throw new Error(
199
+ `POST /gpt/prompt: expected a done event with a history and the fake title, got ${JSON.stringify(done)}`
200
+ );
201
+ }
202
+ console.info("✅ POST /gpt/prompt streams the fake provider's reply");
203
+ } catch (error: unknown) {
204
+ if (server) {
205
+ console.error(`--- server output ---\n${server.output()}`);
206
+ }
207
+ throw error;
208
+ } finally {
209
+ await server?.stop();
210
+ await stopSmokeMongo();
211
+ }
212
+
213
+ console.info(`ai smoke passed in ${DateTime.now().diff(startedAt).toMillis()} ms`);
214
+ };
215
+
216
+ const main = process.argv[2] === SERVE_ARG ? serve : run;
217
+ main().then(
218
+ () => {
219
+ if (main === run) {
220
+ process.exit(0);
221
+ }
222
+ },
223
+ (error: unknown) => {
224
+ console.error(`❌ ${error instanceof Error ? error.message : String(error)}`);
225
+ process.exit(1);
226
+ }
227
+ );