@terreno/ai 58.0.0-beta.2 → 58.0.0-beta.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. package/CHANGELOG.md +146 -0
  2. package/README.md +3 -3
  3. package/dist/harness/commit.d.ts +4 -3
  4. package/dist/harness/commit.js +23 -4
  5. package/dist/index.d.ts +6 -9
  6. package/dist/index.js +14 -25
  7. package/dist/observability/adminScreens.js +5 -1
  8. package/dist/observability/evaluate.d.ts +8 -1
  9. package/dist/observability/evaluate.js +10 -2
  10. package/dist/observability/judgeAgreement.d.ts +53 -0
  11. package/dist/observability/judgeAgreement.js +123 -0
  12. package/dist/observability/local/datasetStore.d.ts +5 -0
  13. package/dist/observability/local/datasetStore.js +7 -2
  14. package/dist/observability/local/evaluatorStore.d.ts +17 -0
  15. package/dist/observability/local/evaluatorStore.js +73 -2
  16. package/dist/observability/local/experimentRunner.d.ts +40 -3
  17. package/dist/observability/local/experimentRunner.js +390 -11
  18. package/dist/observability/local/humanAnswers.d.ts +57 -0
  19. package/dist/observability/local/humanAnswers.js +174 -0
  20. package/dist/observability/local/judgeRun.d.ts +56 -0
  21. package/dist/observability/local/judgeRun.js +111 -0
  22. package/dist/observability/local/localPlugin.js +4 -0
  23. package/dist/observability/local/models/obsDatasetItem.js +4 -0
  24. package/dist/observability/local/models/obsEvaluator.js +4 -0
  25. package/dist/observability/local/models/obsExperiment.js +28 -0
  26. package/dist/observability/local/models/obsExperimentItem.js +4 -0
  27. package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
  28. package/dist/observability/local/models/obsJudgeRun.js +60 -0
  29. package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
  30. package/dist/observability/local/models/obsReviewBatch.js +48 -0
  31. package/dist/observability/local/models/obsReviewItem.js +10 -2
  32. package/dist/observability/local/models/obsTrace.js +4 -0
  33. package/dist/observability/local/promptReferences.d.ts +31 -0
  34. package/dist/observability/local/promptReferences.js +114 -0
  35. package/dist/observability/local/promptStore.d.ts +55 -19
  36. package/dist/observability/local/promptStore.js +171 -7
  37. package/dist/observability/local/reviewBatchStore.d.ts +106 -0
  38. package/dist/observability/local/reviewBatchStore.js +517 -0
  39. package/dist/observability/local/reviewStore.d.ts +24 -3
  40. package/dist/observability/local/reviewStore.js +98 -22
  41. package/dist/observability/local/traceStore.d.ts +14 -1
  42. package/dist/observability/local/traceStore.js +144 -11
  43. package/dist/observability/observabilityApp.js +10 -1
  44. package/dist/observability/questionFingerprint.d.ts +7 -0
  45. package/dist/observability/questionFingerprint.js +23 -0
  46. package/dist/observability/routes/experiments.js +12 -1
  47. package/dist/observability/routes/review.js +3 -1
  48. package/dist/observability/routes/reviewBatches.d.ts +11 -0
  49. package/dist/observability/routes/reviewBatches.js +78 -0
  50. package/dist/observability/routes/traces.d.ts +1 -1
  51. package/dist/observability/routes/traces.js +34 -2
  52. package/dist/observability/types.d.ts +10 -1
  53. package/dist/service/aiService.js +4 -1
  54. package/dist/service/chatTurn.js +38 -31
  55. package/dist/types/harness.d.ts +5 -2
  56. package/dist/types/index.d.ts +8 -3
  57. package/dist/types/observability.d.ts +74 -2
  58. package/package.json +8 -24
  59. package/src/harness/agentLoop.test.ts +0 -3
  60. package/src/harness/agentRecovery.test.ts +0 -3
  61. package/src/harness/commit.ts +24 -4
  62. package/src/harness/extensions.test.ts +0 -3
  63. package/src/harness/subagent.test.ts +27 -3
  64. package/src/index.test.ts +0 -17
  65. package/src/index.ts +23 -26
  66. package/src/isolated/gptStream.isolated.ts +0 -8
  67. package/src/observability/adminScreens.test.ts +7 -0
  68. package/src/observability/adminScreens.ts +6 -1
  69. package/src/observability/evaluate.ts +12 -2
  70. package/src/observability/judgeAgreement.test.ts +111 -0
  71. package/src/observability/judgeAgreement.ts +184 -0
  72. package/src/observability/local/datasetStore.ts +30 -11
  73. package/src/observability/local/evaluatorStore.ts +96 -2
  74. package/src/observability/local/experimentRunner.ts +495 -15
  75. package/src/observability/local/humanAnswers.ts +233 -0
  76. package/src/observability/local/judgePipeline.test.ts +444 -0
  77. package/src/observability/local/judgeRun.test.ts +173 -0
  78. package/src/observability/local/judgeRun.ts +184 -0
  79. package/src/observability/local/localPlugin.ts +4 -0
  80. package/src/observability/local/models/obsDatasetItem.ts +5 -0
  81. package/src/observability/local/models/obsEvaluator.ts +4 -0
  82. package/src/observability/local/models/obsExperiment.ts +29 -0
  83. package/src/observability/local/models/obsExperimentItem.ts +4 -0
  84. package/src/observability/local/models/obsJudgeRun.ts +61 -0
  85. package/src/observability/local/models/obsReviewBatch.ts +52 -0
  86. package/src/observability/local/models/obsReviewItem.ts +11 -2
  87. package/src/observability/local/models/obsTrace.ts +5 -0
  88. package/src/observability/local/obsModels.test.ts +13 -0
  89. package/src/observability/local/promptReferences.test.ts +128 -0
  90. package/src/observability/local/promptReferences.ts +149 -0
  91. package/src/observability/local/promptStore.test.ts +291 -0
  92. package/src/observability/local/promptStore.ts +243 -32
  93. package/src/observability/local/reviewBatchStore.test.ts +251 -0
  94. package/src/observability/local/reviewBatchStore.ts +656 -0
  95. package/src/observability/local/reviewStore.ts +122 -25
  96. package/src/observability/local/traceStore.test.ts +139 -1
  97. package/src/observability/local/traceStore.ts +162 -11
  98. package/src/observability/observabilityApp.test.ts +13 -10
  99. package/src/observability/observabilityApp.ts +10 -1
  100. package/src/observability/questionFingerprint.test.ts +33 -0
  101. package/src/observability/questionFingerprint.ts +24 -0
  102. package/src/observability/routes/experiments.test.ts +37 -0
  103. package/src/observability/routes/experiments.ts +13 -1
  104. package/src/observability/routes/review.ts +3 -1
  105. package/src/observability/routes/reviewBatches.test.ts +209 -0
  106. package/src/observability/routes/reviewBatches.ts +138 -0
  107. package/src/observability/routes/status.test.ts +4 -4
  108. package/src/observability/routes/traces.test.ts +61 -0
  109. package/src/observability/routes/traces.ts +46 -4
  110. package/src/observability/types.ts +5 -2
  111. package/src/routes/routes.test.ts +101 -76
  112. package/src/service/aiService.test.ts +21 -0
  113. package/src/service/aiService.ts +4 -1
  114. package/src/service/chatTurn.ts +42 -33
  115. package/src/tests/bunSetup.ts +0 -54
  116. package/src/types/harness.ts +5 -2
  117. package/src/types/index.ts +8 -3
  118. package/src/types/observability.ts +86 -2
  119. package/dist/langfuseApp.d.ts +0 -9
  120. package/dist/langfuseApp.js +0 -53
  121. package/dist/langfuseCache.d.ts +0 -15
  122. package/dist/langfuseCache.js +0 -54
  123. package/dist/langfuseClient.d.ts +0 -6
  124. package/dist/langfuseClient.js +0 -34
  125. package/dist/langfusePrompts.d.ts +0 -12
  126. package/dist/langfusePrompts.js +0 -89
  127. package/dist/langfuseRoutesEvaluations.d.ts +0 -3
  128. package/dist/langfuseRoutesEvaluations.js +0 -32
  129. package/dist/langfuseRoutesMiddleware.d.ts +0 -2
  130. package/dist/langfuseRoutesMiddleware.js +0 -13
  131. package/dist/langfuseRoutesPlayground.d.ts +0 -2
  132. package/dist/langfuseRoutesPlayground.js +0 -36
  133. package/dist/langfuseRoutesPrompts.d.ts +0 -2
  134. package/dist/langfuseRoutesPrompts.js +0 -56
  135. package/dist/langfuseRoutesTraces.d.ts +0 -2
  136. package/dist/langfuseRoutesTraces.js +0 -30
  137. package/dist/langfuseTracing.d.ts +0 -4
  138. package/dist/langfuseTracing.js +0 -32
  139. package/dist/langfuseTypes.d.ts +0 -96
  140. package/dist/langfuseTypes.js +0 -2
  141. package/dist/langfuseVercelAi.d.ts +0 -12
  142. package/dist/langfuseVercelAi.js +0 -49
  143. package/src/isolated/langfuseClient.isolated.ts +0 -66
  144. package/src/langfuseApp.test.ts +0 -119
  145. package/src/langfuseApp.ts +0 -62
  146. package/src/langfuseCache.test.ts +0 -75
  147. package/src/langfuseCache.ts +0 -72
  148. package/src/langfuseClient.test.ts +0 -69
  149. package/src/langfuseClient.ts +0 -34
  150. package/src/langfusePrompts.test.ts +0 -188
  151. package/src/langfusePrompts.ts +0 -121
  152. package/src/langfuseRoutes.test.ts +0 -261
  153. package/src/langfuseRoutesEvaluations.ts +0 -48
  154. package/src/langfuseRoutesMiddleware.test.ts +0 -57
  155. package/src/langfuseRoutesMiddleware.ts +0 -12
  156. package/src/langfuseRoutesPlayground.ts +0 -50
  157. package/src/langfuseRoutesPrompts.ts +0 -87
  158. package/src/langfuseRoutesTraces.ts +0 -40
  159. package/src/langfuseTracing.test.ts +0 -72
  160. package/src/langfuseTracing.ts +0 -33
  161. package/src/langfuseTypes.ts +0 -103
  162. package/src/langfuseVercelAi.test.ts +0 -82
  163. package/src/langfuseVercelAi.ts +0 -72
  164. package/src/tests/fakeLangfusePrompt.ts +0 -10
@@ -2,6 +2,7 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.addObservabilityTraceRoutes = void 0;
4
4
  const api_1 = require("@terreno/api");
5
+ const traceStore_1 = require("../local/traceStore");
5
6
  const observabilityAppRegistry_1 = require("../observabilityAppRegistry");
6
7
  const observabilityRouteAccess_1 = require("./observabilityRouteAccess");
7
8
  const BASE_PATH = "/ai/observability";
@@ -25,11 +26,34 @@ const parsePromptVersion = (value, prompt) => {
25
26
  if (!Number.isInteger(parsed) || parsed < 1) {
26
27
  throw new api_1.APIError({ status: 400, title: "promptVersion must be a positive integer" });
27
28
  }
28
- if (!prompt) {
29
+ if (!prompt || prompt.length === 0) {
29
30
  throw new api_1.APIError({ status: 400, title: "prompt is required when promptVersion is set" });
30
31
  }
31
32
  return parsed;
32
33
  };
34
+ const parsePrompt = (value) => {
35
+ if (typeof value === "string") {
36
+ return value;
37
+ }
38
+ if (Array.isArray(value) && value.every((entry) => typeof entry === "string")) {
39
+ return value;
40
+ }
41
+ if (value === undefined) {
42
+ return undefined;
43
+ }
44
+ throw new api_1.APIError({ status: 400, title: "prompt must be a string or a list of strings" });
45
+ };
46
+ const parseSort = (value) => {
47
+ if (value === undefined) {
48
+ return undefined;
49
+ }
50
+ if (typeof value !== "string") {
51
+ throw new api_1.APIError({ status: 400, title: "Unknown trace sort key" });
52
+ }
53
+ // Validate here so a bad key fails before the store runs any query.
54
+ (0, traceStore_1.parseTraceListSort)(value);
55
+ return value;
56
+ };
33
57
  const parsePositiveInt = (value, fallback) => {
34
58
  if (value === undefined) {
35
59
  return fallback;
@@ -51,9 +75,16 @@ const addObservabilityTraceRoutes = (router, options) => {
51
75
  .withQueryParameter("prompt", { type: "string" }, { required: false })
52
76
  .withQueryParameter("promptVersion", { type: "number" }, { required: false })
53
77
  .withQueryParameter("scope", { type: "string" }, { required: false })
78
+ .withQueryParameter("sort", {
79
+ enum: traceStore_1.TRACE_LIST_SORT_KEYS.flatMap((key) => [key, `-${key}`]),
80
+ type: "string",
81
+ }, {
82
+ description: `Sort column, \`-\` prefix for descending. Defaults to \`${traceStore_1.DEFAULT_TRACE_LIST_SORT}\`.`,
83
+ required: false,
84
+ })
54
85
  .withResponse(200, { data: { type: "array" } })
55
86
  .build()), (0, api_1.asyncHandler)(async (req, res) => {
56
- const prompt = typeof req.query.prompt === "string" ? req.query.prompt : undefined;
87
+ const prompt = parsePrompt(req.query.prompt);
57
88
  const listed = await options.store.list({
58
89
  flaggedForDataset: parseBoolean(req.query.flaggedForDataset),
59
90
  from: typeof req.query.from === "string" ? req.query.from : undefined,
@@ -65,6 +96,7 @@ const addObservabilityTraceRoutes = (router, options) => {
65
96
  scope: typeof req.query.scope === "string" ? req.query.scope : undefined,
66
97
  sensitive: parseBoolean(req.query.sensitive),
67
98
  sessionId: typeof req.query.sessionId === "string" ? req.query.sessionId : undefined,
99
+ sort: parseSort(req.query.sort),
68
100
  status: req.query.status === "ok" || req.query.status === "error" ? req.query.status : undefined,
69
101
  to: typeof req.query.to === "string" ? req.query.to : undefined,
70
102
  userId: typeof req.query.userId === "string" ? req.query.userId : undefined,
@@ -1,6 +1,6 @@
1
1
  import { type AnyTerrenoAccess } from "@terreno/api";
2
2
  export type ObservabilityCapability = "datasets" | "experiments" | "prompts" | "reviewQueue" | "scores" | "traces";
3
- export type ControlPrimary = "langfuse" | "local";
3
+ export type ControlPrimary = "local";
4
4
  export interface ObservabilityControlConfig {
5
5
  datasets: ControlPrimary;
6
6
  experiments: ControlPrimary;
@@ -39,6 +39,7 @@ export interface TraceRecord {
39
39
  name: string;
40
40
  output?: unknown;
41
41
  prompts: {
42
+ includedBy?: string;
42
43
  label?: string;
43
44
  name: string;
44
45
  version: number;
@@ -68,7 +69,15 @@ export interface ScoreRecord {
68
69
  value: boolean | number | string;
69
70
  }
70
71
  export interface PromptVersionRef {
72
+ /** `body` with every prompt reference resolved. */
71
73
  body: string;
74
+ /** Prompt versions pulled in by references, recorded on the trace after this prompt. */
75
+ includes?: {
76
+ includedBy?: string;
77
+ label?: string;
78
+ name: string;
79
+ version: number;
80
+ }[];
72
81
  label?: string;
73
82
  name: string;
74
83
  sensitive?: boolean;
@@ -267,7 +267,10 @@ class AIService {
267
267
  name: spanName,
268
268
  output: params.response,
269
269
  prompts: promptRef
270
- ? [{ label: promptRef.label, name: promptRef.name, version: promptRef.version }]
270
+ ? [
271
+ { label: promptRef.label, name: promptRef.name, version: promptRef.version },
272
+ ...(promptRef.includes ?? []),
273
+ ]
271
274
  : [],
272
275
  sensitive: params.observability.sensitive,
273
276
  sessionId: params.observability.sessionId,
@@ -6,11 +6,11 @@ const api_1 = require("@terreno/api");
6
6
  const blocks_1 = require("@terreno/blocks");
7
7
  const ai_1 = require("ai");
8
8
  const luxon_1 = require("luxon");
9
- const langfuseVercelAi_1 = require("../langfuseVercelAi");
10
9
  const aiRequest_1 = require("../models/aiRequest");
11
10
  const fileAttachment_1 = require("../models/fileAttachment");
12
11
  const gptHistory_1 = require("../models/gptHistory");
13
12
  const project_1 = require("../models/project");
13
+ const observabilityAppRegistry_1 = require("../observability/observabilityAppRegistry");
14
14
  const aiService_1 = require("./aiService");
15
15
  const askFiles_1 = require("./askFiles");
16
16
  const asks_1 = require("./asks");
@@ -696,8 +696,28 @@ const startTurn = async ({ aiService, askAnswer, body, options, req, userId, })
696
696
  };
697
697
  };
698
698
  const lastUserText = (prompts) => prompts.findLast((prompt) => prompt.type === "user")?.text ?? "";
699
- /** The request's system prompt with project context and the Langfuse prompt prepended. */
700
- const buildSystemPrompt = async ({ history, options, projectId, systemPrompt, userId, }) => {
699
+ /** The route's default registry system prompt, or undefined when it cannot be loaded. */
700
+ const loadRegistrySystemPrompt = async (options) => {
701
+ const registry = (0, observabilityAppRegistry_1.getObservabilityApp)()?.promptRegistry;
702
+ if (!options.systemPromptName || !registry) {
703
+ return undefined;
704
+ }
705
+ try {
706
+ return await registry.get({
707
+ label: options.systemPromptLabel ?? "production",
708
+ name: options.systemPromptName,
709
+ });
710
+ }
711
+ catch (error) {
712
+ api_1.logger.warn("Registry system prompt skipped", {
713
+ error: error instanceof Error ? error.message : String(error),
714
+ promptName: options.systemPromptName,
715
+ });
716
+ return undefined;
717
+ }
718
+ };
719
+ /** The request's system prompt with project context and the registry prompt prepended. */
720
+ const buildSystemPrompt = async ({ history, projectId, registryPrompt, systemPrompt, userId, }) => {
701
721
  const effectiveProjectId = projectId ?? history.projectId;
702
722
  let effectiveSystemPrompt = systemPrompt;
703
723
  if (effectiveProjectId) {
@@ -723,22 +743,10 @@ const buildSystemPrompt = async ({ history, options, projectId, systemPrompt, us
723
743
  // Project loading failure should not block the request
724
744
  }
725
745
  }
726
- if (options.langfuseSystemPromptName) {
727
- try {
728
- const langfuseResult = await (0, langfuseVercelAi_1.preparePromptForAI)({
729
- promptName: options.langfuseSystemPromptName,
730
- userId: userId?.toString(),
731
- });
732
- const langfusePrompt = typeof langfuseResult.prompt === "string" ? langfuseResult.prompt : undefined;
733
- if (langfusePrompt) {
734
- effectiveSystemPrompt = effectiveSystemPrompt
735
- ? `${langfusePrompt}\n\n${effectiveSystemPrompt}`
736
- : langfusePrompt;
737
- }
738
- }
739
- catch (err) {
740
- api_1.logger.debug(`Langfuse system prompt skipped: ${err.message}`);
741
- }
746
+ if (registryPrompt) {
747
+ effectiveSystemPrompt = effectiveSystemPrompt
748
+ ? `${registryPrompt}\n\n${effectiveSystemPrompt}`
749
+ : registryPrompt;
742
750
  }
743
751
  return effectiveSystemPrompt;
744
752
  };
@@ -1197,20 +1205,29 @@ const runChatTurn = async ({ body, options, req, sink, }) => {
1197
1205
  const turn = await startTurn({ aiService, askAnswer, body, options, req, userId });
1198
1206
  const { answer, isNewHistory, logPrompt, messages, replayedMessages, resolvedAsk, titlePrompt } = turn;
1199
1207
  let { history } = turn;
1208
+ // An admin-selected promptName replaces the system prompt, so the route default is skipped.
1209
+ const registryPrompt = promptName ? undefined : await loadRegistrySystemPrompt(options);
1200
1210
  let effectiveSystemPrompt = await buildSystemPrompt({
1201
1211
  history,
1202
- options,
1203
1212
  projectId,
1213
+ registryPrompt: registryPrompt?.body,
1204
1214
  systemPrompt,
1205
1215
  userId,
1206
1216
  });
1207
- const observability = await aiService.resolveGenerateObservability({
1217
+ const resolvedObservability = await aiService.resolveGenerateObservability({
1208
1218
  promptLabel,
1209
1219
  promptName,
1210
1220
  sensitive: body.sensitive === true ? true : undefined,
1211
1221
  sessionId,
1212
1222
  systemPrompt: effectiveSystemPrompt,
1213
1223
  });
1224
+ const observability = registryPrompt
1225
+ ? {
1226
+ ...resolvedObservability,
1227
+ promptRef: registryPrompt,
1228
+ sensitive: resolvedObservability.sensitive || Boolean(registryPrompt.sensitive),
1229
+ }
1230
+ : resolvedObservability;
1214
1231
  effectiveSystemPrompt = observability.systemPrompt;
1215
1232
  // Some models (e.g. gemini-2.5-flash-image) don't support tool calling
1216
1233
  const modelId = aiService.modelId;
@@ -1440,17 +1457,7 @@ const runChatTurn = async ({ body, options, req, sink, }) => {
1440
1457
  sink.emit({ askResolved: { action: resolvedAsk.action, toolCallId: resolvedAsk.toolCallId } });
1441
1458
  }
1442
1459
  api_1.logger.debug("Starting streamText", { model: modelId, supportsTools });
1443
- const telemetry = (0, langfuseVercelAi_1.createTelemetryConfig)({
1444
- functionId: "gpt-prompt",
1445
- metadata: {
1446
- ...(options.langfuseSystemPromptName
1447
- ? { langfusePromptName: options.langfuseSystemPromptName }
1448
- : {}),
1449
- },
1450
- userId: userId?.toString(),
1451
- });
1452
1460
  const result = (0, ai_1.streamText)({
1453
- experimental_telemetry: telemetry,
1454
1461
  messages,
1455
1462
  model: aiService.model,
1456
1463
  providerOptions: !supportsTools
@@ -6,9 +6,12 @@ import type { ExecutionEnv } from "../harness/executionEnv";
6
6
  import type { ObsPromptRef } from "./observability";
7
7
  /**
8
8
  * A prompt version a task used, recorded on its `ObsTrace.prompts`. A `PromptVersionRef`
9
- * from `promptRegistry.get` fits as-is; its `body` is ignored.
9
+ * from `promptRegistry.get` fits as-is; its `body` is ignored and its `includes` (shared
10
+ * prompts it referenced) are recorded right after it.
10
11
  */
11
- export type HarnessPromptRef = ObsPromptRef;
12
+ export interface HarnessPromptRef extends ObsPromptRef {
13
+ includes?: ObsPromptRef[];
14
+ }
12
15
  /** A registry prompt passed as instructions: its `body` is used and its version recorded. */
13
16
  export interface HarnessPromptInstructions extends HarnessPromptRef {
14
17
  body: string;
@@ -388,9 +388,14 @@ export interface GptRouteOptions {
388
388
  * `POST /gpt/actions`. `{repair: true}` runs one repair call when that check fails.
389
389
  */
390
390
  uiBlocks?: boolean | UiBlocksOptions;
391
- /** Langfuse prompt name to load and use as the system prompt. Compiled with no variables.
392
- * Falls back gracefully if Langfuse is not configured or the prompt is not found. */
393
- langfuseSystemPromptName?: string;
391
+ /**
392
+ * Observability prompt registry name used as the default system prompt (prepended to the
393
+ * request's system prompt). The trace records the version. Skipped when no registry is registered, the
394
+ * label has no version, or an admin request selects `promptName`.
395
+ */
396
+ systemPromptName?: string;
397
+ /** Registry label read for `systemPromptName`. Defaults to `production`. */
398
+ systemPromptLabel?: string;
394
399
  /**
395
400
  * When `false` or the function returns `false`, prompts that include attachments are rejected.
396
401
  * Omit or pass `true` to leave uploads enabled.
@@ -1,5 +1,6 @@
1
1
  import type { FindExactlyOnePlugin, FindOneOrNonePlugin } from "@terreno/api";
2
2
  import type mongoose from "mongoose";
3
+ import type { JudgeAgreement } from "../observability/judgeAgreement";
3
4
  export interface ObsPromptVariable {
4
5
  key: string;
5
6
  label?: string;
@@ -13,6 +14,8 @@ export interface ObsPromptUsage {
13
14
  outputTokens?: number;
14
15
  }
15
16
  export interface ObsPromptRef {
17
+ /** Set when this version was pulled in by another prompt's reference, to that prompt's name. */
18
+ includedBy?: string;
16
19
  label?: string;
17
20
  name: string;
18
21
  version: number;
@@ -131,6 +134,8 @@ export interface ObsScoreModel extends mongoose.Model<ObsScoreDocument>, ObsScor
131
134
  export interface EvaluatorDimension {
132
135
  dataType: "boolean" | "categorical" | "numeric";
133
136
  key: string;
137
+ /** Question shown to human reviewers, e.g. "Does the summary mention every todo?" */
138
+ question?: string;
134
139
  range?: string;
135
140
  required: boolean;
136
141
  }
@@ -163,6 +168,7 @@ export interface ObsEvaluatorModel extends mongoose.Model<ObsEvaluatorDocument>,
163
168
  }
164
169
  export interface ObsReviewItemDocument extends mongoose.Document<mongoose.Types.ObjectId> {
165
170
  assigneeId?: mongoose.Types.ObjectId;
171
+ batchId?: mongoose.Types.ObjectId;
166
172
  comment?: string;
167
173
  created: Date;
168
174
  datasetItemId?: mongoose.Types.ObjectId;
@@ -173,13 +179,53 @@ export interface ObsReviewItemDocument extends mongoose.Document<mongoose.Types.
173
179
  scores?: Record<string, boolean | number | string>;
174
180
  spanId?: mongoose.Types.ObjectId;
175
181
  status: "done" | "in_progress" | "pending" | "skipped";
176
- traceId: mongoose.Types.ObjectId;
182
+ /** Trace under review. Unset when reviewers judge a dataset item that has no source trace. */
183
+ traceId?: mongoose.Types.ObjectId;
177
184
  updated: Date;
178
185
  }
179
186
  export interface ObsReviewItemStatics extends FindExactlyOnePlugin<ObsReviewItemDocument>, FindOneOrNonePlugin<ObsReviewItemDocument> {
180
187
  }
181
188
  export interface ObsReviewItemModel extends mongoose.Model<ObsReviewItemDocument>, ObsReviewItemStatics {
182
189
  }
190
+ export interface ObsReviewBatchDocument extends mongoose.Document<mongoose.Types.ObjectId> {
191
+ created: Date;
192
+ /** Dataset the batch was built from, when reviewers judge dataset items. */
193
+ datasetId?: mongoose.Types.ObjectId;
194
+ deleted: boolean;
195
+ description?: string;
196
+ evaluatorId: mongoose.Types.ObjectId;
197
+ name: string;
198
+ promptName?: string;
199
+ /** Question-set fingerprint when the batch was created (see `questionSetFingerprint`). */
200
+ questionsFingerprint?: string;
201
+ traceIds: mongoose.Types.ObjectId[];
202
+ updated: Date;
203
+ }
204
+ export interface ObsReviewBatchStatics extends FindExactlyOnePlugin<ObsReviewBatchDocument>, FindOneOrNonePlugin<ObsReviewBatchDocument> {
205
+ }
206
+ export interface ObsReviewBatchModel extends mongoose.Model<ObsReviewBatchDocument>, ObsReviewBatchStatics {
207
+ }
208
+ export interface ObsJudgeRunResult {
209
+ confidence?: number;
210
+ datasetItemId?: mongoose.Types.ObjectId;
211
+ error?: string;
212
+ scores?: Record<string, boolean | number | string>;
213
+ traceId?: mongoose.Types.ObjectId;
214
+ }
215
+ export interface ObsJudgeRunDocument extends mongoose.Document<mongoose.Types.ObjectId> {
216
+ batchId: mongoose.Types.ObjectId;
217
+ completedAt: Date;
218
+ created: Date;
219
+ deleted: boolean;
220
+ evaluatorId: mongoose.Types.ObjectId;
221
+ modelId?: string;
222
+ results: ObsJudgeRunResult[];
223
+ updated: Date;
224
+ }
225
+ export interface ObsJudgeRunStatics extends FindExactlyOnePlugin<ObsJudgeRunDocument>, FindOneOrNonePlugin<ObsJudgeRunDocument> {
226
+ }
227
+ export interface ObsJudgeRunModel extends mongoose.Model<ObsJudgeRunDocument>, ObsJudgeRunStatics {
228
+ }
183
229
  export interface ObsDatasetAnnotatedBy {
184
230
  label: string;
185
231
  reviewItemId?: string;
@@ -199,12 +245,23 @@ export interface ObsDatasetStatics extends FindExactlyOnePlugin<ObsDatasetDocume
199
245
  }
200
246
  export interface ObsDatasetModel extends mongoose.Model<ObsDatasetDocument>, ObsDatasetStatics {
201
247
  }
248
+ /** One question set's human answers for a dataset item. */
249
+ export interface ObsDatasetHumanAnswer {
250
+ answeredAt: Date;
251
+ /** `questionSetFingerprint` of the question set when the reviewer answered. */
252
+ fingerprint: string;
253
+ reviewItemId: string;
254
+ scores: Record<string, boolean | number | string>;
255
+ userId?: string;
256
+ }
202
257
  export interface ObsDatasetItemDocument extends mongoose.Document<mongoose.Types.ObjectId> {
203
258
  annotatedBy?: ObsDatasetAnnotatedBy;
204
259
  created: Date;
205
260
  datasetId: mongoose.Types.ObjectId;
206
261
  deleted: boolean;
207
262
  expectedOutput?: unknown;
263
+ /** Human answers keyed by question-set (human evaluator) id. */
264
+ humanAnswers?: Record<string, ObsDatasetHumanAnswer>;
208
265
  input: unknown;
209
266
  metadata?: Record<string, unknown>;
210
267
  origin: "manual" | "synthetic" | "trace";
@@ -219,7 +276,8 @@ export interface ObsDatasetItemStatics extends FindExactlyOnePlugin<ObsDatasetIt
219
276
  export interface ObsDatasetItemModel extends mongoose.Model<ObsDatasetItemDocument>, ObsDatasetItemStatics {
220
277
  }
221
278
  export interface ScoreThreshold {
222
- aggregate: "mean" | "trueRate";
279
+ /** `agreement` gates compare a judge prompt version's answers with human answers. */
280
+ aggregate: "agreement" | "mean" | "trueRate";
223
281
  dimension: string;
224
282
  evaluatorName: string;
225
283
  op: "eq" | "gte" | "lte";
@@ -235,7 +293,10 @@ export interface ExperimentEstimate {
235
293
  generations: number;
236
294
  wallClockSeconds: number;
237
295
  }
296
+ export type ExperimentKind = "judge" | "prompt";
238
297
  export interface ExperimentAggregates {
298
+ /** Judge experiments: agreement with the human answers per judge prompt version. */
299
+ agreement?: Record<string, JudgeAgreement>;
239
300
  gates: ExperimentGateResult[];
240
301
  lowConfidenceItemIds: string[];
241
302
  outlierItemIds: string[];
@@ -253,11 +314,20 @@ export interface ObsExperimentDocument extends mongoose.Document<mongoose.Types.
253
314
  estimate?: ExperimentEstimate;
254
315
  evaluatorIds: mongoose.Types.ObjectId[];
255
316
  includeUnproofread: boolean;
317
+ /** Judge experiments: the llm-judge evaluator whose prompt versions are compared. */
318
+ judgeEvaluatorId?: mongoose.Types.ObjectId;
319
+ kind: ExperimentKind;
256
320
  modelOverride?: string;
257
321
  name: string;
258
322
  promptName: string;
323
+ /** Judge experiments: question-set fingerprint of the human answers compared. */
324
+ questionsFingerprint?: string;
325
+ /** Judge experiments: the human question set whose answers are the reference. */
326
+ questionSetId?: mongoose.Types.ObjectId;
259
327
  results?: ExperimentAggregates;
260
328
  status: "completed" | "failed" | "pending" | "running";
329
+ /** Judge experiments: agreement rate (0–1) a version must reach to pass. */
330
+ targetAgreement?: number;
261
331
  thresholds: ScoreThreshold[];
262
332
  updated: Date;
263
333
  versions: number[];
@@ -282,6 +352,8 @@ export interface ObsExperimentItemDocument extends mongoose.Document<mongoose.Ty
282
352
  deleted: boolean;
283
353
  experimentId: mongoose.Types.ObjectId;
284
354
  failed: boolean;
355
+ /** Judge experiments: the human answers the judge versions are compared with. */
356
+ humanScores?: Record<string, boolean | number | string>;
285
357
  updated: Date;
286
358
  versionResults: Record<string, ExperimentVersionResult>;
287
359
  }
package/package.json CHANGED
@@ -6,11 +6,7 @@
6
6
  "dependencies": {
7
7
  "@ai-sdk/mcp": "^2.0.57",
8
8
  "@google-cloud/storage": "^7.15.0",
9
- "@langfuse/client": "^5.0.1",
10
- "@langfuse/otel": "^5.0.1",
11
- "@opentelemetry/api": "^1.9.0",
12
- "@opentelemetry/sdk-node": "0.222.0",
13
- "@terreno/blocks": "58.0.0-beta.2",
9
+ "@terreno/blocks": "58.0.0-beta.4",
14
10
  "ai": "^6.0.97",
15
11
  "ajv": "8.20.0",
16
12
  "csv-parse": "^6.1.0",
@@ -22,10 +18,10 @@
22
18
  "description": "Provider-agnostic AI service with streaming chat, request logging, and admin tools — part of Terreno, the batteries-included TypeScript framework for universal apps.",
23
19
  "devDependencies": {
24
20
  "@biomejs/biome": "^2.3.6",
25
- "@terreno/admin-backend": "58.0.0-beta.2",
26
- "@terreno/api": "58.0.0-beta.2",
27
- "@terreno/jobs": "58.0.0-beta.2",
28
- "@terreno/test": "58.0.0-beta.2",
21
+ "@terreno/admin-backend": "58.0.0-beta.4",
22
+ "@terreno/api": "58.0.0-beta.4",
23
+ "@terreno/jobs": "58.0.0-beta.4",
24
+ "@terreno/test": "58.0.0-beta.4",
29
25
  "@types/bun": "^1.4.0",
30
26
  "@types/express": "^5.0.6",
31
27
  "@types/luxon": "^3.7.1",
@@ -87,18 +83,6 @@
87
83
  "require": "./dist/harness/runners/jobsRunner.js",
88
84
  "types": "./dist/harness/runners/jobsRunner.d.ts"
89
85
  },
90
- "./langfuseApp": {
91
- "default": "./dist/langfuseApp.js",
92
- "import": "./dist/langfuseApp.js",
93
- "require": "./dist/langfuseApp.js",
94
- "types": "./dist/langfuseApp.d.ts"
95
- },
96
- "./langfuseClient": {
97
- "default": "./dist/langfuseClient.js",
98
- "import": "./dist/langfuseClient.js",
99
- "require": "./dist/langfuseClient.js",
100
- "types": "./dist/langfuseClient.d.ts"
101
- },
102
86
  "./observability/localPlugin": {
103
87
  "default": "./dist/observability/local/localPlugin.js",
104
88
  "import": "./dist/observability/local/localPlugin.js",
@@ -137,8 +121,8 @@
137
121
  "peerDependencies": {
138
122
  "@ai-sdk/google-vertex": "^5.0.92",
139
123
  "@ai-sdk/provider": "^4.0.26",
140
- "@terreno/api": "58.0.0-beta.2",
141
- "@terreno/jobs": "58.0.0-beta.2",
124
+ "@terreno/api": "58.0.0-beta.4",
125
+ "@terreno/jobs": "58.0.0-beta.4",
142
126
  "mongoose": "^8.0.0 || ^9.0.0"
143
127
  },
144
128
  "peerDependenciesMeta": {
@@ -171,5 +155,5 @@
171
155
  "typecheck": "tsc -p tsconfig.typecheck.json"
172
156
  },
173
157
  "types": "dist/index.d.ts",
174
- "version": "58.0.0-beta.2"
158
+ "version": "58.0.0-beta.4"
175
159
  }
@@ -178,9 +178,6 @@ const spansOf = (task: HarnessTaskDocument) =>
178
178
 
179
179
  describe("Agent turns", () => {
180
180
  beforeAll(async () => {
181
- // The shared preload's beforeEach imports this lazily under a 5 s hook timeout; a
182
- // cold first import can exceed it, so load it here first.
183
- await import("../langfuseClient.js");
184
181
  createLocalObservabilityPlugin();
185
182
  SpanModel = registerObsSpan();
186
183
  TraceModel = registerObsTrace();
@@ -170,9 +170,6 @@ const toolMessages = async (conversationId: string) => {
170
170
 
171
171
  describe("Agent turns across a crash", () => {
172
172
  beforeAll(async () => {
173
- // The shared preload's beforeEach imports this lazily under a 5 s hook timeout; a
174
- // cold first import can exceed it, so load it here first.
175
- await import("../langfuseClient.js");
176
173
  createLocalObservabilityPlugin();
177
174
  SpanModel = registerObsSpan();
178
175
  TraceModel = registerObsTrace();
@@ -152,9 +152,10 @@ const findByRequestId = async ({
152
152
  };
153
153
 
154
154
  /**
155
- * Prompt refs as stored on `ObsTrace.prompts` (`{label?, name, version}`; a registry
156
- * `body` is dropped), once each. Throws `invalidRequest` for a ref without a name, with a
157
- * non-string label, or without a positive integer version.
155
+ * Prompt refs as stored on `ObsTrace.prompts` (`{includedBy?, label?, name, version}`; a
156
+ * registry `body` is dropped and its `includes` follow it), once each. Throws
157
+ * `invalidRequest` for a ref without a name, with a non-string label or includedBy, with
158
+ * non-array includes, or without a positive integer version.
158
159
  */
159
160
  export const tracePromptRefs = (refs: HarnessPromptRef[] | undefined): HarnessPromptRef[] => {
160
161
  if (refs === undefined) {
@@ -164,7 +165,19 @@ export const tracePromptRefs = (refs: HarnessPromptRef[] | undefined): HarnessPr
164
165
  throw harnessError({detail: "prompts must be an array", kind: "invalidRequest"});
165
166
  }
166
167
  const unique = new Map<string, HarnessPromptRef>();
167
- for (const ref of refs) {
168
+ const expanded = refs.flatMap((ref) => {
169
+ if (ref?.includes !== undefined && !Array.isArray(ref.includes)) {
170
+ throw harnessError({
171
+ detail:
172
+ typeof ref.name === "string" && ref.name.trim()
173
+ ? `Prompt ref "${ref.name}" includes must be an array`
174
+ : "Prompt ref includes must be an array",
175
+ kind: "invalidRequest",
176
+ });
177
+ }
178
+ return [ref, ...(ref?.includes ?? [])];
179
+ });
180
+ for (const ref of expanded) {
168
181
  if (typeof ref?.name !== "string" || !ref.name.trim()) {
169
182
  throw harnessError({detail: "Each prompt ref needs a name", kind: "invalidRequest"});
170
183
  }
@@ -180,7 +193,14 @@ export const tracePromptRefs = (refs: HarnessPromptRef[] | undefined): HarnessPr
180
193
  kind: "invalidRequest",
181
194
  });
182
195
  }
196
+ if (ref.includedBy !== undefined && typeof ref.includedBy !== "string") {
197
+ throw harnessError({
198
+ detail: `Prompt ref "${ref.name}" includedBy must be a string`,
199
+ kind: "invalidRequest",
200
+ });
201
+ }
183
202
  const normalized = {
203
+ ...(ref.includedBy !== undefined ? {includedBy: ref.includedBy} : {}),
184
204
  ...(ref.label !== undefined ? {label: ref.label} : {}),
185
205
  name: ref.name,
186
206
  version: ref.version,
@@ -198,9 +198,6 @@ const llmSpans = (task: HarnessTaskDocument) =>
198
198
 
199
199
  describe("Extensions, hooks, wraps, and memos", () => {
200
200
  beforeAll(async () => {
201
- // The shared preload's beforeEach imports this lazily under a 5 s hook timeout; a
202
- // cold first import can exceed it, so load it here first.
203
- await import("../langfuseClient.js");
204
201
  createLocalObservabilityPlugin();
205
202
  SpanModel = registerObsSpan();
206
203
  TraceModel = registerObsTrace();
@@ -247,9 +247,6 @@ const parentTask = ({
247
247
 
248
248
  describe("rt.runAgent (subagents)", () => {
249
249
  beforeAll(async () => {
250
- // The shared preload's beforeEach imports this lazily under a 5 s hook timeout; a
251
- // cold first import can exceed it, so load it here first.
252
- await import("../langfuseClient.js");
253
250
  createLocalObservabilityPlugin();
254
251
  SpanModel = registerObsSpan();
255
252
  TraceModel = registerObsTrace();
@@ -378,6 +375,16 @@ describe("rt.runAgent (subagents)", () => {
378
375
  ).rejects.toThrow(
379
376
  harnessErrorMatching("invalidRequest", 'Prompt ref "plan" label must be a string')
380
377
  );
378
+ await expect(
379
+ harness.createTask(parent, {}, {prompts: [{includes: {}, name: "plan", version: 1} as never]})
380
+ ).rejects.toThrow(
381
+ harnessErrorMatching("invalidRequest", 'Prompt ref "plan" includes must be an array')
382
+ );
383
+ await expect(
384
+ harness.createTask(parent, {}, {prompts: [{includes: {}, version: 1} as never]})
385
+ ).rejects.toThrow(
386
+ harnessErrorMatching("invalidRequest", "Prompt ref includes must be an array")
387
+ );
381
388
  await expect(harness.createTask(parent, {}, {trace: {tags: ["ok", " "]}})).rejects.toThrow(
382
389
  harnessErrorMatching("invalidRequest", "trace.tags must be an array of non-empty strings")
383
390
  );
@@ -397,6 +404,23 @@ describe("rt.runAgent (subagents)", () => {
397
404
  expect(plain(trace.prompts)).toEqual([plan]);
398
405
  });
399
406
 
407
+ it("records a registry prompt's shared includes after it", async () => {
408
+ const agent = summarizerWith();
409
+ const parent = parentTask({agent, inputs: []});
410
+ const {model} = scriptedModel([]);
411
+ const {harness} = await openProcess({model, registry: [agent, parent]});
412
+
413
+ const include = {includedBy: "plan", label: "production", name: "base-rules", version: 4};
414
+ const created = await harness.createTask(
415
+ parent,
416
+ {},
417
+ {prompts: [{includes: [include], name: "plan", version: 2}]}
418
+ );
419
+
420
+ const trace = await TraceModel.findExactlyOne({_id: created.traceId});
421
+ expect(plain(trace.prompts)).toEqual([{name: "plan", version: 2}, include]);
422
+ });
423
+
400
424
  it("fails the caller at once for an instructions prompt without a body or a bad prompt ref", async () => {
401
425
  const agent = summarizerWith();
402
426
  const noBody = parentTask({