@terreno/ai 57.20.0 → 57.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. package/CHANGELOG.md +81 -0
  2. package/README.md +3 -3
  3. package/biome.jsonc +3 -2
  4. package/dist/harness/agentLoop.js +1 -0
  5. package/dist/harness/commit.d.ts +4 -3
  6. package/dist/harness/commit.js +23 -4
  7. package/dist/index.d.ts +6 -9
  8. package/dist/index.js +14 -25
  9. package/dist/observability/adminScreens.js +5 -1
  10. package/dist/observability/evaluate.d.ts +8 -1
  11. package/dist/observability/evaluate.js +10 -2
  12. package/dist/observability/judgeAgreement.d.ts +53 -0
  13. package/dist/observability/judgeAgreement.js +123 -0
  14. package/dist/observability/local/datasetStore.d.ts +5 -0
  15. package/dist/observability/local/datasetStore.js +7 -2
  16. package/dist/observability/local/evaluatorStore.d.ts +17 -0
  17. package/dist/observability/local/evaluatorStore.js +73 -2
  18. package/dist/observability/local/experimentRunner.d.ts +42 -8
  19. package/dist/observability/local/experimentRunner.js +390 -11
  20. package/dist/observability/local/humanAnswers.d.ts +57 -0
  21. package/dist/observability/local/humanAnswers.js +174 -0
  22. package/dist/observability/local/judgeRun.d.ts +56 -0
  23. package/dist/observability/local/judgeRun.js +111 -0
  24. package/dist/observability/local/localPlugin.js +4 -0
  25. package/dist/observability/local/models/obsDatasetItem.js +4 -0
  26. package/dist/observability/local/models/obsEvaluator.js +4 -0
  27. package/dist/observability/local/models/obsExperiment.js +28 -0
  28. package/dist/observability/local/models/obsExperimentItem.js +4 -0
  29. package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
  30. package/dist/observability/local/models/obsJudgeRun.js +60 -0
  31. package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
  32. package/dist/observability/local/models/obsReviewBatch.js +48 -0
  33. package/dist/observability/local/models/obsReviewItem.js +10 -2
  34. package/dist/observability/local/models/obsTrace.js +4 -0
  35. package/dist/observability/local/promptReferences.d.ts +31 -0
  36. package/dist/observability/local/promptReferences.js +114 -0
  37. package/dist/observability/local/promptStore.d.ts +55 -19
  38. package/dist/observability/local/promptStore.js +171 -7
  39. package/dist/observability/local/reviewBatchStore.d.ts +106 -0
  40. package/dist/observability/local/reviewBatchStore.js +517 -0
  41. package/dist/observability/local/reviewStore.d.ts +24 -3
  42. package/dist/observability/local/reviewStore.js +98 -22
  43. package/dist/observability/local/traceStore.d.ts +14 -1
  44. package/dist/observability/local/traceStore.js +144 -11
  45. package/dist/observability/observabilityApp.js +10 -1
  46. package/dist/observability/questionFingerprint.d.ts +7 -0
  47. package/dist/observability/questionFingerprint.js +23 -0
  48. package/dist/observability/routes/experiments.js +12 -1
  49. package/dist/observability/routes/review.js +3 -1
  50. package/dist/observability/routes/reviewBatches.d.ts +11 -0
  51. package/dist/observability/routes/reviewBatches.js +78 -0
  52. package/dist/observability/routes/traces.d.ts +1 -1
  53. package/dist/observability/routes/traces.js +34 -2
  54. package/dist/observability/types.d.ts +10 -1
  55. package/dist/service/aiService.js +4 -1
  56. package/dist/service/chatTurn.js +38 -31
  57. package/dist/types/harness.d.ts +10 -4
  58. package/dist/types/index.d.ts +8 -3
  59. package/dist/types/observability.d.ts +74 -2
  60. package/e2e/smoke.ts +227 -0
  61. package/package.json +14 -28
  62. package/src/aiApp.test.ts +22 -17
  63. package/src/harness/agentLoop.test.ts +0 -3
  64. package/src/harness/agentLoop.ts +2 -1
  65. package/src/harness/agentRecovery.test.ts +30 -30
  66. package/src/harness/approvals.test.ts +11 -9
  67. package/src/harness/commit.ts +24 -4
  68. package/src/harness/events.test.ts +10 -6
  69. package/src/harness/extensions.test.ts +6 -8
  70. package/src/harness/ownership.test.ts +20 -9
  71. package/src/harness/retries.test.ts +6 -3
  72. package/src/harness/routes/conversations.test.ts +2 -2
  73. package/src/harness/routes/tasks.test.ts +2 -2
  74. package/src/harness/runners/jobsRunner.test.ts +1 -1
  75. package/src/harness/subagent.test.ts +29 -4
  76. package/src/harness/waits.test.ts +6 -4
  77. package/src/index.test.ts +3 -20
  78. package/src/index.ts +23 -26
  79. package/src/isolated/gptStream.isolated.ts +25 -28
  80. package/src/isolated/lazyProjectModel.isolated.ts +2 -2
  81. package/src/models/gptHistory.test.ts +10 -6
  82. package/src/observability/adminScreens.test.ts +7 -0
  83. package/src/observability/adminScreens.ts +6 -1
  84. package/src/observability/evaluate.test.ts +12 -10
  85. package/src/observability/evaluate.ts +12 -2
  86. package/src/observability/judgeAgreement.test.ts +111 -0
  87. package/src/observability/judgeAgreement.ts +184 -0
  88. package/src/observability/local/datasetStore.ts +30 -11
  89. package/src/observability/local/evaluatorStore.ts +96 -2
  90. package/src/observability/local/experimentRunner.test.ts +9 -12
  91. package/src/observability/local/experimentRunner.ts +497 -17
  92. package/src/observability/local/humanAnswers.ts +233 -0
  93. package/src/observability/local/judgePipeline.test.ts +444 -0
  94. package/src/observability/local/judgeRun.test.ts +173 -0
  95. package/src/observability/local/judgeRun.ts +184 -0
  96. package/src/observability/local/localPlugin.ts +4 -0
  97. package/src/observability/local/models/obsDatasetItem.ts +5 -0
  98. package/src/observability/local/models/obsEvaluator.ts +4 -0
  99. package/src/observability/local/models/obsExperiment.ts +29 -0
  100. package/src/observability/local/models/obsExperimentItem.ts +4 -0
  101. package/src/observability/local/models/obsJudgeRun.ts +61 -0
  102. package/src/observability/local/models/obsReviewBatch.ts +52 -0
  103. package/src/observability/local/models/obsReviewItem.ts +11 -2
  104. package/src/observability/local/models/obsTrace.ts +5 -0
  105. package/src/observability/local/obsModels.test.ts +13 -0
  106. package/src/observability/local/promptReferences.test.ts +128 -0
  107. package/src/observability/local/promptReferences.ts +149 -0
  108. package/src/observability/local/promptStore.test.ts +295 -3
  109. package/src/observability/local/promptStore.ts +243 -32
  110. package/src/observability/local/reviewBatchStore.test.ts +251 -0
  111. package/src/observability/local/reviewBatchStore.ts +656 -0
  112. package/src/observability/local/reviewStore.ts +122 -25
  113. package/src/observability/local/traceStore.test.ts +139 -1
  114. package/src/observability/local/traceStore.ts +162 -11
  115. package/src/observability/observabilityApp.test.ts +13 -10
  116. package/src/observability/observabilityApp.ts +10 -1
  117. package/src/observability/questionFingerprint.test.ts +33 -0
  118. package/src/observability/questionFingerprint.ts +24 -0
  119. package/src/observability/routes/datasets.test.ts +2 -2
  120. package/src/observability/routes/evaluators.test.ts +2 -2
  121. package/src/observability/routes/experiments.test.ts +39 -2
  122. package/src/observability/routes/experiments.ts +13 -1
  123. package/src/observability/routes/prompts.test.ts +3 -3
  124. package/src/observability/routes/review.test.ts +2 -2
  125. package/src/observability/routes/review.ts +3 -1
  126. package/src/observability/routes/reviewBatches.test.ts +209 -0
  127. package/src/observability/routes/reviewBatches.ts +138 -0
  128. package/src/observability/routes/status.test.ts +8 -8
  129. package/src/observability/routes/testMultiStage.test.ts +6 -6
  130. package/src/observability/routes/traces.test.ts +63 -2
  131. package/src/observability/routes/traces.ts +46 -4
  132. package/src/observability/types.ts +5 -2
  133. package/src/routes/files.test.ts +5 -3
  134. package/src/routes/gpt.test.ts +21 -14
  135. package/src/routes/gptActions.test.ts +7 -8
  136. package/src/routes/gptDatasets.test.ts +1 -1
  137. package/src/routes/gptFiles.test.ts +1 -1
  138. package/src/routes/gptHistories.test.ts +15 -9
  139. package/src/routes/mcp.test.ts +3 -3
  140. package/src/routes/projects.test.ts +5 -3
  141. package/src/routes/routes.test.ts +156 -115
  142. package/src/service/aiService.test.ts +22 -1
  143. package/src/service/aiService.ts +4 -1
  144. package/src/service/asks.test.ts +14 -6
  145. package/src/service/chatTurn.ts +42 -33
  146. package/src/service/fileStorage.test.ts +5 -2
  147. package/src/service/getMCPTools.test.ts +9 -5
  148. package/src/service/mcpService.test.ts +6 -3
  149. package/src/tests/bunSetup.ts +0 -54
  150. package/src/tests/chatHarness.ts +4 -4
  151. package/src/tests/fakeJudge.ts +8 -0
  152. package/src/tests/helpers.ts +26 -4
  153. package/src/types/harness.ts +11 -4
  154. package/src/types/index.ts +8 -3
  155. package/src/types/observability.ts +86 -2
  156. package/tsconfig.typecheck.json +18 -0
  157. package/dist/langfuseApp.d.ts +0 -9
  158. package/dist/langfuseApp.js +0 -53
  159. package/dist/langfuseCache.d.ts +0 -15
  160. package/dist/langfuseCache.js +0 -54
  161. package/dist/langfuseClient.d.ts +0 -6
  162. package/dist/langfuseClient.js +0 -34
  163. package/dist/langfusePrompts.d.ts +0 -12
  164. package/dist/langfusePrompts.js +0 -89
  165. package/dist/langfuseRoutesEvaluations.d.ts +0 -3
  166. package/dist/langfuseRoutesEvaluations.js +0 -32
  167. package/dist/langfuseRoutesMiddleware.d.ts +0 -2
  168. package/dist/langfuseRoutesMiddleware.js +0 -13
  169. package/dist/langfuseRoutesPlayground.d.ts +0 -2
  170. package/dist/langfuseRoutesPlayground.js +0 -36
  171. package/dist/langfuseRoutesPrompts.d.ts +0 -2
  172. package/dist/langfuseRoutesPrompts.js +0 -56
  173. package/dist/langfuseRoutesTraces.d.ts +0 -2
  174. package/dist/langfuseRoutesTraces.js +0 -30
  175. package/dist/langfuseTracing.d.ts +0 -4
  176. package/dist/langfuseTracing.js +0 -32
  177. package/dist/langfuseTypes.d.ts +0 -96
  178. package/dist/langfuseTypes.js +0 -2
  179. package/dist/langfuseVercelAi.d.ts +0 -12
  180. package/dist/langfuseVercelAi.js +0 -49
  181. package/src/isolated/langfuseClient.isolated.ts +0 -60
  182. package/src/langfuseApp.test.ts +0 -135
  183. package/src/langfuseApp.ts +0 -62
  184. package/src/langfuseCache.test.ts +0 -75
  185. package/src/langfuseCache.ts +0 -72
  186. package/src/langfuseClient.test.ts +0 -69
  187. package/src/langfuseClient.ts +0 -34
  188. package/src/langfusePrompts.test.ts +0 -188
  189. package/src/langfusePrompts.ts +0 -121
  190. package/src/langfuseRoutes.test.ts +0 -256
  191. package/src/langfuseRoutesEvaluations.ts +0 -48
  192. package/src/langfuseRoutesMiddleware.test.ts +0 -57
  193. package/src/langfuseRoutesMiddleware.ts +0 -12
  194. package/src/langfuseRoutesPlayground.ts +0 -50
  195. package/src/langfuseRoutesPrompts.ts +0 -87
  196. package/src/langfuseRoutesTraces.ts +0 -40
  197. package/src/langfuseTracing.test.ts +0 -68
  198. package/src/langfuseTracing.ts +0 -33
  199. package/src/langfuseTypes.ts +0 -103
  200. package/src/langfuseVercelAi.test.ts +0 -79
  201. package/src/langfuseVercelAi.ts +0 -72
@@ -26,12 +26,15 @@ const createLocalPlugin = (): ObservabilityPlugin => {
26
26
  };
27
27
  };
28
28
 
29
- const createLangfusePlugin = (): ObservabilityPlugin => {
29
+ // Only "local" is a typed primary; these cases check runtime validation of other ids.
30
+ const REMOTE = "remote" as never;
31
+
32
+ const createRemotePlugin = (): ObservabilityPlugin => {
30
33
  return {
31
34
  capabilities: new Set(["datasets", "experiments", "prompts", "scores", "traces"]),
32
35
  datasetStore: {},
33
36
  experimentRunner: {},
34
- id: "langfuse",
37
+ id: "remote",
35
38
  promptRegistry: {get: async () => undefined},
36
39
  scoreSink: {export: async () => {}},
37
40
  traceSink: {export: async () => {}},
@@ -55,16 +58,16 @@ describe("ObservabilityApp config", () => {
55
58
  it("rejects experiments.primary that does not equal datasets.primary", () => {
56
59
  expect(() => {
57
60
  new ObservabilityApp({
58
- control: {datasets: "local", experiments: "langfuse"},
59
- plugins: [createLocalPlugin(), createLangfusePlugin()],
61
+ control: {datasets: "local", experiments: REMOTE},
62
+ plugins: [createLocalPlugin(), createRemotePlugin()],
60
63
  });
61
64
  }).toThrow(/experiments\.primary must equal datasets\.primary/);
62
65
  });
63
66
 
64
- it("rejects reviewQueue.primary langfuse", () => {
67
+ it("rejects a reviewQueue.primary other than local", () => {
65
68
  expect(() => {
66
69
  new ObservabilityApp({
67
- control: {reviewQueue: "langfuse"},
70
+ control: {reviewQueue: REMOTE},
68
71
  plugins: [createLocalPlugin()],
69
72
  });
70
73
  }).toThrow(/reviewQueue\.primary must be local/);
@@ -81,12 +84,12 @@ describe("ObservabilityApp config", () => {
81
84
  expect(() => {
82
85
  new ObservabilityApp({
83
86
  control: {
84
- datasets: "langfuse",
85
- experiments: "langfuse",
86
- prompts: "langfuse",
87
+ datasets: REMOTE,
88
+ experiments: REMOTE,
89
+ prompts: REMOTE,
87
90
  },
88
91
  plugins: [createLocalPlugin()],
89
92
  });
90
- }).toThrow(/prompts primary "langfuse" has no plugin/);
93
+ }).toThrow(/prompts primary "remote" has no plugin/);
91
94
  });
92
95
  });
@@ -6,6 +6,7 @@ import {LocalDatasetStore} from "./local/datasetStore";
6
6
  import {LocalEvaluatorStore} from "./local/evaluatorStore";
7
7
  import {LocalExperimentRunner} from "./local/experimentRunner";
8
8
  import {LocalPromptStore} from "./local/promptStore";
9
+ import {LocalReviewBatchStore} from "./local/reviewBatchStore";
9
10
  import {LocalReviewStore} from "./local/reviewStore";
10
11
  import {LocalTraceSink} from "./local/traceStore";
11
12
  import {
@@ -18,6 +19,7 @@ import {addObservabilityEvaluatorRoutes} from "./routes/evaluators";
18
19
  import {addObservabilityExperimentRoutes} from "./routes/experiments";
19
20
  import {addObservabilityPromptRoutes} from "./routes/prompts";
20
21
  import {addObservabilityReviewRoutes} from "./routes/review";
22
+ import {addObservabilityReviewBatchRoutes} from "./routes/reviewBatches";
21
23
  import {addObservabilityStatusRoutes} from "./routes/status";
22
24
  import {addObservabilityTestMultiStageRoutes} from "./routes/testMultiStage";
23
25
  import {addObservabilityTraceRoutes} from "./routes/traces";
@@ -149,9 +151,16 @@ export class ObservabilityApp implements TerrenoPlugin {
149
151
  ...routeAccess,
150
152
  store: evaluatorStore,
151
153
  });
154
+ const reviewStore = new LocalReviewStore();
155
+ addObservabilityReviewBatchRoutes(app, {
156
+ ...routeAccess,
157
+ aiService: this.aiService,
158
+ requestAiServiceFactory: this.requestAiServiceFactory,
159
+ store: new LocalReviewBatchStore(promptStore ?? new LocalPromptStore(), reviewStore),
160
+ });
152
161
  addObservabilityReviewRoutes(app, {
153
162
  ...routeAccess,
154
- store: new LocalReviewStore(),
163
+ store: reviewStore,
155
164
  });
156
165
  if (localPlugin.datasetStore instanceof LocalDatasetStore) {
157
166
  addObservabilityDatasetRoutes(app, {
@@ -0,0 +1,33 @@
1
+ import {describe, expect, it} from "bun:test";
2
+
3
+ import {questionSetFingerprint} from "./questionFingerprint";
4
+
5
+ const QUESTIONS = [
6
+ {dataType: "boolean" as const, key: "faithful", question: "Is it faithful?", required: true},
7
+ {dataType: "numeric" as const, key: "score", range: "1-5", required: false},
8
+ ];
9
+
10
+ describe("questionSetFingerprint", () => {
11
+ it("is a stable sha256 that ignores dimension order and the required flag", () => {
12
+ const fingerprint = questionSetFingerprint(QUESTIONS);
13
+ expect(fingerprint).toMatch(/^[0-9a-f]{64}$/);
14
+ expect(questionSetFingerprint([...QUESTIONS].reverse())).toBe(fingerprint);
15
+ expect(
16
+ questionSetFingerprint(QUESTIONS.map((row) => ({...row, required: !row.required})))
17
+ ).toBe(fingerprint);
18
+ });
19
+
20
+ it("changes when a key, data type, question, or range changes", () => {
21
+ const fingerprint = questionSetFingerprint(QUESTIONS);
22
+ const [first, second] = QUESTIONS;
23
+ const variants = [
24
+ [{...first, key: "grounded"}, second],
25
+ [{...first, dataType: "categorical" as const}, second],
26
+ [{...first, question: "Is every claim backed by the input?"}, second],
27
+ [first, {...second, range: "1-10"}],
28
+ ];
29
+ for (const variant of variants) {
30
+ expect(questionSetFingerprint(variant)).not.toBe(fingerprint);
31
+ }
32
+ });
33
+ });
@@ -0,0 +1,24 @@
1
+ import {createHash} from "node:crypto";
2
+
3
+ import type {EvaluatorDimension} from "../types/observability";
4
+
5
+ /**
6
+ * Stable fingerprint of a question set's questions. Human answers stay reusable while the
7
+ * fingerprint matches; editing a question's key, type, wording, or range makes earlier answers
8
+ * stale. Dimension order and the `required` flag do not change the fingerprint.
9
+ */
10
+ export const questionSetFingerprint = (
11
+ dimensions: Pick<EvaluatorDimension, "dataType" | "key" | "question" | "range">[]
12
+ ): string => {
13
+ const canonical = [...dimensions]
14
+ .map((dimension) => {
15
+ return {
16
+ dataType: dimension.dataType,
17
+ key: dimension.key,
18
+ question: dimension.question?.trim() ?? null,
19
+ range: dimension.range?.trim() ?? null,
20
+ };
21
+ })
22
+ .sort((left, right) => left.key.localeCompare(right.key));
23
+ return createHash("sha256").update(JSON.stringify(canonical)).digest("hex");
24
+ };
@@ -2,7 +2,7 @@ import {afterEach, beforeAll, beforeEach, describe, expect, it} from "bun:test";
2
2
  import {TerrenoApp} from "@terreno/api";
3
3
  import type express from "express";
4
4
 
5
- import {authAsUser, ensureTestUsers, UserModel} from "../../tests/helpers";
5
+ import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
6
6
  import {createLocalObservabilityPlugin} from "../local/localPlugin";
7
7
  import {registerObsDataset} from "../local/models/obsDataset";
8
8
  import {registerObsDatasetItem} from "../local/models/obsDatasetItem";
@@ -32,7 +32,7 @@ describe("observability dataset routes", () => {
32
32
  await registerObsPromptLabel().deleteMany({});
33
33
  promptStore = new LocalPromptStore();
34
34
  const plugin = createLocalObservabilityPlugin();
35
- app = new TerrenoApp({skipListen: true, userModel: UserModel})
35
+ app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
36
36
  .register(new ObservabilityApp({plugins: [plugin]}))
37
37
  .build();
38
38
  });
@@ -2,7 +2,7 @@ import {afterEach, beforeAll, beforeEach, describe, expect, it} from "bun:test";
2
2
  import {TerrenoApp} from "@terreno/api";
3
3
  import type express from "express";
4
4
 
5
- import {authAsUser, ensureTestUsers, UserModel} from "../../tests/helpers";
5
+ import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
6
6
  import {createLocalObservabilityPlugin} from "../local/localPlugin";
7
7
  import {registerObsEvaluator} from "../local/models/obsEvaluator";
8
8
  import {registerObsPrompt} from "../local/models/obsPrompt";
@@ -40,7 +40,7 @@ describe("observability evaluator routes", () => {
40
40
  type: "text",
41
41
  });
42
42
  await promptStore.moveLabel("eval-judge-correctness", {label: "production", version: 1});
43
- app = new TerrenoApp({skipListen: true, userModel: UserModel})
43
+ app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
44
44
  .register(new ObservabilityApp({plugins: [createLocalObservabilityPlugin()]}))
45
45
  .build();
46
46
  });
@@ -4,7 +4,7 @@ import type {LanguageModel} from "ai";
4
4
  import type express from "express";
5
5
 
6
6
  import {AIService} from "../../service/aiService";
7
- import {authAsUser, ensureTestUsers, UserModel} from "../../tests/helpers";
7
+ import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
8
8
  import {createLocalObservabilityPlugin} from "../local/localPlugin";
9
9
  import {registerObsDataset} from "../local/models/obsDataset";
10
10
  import {registerObsDatasetItem} from "../local/models/obsDatasetItem";
@@ -70,7 +70,7 @@ describe("observability experiment routes", () => {
70
70
  },
71
71
  },
72
72
  });
73
- app = new TerrenoApp({skipListen: true, userModel: UserModel})
73
+ app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
74
74
  .register(
75
75
  new ObservabilityApp({
76
76
  aiService,
@@ -240,6 +240,43 @@ describe("observability experiment routes", () => {
240
240
  expect(emptyPromptName.body.total).toBeGreaterThanOrEqual(2);
241
241
  });
242
242
 
243
+ it("filters experiments by kind and validates judge experiments", async () => {
244
+ const admin = await authAsUser(app, "admin");
245
+ const fixtures = await seedExperimentFixtures(admin);
246
+ const run = await admin.post("/ai/observability/experiments").send({
247
+ datasetId: fixtures.datasetId,
248
+ evaluatorIds: [fixtures.evaluatorId],
249
+ name: "prompt-run",
250
+ promptName: fixtures.promptName,
251
+ versions: [1, 2],
252
+ });
253
+ expect(run.status).toBe(201);
254
+ expect(run.body.data.kind).toBe("prompt");
255
+ expect(run.body.data.datasetName).toBe("route-exp-dataset");
256
+
257
+ const prompts = await admin.get("/ai/observability/experiments?kind=prompt");
258
+ expect(prompts.body.total).toBe(1);
259
+ const judges = await admin.get("/ai/observability/experiments?kind=judge");
260
+ expect(judges.body.total).toBe(0);
261
+ const badKind = await admin.get("/ai/observability/experiments?kind=other");
262
+ expect(badKind.status).toBe(400);
263
+
264
+ const notHuman = await admin.post("/ai/observability/experiments").send({
265
+ datasetId: fixtures.datasetId,
266
+ judgeEvaluatorId: fixtures.evaluatorId,
267
+ kind: "judge",
268
+ name: "judge-run",
269
+ questionSetId: fixtures.evaluatorId,
270
+ versions: [1, 2],
271
+ });
272
+ expect(notHuman.status).toBe(400);
273
+ expect(notHuman.body.title).toBe("questionSetId must be a human question set");
274
+ const unknownKind = await admin
275
+ .post("/ai/observability/experiments")
276
+ .send({datasetId: fixtures.datasetId, kind: "other", name: "x", versions: [1, 2]});
277
+ expect(unknownKind.status).toBe(400);
278
+ });
279
+
243
280
  it("lists experiments and returns estimates", async () => {
244
281
  const admin = await authAsUser(app, "admin");
245
282
  const fixtures = await seedExperimentFixtures(admin);
@@ -27,6 +27,16 @@ const parseOptionalPromptName = (value: unknown): string | undefined => {
27
27
  return trimmed;
28
28
  };
29
29
 
30
+ const parseKind = (value: unknown): "judge" | "prompt" | undefined => {
31
+ if (value === undefined || value === "") {
32
+ return undefined;
33
+ }
34
+ if (value !== "judge" && value !== "prompt") {
35
+ throw new APIError({status: 400, title: "kind must be prompt or judge"});
36
+ }
37
+ return value;
38
+ };
39
+
30
40
  const parsePositiveInt = (value: unknown, fallback: number): number => {
31
41
  if (value === undefined) {
32
42
  return fallback;
@@ -73,6 +83,7 @@ export const addObservabilityExperimentRoutes = (
73
83
  .withTags(["observability"])
74
84
  .withSummary("List experiments")
75
85
  .withQueryParameter("promptName", {type: "string"}, {required: false})
86
+ .withQueryParameter("kind", {enum: ["judge", "prompt"], type: "string"}, {required: false})
76
87
  .withQueryParameter("page", {type: "number"}, {required: false})
77
88
  .withQueryParameter("limit", {type: "number"}, {required: false})
78
89
  .withResponse(200, {})
@@ -80,6 +91,7 @@ export const addObservabilityExperimentRoutes = (
80
91
  ),
81
92
  asyncHandler(async (req, res) => {
82
93
  const listed = await options.runner.list({
94
+ kind: parseKind(req.query.kind),
83
95
  limit: parsePositiveInt(req.query.limit, EXPERIMENT_LIST_DEFAULT_LIMIT),
84
96
  page: parsePositiveInt(req.query.page, 1),
85
97
  promptName: parseOptionalPromptName(req.query.promptName),
@@ -102,7 +114,7 @@ export const addObservabilityExperimentRoutes = (
102
114
  {action: "create", resource: "aiExperiment"},
103
115
  builder()
104
116
  .withTags(["observability"])
105
- .withSummary("Create an experiment")
117
+ .withSummary("Create a prompt comparison or judge calibration experiment")
106
118
  .withResponse(201, {})
107
119
  .build()
108
120
  ),
@@ -8,7 +8,7 @@ import mongoose from "mongoose";
8
8
 
9
9
  import {AIRequest} from "../../models/aiRequest";
10
10
  import {AIService} from "../../service/aiService";
11
- import {authAsUser, ensureTestUsers, UserModel} from "../../tests/helpers";
11
+ import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
12
12
  import {createLocalObservabilityPlugin} from "../local/localPlugin";
13
13
  import {registerObsExperiment} from "../local/models/obsExperiment";
14
14
  import {registerObsPrompt} from "../local/models/obsPrompt";
@@ -61,7 +61,7 @@ describe("observability prompt routes", () => {
61
61
  const model = createMockModel();
62
62
  doGenerate = model.doGenerate;
63
63
  aiService = new AIService({model: model as unknown as LanguageModel});
64
- app = new TerrenoApp({skipListen: true, userModel: UserModel})
64
+ app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
65
65
  .register(
66
66
  new ObservabilityApp({
67
67
  aiService,
@@ -134,7 +134,7 @@ describe("observability prompt routes", () => {
134
134
  const requestAiServiceFactory = mock(({apiKey}: {apiKey?: string}) => {
135
135
  return apiKey === "request-key" ? aiService : undefined;
136
136
  });
137
- const requestServiceApp = new TerrenoApp({skipListen: true, userModel: UserModel})
137
+ const requestServiceApp = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
138
138
  .register(
139
139
  new ObservabilityApp({
140
140
  plugins: [createLocalObservabilityPlugin()],
@@ -4,7 +4,7 @@ import {assert} from "chai";
4
4
  import type express from "express";
5
5
  import {DateTime} from "luxon";
6
6
 
7
- import {authAsUser, ensureTestUsers, UserModel} from "../../tests/helpers";
7
+ import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
8
8
  import {LocalEvaluatorStore} from "../local/evaluatorStore";
9
9
  import {createLocalObservabilityPlugin} from "../local/localPlugin";
10
10
  import {registerObsEvaluator} from "../local/models/obsEvaluator";
@@ -28,7 +28,7 @@ describe("observability review routes", () => {
28
28
  await registerObsReviewItem().deleteMany({});
29
29
  await registerObsEvaluator().deleteMany({});
30
30
  await registerObsTrace().deleteMany({});
31
- app = new TerrenoApp({skipListen: true, userModel: UserModel})
31
+ app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
32
32
  .register(new ObservabilityApp({plugins: [createLocalObservabilityPlugin()]}))
33
33
  .build();
34
34
  });
@@ -62,7 +62,8 @@ export const addObservabilityReviewRoutes = (
62
62
  ),
63
63
  asyncHandler(async (req, res) => {
64
64
  const status = req.query.status as ReviewStatus | undefined;
65
- return res.json({...(await options.store.list(status)), more: false});
65
+ const batchId = typeof req.query.batchId === "string" ? req.query.batchId : undefined;
66
+ return res.json({...(await options.store.list(status, {batchId})), more: false});
66
67
  })
67
68
  );
68
69
 
@@ -108,6 +109,7 @@ export const addObservabilityReviewRoutes = (
108
109
  id: req.params.id,
109
110
  scores: body.scores ?? {},
110
111
  sinks: getObservabilityApp()?.scoreSinks ?? [],
112
+ userId: req.user?._id ? String(req.user._id) : undefined,
111
113
  });
112
114
  } else if (body.action === "skip") {
113
115
  await options.store.skip(req.params.id);
@@ -0,0 +1,209 @@
1
+ import {afterEach, beforeAll, beforeEach, describe, expect, it} from "bun:test";
2
+ import {TerrenoApp} from "@terreno/api";
3
+ import type express from "express";
4
+ import {DateTime} from "luxon";
5
+
6
+ import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
7
+ import {LocalEvaluatorStore} from "../local/evaluatorStore";
8
+ import {createLocalObservabilityPlugin} from "../local/localPlugin";
9
+ import {registerObsDataset} from "../local/models/obsDataset";
10
+ import {registerObsDatasetItem} from "../local/models/obsDatasetItem";
11
+ import {registerObsEvaluator} from "../local/models/obsEvaluator";
12
+ import {registerObsJudgeRun} from "../local/models/obsJudgeRun";
13
+ import {registerObsReviewBatch} from "../local/models/obsReviewBatch";
14
+ import {registerObsReviewItem} from "../local/models/obsReviewItem";
15
+ import {registerObsTrace} from "../local/models/obsTrace";
16
+ import {LocalPromptStore} from "../local/promptStore";
17
+ import {LocalTraceStore} from "../local/traceStore";
18
+ import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
19
+ import type {ObservabilityGenerateClient} from "../types";
20
+
21
+ const judgeClient: ObservabilityGenerateClient = {
22
+ generateJsonObject: async <OBJECT>() => ({confidence: 0.9, faithful: true}) as OBJECT,
23
+ generateText: async () => "",
24
+ modelId: "route-judge",
25
+ };
26
+
27
+ const buildApp = (aiService?: ObservabilityGenerateClient): express.Application => {
28
+ return new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
29
+ .register(new ObservabilityApp({aiService, plugins: [createLocalObservabilityPlugin()]}))
30
+ .build();
31
+ };
32
+
33
+ describe("observability review batch routes", () => {
34
+ let questionSetId: string;
35
+ let judgeId: string;
36
+
37
+ beforeAll(async () => {
38
+ await ensureTestUsers();
39
+ });
40
+
41
+ afterEach(() => {
42
+ resetObservabilityApp();
43
+ });
44
+
45
+ beforeEach(async () => {
46
+ await registerObsReviewItem().deleteMany({});
47
+ await registerObsReviewBatch().deleteMany({});
48
+ await registerObsDataset().deleteMany({});
49
+ await registerObsDatasetItem().deleteMany({});
50
+ await registerObsJudgeRun().deleteMany({});
51
+ await registerObsEvaluator().deleteMany({});
52
+ await registerObsTrace().deleteMany({});
53
+ // Register the models and default local stores before seeding fixtures.
54
+ buildApp();
55
+ resetObservabilityApp();
56
+ const promptStore = new LocalPromptStore();
57
+ if (!(await promptStore.getVersionByLabel("route-judge-prompt"))) {
58
+ await promptStore.create({
59
+ folder: "evaluators",
60
+ name: "route-judge-prompt",
61
+ outputSchema: {
62
+ properties: {confidence: {type: "number"}, faithful: {type: "boolean"}},
63
+ required: ["faithful"],
64
+ type: "object",
65
+ },
66
+ type: "chat",
67
+ });
68
+ await promptStore.moveLabel("route-judge-prompt", {label: "production", version: 1});
69
+ }
70
+ const evaluators = new LocalEvaluatorStore(promptStore);
71
+ questionSetId = (
72
+ await evaluators.create({
73
+ dimensions: [
74
+ {dataType: "boolean", key: "faithful", question: "Is it faithful?", required: true},
75
+ ],
76
+ name: "route-questions",
77
+ target: "full trace",
78
+ type: "human",
79
+ })
80
+ ).id;
81
+ judgeId = (
82
+ await evaluators.create({
83
+ dimensions: [{dataType: "boolean", key: "faithful", required: true}],
84
+ judgePromptName: "route-judge-prompt",
85
+ name: "route-judge",
86
+ target: "full trace",
87
+ type: "llm-judge",
88
+ })
89
+ ).id;
90
+ for (let index = 0; index < 3; index += 1) {
91
+ const startedAt = DateTime.utc().minus({hours: index}).toISO() ?? "";
92
+ await new LocalTraceStore().exportTrace({
93
+ id: `t${index}`,
94
+ input: `in ${index}`,
95
+ name: "summarize",
96
+ output: `out ${index}`,
97
+ prompts: [{name: "summarize", version: 1}],
98
+ sensitive: false,
99
+ spans: [{id: "s", kind: "LLM", name: "llm", startedAt, status: "ok"}],
100
+ startedAt,
101
+ status: "ok",
102
+ });
103
+ }
104
+ });
105
+
106
+ it("creates a batch from recent traces, answers it, and compares a judge", async () => {
107
+ const app = buildApp(judgeClient);
108
+ const agent = await authAsUser(app, "admin");
109
+ const created = await agent
110
+ .post("/ai/observability/review/batches")
111
+ .send({evaluatorId: questionSetId, limit: 2, promptName: "summarize"});
112
+ expect(created.status).toBe(201);
113
+ expect(created.body.data.counts.total).toBe(2);
114
+ const batchId = created.body.data.id as string;
115
+
116
+ const listed = await agent.get("/ai/observability/review/batches");
117
+ expect(listed.status).toBe(200);
118
+ expect(listed.body.data.map((row: {id: string}) => row.id)).toEqual([batchId]);
119
+
120
+ const pending = await agent.get(`/ai/observability/review?status=pending&batchId=${batchId}`);
121
+ expect(pending.body.data).toHaveLength(2);
122
+ for (const item of pending.body.data as Array<{id: string}>) {
123
+ const submitted = await agent
124
+ .post(`/ai/observability/review/${item.id}`)
125
+ .send({action: "submit", scores: {faithful: true}});
126
+ expect(submitted.status).toBe(200);
127
+ }
128
+
129
+ const judged = await agent
130
+ .post(`/ai/observability/review/batches/${batchId}/judgeRuns`)
131
+ .send({evaluatorId: judgeId});
132
+ expect(judged.status).toBe(201);
133
+ expect(judged.body.data.judges[0].agreement.rate).toBe(1);
134
+ expect(judged.body.data.judges[0].modelId).toBe("route-judge");
135
+
136
+ const detail = await agent.get(`/ai/observability/review/batches/${batchId}`);
137
+ expect(detail.status).toBe(200);
138
+ expect(detail.body.data.counts.done).toBe(2);
139
+ });
140
+
141
+ it("requires an evaluator and an AI client for judge runs", async () => {
142
+ const app = buildApp();
143
+ const agent = await authAsUser(app, "admin");
144
+ const missing = await agent.post("/ai/observability/review/batches").send({});
145
+ expect(missing.status).toBe(400);
146
+ const created = await agent
147
+ .post("/ai/observability/review/batches")
148
+ .send({evaluatorId: questionSetId});
149
+ expect(created.status).toBe(201);
150
+ const judged = await agent
151
+ .post(`/ai/observability/review/batches/${created.body.data.id}/judgeRuns`)
152
+ .send({evaluatorId: judgeId});
153
+ expect(judged.status).toBe(503);
154
+ });
155
+
156
+ it("builds a batch from a dataset's unanswered items and records answers on them", async () => {
157
+ const app = buildApp(judgeClient);
158
+ const agent = await authAsUser(app, "admin");
159
+ const dataset = await agent.post("/ai/observability/datasets").send({name: "route-set"});
160
+ const datasetId = dataset.body.data.id as string;
161
+ await agent
162
+ .post(`/ai/observability/datasets/${datasetId}/items`)
163
+ .send({expectedOutput: "summary", input: {text: "todo"}});
164
+
165
+ const mixed = await agent
166
+ .post("/ai/observability/review/batches")
167
+ .send({datasetId, evaluatorId: questionSetId, traceIds: []});
168
+ expect(mixed.status).toBe(400);
169
+
170
+ const created = await agent
171
+ .post("/ai/observability/review/batches")
172
+ .send({datasetId, evaluatorId: questionSetId});
173
+ expect(created.status).toBe(201);
174
+ expect(created.body.data.datasetId).toBe(datasetId);
175
+ const [item] = created.body.data.items as Array<{datasetItemId: string; reviewItemId: string}>;
176
+ const submitted = await agent
177
+ .post(`/ai/observability/review/${item?.reviewItemId}`)
178
+ .send({action: "submit", scores: {faithful: true}});
179
+ expect(submitted.status).toBe(200);
180
+ expect(submitted.body.data.datasetItemId).toBe(item?.datasetItemId);
181
+
182
+ const again = await agent
183
+ .post("/ai/observability/review/batches")
184
+ .send({datasetId, evaluatorId: questionSetId});
185
+ expect(again.status).toBe(400);
186
+ expect(again.body.title).toBe("Every item already has current answers");
187
+
188
+ const detail = await agent.get(`/ai/observability/datasets/${datasetId}`);
189
+ expect(detail.body.data.judgments).toEqual([
190
+ {
191
+ answered: 1,
192
+ current: 1,
193
+ evaluatorId: questionSetId,
194
+ evaluatorName: "route-questions",
195
+ stale: 0,
196
+ total: 1,
197
+ },
198
+ ]);
199
+ const items = await agent.get(`/ai/observability/datasets/${datasetId}/items`);
200
+ expect(items.body.data[0].humanAnswers[questionSetId].scores).toEqual({faithful: true});
201
+ expect(items.body.data[0].humanAnswers[questionSetId].userId).toBeDefined();
202
+ });
203
+
204
+ it("forbids non-admin users", async () => {
205
+ const agent = await authAsUser(buildApp(judgeClient), "notAdmin");
206
+ const listed = await agent.get("/ai/observability/review/batches");
207
+ expect(listed.status).toBe(403);
208
+ });
209
+ });