@terreno/ai 58.0.0-beta.3 → 58.0.0-beta.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. package/CHANGELOG.md +99 -0
  2. package/README.md +3 -3
  3. package/dist/harness/commit.d.ts +4 -3
  4. package/dist/harness/commit.js +23 -4
  5. package/dist/index.d.ts +6 -9
  6. package/dist/index.js +14 -25
  7. package/dist/observability/adminScreens.js +5 -1
  8. package/dist/observability/evaluate.d.ts +8 -1
  9. package/dist/observability/evaluate.js +10 -2
  10. package/dist/observability/judgeAgreement.d.ts +53 -0
  11. package/dist/observability/judgeAgreement.js +123 -0
  12. package/dist/observability/local/datasetStore.d.ts +5 -0
  13. package/dist/observability/local/datasetStore.js +7 -2
  14. package/dist/observability/local/evaluatorStore.d.ts +17 -0
  15. package/dist/observability/local/evaluatorStore.js +73 -2
  16. package/dist/observability/local/experimentRunner.d.ts +40 -3
  17. package/dist/observability/local/experimentRunner.js +390 -11
  18. package/dist/observability/local/humanAnswers.d.ts +57 -0
  19. package/dist/observability/local/humanAnswers.js +174 -0
  20. package/dist/observability/local/judgeRun.d.ts +56 -0
  21. package/dist/observability/local/judgeRun.js +111 -0
  22. package/dist/observability/local/localPlugin.js +4 -0
  23. package/dist/observability/local/models/obsDatasetItem.js +4 -0
  24. package/dist/observability/local/models/obsEvaluator.js +4 -0
  25. package/dist/observability/local/models/obsExperiment.js +28 -0
  26. package/dist/observability/local/models/obsExperimentItem.js +4 -0
  27. package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
  28. package/dist/observability/local/models/obsJudgeRun.js +60 -0
  29. package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
  30. package/dist/observability/local/models/obsReviewBatch.js +48 -0
  31. package/dist/observability/local/models/obsReviewItem.js +10 -2
  32. package/dist/observability/local/models/obsTrace.js +4 -0
  33. package/dist/observability/local/promptReferences.d.ts +31 -0
  34. package/dist/observability/local/promptReferences.js +114 -0
  35. package/dist/observability/local/promptStore.d.ts +55 -19
  36. package/dist/observability/local/promptStore.js +171 -7
  37. package/dist/observability/local/reviewBatchStore.d.ts +106 -0
  38. package/dist/observability/local/reviewBatchStore.js +517 -0
  39. package/dist/observability/local/reviewStore.d.ts +24 -3
  40. package/dist/observability/local/reviewStore.js +98 -22
  41. package/dist/observability/local/traceStore.d.ts +14 -1
  42. package/dist/observability/local/traceStore.js +144 -11
  43. package/dist/observability/observabilityApp.js +10 -1
  44. package/dist/observability/questionFingerprint.d.ts +7 -0
  45. package/dist/observability/questionFingerprint.js +23 -0
  46. package/dist/observability/routes/experiments.js +12 -1
  47. package/dist/observability/routes/review.js +3 -1
  48. package/dist/observability/routes/reviewBatches.d.ts +11 -0
  49. package/dist/observability/routes/reviewBatches.js +78 -0
  50. package/dist/observability/routes/traces.d.ts +1 -1
  51. package/dist/observability/routes/traces.js +34 -2
  52. package/dist/observability/types.d.ts +10 -1
  53. package/dist/service/aiService.js +4 -1
  54. package/dist/service/chatTurn.js +38 -31
  55. package/dist/types/harness.d.ts +5 -2
  56. package/dist/types/index.d.ts +8 -3
  57. package/dist/types/observability.d.ts +74 -2
  58. package/package.json +8 -24
  59. package/src/harness/agentLoop.test.ts +0 -3
  60. package/src/harness/agentRecovery.test.ts +0 -3
  61. package/src/harness/commit.ts +24 -4
  62. package/src/harness/extensions.test.ts +0 -3
  63. package/src/harness/subagent.test.ts +27 -3
  64. package/src/index.test.ts +0 -17
  65. package/src/index.ts +23 -26
  66. package/src/isolated/gptStream.isolated.ts +0 -8
  67. package/src/observability/adminScreens.test.ts +7 -0
  68. package/src/observability/adminScreens.ts +6 -1
  69. package/src/observability/evaluate.ts +12 -2
  70. package/src/observability/judgeAgreement.test.ts +111 -0
  71. package/src/observability/judgeAgreement.ts +184 -0
  72. package/src/observability/local/datasetStore.ts +30 -11
  73. package/src/observability/local/evaluatorStore.ts +96 -2
  74. package/src/observability/local/experimentRunner.ts +495 -15
  75. package/src/observability/local/humanAnswers.ts +233 -0
  76. package/src/observability/local/judgePipeline.test.ts +444 -0
  77. package/src/observability/local/judgeRun.test.ts +173 -0
  78. package/src/observability/local/judgeRun.ts +184 -0
  79. package/src/observability/local/localPlugin.ts +4 -0
  80. package/src/observability/local/models/obsDatasetItem.ts +5 -0
  81. package/src/observability/local/models/obsEvaluator.ts +4 -0
  82. package/src/observability/local/models/obsExperiment.ts +29 -0
  83. package/src/observability/local/models/obsExperimentItem.ts +4 -0
  84. package/src/observability/local/models/obsJudgeRun.ts +61 -0
  85. package/src/observability/local/models/obsReviewBatch.ts +52 -0
  86. package/src/observability/local/models/obsReviewItem.ts +11 -2
  87. package/src/observability/local/models/obsTrace.ts +5 -0
  88. package/src/observability/local/obsModels.test.ts +13 -0
  89. package/src/observability/local/promptReferences.test.ts +128 -0
  90. package/src/observability/local/promptReferences.ts +149 -0
  91. package/src/observability/local/promptStore.test.ts +291 -0
  92. package/src/observability/local/promptStore.ts +243 -32
  93. package/src/observability/local/reviewBatchStore.test.ts +251 -0
  94. package/src/observability/local/reviewBatchStore.ts +656 -0
  95. package/src/observability/local/reviewStore.ts +122 -25
  96. package/src/observability/local/traceStore.test.ts +139 -1
  97. package/src/observability/local/traceStore.ts +162 -11
  98. package/src/observability/observabilityApp.test.ts +13 -10
  99. package/src/observability/observabilityApp.ts +10 -1
  100. package/src/observability/questionFingerprint.test.ts +33 -0
  101. package/src/observability/questionFingerprint.ts +24 -0
  102. package/src/observability/routes/experiments.test.ts +37 -0
  103. package/src/observability/routes/experiments.ts +13 -1
  104. package/src/observability/routes/review.ts +3 -1
  105. package/src/observability/routes/reviewBatches.test.ts +209 -0
  106. package/src/observability/routes/reviewBatches.ts +138 -0
  107. package/src/observability/routes/status.test.ts +4 -4
  108. package/src/observability/routes/traces.test.ts +61 -0
  109. package/src/observability/routes/traces.ts +46 -4
  110. package/src/observability/types.ts +5 -2
  111. package/src/routes/routes.test.ts +101 -76
  112. package/src/service/aiService.test.ts +21 -0
  113. package/src/service/aiService.ts +4 -1
  114. package/src/service/chatTurn.ts +42 -33
  115. package/src/tests/bunSetup.ts +0 -54
  116. package/src/types/harness.ts +5 -2
  117. package/src/types/index.ts +8 -3
  118. package/src/types/observability.ts +86 -2
  119. package/dist/langfuseApp.d.ts +0 -9
  120. package/dist/langfuseApp.js +0 -53
  121. package/dist/langfuseCache.d.ts +0 -15
  122. package/dist/langfuseCache.js +0 -54
  123. package/dist/langfuseClient.d.ts +0 -6
  124. package/dist/langfuseClient.js +0 -34
  125. package/dist/langfusePrompts.d.ts +0 -12
  126. package/dist/langfusePrompts.js +0 -89
  127. package/dist/langfuseRoutesEvaluations.d.ts +0 -3
  128. package/dist/langfuseRoutesEvaluations.js +0 -32
  129. package/dist/langfuseRoutesMiddleware.d.ts +0 -2
  130. package/dist/langfuseRoutesMiddleware.js +0 -13
  131. package/dist/langfuseRoutesPlayground.d.ts +0 -2
  132. package/dist/langfuseRoutesPlayground.js +0 -36
  133. package/dist/langfuseRoutesPrompts.d.ts +0 -2
  134. package/dist/langfuseRoutesPrompts.js +0 -56
  135. package/dist/langfuseRoutesTraces.d.ts +0 -2
  136. package/dist/langfuseRoutesTraces.js +0 -30
  137. package/dist/langfuseTracing.d.ts +0 -4
  138. package/dist/langfuseTracing.js +0 -32
  139. package/dist/langfuseTypes.d.ts +0 -96
  140. package/dist/langfuseTypes.js +0 -2
  141. package/dist/langfuseVercelAi.d.ts +0 -12
  142. package/dist/langfuseVercelAi.js +0 -49
  143. package/src/isolated/langfuseClient.isolated.ts +0 -66
  144. package/src/langfuseApp.test.ts +0 -119
  145. package/src/langfuseApp.ts +0 -62
  146. package/src/langfuseCache.test.ts +0 -75
  147. package/src/langfuseCache.ts +0 -72
  148. package/src/langfuseClient.test.ts +0 -69
  149. package/src/langfuseClient.ts +0 -34
  150. package/src/langfusePrompts.test.ts +0 -188
  151. package/src/langfusePrompts.ts +0 -121
  152. package/src/langfuseRoutes.test.ts +0 -261
  153. package/src/langfuseRoutesEvaluations.ts +0 -48
  154. package/src/langfuseRoutesMiddleware.test.ts +0 -57
  155. package/src/langfuseRoutesMiddleware.ts +0 -12
  156. package/src/langfuseRoutesPlayground.ts +0 -50
  157. package/src/langfuseRoutesPrompts.ts +0 -87
  158. package/src/langfuseRoutesTraces.ts +0 -40
  159. package/src/langfuseTracing.test.ts +0 -72
  160. package/src/langfuseTracing.ts +0 -33
  161. package/src/langfuseTypes.ts +0 -103
  162. package/src/langfuseVercelAi.test.ts +0 -82
  163. package/src/langfuseVercelAi.ts +0 -72
  164. package/src/tests/fakeLangfusePrompt.ts +0 -10
@@ -26,12 +26,15 @@ const createLocalPlugin = (): ObservabilityPlugin => {
26
26
  };
27
27
  };
28
28
 
29
- const createLangfusePlugin = (): ObservabilityPlugin => {
29
+ // Only "local" is a typed primary; these cases check runtime validation of other ids.
30
+ const REMOTE = "remote" as never;
31
+
32
+ const createRemotePlugin = (): ObservabilityPlugin => {
30
33
  return {
31
34
  capabilities: new Set(["datasets", "experiments", "prompts", "scores", "traces"]),
32
35
  datasetStore: {},
33
36
  experimentRunner: {},
34
- id: "langfuse",
37
+ id: "remote",
35
38
  promptRegistry: {get: async () => undefined},
36
39
  scoreSink: {export: async () => {}},
37
40
  traceSink: {export: async () => {}},
@@ -55,16 +58,16 @@ describe("ObservabilityApp config", () => {
55
58
  it("rejects experiments.primary that does not equal datasets.primary", () => {
56
59
  expect(() => {
57
60
  new ObservabilityApp({
58
- control: {datasets: "local", experiments: "langfuse"},
59
- plugins: [createLocalPlugin(), createLangfusePlugin()],
61
+ control: {datasets: "local", experiments: REMOTE},
62
+ plugins: [createLocalPlugin(), createRemotePlugin()],
60
63
  });
61
64
  }).toThrow(/experiments\.primary must equal datasets\.primary/);
62
65
  });
63
66
 
64
- it("rejects reviewQueue.primary langfuse", () => {
67
+ it("rejects a reviewQueue.primary other than local", () => {
65
68
  expect(() => {
66
69
  new ObservabilityApp({
67
- control: {reviewQueue: "langfuse"},
70
+ control: {reviewQueue: REMOTE},
68
71
  plugins: [createLocalPlugin()],
69
72
  });
70
73
  }).toThrow(/reviewQueue\.primary must be local/);
@@ -81,12 +84,12 @@ describe("ObservabilityApp config", () => {
81
84
  expect(() => {
82
85
  new ObservabilityApp({
83
86
  control: {
84
- datasets: "langfuse",
85
- experiments: "langfuse",
86
- prompts: "langfuse",
87
+ datasets: REMOTE,
88
+ experiments: REMOTE,
89
+ prompts: REMOTE,
87
90
  },
88
91
  plugins: [createLocalPlugin()],
89
92
  });
90
- }).toThrow(/prompts primary "langfuse" has no plugin/);
93
+ }).toThrow(/prompts primary "remote" has no plugin/);
91
94
  });
92
95
  });
@@ -6,6 +6,7 @@ import {LocalDatasetStore} from "./local/datasetStore";
6
6
  import {LocalEvaluatorStore} from "./local/evaluatorStore";
7
7
  import {LocalExperimentRunner} from "./local/experimentRunner";
8
8
  import {LocalPromptStore} from "./local/promptStore";
9
+ import {LocalReviewBatchStore} from "./local/reviewBatchStore";
9
10
  import {LocalReviewStore} from "./local/reviewStore";
10
11
  import {LocalTraceSink} from "./local/traceStore";
11
12
  import {
@@ -18,6 +19,7 @@ import {addObservabilityEvaluatorRoutes} from "./routes/evaluators";
18
19
  import {addObservabilityExperimentRoutes} from "./routes/experiments";
19
20
  import {addObservabilityPromptRoutes} from "./routes/prompts";
20
21
  import {addObservabilityReviewRoutes} from "./routes/review";
22
+ import {addObservabilityReviewBatchRoutes} from "./routes/reviewBatches";
21
23
  import {addObservabilityStatusRoutes} from "./routes/status";
22
24
  import {addObservabilityTestMultiStageRoutes} from "./routes/testMultiStage";
23
25
  import {addObservabilityTraceRoutes} from "./routes/traces";
@@ -149,9 +151,16 @@ export class ObservabilityApp implements TerrenoPlugin {
149
151
  ...routeAccess,
150
152
  store: evaluatorStore,
151
153
  });
154
+ const reviewStore = new LocalReviewStore();
155
+ addObservabilityReviewBatchRoutes(app, {
156
+ ...routeAccess,
157
+ aiService: this.aiService,
158
+ requestAiServiceFactory: this.requestAiServiceFactory,
159
+ store: new LocalReviewBatchStore(promptStore ?? new LocalPromptStore(), reviewStore),
160
+ });
152
161
  addObservabilityReviewRoutes(app, {
153
162
  ...routeAccess,
154
- store: new LocalReviewStore(),
163
+ store: reviewStore,
155
164
  });
156
165
  if (localPlugin.datasetStore instanceof LocalDatasetStore) {
157
166
  addObservabilityDatasetRoutes(app, {
@@ -0,0 +1,33 @@
1
+ import {describe, expect, it} from "bun:test";
2
+
3
+ import {questionSetFingerprint} from "./questionFingerprint";
4
+
5
+ const QUESTIONS = [
6
+ {dataType: "boolean" as const, key: "faithful", question: "Is it faithful?", required: true},
7
+ {dataType: "numeric" as const, key: "score", range: "1-5", required: false},
8
+ ];
9
+
10
+ describe("questionSetFingerprint", () => {
11
+ it("is a stable sha256 that ignores dimension order and the required flag", () => {
12
+ const fingerprint = questionSetFingerprint(QUESTIONS);
13
+ expect(fingerprint).toMatch(/^[0-9a-f]{64}$/);
14
+ expect(questionSetFingerprint([...QUESTIONS].reverse())).toBe(fingerprint);
15
+ expect(
16
+ questionSetFingerprint(QUESTIONS.map((row) => ({...row, required: !row.required})))
17
+ ).toBe(fingerprint);
18
+ });
19
+
20
+ it("changes when a key, data type, question, or range changes", () => {
21
+ const fingerprint = questionSetFingerprint(QUESTIONS);
22
+ const [first, second] = QUESTIONS;
23
+ const variants = [
24
+ [{...first, key: "grounded"}, second],
25
+ [{...first, dataType: "categorical" as const}, second],
26
+ [{...first, question: "Is every claim backed by the input?"}, second],
27
+ [first, {...second, range: "1-10"}],
28
+ ];
29
+ for (const variant of variants) {
30
+ expect(questionSetFingerprint(variant)).not.toBe(fingerprint);
31
+ }
32
+ });
33
+ });
@@ -0,0 +1,24 @@
1
+ import {createHash} from "node:crypto";
2
+
3
+ import type {EvaluatorDimension} from "../types/observability";
4
+
5
+ /**
6
+ * Stable fingerprint of a question set's questions. Human answers stay reusable while the
7
+ * fingerprint matches; editing a question's key, type, wording, or range makes earlier answers
8
+ * stale. Dimension order and the `required` flag do not change the fingerprint.
9
+ */
10
+ export const questionSetFingerprint = (
11
+ dimensions: Pick<EvaluatorDimension, "dataType" | "key" | "question" | "range">[]
12
+ ): string => {
13
+ const canonical = [...dimensions]
14
+ .map((dimension) => {
15
+ return {
16
+ dataType: dimension.dataType,
17
+ key: dimension.key,
18
+ question: dimension.question?.trim() ?? null,
19
+ range: dimension.range?.trim() ?? null,
20
+ };
21
+ })
22
+ .sort((left, right) => left.key.localeCompare(right.key));
23
+ return createHash("sha256").update(JSON.stringify(canonical)).digest("hex");
24
+ };
@@ -240,6 +240,43 @@ describe("observability experiment routes", () => {
240
240
  expect(emptyPromptName.body.total).toBeGreaterThanOrEqual(2);
241
241
  });
242
242
 
243
+ it("filters experiments by kind and validates judge experiments", async () => {
244
+ const admin = await authAsUser(app, "admin");
245
+ const fixtures = await seedExperimentFixtures(admin);
246
+ const run = await admin.post("/ai/observability/experiments").send({
247
+ datasetId: fixtures.datasetId,
248
+ evaluatorIds: [fixtures.evaluatorId],
249
+ name: "prompt-run",
250
+ promptName: fixtures.promptName,
251
+ versions: [1, 2],
252
+ });
253
+ expect(run.status).toBe(201);
254
+ expect(run.body.data.kind).toBe("prompt");
255
+ expect(run.body.data.datasetName).toBe("route-exp-dataset");
256
+
257
+ const prompts = await admin.get("/ai/observability/experiments?kind=prompt");
258
+ expect(prompts.body.total).toBe(1);
259
+ const judges = await admin.get("/ai/observability/experiments?kind=judge");
260
+ expect(judges.body.total).toBe(0);
261
+ const badKind = await admin.get("/ai/observability/experiments?kind=other");
262
+ expect(badKind.status).toBe(400);
263
+
264
+ const notHuman = await admin.post("/ai/observability/experiments").send({
265
+ datasetId: fixtures.datasetId,
266
+ judgeEvaluatorId: fixtures.evaluatorId,
267
+ kind: "judge",
268
+ name: "judge-run",
269
+ questionSetId: fixtures.evaluatorId,
270
+ versions: [1, 2],
271
+ });
272
+ expect(notHuman.status).toBe(400);
273
+ expect(notHuman.body.title).toBe("questionSetId must be a human question set");
274
+ const unknownKind = await admin
275
+ .post("/ai/observability/experiments")
276
+ .send({datasetId: fixtures.datasetId, kind: "other", name: "x", versions: [1, 2]});
277
+ expect(unknownKind.status).toBe(400);
278
+ });
279
+
243
280
  it("lists experiments and returns estimates", async () => {
244
281
  const admin = await authAsUser(app, "admin");
245
282
  const fixtures = await seedExperimentFixtures(admin);
@@ -27,6 +27,16 @@ const parseOptionalPromptName = (value: unknown): string | undefined => {
27
27
  return trimmed;
28
28
  };
29
29
 
30
+ const parseKind = (value: unknown): "judge" | "prompt" | undefined => {
31
+ if (value === undefined || value === "") {
32
+ return undefined;
33
+ }
34
+ if (value !== "judge" && value !== "prompt") {
35
+ throw new APIError({status: 400, title: "kind must be prompt or judge"});
36
+ }
37
+ return value;
38
+ };
39
+
30
40
  const parsePositiveInt = (value: unknown, fallback: number): number => {
31
41
  if (value === undefined) {
32
42
  return fallback;
@@ -73,6 +83,7 @@ export const addObservabilityExperimentRoutes = (
73
83
  .withTags(["observability"])
74
84
  .withSummary("List experiments")
75
85
  .withQueryParameter("promptName", {type: "string"}, {required: false})
86
+ .withQueryParameter("kind", {enum: ["judge", "prompt"], type: "string"}, {required: false})
76
87
  .withQueryParameter("page", {type: "number"}, {required: false})
77
88
  .withQueryParameter("limit", {type: "number"}, {required: false})
78
89
  .withResponse(200, {})
@@ -80,6 +91,7 @@ export const addObservabilityExperimentRoutes = (
80
91
  ),
81
92
  asyncHandler(async (req, res) => {
82
93
  const listed = await options.runner.list({
94
+ kind: parseKind(req.query.kind),
83
95
  limit: parsePositiveInt(req.query.limit, EXPERIMENT_LIST_DEFAULT_LIMIT),
84
96
  page: parsePositiveInt(req.query.page, 1),
85
97
  promptName: parseOptionalPromptName(req.query.promptName),
@@ -102,7 +114,7 @@ export const addObservabilityExperimentRoutes = (
102
114
  {action: "create", resource: "aiExperiment"},
103
115
  builder()
104
116
  .withTags(["observability"])
105
- .withSummary("Create an experiment")
117
+ .withSummary("Create a prompt comparison or judge calibration experiment")
106
118
  .withResponse(201, {})
107
119
  .build()
108
120
  ),
@@ -62,7 +62,8 @@ export const addObservabilityReviewRoutes = (
62
62
  ),
63
63
  asyncHandler(async (req, res) => {
64
64
  const status = req.query.status as ReviewStatus | undefined;
65
- return res.json({...(await options.store.list(status)), more: false});
65
+ const batchId = typeof req.query.batchId === "string" ? req.query.batchId : undefined;
66
+ return res.json({...(await options.store.list(status, {batchId})), more: false});
66
67
  })
67
68
  );
68
69
 
@@ -108,6 +109,7 @@ export const addObservabilityReviewRoutes = (
108
109
  id: req.params.id,
109
110
  scores: body.scores ?? {},
110
111
  sinks: getObservabilityApp()?.scoreSinks ?? [],
112
+ userId: req.user?._id ? String(req.user._id) : undefined,
111
113
  });
112
114
  } else if (body.action === "skip") {
113
115
  await options.store.skip(req.params.id);
@@ -0,0 +1,209 @@
1
+ import {afterEach, beforeAll, beforeEach, describe, expect, it} from "bun:test";
2
+ import {TerrenoApp} from "@terreno/api";
3
+ import type express from "express";
4
+ import {DateTime} from "luxon";
5
+
6
+ import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
7
+ import {LocalEvaluatorStore} from "../local/evaluatorStore";
8
+ import {createLocalObservabilityPlugin} from "../local/localPlugin";
9
+ import {registerObsDataset} from "../local/models/obsDataset";
10
+ import {registerObsDatasetItem} from "../local/models/obsDatasetItem";
11
+ import {registerObsEvaluator} from "../local/models/obsEvaluator";
12
+ import {registerObsJudgeRun} from "../local/models/obsJudgeRun";
13
+ import {registerObsReviewBatch} from "../local/models/obsReviewBatch";
14
+ import {registerObsReviewItem} from "../local/models/obsReviewItem";
15
+ import {registerObsTrace} from "../local/models/obsTrace";
16
+ import {LocalPromptStore} from "../local/promptStore";
17
+ import {LocalTraceStore} from "../local/traceStore";
18
+ import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
19
+ import type {ObservabilityGenerateClient} from "../types";
20
+
21
+ const judgeClient: ObservabilityGenerateClient = {
22
+ generateJsonObject: async <OBJECT>() => ({confidence: 0.9, faithful: true}) as OBJECT,
23
+ generateText: async () => "",
24
+ modelId: "route-judge",
25
+ };
26
+
27
+ const buildApp = (aiService?: ObservabilityGenerateClient): express.Application => {
28
+ return new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
29
+ .register(new ObservabilityApp({aiService, plugins: [createLocalObservabilityPlugin()]}))
30
+ .build();
31
+ };
32
+
33
+ describe("observability review batch routes", () => {
34
+ let questionSetId: string;
35
+ let judgeId: string;
36
+
37
+ beforeAll(async () => {
38
+ await ensureTestUsers();
39
+ });
40
+
41
+ afterEach(() => {
42
+ resetObservabilityApp();
43
+ });
44
+
45
+ beforeEach(async () => {
46
+ await registerObsReviewItem().deleteMany({});
47
+ await registerObsReviewBatch().deleteMany({});
48
+ await registerObsDataset().deleteMany({});
49
+ await registerObsDatasetItem().deleteMany({});
50
+ await registerObsJudgeRun().deleteMany({});
51
+ await registerObsEvaluator().deleteMany({});
52
+ await registerObsTrace().deleteMany({});
53
+ // Register the models and default local stores before seeding fixtures.
54
+ buildApp();
55
+ resetObservabilityApp();
56
+ const promptStore = new LocalPromptStore();
57
+ if (!(await promptStore.getVersionByLabel("route-judge-prompt"))) {
58
+ await promptStore.create({
59
+ folder: "evaluators",
60
+ name: "route-judge-prompt",
61
+ outputSchema: {
62
+ properties: {confidence: {type: "number"}, faithful: {type: "boolean"}},
63
+ required: ["faithful"],
64
+ type: "object",
65
+ },
66
+ type: "chat",
67
+ });
68
+ await promptStore.moveLabel("route-judge-prompt", {label: "production", version: 1});
69
+ }
70
+ const evaluators = new LocalEvaluatorStore(promptStore);
71
+ questionSetId = (
72
+ await evaluators.create({
73
+ dimensions: [
74
+ {dataType: "boolean", key: "faithful", question: "Is it faithful?", required: true},
75
+ ],
76
+ name: "route-questions",
77
+ target: "full trace",
78
+ type: "human",
79
+ })
80
+ ).id;
81
+ judgeId = (
82
+ await evaluators.create({
83
+ dimensions: [{dataType: "boolean", key: "faithful", required: true}],
84
+ judgePromptName: "route-judge-prompt",
85
+ name: "route-judge",
86
+ target: "full trace",
87
+ type: "llm-judge",
88
+ })
89
+ ).id;
90
+ for (let index = 0; index < 3; index += 1) {
91
+ const startedAt = DateTime.utc().minus({hours: index}).toISO() ?? "";
92
+ await new LocalTraceStore().exportTrace({
93
+ id: `t${index}`,
94
+ input: `in ${index}`,
95
+ name: "summarize",
96
+ output: `out ${index}`,
97
+ prompts: [{name: "summarize", version: 1}],
98
+ sensitive: false,
99
+ spans: [{id: "s", kind: "LLM", name: "llm", startedAt, status: "ok"}],
100
+ startedAt,
101
+ status: "ok",
102
+ });
103
+ }
104
+ });
105
+
106
+ it("creates a batch from recent traces, answers it, and compares a judge", async () => {
107
+ const app = buildApp(judgeClient);
108
+ const agent = await authAsUser(app, "admin");
109
+ const created = await agent
110
+ .post("/ai/observability/review/batches")
111
+ .send({evaluatorId: questionSetId, limit: 2, promptName: "summarize"});
112
+ expect(created.status).toBe(201);
113
+ expect(created.body.data.counts.total).toBe(2);
114
+ const batchId = created.body.data.id as string;
115
+
116
+ const listed = await agent.get("/ai/observability/review/batches");
117
+ expect(listed.status).toBe(200);
118
+ expect(listed.body.data.map((row: {id: string}) => row.id)).toEqual([batchId]);
119
+
120
+ const pending = await agent.get(`/ai/observability/review?status=pending&batchId=${batchId}`);
121
+ expect(pending.body.data).toHaveLength(2);
122
+ for (const item of pending.body.data as Array<{id: string}>) {
123
+ const submitted = await agent
124
+ .post(`/ai/observability/review/${item.id}`)
125
+ .send({action: "submit", scores: {faithful: true}});
126
+ expect(submitted.status).toBe(200);
127
+ }
128
+
129
+ const judged = await agent
130
+ .post(`/ai/observability/review/batches/${batchId}/judgeRuns`)
131
+ .send({evaluatorId: judgeId});
132
+ expect(judged.status).toBe(201);
133
+ expect(judged.body.data.judges[0].agreement.rate).toBe(1);
134
+ expect(judged.body.data.judges[0].modelId).toBe("route-judge");
135
+
136
+ const detail = await agent.get(`/ai/observability/review/batches/${batchId}`);
137
+ expect(detail.status).toBe(200);
138
+ expect(detail.body.data.counts.done).toBe(2);
139
+ });
140
+
141
+ it("requires an evaluator and an AI client for judge runs", async () => {
142
+ const app = buildApp();
143
+ const agent = await authAsUser(app, "admin");
144
+ const missing = await agent.post("/ai/observability/review/batches").send({});
145
+ expect(missing.status).toBe(400);
146
+ const created = await agent
147
+ .post("/ai/observability/review/batches")
148
+ .send({evaluatorId: questionSetId});
149
+ expect(created.status).toBe(201);
150
+ const judged = await agent
151
+ .post(`/ai/observability/review/batches/${created.body.data.id}/judgeRuns`)
152
+ .send({evaluatorId: judgeId});
153
+ expect(judged.status).toBe(503);
154
+ });
155
+
156
+ it("builds a batch from a dataset's unanswered items and records answers on them", async () => {
157
+ const app = buildApp(judgeClient);
158
+ const agent = await authAsUser(app, "admin");
159
+ const dataset = await agent.post("/ai/observability/datasets").send({name: "route-set"});
160
+ const datasetId = dataset.body.data.id as string;
161
+ await agent
162
+ .post(`/ai/observability/datasets/${datasetId}/items`)
163
+ .send({expectedOutput: "summary", input: {text: "todo"}});
164
+
165
+ const mixed = await agent
166
+ .post("/ai/observability/review/batches")
167
+ .send({datasetId, evaluatorId: questionSetId, traceIds: []});
168
+ expect(mixed.status).toBe(400);
169
+
170
+ const created = await agent
171
+ .post("/ai/observability/review/batches")
172
+ .send({datasetId, evaluatorId: questionSetId});
173
+ expect(created.status).toBe(201);
174
+ expect(created.body.data.datasetId).toBe(datasetId);
175
+ const [item] = created.body.data.items as Array<{datasetItemId: string; reviewItemId: string}>;
176
+ const submitted = await agent
177
+ .post(`/ai/observability/review/${item?.reviewItemId}`)
178
+ .send({action: "submit", scores: {faithful: true}});
179
+ expect(submitted.status).toBe(200);
180
+ expect(submitted.body.data.datasetItemId).toBe(item?.datasetItemId);
181
+
182
+ const again = await agent
183
+ .post("/ai/observability/review/batches")
184
+ .send({datasetId, evaluatorId: questionSetId});
185
+ expect(again.status).toBe(400);
186
+ expect(again.body.title).toBe("Every item already has current answers");
187
+
188
+ const detail = await agent.get(`/ai/observability/datasets/${datasetId}`);
189
+ expect(detail.body.data.judgments).toEqual([
190
+ {
191
+ answered: 1,
192
+ current: 1,
193
+ evaluatorId: questionSetId,
194
+ evaluatorName: "route-questions",
195
+ stale: 0,
196
+ total: 1,
197
+ },
198
+ ]);
199
+ const items = await agent.get(`/ai/observability/datasets/${datasetId}/items`);
200
+ expect(items.body.data[0].humanAnswers[questionSetId].scores).toEqual({faithful: true});
201
+ expect(items.body.data[0].humanAnswers[questionSetId].userId).toBeDefined();
202
+ });
203
+
204
+ it("forbids non-admin users", async () => {
205
+ const agent = await authAsUser(buildApp(judgeClient), "notAdmin");
206
+ const listed = await agent.get("/ai/observability/review/batches");
207
+ expect(listed.status).toBe(403);
208
+ });
209
+ });
@@ -0,0 +1,138 @@
1
+ import {APIError, asyncHandler, createOpenApiBuilder} from "@terreno/api";
2
+ import type express from "express";
3
+
4
+ import type {LocalReviewBatchStore} from "../local/reviewBatchStore";
5
+ import type {ObservabilityGenerateClient, ObservabilityRequestAiServiceFactory} from "../types";
6
+ import {
7
+ type ObservabilityRouteAccessOptions,
8
+ observabilityRouteMiddleware,
9
+ } from "./observabilityRouteAccess";
10
+ import {resolveRequestAiService} from "./requestAiService";
11
+
12
+ const BASE_PATH = "/ai/observability";
13
+
14
+ export interface ObservabilityReviewBatchRouteOptions extends ObservabilityRouteAccessOptions {
15
+ aiService?: ObservabilityGenerateClient;
16
+ requestAiServiceFactory?: ObservabilityRequestAiServiceFactory;
17
+ store: LocalReviewBatchStore;
18
+ }
19
+
20
+ /** Register before the review item routes so `/review/batches` is not read as an item id. */
21
+ export const addObservabilityReviewBatchRoutes = (
22
+ router: express.Application,
23
+ options: ObservabilityReviewBatchRouteOptions
24
+ ): void => {
25
+ const openApiOptions = options.openApi ? {openApi: options.openApi} : {};
26
+ const builder = (): ReturnType<typeof createOpenApiBuilder> => {
27
+ return createOpenApiBuilder(openApiOptions as Parameters<typeof createOpenApiBuilder>[0]);
28
+ };
29
+
30
+ router.get(
31
+ `${BASE_PATH}/review/batches`,
32
+ observabilityRouteMiddleware(
33
+ options.accessControl,
34
+ {action: "list", resource: "aiReview"},
35
+ builder()
36
+ .withTags(["observability"])
37
+ .withSummary("List human review batches with judge agreement")
38
+ .withResponse(200, {data: {type: "array"}})
39
+ .build()
40
+ ),
41
+ asyncHandler(async (_req, res) => {
42
+ return res.json({data: await options.store.list(), more: false});
43
+ })
44
+ );
45
+
46
+ router.post(
47
+ `${BASE_PATH}/review/batches`,
48
+ observabilityRouteMiddleware(
49
+ options.accessControl,
50
+ {action: "assign", resource: "aiReview"},
51
+ builder()
52
+ .withTags(["observability"])
53
+ .withSummary(
54
+ "Create a review batch from traces, the most recent traces, or a dataset's unanswered items"
55
+ )
56
+ .withResponse(201, {data: {type: "object"}})
57
+ .build()
58
+ ),
59
+ asyncHandler(async (req, res) => {
60
+ const body = req.body as {
61
+ datasetId?: string;
62
+ description?: string;
63
+ evaluatorId?: string;
64
+ limit?: number;
65
+ name?: string;
66
+ promptName?: string;
67
+ traceIds?: string[];
68
+ };
69
+ if (!body.evaluatorId) {
70
+ throw new APIError({status: 400, title: "evaluatorId is required"});
71
+ }
72
+ if (body.traceIds !== undefined && !Array.isArray(body.traceIds)) {
73
+ throw new APIError({status: 400, title: "traceIds must be an array"});
74
+ }
75
+ if (body.datasetId !== undefined && typeof body.datasetId !== "string") {
76
+ throw new APIError({status: 400, title: "datasetId must be a string"});
77
+ }
78
+ const data = await options.store.create({
79
+ datasetId: body.datasetId || undefined,
80
+ description: body.description,
81
+ evaluatorId: body.evaluatorId,
82
+ limit: body.limit,
83
+ name: body.name,
84
+ promptName: body.promptName || undefined,
85
+ traceIds: body.traceIds,
86
+ });
87
+ return res.status(201).json({data});
88
+ })
89
+ );
90
+
91
+ router.get(
92
+ `${BASE_PATH}/review/batches/:id`,
93
+ observabilityRouteMiddleware(
94
+ options.accessControl,
95
+ {action: "read", resource: "aiReview"},
96
+ builder()
97
+ .withTags(["observability"])
98
+ .withSummary("Get a review batch with human answers and judge agreement")
99
+ .withPathParameter("id", {type: "string"})
100
+ .withResponse(200, {data: {type: "object"}})
101
+ .build()
102
+ ),
103
+ asyncHandler(async (req, res) => {
104
+ return res.json({data: await options.store.get(req.params.id)});
105
+ })
106
+ );
107
+
108
+ router.post(
109
+ `${BASE_PATH}/review/batches/:id/judgeRuns`,
110
+ observabilityRouteMiddleware(
111
+ options.accessControl,
112
+ {action: "assign", resource: "aiReview"},
113
+ builder()
114
+ .withTags(["observability"])
115
+ .withSummary("Run an LLM judge over a review batch")
116
+ .withPathParameter("id", {type: "string"})
117
+ .withResponse(201, {data: {type: "object"}})
118
+ .build()
119
+ ),
120
+ asyncHandler(async (req, res) => {
121
+ const body = req.body as {evaluatorId?: string};
122
+ if (!body.evaluatorId) {
123
+ throw new APIError({status: 400, title: "evaluatorId is required"});
124
+ }
125
+ const ai = resolveRequestAiService({
126
+ aiService: options.aiService,
127
+ req,
128
+ requestAiServiceFactory: options.requestAiServiceFactory,
129
+ });
130
+ const data = await options.store.runJudge({
131
+ ai,
132
+ batchId: req.params.id,
133
+ evaluatorId: body.evaluatorId,
134
+ });
135
+ return res.status(201).json({data});
136
+ })
137
+ );
138
+ };
@@ -9,12 +9,12 @@ import {createLocalObservabilityPlugin} from "../local/localPlugin";
9
9
  import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
10
10
  import type {ObservabilityPlugin} from "../types";
11
11
 
12
- const createLangfusePlugin = (): ObservabilityPlugin => {
12
+ const createRemotePlugin = (): ObservabilityPlugin => {
13
13
  return {
14
14
  capabilities: new Set(["datasets", "experiments", "prompts", "scores", "traces"]),
15
15
  datasetStore: {},
16
16
  experimentRunner: {},
17
- id: "langfuse",
17
+ id: "remote",
18
18
  promptRegistry: {get: async () => undefined},
19
19
  scoreSink: {export: async () => {}},
20
20
  traceSink: {export: async () => {}},
@@ -36,7 +36,7 @@ describe("observability status routes", () => {
36
36
  app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
37
37
  .register(
38
38
  new ObservabilityApp({
39
- plugins: [createLocalObservabilityPlugin(), createLangfusePlugin()],
39
+ plugins: [createLocalObservabilityPlugin(), createRemotePlugin()],
40
40
  })
41
41
  )
42
42
  .build();
@@ -55,7 +55,7 @@ describe("observability status routes", () => {
55
55
  });
56
56
  expect(response.body.data.plugins.map((plugin: {id: string}) => plugin.id)).toEqual([
57
57
  "local",
58
- "langfuse",
58
+ "remote",
59
59
  ]);
60
60
  expect(response.body.data.plugins[0].capabilities).toContain("reviewQueue");
61
61
  });