@terreno/ai 58.0.0-beta.3 → 58.0.0-beta.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. package/CHANGELOG.md +99 -0
  2. package/README.md +3 -3
  3. package/dist/harness/commit.d.ts +4 -3
  4. package/dist/harness/commit.js +23 -4
  5. package/dist/index.d.ts +6 -9
  6. package/dist/index.js +14 -25
  7. package/dist/observability/adminScreens.js +5 -1
  8. package/dist/observability/evaluate.d.ts +8 -1
  9. package/dist/observability/evaluate.js +10 -2
  10. package/dist/observability/judgeAgreement.d.ts +53 -0
  11. package/dist/observability/judgeAgreement.js +123 -0
  12. package/dist/observability/local/datasetStore.d.ts +5 -0
  13. package/dist/observability/local/datasetStore.js +7 -2
  14. package/dist/observability/local/evaluatorStore.d.ts +17 -0
  15. package/dist/observability/local/evaluatorStore.js +73 -2
  16. package/dist/observability/local/experimentRunner.d.ts +40 -3
  17. package/dist/observability/local/experimentRunner.js +390 -11
  18. package/dist/observability/local/humanAnswers.d.ts +57 -0
  19. package/dist/observability/local/humanAnswers.js +174 -0
  20. package/dist/observability/local/judgeRun.d.ts +56 -0
  21. package/dist/observability/local/judgeRun.js +111 -0
  22. package/dist/observability/local/localPlugin.js +4 -0
  23. package/dist/observability/local/models/obsDatasetItem.js +4 -0
  24. package/dist/observability/local/models/obsEvaluator.js +4 -0
  25. package/dist/observability/local/models/obsExperiment.js +28 -0
  26. package/dist/observability/local/models/obsExperimentItem.js +4 -0
  27. package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
  28. package/dist/observability/local/models/obsJudgeRun.js +60 -0
  29. package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
  30. package/dist/observability/local/models/obsReviewBatch.js +48 -0
  31. package/dist/observability/local/models/obsReviewItem.js +10 -2
  32. package/dist/observability/local/models/obsTrace.js +4 -0
  33. package/dist/observability/local/promptReferences.d.ts +31 -0
  34. package/dist/observability/local/promptReferences.js +114 -0
  35. package/dist/observability/local/promptStore.d.ts +55 -19
  36. package/dist/observability/local/promptStore.js +171 -7
  37. package/dist/observability/local/reviewBatchStore.d.ts +106 -0
  38. package/dist/observability/local/reviewBatchStore.js +517 -0
  39. package/dist/observability/local/reviewStore.d.ts +24 -3
  40. package/dist/observability/local/reviewStore.js +98 -22
  41. package/dist/observability/local/traceStore.d.ts +14 -1
  42. package/dist/observability/local/traceStore.js +144 -11
  43. package/dist/observability/observabilityApp.js +10 -1
  44. package/dist/observability/questionFingerprint.d.ts +7 -0
  45. package/dist/observability/questionFingerprint.js +23 -0
  46. package/dist/observability/routes/experiments.js +12 -1
  47. package/dist/observability/routes/review.js +3 -1
  48. package/dist/observability/routes/reviewBatches.d.ts +11 -0
  49. package/dist/observability/routes/reviewBatches.js +78 -0
  50. package/dist/observability/routes/traces.d.ts +1 -1
  51. package/dist/observability/routes/traces.js +34 -2
  52. package/dist/observability/types.d.ts +10 -1
  53. package/dist/service/aiService.js +4 -1
  54. package/dist/service/chatTurn.js +38 -31
  55. package/dist/types/harness.d.ts +5 -2
  56. package/dist/types/index.d.ts +8 -3
  57. package/dist/types/observability.d.ts +74 -2
  58. package/package.json +8 -24
  59. package/src/harness/agentLoop.test.ts +0 -3
  60. package/src/harness/agentRecovery.test.ts +0 -3
  61. package/src/harness/commit.ts +24 -4
  62. package/src/harness/extensions.test.ts +0 -3
  63. package/src/harness/subagent.test.ts +27 -3
  64. package/src/index.test.ts +0 -17
  65. package/src/index.ts +23 -26
  66. package/src/isolated/gptStream.isolated.ts +0 -8
  67. package/src/observability/adminScreens.test.ts +7 -0
  68. package/src/observability/adminScreens.ts +6 -1
  69. package/src/observability/evaluate.ts +12 -2
  70. package/src/observability/judgeAgreement.test.ts +111 -0
  71. package/src/observability/judgeAgreement.ts +184 -0
  72. package/src/observability/local/datasetStore.ts +30 -11
  73. package/src/observability/local/evaluatorStore.ts +96 -2
  74. package/src/observability/local/experimentRunner.ts +495 -15
  75. package/src/observability/local/humanAnswers.ts +233 -0
  76. package/src/observability/local/judgePipeline.test.ts +444 -0
  77. package/src/observability/local/judgeRun.test.ts +173 -0
  78. package/src/observability/local/judgeRun.ts +184 -0
  79. package/src/observability/local/localPlugin.ts +4 -0
  80. package/src/observability/local/models/obsDatasetItem.ts +5 -0
  81. package/src/observability/local/models/obsEvaluator.ts +4 -0
  82. package/src/observability/local/models/obsExperiment.ts +29 -0
  83. package/src/observability/local/models/obsExperimentItem.ts +4 -0
  84. package/src/observability/local/models/obsJudgeRun.ts +61 -0
  85. package/src/observability/local/models/obsReviewBatch.ts +52 -0
  86. package/src/observability/local/models/obsReviewItem.ts +11 -2
  87. package/src/observability/local/models/obsTrace.ts +5 -0
  88. package/src/observability/local/obsModels.test.ts +13 -0
  89. package/src/observability/local/promptReferences.test.ts +128 -0
  90. package/src/observability/local/promptReferences.ts +149 -0
  91. package/src/observability/local/promptStore.test.ts +291 -0
  92. package/src/observability/local/promptStore.ts +243 -32
  93. package/src/observability/local/reviewBatchStore.test.ts +251 -0
  94. package/src/observability/local/reviewBatchStore.ts +656 -0
  95. package/src/observability/local/reviewStore.ts +122 -25
  96. package/src/observability/local/traceStore.test.ts +139 -1
  97. package/src/observability/local/traceStore.ts +162 -11
  98. package/src/observability/observabilityApp.test.ts +13 -10
  99. package/src/observability/observabilityApp.ts +10 -1
  100. package/src/observability/questionFingerprint.test.ts +33 -0
  101. package/src/observability/questionFingerprint.ts +24 -0
  102. package/src/observability/routes/experiments.test.ts +37 -0
  103. package/src/observability/routes/experiments.ts +13 -1
  104. package/src/observability/routes/review.ts +3 -1
  105. package/src/observability/routes/reviewBatches.test.ts +209 -0
  106. package/src/observability/routes/reviewBatches.ts +138 -0
  107. package/src/observability/routes/status.test.ts +4 -4
  108. package/src/observability/routes/traces.test.ts +61 -0
  109. package/src/observability/routes/traces.ts +46 -4
  110. package/src/observability/types.ts +5 -2
  111. package/src/routes/routes.test.ts +101 -76
  112. package/src/service/aiService.test.ts +21 -0
  113. package/src/service/aiService.ts +4 -1
  114. package/src/service/chatTurn.ts +42 -33
  115. package/src/tests/bunSetup.ts +0 -54
  116. package/src/types/harness.ts +5 -2
  117. package/src/types/index.ts +8 -3
  118. package/src/types/observability.ts +86 -2
  119. package/dist/langfuseApp.d.ts +0 -9
  120. package/dist/langfuseApp.js +0 -53
  121. package/dist/langfuseCache.d.ts +0 -15
  122. package/dist/langfuseCache.js +0 -54
  123. package/dist/langfuseClient.d.ts +0 -6
  124. package/dist/langfuseClient.js +0 -34
  125. package/dist/langfusePrompts.d.ts +0 -12
  126. package/dist/langfusePrompts.js +0 -89
  127. package/dist/langfuseRoutesEvaluations.d.ts +0 -3
  128. package/dist/langfuseRoutesEvaluations.js +0 -32
  129. package/dist/langfuseRoutesMiddleware.d.ts +0 -2
  130. package/dist/langfuseRoutesMiddleware.js +0 -13
  131. package/dist/langfuseRoutesPlayground.d.ts +0 -2
  132. package/dist/langfuseRoutesPlayground.js +0 -36
  133. package/dist/langfuseRoutesPrompts.d.ts +0 -2
  134. package/dist/langfuseRoutesPrompts.js +0 -56
  135. package/dist/langfuseRoutesTraces.d.ts +0 -2
  136. package/dist/langfuseRoutesTraces.js +0 -30
  137. package/dist/langfuseTracing.d.ts +0 -4
  138. package/dist/langfuseTracing.js +0 -32
  139. package/dist/langfuseTypes.d.ts +0 -96
  140. package/dist/langfuseTypes.js +0 -2
  141. package/dist/langfuseVercelAi.d.ts +0 -12
  142. package/dist/langfuseVercelAi.js +0 -49
  143. package/src/isolated/langfuseClient.isolated.ts +0 -66
  144. package/src/langfuseApp.test.ts +0 -119
  145. package/src/langfuseApp.ts +0 -62
  146. package/src/langfuseCache.test.ts +0 -75
  147. package/src/langfuseCache.ts +0 -72
  148. package/src/langfuseClient.test.ts +0 -69
  149. package/src/langfuseClient.ts +0 -34
  150. package/src/langfusePrompts.test.ts +0 -188
  151. package/src/langfusePrompts.ts +0 -121
  152. package/src/langfuseRoutes.test.ts +0 -261
  153. package/src/langfuseRoutesEvaluations.ts +0 -48
  154. package/src/langfuseRoutesMiddleware.test.ts +0 -57
  155. package/src/langfuseRoutesMiddleware.ts +0 -12
  156. package/src/langfuseRoutesPlayground.ts +0 -50
  157. package/src/langfuseRoutesPrompts.ts +0 -87
  158. package/src/langfuseRoutesTraces.ts +0 -40
  159. package/src/langfuseTracing.test.ts +0 -72
  160. package/src/langfuseTracing.ts +0 -33
  161. package/src/langfuseTypes.ts +0 -103
  162. package/src/langfuseVercelAi.test.ts +0 -82
  163. package/src/langfuseVercelAi.ts +0 -72
  164. package/src/tests/fakeLangfusePrompt.ts +0 -10
package/src/index.test.ts CHANGED
@@ -18,21 +18,10 @@ describe("@terreno/ai public exports", () => {
18
18
  "addMcpRoutes",
19
19
  "addProjectRoutes",
20
20
  "assertVertexModelsEnabled",
21
- "compilePrompt",
22
21
  "createAskTools",
23
- "createPrompt",
24
- "createTelemetryConfig",
25
22
  "createVertexProvider",
26
23
  "findAgentBlock",
27
- "getCached",
28
24
  "getMCPTools",
29
- "getLangfuseClient",
30
- "getPrompt",
31
- "initLangfuseClient",
32
- "initTracing",
33
- "invalidateCache",
34
- "invalidatePromptCache",
35
- "isLangfuseInitialized",
36
25
  "isVertexModelAllowed",
37
26
  "jsonSchema",
38
27
  "listEnabledVertexModels",
@@ -41,15 +30,11 @@ describe("@terreno/ai public exports", () => {
41
30
  "normalizeLlmJsonTextForStructuredOutput",
42
31
  "normalizeVertexModelId",
43
32
  "parseAiJson",
44
- "preparePromptForAI",
45
33
  "getObservabilityApp",
46
34
  "resetObservabilityApp",
47
35
  "resolveObservabilityControl",
48
36
  "runBufferedChatTurn",
49
37
  "validateObservabilityConfig",
50
- "setCached",
51
- "shutdownLangfuseClient",
52
- "shutdownTracing",
53
38
  "verifyVertexModelsEnabled",
54
39
  ];
55
40
 
@@ -60,8 +45,6 @@ describe("@terreno/ai public exports", () => {
60
45
  "FileAttachment",
61
46
  "FileStorageService",
62
47
  "GptHistory",
63
- "LangfuseApp",
64
- "LangfuseCache",
65
48
  "MCPService",
66
49
  "MemoryScoreSink",
67
50
  "MemoryTraceSink",
package/src/index.ts CHANGED
@@ -12,44 +12,40 @@ export type {AIAdminAppOptions} from "./aiAdminApp";
12
12
  export {AIAdminApp} from "./aiAdminApp";
13
13
  export type {AiAppOptions} from "./aiApp";
14
14
  export {AiApp} from "./aiApp";
15
- export {LangfuseApp} from "./langfuseApp";
16
- export {getCached, invalidateCache, LangfuseCache, setCached} from "./langfuseCache";
17
- export {
18
- getLangfuseClient,
19
- initLangfuseClient,
20
- isLangfuseInitialized,
21
- shutdownLangfuseClient,
22
- } from "./langfuseClient";
23
- export {compilePrompt, createPrompt, getPrompt, invalidatePromptCache} from "./langfusePrompts";
24
- export {initTracing, shutdownTracing} from "./langfuseTracing";
25
- export type {
26
- ChatMessage,
27
- GetPromptOptions,
28
- LangfuseAppOptions,
29
- LangfuseCachedPrompt,
30
- PaginatedResult,
31
- PreparePromptResult,
32
- PromptListItem,
33
- ScoreSubmission,
34
- ScoringFunction,
35
- TelemetrySettings,
36
- TraceListItem,
37
- } from "./langfuseTypes";
38
- export {createTelemetryConfig, preparePromptForAI} from "./langfuseVercelAi";
39
15
  export {AIDataset} from "./models/aiDataset";
40
16
  export {AIRequest} from "./models/aiRequest";
41
17
  export {FileAttachment} from "./models/fileAttachment";
42
18
  export {GptHistory} from "./models/gptHistory";
43
19
  export {getProjectModel, Project} from "./models/project";
44
20
  export {AI_OBSERVABILITY_GROUP, observabilityAdminScreens} from "./observability/adminScreens";
21
+ export {
22
+ answersAgree,
23
+ computeJudgeAgreement,
24
+ type JudgeAgreement,
25
+ } from "./observability/judgeAgreement";
45
26
  export {LocalDatasetStore} from "./observability/local/datasetStore";
46
- export {LocalEvaluatorStore} from "./observability/local/evaluatorStore";
47
- export {LocalExperimentRunner} from "./observability/local/experimentRunner";
27
+ export {type EvaluatorCalibration, LocalEvaluatorStore} from "./observability/local/evaluatorStore";
28
+ export {
29
+ DEFAULT_TARGET_AGREEMENT,
30
+ type JudgeExperimentCreateInput,
31
+ LocalExperimentRunner,
32
+ } from "./observability/local/experimentRunner";
33
+ export {
34
+ type DatasetJudgmentSummary,
35
+ loadCurrentHumanAnswers,
36
+ } from "./observability/local/humanAnswers";
48
37
  export {
49
38
  createLocalObservabilityBundle,
50
39
  createLocalObservabilityPlugin,
51
40
  } from "./observability/local/localPlugin";
52
41
  export {LocalPromptStore} from "./observability/local/promptStore";
42
+ export {
43
+ DEFAULT_REVIEW_BATCH_SIZE,
44
+ LocalReviewBatchStore,
45
+ MAX_REVIEW_BATCH_SIZE,
46
+ type ReviewBatchDetail,
47
+ type ReviewBatchListItem,
48
+ } from "./observability/local/reviewBatchStore";
53
49
  export {LocalReviewStore} from "./observability/local/reviewStore";
54
50
  export {
55
51
  LocalScoreSink,
@@ -63,6 +59,7 @@ export {
63
59
  ObservabilityApp,
64
60
  resetObservabilityApp,
65
61
  } from "./observability/observabilityApp";
62
+ export {questionSetFingerprint} from "./observability/questionFingerprint";
66
63
  export type {
67
64
  ObservabilityPluginStatus,
68
65
  ObservabilityStatus,
@@ -17,14 +17,6 @@ let addGptRoutes: typeof import("../routes/gpt.js").addGptRoutes;
17
17
  let AIService: typeof import("../service/aiService.js").AIService;
18
18
 
19
19
  beforeAll(async () => {
20
- // Mock langfuseVercelAi to avoid transitive langfuse SDK imports.
21
- const realLangfuseVercelAi = await import("../langfuseVercelAi.js");
22
- mock.module("../langfuseVercelAi", () => ({
23
- ...realLangfuseVercelAi,
24
- createTelemetryConfig: () => ({functionId: "test", isEnabled: false}),
25
- preparePromptForAI: async () => ({config: {}, prompt: "test", telemetry: {isEnabled: false}}),
26
- }));
27
-
28
20
  // Mock the AI SDK so we control exactly which fullStream parts the GPT prompt
29
21
  // handler iterates over. All other exports are preserved so AIService and the
30
22
  // route helpers keep working.
@@ -24,6 +24,7 @@ describe("observabilityAdminScreens", () => {
24
24
  "ai-datasets",
25
25
  "ai-experiments",
26
26
  "ai-review",
27
+ "ai-review-batches",
27
28
  ]);
28
29
  });
29
30
 
@@ -66,6 +67,12 @@ describe("observabilityAdminScreens", () => {
66
67
  group: AI_OBSERVABILITY_GROUP,
67
68
  name: "ai-review",
68
69
  },
70
+ {
71
+ adminAccess: {action: "list", resource: "aiReview"},
72
+ displayName: "Review batches",
73
+ group: AI_OBSERVABILITY_GROUP,
74
+ name: "ai-review-batches",
75
+ },
69
76
  ]);
70
77
  });
71
78
  });
@@ -35,7 +35,12 @@ export const observabilityAdminScreens = ({localOn}: {localOn: boolean}): AdminC
35
35
  ];
36
36
  if (localOn) {
37
37
  screens.push(
38
- observabilityScreen({displayName: "Review queue", name: "ai-review", resource: "aiReview"})
38
+ observabilityScreen({displayName: "Review queue", name: "ai-review", resource: "aiReview"}),
39
+ observabilityScreen({
40
+ displayName: "Review batches",
41
+ name: "ai-review-batches",
42
+ resource: "aiReview",
43
+ })
39
44
  );
40
45
  }
41
46
  return screens;
@@ -170,10 +170,16 @@ export const runJsonAssertEvaluator = (
170
170
  };
171
171
  };
172
172
 
173
+ /**
174
+ * Run an llm-judge evaluator. By default the AI client resolves the judge prompt's production
175
+ * version by name. Pass `judgeSystemPrompt` (a specific version's resolved body) to run that
176
+ * version instead, as judge calibration experiments do.
177
+ */
173
178
  export const runLlmJudgeEvaluator = async (
174
179
  context: EvaluatorExecutionContext,
175
180
  ai: EvaluatorAiClient,
176
- judgeOutputSchema: Record<string, unknown>
181
+ judgeOutputSchema: Record<string, unknown>,
182
+ options: {judgeSystemPrompt?: string} = {}
177
183
  ): Promise<EvaluatorExecutionResult> => {
178
184
  const judgePromptName = context.evaluator.judgePromptName;
179
185
  if (!judgePromptName) {
@@ -181,9 +187,13 @@ export const runLlmJudgeEvaluator = async (
181
187
  }
182
188
 
183
189
  try {
190
+ const promptSource =
191
+ options.judgeSystemPrompt === undefined
192
+ ? {promptName: judgePromptName}
193
+ : {systemPrompt: options.judgeSystemPrompt};
184
194
  const payload = (await ai.generateJsonObject({
185
195
  prompt: buildJudgePrompt(context),
186
- promptName: judgePromptName,
196
+ ...promptSource,
187
197
  schema: jsonSchema(judgeOutputSchema) as FlexibleSchema<Record<string, unknown>>,
188
198
  schemaName: `${context.evaluator.name}-judge`,
189
199
  skipTrace: true,
@@ -0,0 +1,111 @@
1
+ import {describe, expect, it} from "bun:test";
2
+
3
+ import type {EvaluatorDimension} from "../types/observability";
4
+ import {answersAgree, computeJudgeAgreement} from "./judgeAgreement";
5
+
6
+ const QUESTIONS: EvaluatorDimension[] = [
7
+ {dataType: "boolean", key: "faithful", question: "Is it faithful?", required: true},
8
+ {dataType: "boolean", key: "complete", question: "Is it complete?", required: true},
9
+ {dataType: "numeric", key: "tone", range: "0-10", required: false},
10
+ ];
11
+
12
+ describe("answersAgree", () => {
13
+ it("matches booleans, including yes/no strings", () => {
14
+ expect(answersAgree(QUESTIONS[0] as EvaluatorDimension, true, true)).toBe(true);
15
+ expect(answersAgree(QUESTIONS[0] as EvaluatorDimension, true, "yes")).toBe(true);
16
+ expect(answersAgree(QUESTIONS[0] as EvaluatorDimension, false, true)).toBe(false);
17
+ expect(answersAgree(QUESTIONS[0] as EvaluatorDimension, true, undefined)).toBe(false);
18
+ });
19
+
20
+ it("allows numeric answers within 10% of the range", () => {
21
+ const tone = QUESTIONS[2] as EvaluatorDimension;
22
+ expect(answersAgree(tone, 7, 8)).toBe(true);
23
+ expect(answersAgree(tone, 7, 8.5)).toBe(false);
24
+ expect(answersAgree({...tone, range: undefined}, 0.5, 0.55)).toBe(true);
25
+ });
26
+
27
+ it("matches categorical answers case-insensitively", () => {
28
+ const dimension: EvaluatorDimension = {dataType: "categorical", key: "c", required: true};
29
+ expect(answersAgree(dimension, "Warm", " warm ")).toBe(true);
30
+ expect(answersAgree(dimension, "warm", "cold")).toBe(false);
31
+ });
32
+ });
33
+
34
+ describe("computeJudgeAgreement", () => {
35
+ it("pairs answers by dataset item before trace", () => {
36
+ const agreement = computeJudgeAgreement({
37
+ dimensions: QUESTIONS,
38
+ humanAnswers: [
39
+ {datasetItemId: "item-1", scores: {faithful: false}, traceId: "shared-trace"},
40
+ {datasetItemId: "item-2", scores: {faithful: true}},
41
+ ],
42
+ judgeAnswers: [
43
+ {datasetItemId: "item-2", scores: {faithful: true}},
44
+ {datasetItemId: "item-1", scores: {faithful: true}, traceId: "shared-trace"},
45
+ {scores: {faithful: false}, traceId: "shared-trace"},
46
+ ],
47
+ judgeKeys: ["faithful"],
48
+ });
49
+ expect(agreement.rate).toBe(0.5);
50
+ expect(agreement.disagreements).toEqual([
51
+ {
52
+ datasetItemId: "item-1",
53
+ human: false,
54
+ judge: true,
55
+ key: "faithful",
56
+ traceId: "shared-trace",
57
+ },
58
+ ]);
59
+ });
60
+
61
+ it("scores only covered questions and lists disagreements", () => {
62
+ const agreement = computeJudgeAgreement({
63
+ dimensions: QUESTIONS,
64
+ humanAnswers: [
65
+ {scores: {complete: true, faithful: true, tone: 5}, traceId: "a"},
66
+ {scores: {complete: false, faithful: true}, traceId: "b"},
67
+ {scores: {complete: true, faithful: false}, traceId: "unjudged"},
68
+ ],
69
+ judgeAnswers: [
70
+ {scores: {complete: true, faithful: true}, traceId: "a"},
71
+ {scores: {complete: true, faithful: true}, traceId: "b"},
72
+ ],
73
+ judgeKeys: ["faithful", "complete"],
74
+ });
75
+ expect(agreement.compared).toBe(4);
76
+ expect(agreement.matches).toBe(3);
77
+ expect(agreement.rate).toBe(0.75);
78
+ expect(agreement.itemsCompared).toBe(2);
79
+ expect(agreement.questions.map((row) => [row.key, row.covered, row.rate])).toEqual([
80
+ ["faithful", true, 1],
81
+ ["complete", true, 0.5],
82
+ ["tone", false, undefined],
83
+ ]);
84
+ expect(agreement.disagreements).toEqual([
85
+ {human: false, judge: true, key: "complete", traceId: "b"},
86
+ ]);
87
+ });
88
+
89
+ it("counts judge errors as disagreements", () => {
90
+ const agreement = computeJudgeAgreement({
91
+ dimensions: QUESTIONS.slice(0, 1),
92
+ humanAnswers: [{scores: {faithful: true}, traceId: "a"}],
93
+ judgeAnswers: [{error: "timeout", traceId: "a"}],
94
+ judgeKeys: ["faithful"],
95
+ });
96
+ expect(agreement.errors).toBe(1);
97
+ expect(agreement.rate).toBe(0);
98
+ expect(agreement.disagreements[0]?.judge).toBeUndefined();
99
+ });
100
+
101
+ it("has no rate before any human answers", () => {
102
+ const agreement = computeJudgeAgreement({
103
+ dimensions: QUESTIONS,
104
+ humanAnswers: [],
105
+ judgeAnswers: [{scores: {faithful: true}, traceId: "a"}],
106
+ judgeKeys: ["faithful"],
107
+ });
108
+ expect(agreement.rate).toBeUndefined();
109
+ expect(agreement.compared).toBe(0);
110
+ });
111
+ });
@@ -0,0 +1,184 @@
1
+ import type {EvaluatorDimension} from "../types/observability";
2
+
3
+ export type ScoreValue = boolean | number | string;
4
+
5
+ /**
6
+ * Answers for one reviewed item. Human and judge answers pair up by `datasetItemId` when set,
7
+ * otherwise by `traceId`.
8
+ */
9
+ export interface AgreementAnswerSet {
10
+ datasetItemId?: string;
11
+ scores?: Record<string, ScoreValue>;
12
+ traceId?: string;
13
+ }
14
+
15
+ export interface JudgeAnswerSet extends AgreementAnswerSet {
16
+ error?: string;
17
+ }
18
+
19
+ export interface JudgeQuestionAgreement {
20
+ compared: number;
21
+ /** False when the judge does not answer this question, so it is left out of the totals. */
22
+ covered: boolean;
23
+ key: string;
24
+ matches: number;
25
+ question?: string;
26
+ rate?: number;
27
+ }
28
+
29
+ export interface JudgeDisagreement {
30
+ datasetItemId?: string;
31
+ human: ScoreValue;
32
+ judge?: ScoreValue;
33
+ key: string;
34
+ traceId?: string;
35
+ }
36
+
37
+ export interface JudgeAgreement {
38
+ compared: number;
39
+ disagreements: JudgeDisagreement[];
40
+ errors: number;
41
+ itemsCompared: number;
42
+ matches: number;
43
+ questions: JudgeQuestionAgreement[];
44
+ rate?: number;
45
+ }
46
+
47
+ // Numeric answers agree when they land within this share of the dimension's range.
48
+ const NUMERIC_TOLERANCE = 0.1;
49
+
50
+ const numericSpan = (range?: string): number => {
51
+ const matches = range?.match(/(-?\d+(?:\.\d+)?)\s*(?:-|\.\.)\s*(-?\d+(?:\.\d+)?)/);
52
+ if (!matches) {
53
+ return 1;
54
+ }
55
+ const span = Math.abs(Number(matches[2]) - Number(matches[1]));
56
+ return span > 0 ? span : 1;
57
+ };
58
+
59
+ const toBoolean = (value: ScoreValue): boolean | undefined => {
60
+ if (typeof value === "boolean") {
61
+ return value;
62
+ }
63
+ if (value === "true" || value === "yes") {
64
+ return true;
65
+ }
66
+ if (value === "false" || value === "no") {
67
+ return false;
68
+ }
69
+ return undefined;
70
+ };
71
+
72
+ /** True when a judge answer counts as the same answer a human gave for this dimension. */
73
+ export const answersAgree = (
74
+ dimension: EvaluatorDimension,
75
+ human: ScoreValue,
76
+ judge: ScoreValue | undefined
77
+ ): boolean => {
78
+ if (judge === undefined) {
79
+ return false;
80
+ }
81
+ if (dimension.dataType === "boolean") {
82
+ const humanBool = toBoolean(human);
83
+ return humanBool !== undefined && humanBool === toBoolean(judge);
84
+ }
85
+ if (dimension.dataType === "numeric") {
86
+ const humanNumber = Number(human);
87
+ const judgeNumber = Number(judge);
88
+ if (Number.isNaN(humanNumber) || Number.isNaN(judgeNumber)) {
89
+ return false;
90
+ }
91
+ return (
92
+ Math.abs(humanNumber - judgeNumber) <= numericSpan(dimension.range) * NUMERIC_TOLERANCE + 1e-9
93
+ );
94
+ }
95
+ return String(human).trim().toLowerCase() === String(judge).trim().toLowerCase();
96
+ };
97
+
98
+ /** Key that pairs human and judge answers for the same item. */
99
+ export const agreementAnswerKey = (answer: AgreementAnswerSet): string => {
100
+ return answer.datasetItemId ? `item:${answer.datasetItemId}` : `trace:${answer.traceId ?? ""}`;
101
+ };
102
+
103
+ const rateOf = (matches: number, compared: number): number | undefined => {
104
+ return compared > 0 ? matches / compared : undefined;
105
+ };
106
+
107
+ /**
108
+ * Compare one judge's answers with human answers for the same traces or dataset items. Only questions the judge
109
+ * answers (judgeKeys) count; a judge error or a missing answer counts as a disagreement.
110
+ */
111
+ export const computeJudgeAgreement = (params: {
112
+ dimensions: EvaluatorDimension[];
113
+ humanAnswers: AgreementAnswerSet[];
114
+ judgeAnswers: JudgeAnswerSet[];
115
+ judgeKeys: string[];
116
+ }): JudgeAgreement => {
117
+ const judgeKeys = new Set(params.judgeKeys);
118
+ const judgeByKey = new Map(
119
+ params.judgeAnswers.map((answer) => {
120
+ return [agreementAnswerKey(answer), answer];
121
+ })
122
+ );
123
+ const questions: JudgeQuestionAgreement[] = params.dimensions.map((dimension) => {
124
+ return {
125
+ compared: 0,
126
+ covered: judgeKeys.has(dimension.key),
127
+ key: dimension.key,
128
+ matches: 0,
129
+ question: dimension.question,
130
+ };
131
+ });
132
+ const disagreements: JudgeDisagreement[] = [];
133
+ let itemsCompared = 0;
134
+ let errors = 0;
135
+
136
+ for (const human of params.humanAnswers) {
137
+ const judge = judgeByKey.get(agreementAnswerKey(human));
138
+ if (!judge || !human.scores) {
139
+ continue;
140
+ }
141
+ let comparedThisItem = false;
142
+ if (judge.error) {
143
+ errors += 1;
144
+ }
145
+ params.dimensions.forEach((dimension, index) => {
146
+ const humanValue = human.scores?.[dimension.key];
147
+ const question = questions[index];
148
+ if (!question?.covered || humanValue === undefined) {
149
+ return;
150
+ }
151
+ comparedThisItem = true;
152
+ question.compared += 1;
153
+ const judgeValue = judge.error ? undefined : judge.scores?.[dimension.key];
154
+ if (answersAgree(dimension, humanValue, judgeValue)) {
155
+ question.matches += 1;
156
+ return;
157
+ }
158
+ disagreements.push({
159
+ ...(human.datasetItemId ? {datasetItemId: human.datasetItemId} : {}),
160
+ human: humanValue,
161
+ judge: judgeValue,
162
+ key: dimension.key,
163
+ ...(human.traceId ? {traceId: human.traceId} : {}),
164
+ });
165
+ });
166
+ if (comparedThisItem) {
167
+ itemsCompared += 1;
168
+ }
169
+ }
170
+
171
+ const compared = questions.reduce((total, question) => total + question.compared, 0);
172
+ const matches = questions.reduce((total, question) => total + question.matches, 0);
173
+ return {
174
+ compared,
175
+ disagreements,
176
+ errors,
177
+ itemsCompared,
178
+ matches,
179
+ questions: questions.map((question) => {
180
+ return {...question, rate: rateOf(question.matches, question.compared)};
181
+ }),
182
+ rate: rateOf(matches, compared),
183
+ };
184
+ };
@@ -10,6 +10,12 @@ import {
10
10
  parseDatasetJsonImport,
11
11
  } from "../datasetImport";
12
12
  import {validateAgainstSchema} from "../schemaValidation";
13
+ import {
14
+ type DatasetHumanAnswerView,
15
+ type DatasetJudgmentSummary,
16
+ summarizeDatasetJudgments,
17
+ toHumanAnswerViews,
18
+ } from "./humanAnswers";
13
19
  import {registerObsDataset} from "./models/obsDataset";
14
20
  import {registerObsDatasetItem} from "./models/obsDatasetItem";
15
21
  import {registerObsTrace} from "./models/obsTrace";
@@ -48,6 +54,8 @@ export interface DatasetView {
48
54
  expectedOutputSchema?: Record<string, unknown>;
49
55
  id: string;
50
56
  inputSchemaPromptName?: string;
57
+ /** Human judging progress per question set (see `summarizeDatasetJudgments`). */
58
+ judgments: DatasetJudgmentSummary[];
51
59
  name: string;
52
60
  tags: string[];
53
61
  updated: string;
@@ -58,6 +66,8 @@ export interface DatasetItemView {
58
66
  created: string;
59
67
  datasetId: string;
60
68
  expectedOutput?: unknown;
69
+ /** Human answers keyed by question-set evaluator id. */
70
+ humanAnswers?: Record<string, DatasetHumanAnswerView>;
61
71
  id: string;
62
72
  input: unknown;
63
73
  metadata?: Record<string, unknown>;
@@ -85,17 +95,22 @@ export const throwOnDatasetImportErrors = (result: DatasetImportResult): void =>
85
95
  });
86
96
  };
87
97
 
88
- const toDatasetView = async (doc: {
89
- _id: mongoose.Types.ObjectId;
90
- created: Date;
91
- description?: string;
92
- expectedOutputSchema?: Record<string, unknown>;
93
- inputSchemaPromptName?: string;
94
- name: string;
95
- tags: string[];
96
- updated: Date;
97
- }): Promise<DatasetView> => {
98
+ const toDatasetView = async (
99
+ doc: {
100
+ _id: mongoose.Types.ObjectId;
101
+ created: Date;
102
+ description?: string;
103
+ expectedOutputSchema?: Record<string, unknown>;
104
+ inputSchemaPromptName?: string;
105
+ name: string;
106
+ tags: string[];
107
+ updated: Date;
108
+ },
109
+ judgments?: DatasetJudgmentSummary[]
110
+ ): Promise<DatasetView> => {
98
111
  const counts = await computeDatasetCounts(String(doc._id));
112
+ const resolvedJudgments =
113
+ judgments ?? (await summarizeDatasetJudgments([doc._id])).get(String(doc._id)) ?? [];
99
114
  return {
100
115
  counts,
101
116
  created: DateTime.fromJSDate(doc.created).toUTC().toISO() ?? "",
@@ -103,6 +118,7 @@ const toDatasetView = async (doc: {
103
118
  expectedOutputSchema: doc.expectedOutputSchema,
104
119
  id: String(doc._id),
105
120
  inputSchemaPromptName: doc.inputSchemaPromptName,
121
+ judgments: resolvedJudgments,
106
122
  name: doc.name,
107
123
  tags: doc.tags,
108
124
  updated: DateTime.fromJSDate(doc.updated).toUTC().toISO() ?? "",
@@ -115,6 +131,7 @@ const toItemView = (doc: {
115
131
  created: Date;
116
132
  datasetId: mongoose.Types.ObjectId;
117
133
  expectedOutput?: unknown;
134
+ humanAnswers?: ObsDatasetItemDocument["humanAnswers"];
118
135
  input: unknown;
119
136
  metadata?: Record<string, unknown>;
120
137
  origin: "manual" | "synthetic" | "trace";
@@ -129,6 +146,7 @@ const toItemView = (doc: {
129
146
  created: DateTime.fromJSDate(doc.created).toUTC().toISO() ?? "",
130
147
  datasetId: String(doc.datasetId),
131
148
  expectedOutput: doc.expectedOutput,
149
+ humanAnswers: toHumanAnswerViews(doc.humanAnswers),
132
150
  id: String(doc._id),
133
151
  input: doc.input,
134
152
  metadata: doc.metadata,
@@ -193,9 +211,10 @@ export class LocalDatasetStore {
193
211
 
194
212
  async list(): Promise<DatasetView[]> {
195
213
  const rows = await registerObsDataset().find({}).sort({name: 1});
214
+ const judgments = await summarizeDatasetJudgments(rows.map((row) => row._id));
196
215
  return Promise.all(
197
216
  rows.map((row) => {
198
- return toDatasetView(row);
217
+ return toDatasetView(row, judgments.get(String(row._id)) ?? []);
199
218
  })
200
219
  );
201
220
  }