@terreno/ai 57.20.0 → 57.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. package/CHANGELOG.md +81 -0
  2. package/README.md +3 -3
  3. package/biome.jsonc +3 -2
  4. package/dist/harness/agentLoop.js +1 -0
  5. package/dist/harness/commit.d.ts +4 -3
  6. package/dist/harness/commit.js +23 -4
  7. package/dist/index.d.ts +6 -9
  8. package/dist/index.js +14 -25
  9. package/dist/observability/adminScreens.js +5 -1
  10. package/dist/observability/evaluate.d.ts +8 -1
  11. package/dist/observability/evaluate.js +10 -2
  12. package/dist/observability/judgeAgreement.d.ts +53 -0
  13. package/dist/observability/judgeAgreement.js +123 -0
  14. package/dist/observability/local/datasetStore.d.ts +5 -0
  15. package/dist/observability/local/datasetStore.js +7 -2
  16. package/dist/observability/local/evaluatorStore.d.ts +17 -0
  17. package/dist/observability/local/evaluatorStore.js +73 -2
  18. package/dist/observability/local/experimentRunner.d.ts +42 -8
  19. package/dist/observability/local/experimentRunner.js +390 -11
  20. package/dist/observability/local/humanAnswers.d.ts +57 -0
  21. package/dist/observability/local/humanAnswers.js +174 -0
  22. package/dist/observability/local/judgeRun.d.ts +56 -0
  23. package/dist/observability/local/judgeRun.js +111 -0
  24. package/dist/observability/local/localPlugin.js +4 -0
  25. package/dist/observability/local/models/obsDatasetItem.js +4 -0
  26. package/dist/observability/local/models/obsEvaluator.js +4 -0
  27. package/dist/observability/local/models/obsExperiment.js +28 -0
  28. package/dist/observability/local/models/obsExperimentItem.js +4 -0
  29. package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
  30. package/dist/observability/local/models/obsJudgeRun.js +60 -0
  31. package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
  32. package/dist/observability/local/models/obsReviewBatch.js +48 -0
  33. package/dist/observability/local/models/obsReviewItem.js +10 -2
  34. package/dist/observability/local/models/obsTrace.js +4 -0
  35. package/dist/observability/local/promptReferences.d.ts +31 -0
  36. package/dist/observability/local/promptReferences.js +114 -0
  37. package/dist/observability/local/promptStore.d.ts +55 -19
  38. package/dist/observability/local/promptStore.js +171 -7
  39. package/dist/observability/local/reviewBatchStore.d.ts +106 -0
  40. package/dist/observability/local/reviewBatchStore.js +517 -0
  41. package/dist/observability/local/reviewStore.d.ts +24 -3
  42. package/dist/observability/local/reviewStore.js +98 -22
  43. package/dist/observability/local/traceStore.d.ts +14 -1
  44. package/dist/observability/local/traceStore.js +144 -11
  45. package/dist/observability/observabilityApp.js +10 -1
  46. package/dist/observability/questionFingerprint.d.ts +7 -0
  47. package/dist/observability/questionFingerprint.js +23 -0
  48. package/dist/observability/routes/experiments.js +12 -1
  49. package/dist/observability/routes/review.js +3 -1
  50. package/dist/observability/routes/reviewBatches.d.ts +11 -0
  51. package/dist/observability/routes/reviewBatches.js +78 -0
  52. package/dist/observability/routes/traces.d.ts +1 -1
  53. package/dist/observability/routes/traces.js +34 -2
  54. package/dist/observability/types.d.ts +10 -1
  55. package/dist/service/aiService.js +4 -1
  56. package/dist/service/chatTurn.js +38 -31
  57. package/dist/types/harness.d.ts +10 -4
  58. package/dist/types/index.d.ts +8 -3
  59. package/dist/types/observability.d.ts +74 -2
  60. package/e2e/smoke.ts +227 -0
  61. package/package.json +14 -28
  62. package/src/aiApp.test.ts +22 -17
  63. package/src/harness/agentLoop.test.ts +0 -3
  64. package/src/harness/agentLoop.ts +2 -1
  65. package/src/harness/agentRecovery.test.ts +30 -30
  66. package/src/harness/approvals.test.ts +11 -9
  67. package/src/harness/commit.ts +24 -4
  68. package/src/harness/events.test.ts +10 -6
  69. package/src/harness/extensions.test.ts +6 -8
  70. package/src/harness/ownership.test.ts +20 -9
  71. package/src/harness/retries.test.ts +6 -3
  72. package/src/harness/routes/conversations.test.ts +2 -2
  73. package/src/harness/routes/tasks.test.ts +2 -2
  74. package/src/harness/runners/jobsRunner.test.ts +1 -1
  75. package/src/harness/subagent.test.ts +29 -4
  76. package/src/harness/waits.test.ts +6 -4
  77. package/src/index.test.ts +3 -20
  78. package/src/index.ts +23 -26
  79. package/src/isolated/gptStream.isolated.ts +25 -28
  80. package/src/isolated/lazyProjectModel.isolated.ts +2 -2
  81. package/src/models/gptHistory.test.ts +10 -6
  82. package/src/observability/adminScreens.test.ts +7 -0
  83. package/src/observability/adminScreens.ts +6 -1
  84. package/src/observability/evaluate.test.ts +12 -10
  85. package/src/observability/evaluate.ts +12 -2
  86. package/src/observability/judgeAgreement.test.ts +111 -0
  87. package/src/observability/judgeAgreement.ts +184 -0
  88. package/src/observability/local/datasetStore.ts +30 -11
  89. package/src/observability/local/evaluatorStore.ts +96 -2
  90. package/src/observability/local/experimentRunner.test.ts +9 -12
  91. package/src/observability/local/experimentRunner.ts +497 -17
  92. package/src/observability/local/humanAnswers.ts +233 -0
  93. package/src/observability/local/judgePipeline.test.ts +444 -0
  94. package/src/observability/local/judgeRun.test.ts +173 -0
  95. package/src/observability/local/judgeRun.ts +184 -0
  96. package/src/observability/local/localPlugin.ts +4 -0
  97. package/src/observability/local/models/obsDatasetItem.ts +5 -0
  98. package/src/observability/local/models/obsEvaluator.ts +4 -0
  99. package/src/observability/local/models/obsExperiment.ts +29 -0
  100. package/src/observability/local/models/obsExperimentItem.ts +4 -0
  101. package/src/observability/local/models/obsJudgeRun.ts +61 -0
  102. package/src/observability/local/models/obsReviewBatch.ts +52 -0
  103. package/src/observability/local/models/obsReviewItem.ts +11 -2
  104. package/src/observability/local/models/obsTrace.ts +5 -0
  105. package/src/observability/local/obsModels.test.ts +13 -0
  106. package/src/observability/local/promptReferences.test.ts +128 -0
  107. package/src/observability/local/promptReferences.ts +149 -0
  108. package/src/observability/local/promptStore.test.ts +295 -3
  109. package/src/observability/local/promptStore.ts +243 -32
  110. package/src/observability/local/reviewBatchStore.test.ts +251 -0
  111. package/src/observability/local/reviewBatchStore.ts +656 -0
  112. package/src/observability/local/reviewStore.ts +122 -25
  113. package/src/observability/local/traceStore.test.ts +139 -1
  114. package/src/observability/local/traceStore.ts +162 -11
  115. package/src/observability/observabilityApp.test.ts +13 -10
  116. package/src/observability/observabilityApp.ts +10 -1
  117. package/src/observability/questionFingerprint.test.ts +33 -0
  118. package/src/observability/questionFingerprint.ts +24 -0
  119. package/src/observability/routes/datasets.test.ts +2 -2
  120. package/src/observability/routes/evaluators.test.ts +2 -2
  121. package/src/observability/routes/experiments.test.ts +39 -2
  122. package/src/observability/routes/experiments.ts +13 -1
  123. package/src/observability/routes/prompts.test.ts +3 -3
  124. package/src/observability/routes/review.test.ts +2 -2
  125. package/src/observability/routes/review.ts +3 -1
  126. package/src/observability/routes/reviewBatches.test.ts +209 -0
  127. package/src/observability/routes/reviewBatches.ts +138 -0
  128. package/src/observability/routes/status.test.ts +8 -8
  129. package/src/observability/routes/testMultiStage.test.ts +6 -6
  130. package/src/observability/routes/traces.test.ts +63 -2
  131. package/src/observability/routes/traces.ts +46 -4
  132. package/src/observability/types.ts +5 -2
  133. package/src/routes/files.test.ts +5 -3
  134. package/src/routes/gpt.test.ts +21 -14
  135. package/src/routes/gptActions.test.ts +7 -8
  136. package/src/routes/gptDatasets.test.ts +1 -1
  137. package/src/routes/gptFiles.test.ts +1 -1
  138. package/src/routes/gptHistories.test.ts +15 -9
  139. package/src/routes/mcp.test.ts +3 -3
  140. package/src/routes/projects.test.ts +5 -3
  141. package/src/routes/routes.test.ts +156 -115
  142. package/src/service/aiService.test.ts +22 -1
  143. package/src/service/aiService.ts +4 -1
  144. package/src/service/asks.test.ts +14 -6
  145. package/src/service/chatTurn.ts +42 -33
  146. package/src/service/fileStorage.test.ts +5 -2
  147. package/src/service/getMCPTools.test.ts +9 -5
  148. package/src/service/mcpService.test.ts +6 -3
  149. package/src/tests/bunSetup.ts +0 -54
  150. package/src/tests/chatHarness.ts +4 -4
  151. package/src/tests/fakeJudge.ts +8 -0
  152. package/src/tests/helpers.ts +26 -4
  153. package/src/types/harness.ts +11 -4
  154. package/src/types/index.ts +8 -3
  155. package/src/types/observability.ts +86 -2
  156. package/tsconfig.typecheck.json +18 -0
  157. package/dist/langfuseApp.d.ts +0 -9
  158. package/dist/langfuseApp.js +0 -53
  159. package/dist/langfuseCache.d.ts +0 -15
  160. package/dist/langfuseCache.js +0 -54
  161. package/dist/langfuseClient.d.ts +0 -6
  162. package/dist/langfuseClient.js +0 -34
  163. package/dist/langfusePrompts.d.ts +0 -12
  164. package/dist/langfusePrompts.js +0 -89
  165. package/dist/langfuseRoutesEvaluations.d.ts +0 -3
  166. package/dist/langfuseRoutesEvaluations.js +0 -32
  167. package/dist/langfuseRoutesMiddleware.d.ts +0 -2
  168. package/dist/langfuseRoutesMiddleware.js +0 -13
  169. package/dist/langfuseRoutesPlayground.d.ts +0 -2
  170. package/dist/langfuseRoutesPlayground.js +0 -36
  171. package/dist/langfuseRoutesPrompts.d.ts +0 -2
  172. package/dist/langfuseRoutesPrompts.js +0 -56
  173. package/dist/langfuseRoutesTraces.d.ts +0 -2
  174. package/dist/langfuseRoutesTraces.js +0 -30
  175. package/dist/langfuseTracing.d.ts +0 -4
  176. package/dist/langfuseTracing.js +0 -32
  177. package/dist/langfuseTypes.d.ts +0 -96
  178. package/dist/langfuseTypes.js +0 -2
  179. package/dist/langfuseVercelAi.d.ts +0 -12
  180. package/dist/langfuseVercelAi.js +0 -49
  181. package/src/isolated/langfuseClient.isolated.ts +0 -60
  182. package/src/langfuseApp.test.ts +0 -135
  183. package/src/langfuseApp.ts +0 -62
  184. package/src/langfuseCache.test.ts +0 -75
  185. package/src/langfuseCache.ts +0 -72
  186. package/src/langfuseClient.test.ts +0 -69
  187. package/src/langfuseClient.ts +0 -34
  188. package/src/langfusePrompts.test.ts +0 -188
  189. package/src/langfusePrompts.ts +0 -121
  190. package/src/langfuseRoutes.test.ts +0 -256
  191. package/src/langfuseRoutesEvaluations.ts +0 -48
  192. package/src/langfuseRoutesMiddleware.test.ts +0 -57
  193. package/src/langfuseRoutesMiddleware.ts +0 -12
  194. package/src/langfuseRoutesPlayground.ts +0 -50
  195. package/src/langfuseRoutesPrompts.ts +0 -87
  196. package/src/langfuseRoutesTraces.ts +0 -40
  197. package/src/langfuseTracing.test.ts +0 -68
  198. package/src/langfuseTracing.ts +0 -33
  199. package/src/langfuseTypes.ts +0 -103
  200. package/src/langfuseVercelAi.test.ts +0 -79
  201. package/src/langfuseVercelAi.ts +0 -72
@@ -0,0 +1,111 @@
1
+ import {describe, expect, it} from "bun:test";
2
+
3
+ import type {EvaluatorDimension} from "../types/observability";
4
+ import {answersAgree, computeJudgeAgreement} from "./judgeAgreement";
5
+
6
+ const QUESTIONS: EvaluatorDimension[] = [
7
+ {dataType: "boolean", key: "faithful", question: "Is it faithful?", required: true},
8
+ {dataType: "boolean", key: "complete", question: "Is it complete?", required: true},
9
+ {dataType: "numeric", key: "tone", range: "0-10", required: false},
10
+ ];
11
+
12
+ describe("answersAgree", () => {
13
+ it("matches booleans, including yes/no strings", () => {
14
+ expect(answersAgree(QUESTIONS[0] as EvaluatorDimension, true, true)).toBe(true);
15
+ expect(answersAgree(QUESTIONS[0] as EvaluatorDimension, true, "yes")).toBe(true);
16
+ expect(answersAgree(QUESTIONS[0] as EvaluatorDimension, false, true)).toBe(false);
17
+ expect(answersAgree(QUESTIONS[0] as EvaluatorDimension, true, undefined)).toBe(false);
18
+ });
19
+
20
+ it("allows numeric answers within 10% of the range", () => {
21
+ const tone = QUESTIONS[2] as EvaluatorDimension;
22
+ expect(answersAgree(tone, 7, 8)).toBe(true);
23
+ expect(answersAgree(tone, 7, 8.5)).toBe(false);
24
+ expect(answersAgree({...tone, range: undefined}, 0.5, 0.55)).toBe(true);
25
+ });
26
+
27
+ it("matches categorical answers case-insensitively", () => {
28
+ const dimension: EvaluatorDimension = {dataType: "categorical", key: "c", required: true};
29
+ expect(answersAgree(dimension, "Warm", " warm ")).toBe(true);
30
+ expect(answersAgree(dimension, "warm", "cold")).toBe(false);
31
+ });
32
+ });
33
+
34
+ describe("computeJudgeAgreement", () => {
35
+ it("pairs answers by dataset item before trace", () => {
36
+ const agreement = computeJudgeAgreement({
37
+ dimensions: QUESTIONS,
38
+ humanAnswers: [
39
+ {datasetItemId: "item-1", scores: {faithful: false}, traceId: "shared-trace"},
40
+ {datasetItemId: "item-2", scores: {faithful: true}},
41
+ ],
42
+ judgeAnswers: [
43
+ {datasetItemId: "item-2", scores: {faithful: true}},
44
+ {datasetItemId: "item-1", scores: {faithful: true}, traceId: "shared-trace"},
45
+ {scores: {faithful: false}, traceId: "shared-trace"},
46
+ ],
47
+ judgeKeys: ["faithful"],
48
+ });
49
+ expect(agreement.rate).toBe(0.5);
50
+ expect(agreement.disagreements).toEqual([
51
+ {
52
+ datasetItemId: "item-1",
53
+ human: false,
54
+ judge: true,
55
+ key: "faithful",
56
+ traceId: "shared-trace",
57
+ },
58
+ ]);
59
+ });
60
+
61
+ it("scores only covered questions and lists disagreements", () => {
62
+ const agreement = computeJudgeAgreement({
63
+ dimensions: QUESTIONS,
64
+ humanAnswers: [
65
+ {scores: {complete: true, faithful: true, tone: 5}, traceId: "a"},
66
+ {scores: {complete: false, faithful: true}, traceId: "b"},
67
+ {scores: {complete: true, faithful: false}, traceId: "unjudged"},
68
+ ],
69
+ judgeAnswers: [
70
+ {scores: {complete: true, faithful: true}, traceId: "a"},
71
+ {scores: {complete: true, faithful: true}, traceId: "b"},
72
+ ],
73
+ judgeKeys: ["faithful", "complete"],
74
+ });
75
+ expect(agreement.compared).toBe(4);
76
+ expect(agreement.matches).toBe(3);
77
+ expect(agreement.rate).toBe(0.75);
78
+ expect(agreement.itemsCompared).toBe(2);
79
+ expect(agreement.questions.map((row) => [row.key, row.covered, row.rate])).toEqual([
80
+ ["faithful", true, 1],
81
+ ["complete", true, 0.5],
82
+ ["tone", false, undefined],
83
+ ]);
84
+ expect(agreement.disagreements).toEqual([
85
+ {human: false, judge: true, key: "complete", traceId: "b"},
86
+ ]);
87
+ });
88
+
89
+ it("counts judge errors as disagreements", () => {
90
+ const agreement = computeJudgeAgreement({
91
+ dimensions: QUESTIONS.slice(0, 1),
92
+ humanAnswers: [{scores: {faithful: true}, traceId: "a"}],
93
+ judgeAnswers: [{error: "timeout", traceId: "a"}],
94
+ judgeKeys: ["faithful"],
95
+ });
96
+ expect(agreement.errors).toBe(1);
97
+ expect(agreement.rate).toBe(0);
98
+ expect(agreement.disagreements[0]?.judge).toBeUndefined();
99
+ });
100
+
101
+ it("has no rate before any human answers", () => {
102
+ const agreement = computeJudgeAgreement({
103
+ dimensions: QUESTIONS,
104
+ humanAnswers: [],
105
+ judgeAnswers: [{scores: {faithful: true}, traceId: "a"}],
106
+ judgeKeys: ["faithful"],
107
+ });
108
+ expect(agreement.rate).toBeUndefined();
109
+ expect(agreement.compared).toBe(0);
110
+ });
111
+ });
@@ -0,0 +1,184 @@
1
+ import type {EvaluatorDimension} from "../types/observability";
2
+
3
+ export type ScoreValue = boolean | number | string;
4
+
5
+ /**
6
+ * Answers for one reviewed item. Human and judge answers pair up by `datasetItemId` when set,
7
+ * otherwise by `traceId`.
8
+ */
9
+ export interface AgreementAnswerSet {
10
+ datasetItemId?: string;
11
+ scores?: Record<string, ScoreValue>;
12
+ traceId?: string;
13
+ }
14
+
15
+ export interface JudgeAnswerSet extends AgreementAnswerSet {
16
+ error?: string;
17
+ }
18
+
19
+ export interface JudgeQuestionAgreement {
20
+ compared: number;
21
+ /** False when the judge does not answer this question, so it is left out of the totals. */
22
+ covered: boolean;
23
+ key: string;
24
+ matches: number;
25
+ question?: string;
26
+ rate?: number;
27
+ }
28
+
29
+ export interface JudgeDisagreement {
30
+ datasetItemId?: string;
31
+ human: ScoreValue;
32
+ judge?: ScoreValue;
33
+ key: string;
34
+ traceId?: string;
35
+ }
36
+
37
+ export interface JudgeAgreement {
38
+ compared: number;
39
+ disagreements: JudgeDisagreement[];
40
+ errors: number;
41
+ itemsCompared: number;
42
+ matches: number;
43
+ questions: JudgeQuestionAgreement[];
44
+ rate?: number;
45
+ }
46
+
47
+ // Numeric answers agree when they land within this share of the dimension's range.
48
+ const NUMERIC_TOLERANCE = 0.1;
49
+
50
+ const numericSpan = (range?: string): number => {
51
+ const matches = range?.match(/(-?\d+(?:\.\d+)?)\s*(?:-|\.\.)\s*(-?\d+(?:\.\d+)?)/);
52
+ if (!matches) {
53
+ return 1;
54
+ }
55
+ const span = Math.abs(Number(matches[2]) - Number(matches[1]));
56
+ return span > 0 ? span : 1;
57
+ };
58
+
59
+ const toBoolean = (value: ScoreValue): boolean | undefined => {
60
+ if (typeof value === "boolean") {
61
+ return value;
62
+ }
63
+ if (value === "true" || value === "yes") {
64
+ return true;
65
+ }
66
+ if (value === "false" || value === "no") {
67
+ return false;
68
+ }
69
+ return undefined;
70
+ };
71
+
72
+ /** True when a judge answer counts as the same answer a human gave for this dimension. */
73
+ export const answersAgree = (
74
+ dimension: EvaluatorDimension,
75
+ human: ScoreValue,
76
+ judge: ScoreValue | undefined
77
+ ): boolean => {
78
+ if (judge === undefined) {
79
+ return false;
80
+ }
81
+ if (dimension.dataType === "boolean") {
82
+ const humanBool = toBoolean(human);
83
+ return humanBool !== undefined && humanBool === toBoolean(judge);
84
+ }
85
+ if (dimension.dataType === "numeric") {
86
+ const humanNumber = Number(human);
87
+ const judgeNumber = Number(judge);
88
+ if (Number.isNaN(humanNumber) || Number.isNaN(judgeNumber)) {
89
+ return false;
90
+ }
91
+ return (
92
+ Math.abs(humanNumber - judgeNumber) <= numericSpan(dimension.range) * NUMERIC_TOLERANCE + 1e-9
93
+ );
94
+ }
95
+ return String(human).trim().toLowerCase() === String(judge).trim().toLowerCase();
96
+ };
97
+
98
+ /** Key that pairs human and judge answers for the same item. */
99
+ export const agreementAnswerKey = (answer: AgreementAnswerSet): string => {
100
+ return answer.datasetItemId ? `item:${answer.datasetItemId}` : `trace:${answer.traceId ?? ""}`;
101
+ };
102
+
103
+ const rateOf = (matches: number, compared: number): number | undefined => {
104
+ return compared > 0 ? matches / compared : undefined;
105
+ };
106
+
107
+ /**
108
+ * Compare one judge's answers with human answers for the same traces or dataset items. Only questions the judge
109
+ * answers (judgeKeys) count; a judge error or a missing answer counts as a disagreement.
110
+ */
111
+ export const computeJudgeAgreement = (params: {
112
+ dimensions: EvaluatorDimension[];
113
+ humanAnswers: AgreementAnswerSet[];
114
+ judgeAnswers: JudgeAnswerSet[];
115
+ judgeKeys: string[];
116
+ }): JudgeAgreement => {
117
+ const judgeKeys = new Set(params.judgeKeys);
118
+ const judgeByKey = new Map(
119
+ params.judgeAnswers.map((answer) => {
120
+ return [agreementAnswerKey(answer), answer];
121
+ })
122
+ );
123
+ const questions: JudgeQuestionAgreement[] = params.dimensions.map((dimension) => {
124
+ return {
125
+ compared: 0,
126
+ covered: judgeKeys.has(dimension.key),
127
+ key: dimension.key,
128
+ matches: 0,
129
+ question: dimension.question,
130
+ };
131
+ });
132
+ const disagreements: JudgeDisagreement[] = [];
133
+ let itemsCompared = 0;
134
+ let errors = 0;
135
+
136
+ for (const human of params.humanAnswers) {
137
+ const judge = judgeByKey.get(agreementAnswerKey(human));
138
+ if (!judge || !human.scores) {
139
+ continue;
140
+ }
141
+ let comparedThisItem = false;
142
+ if (judge.error) {
143
+ errors += 1;
144
+ }
145
+ params.dimensions.forEach((dimension, index) => {
146
+ const humanValue = human.scores?.[dimension.key];
147
+ const question = questions[index];
148
+ if (!question?.covered || humanValue === undefined) {
149
+ return;
150
+ }
151
+ comparedThisItem = true;
152
+ question.compared += 1;
153
+ const judgeValue = judge.error ? undefined : judge.scores?.[dimension.key];
154
+ if (answersAgree(dimension, humanValue, judgeValue)) {
155
+ question.matches += 1;
156
+ return;
157
+ }
158
+ disagreements.push({
159
+ ...(human.datasetItemId ? {datasetItemId: human.datasetItemId} : {}),
160
+ human: humanValue,
161
+ judge: judgeValue,
162
+ key: dimension.key,
163
+ ...(human.traceId ? {traceId: human.traceId} : {}),
164
+ });
165
+ });
166
+ if (comparedThisItem) {
167
+ itemsCompared += 1;
168
+ }
169
+ }
170
+
171
+ const compared = questions.reduce((total, question) => total + question.compared, 0);
172
+ const matches = questions.reduce((total, question) => total + question.matches, 0);
173
+ return {
174
+ compared,
175
+ disagreements,
176
+ errors,
177
+ itemsCompared,
178
+ matches,
179
+ questions: questions.map((question) => {
180
+ return {...question, rate: rateOf(question.matches, question.compared)};
181
+ }),
182
+ rate: rateOf(matches, compared),
183
+ };
184
+ };
@@ -10,6 +10,12 @@ import {
10
10
  parseDatasetJsonImport,
11
11
  } from "../datasetImport";
12
12
  import {validateAgainstSchema} from "../schemaValidation";
13
+ import {
14
+ type DatasetHumanAnswerView,
15
+ type DatasetJudgmentSummary,
16
+ summarizeDatasetJudgments,
17
+ toHumanAnswerViews,
18
+ } from "./humanAnswers";
13
19
  import {registerObsDataset} from "./models/obsDataset";
14
20
  import {registerObsDatasetItem} from "./models/obsDatasetItem";
15
21
  import {registerObsTrace} from "./models/obsTrace";
@@ -48,6 +54,8 @@ export interface DatasetView {
48
54
  expectedOutputSchema?: Record<string, unknown>;
49
55
  id: string;
50
56
  inputSchemaPromptName?: string;
57
+ /** Human judging progress per question set (see `summarizeDatasetJudgments`). */
58
+ judgments: DatasetJudgmentSummary[];
51
59
  name: string;
52
60
  tags: string[];
53
61
  updated: string;
@@ -58,6 +66,8 @@ export interface DatasetItemView {
58
66
  created: string;
59
67
  datasetId: string;
60
68
  expectedOutput?: unknown;
69
+ /** Human answers keyed by question-set evaluator id. */
70
+ humanAnswers?: Record<string, DatasetHumanAnswerView>;
61
71
  id: string;
62
72
  input: unknown;
63
73
  metadata?: Record<string, unknown>;
@@ -85,17 +95,22 @@ export const throwOnDatasetImportErrors = (result: DatasetImportResult): void =>
85
95
  });
86
96
  };
87
97
 
88
- const toDatasetView = async (doc: {
89
- _id: mongoose.Types.ObjectId;
90
- created: Date;
91
- description?: string;
92
- expectedOutputSchema?: Record<string, unknown>;
93
- inputSchemaPromptName?: string;
94
- name: string;
95
- tags: string[];
96
- updated: Date;
97
- }): Promise<DatasetView> => {
98
+ const toDatasetView = async (
99
+ doc: {
100
+ _id: mongoose.Types.ObjectId;
101
+ created: Date;
102
+ description?: string;
103
+ expectedOutputSchema?: Record<string, unknown>;
104
+ inputSchemaPromptName?: string;
105
+ name: string;
106
+ tags: string[];
107
+ updated: Date;
108
+ },
109
+ judgments?: DatasetJudgmentSummary[]
110
+ ): Promise<DatasetView> => {
98
111
  const counts = await computeDatasetCounts(String(doc._id));
112
+ const resolvedJudgments =
113
+ judgments ?? (await summarizeDatasetJudgments([doc._id])).get(String(doc._id)) ?? [];
99
114
  return {
100
115
  counts,
101
116
  created: DateTime.fromJSDate(doc.created).toUTC().toISO() ?? "",
@@ -103,6 +118,7 @@ const toDatasetView = async (doc: {
103
118
  expectedOutputSchema: doc.expectedOutputSchema,
104
119
  id: String(doc._id),
105
120
  inputSchemaPromptName: doc.inputSchemaPromptName,
121
+ judgments: resolvedJudgments,
106
122
  name: doc.name,
107
123
  tags: doc.tags,
108
124
  updated: DateTime.fromJSDate(doc.updated).toUTC().toISO() ?? "",
@@ -115,6 +131,7 @@ const toItemView = (doc: {
115
131
  created: Date;
116
132
  datasetId: mongoose.Types.ObjectId;
117
133
  expectedOutput?: unknown;
134
+ humanAnswers?: ObsDatasetItemDocument["humanAnswers"];
118
135
  input: unknown;
119
136
  metadata?: Record<string, unknown>;
120
137
  origin: "manual" | "synthetic" | "trace";
@@ -129,6 +146,7 @@ const toItemView = (doc: {
129
146
  created: DateTime.fromJSDate(doc.created).toUTC().toISO() ?? "",
130
147
  datasetId: String(doc.datasetId),
131
148
  expectedOutput: doc.expectedOutput,
149
+ humanAnswers: toHumanAnswerViews(doc.humanAnswers),
132
150
  id: String(doc._id),
133
151
  input: doc.input,
134
152
  metadata: doc.metadata,
@@ -193,9 +211,10 @@ export class LocalDatasetStore {
193
211
 
194
212
  async list(): Promise<DatasetView[]> {
195
213
  const rows = await registerObsDataset().find({}).sort({name: 1});
214
+ const judgments = await summarizeDatasetJudgments(rows.map((row) => row._id));
196
215
  return Promise.all(
197
216
  rows.map((row) => {
198
- return toDatasetView(row);
217
+ return toDatasetView(row, judgments.get(String(row._id)) ?? []);
199
218
  })
200
219
  );
201
220
  }
@@ -4,10 +4,14 @@ import type {
4
4
  EvaluatorDimension,
5
5
  EvaluatorRunModes,
6
6
  ObsEvaluatorDocument,
7
+ ObsExperimentDocument,
7
8
  } from "../../types/observability";
8
9
  import {judgeSchemaMissingDimensions} from "../evaluate";
9
10
  import {getEvaluatorTemplate} from "../evaluatorTemplates";
11
+ import {questionSetFingerprint} from "../questionFingerprint";
12
+ import {registerObsDataset} from "./models/obsDataset";
10
13
  import {registerObsEvaluator} from "./models/obsEvaluator";
14
+ import {registerObsExperiment} from "./models/obsExperiment";
11
15
  import {LocalPromptStore} from "./promptStore";
12
16
 
13
17
  export interface EvaluatorWriteInput {
@@ -23,8 +27,25 @@ export interface EvaluatorWriteInput {
23
27
  type: "human" | "json-assert" | "llm-judge";
24
28
  }
25
29
 
30
+ /** How well an llm-judge agreed with human answers in its latest judge calibration experiment. */
31
+ export interface EvaluatorCalibration {
32
+ datasetName?: string;
33
+ experimentId: string;
34
+ experimentName: string;
35
+ passed: boolean;
36
+ questionSetName?: string;
37
+ /** Agreement rate (0–1) of `version`; unset when nothing could be compared. */
38
+ rate?: number;
39
+ /** True when the question set's questions changed since the experiment ran. */
40
+ stale: boolean;
41
+ targetAgreement?: number;
42
+ /** The judge prompt's production version when it was compared, else the best version. */
43
+ version: number;
44
+ }
45
+
26
46
  export interface EvaluatorView {
27
47
  assertion?: {constraint: string; path: string};
48
+ calibration?: EvaluatorCalibration;
28
49
  confidenceAlertBelow: number;
29
50
  description?: string;
30
51
  dimensions: EvaluatorDimension[];
@@ -140,8 +161,9 @@ export class LocalEvaluatorStore {
140
161
 
141
162
  async list(): Promise<EvaluatorView[]> {
142
163
  const rows = await registerObsEvaluator().find({}).sort({name: 1});
164
+ const calibrations = await this.loadCalibrations(rows);
143
165
  return rows.map((row) => {
144
- return toView(row);
166
+ return {...toView(row), calibration: calibrations.get(String(row._id))};
145
167
  });
146
168
  }
147
169
 
@@ -150,7 +172,79 @@ export class LocalEvaluatorStore {
150
172
  if (!doc) {
151
173
  throw new APIError({status: 404, title: `Unknown evaluator "${id}"`});
152
174
  }
153
- return toView(doc);
175
+ const calibrations = await this.loadCalibrations([doc]);
176
+ return {...toView(doc), calibration: calibrations.get(String(doc._id))};
177
+ }
178
+
179
+ // Latest completed judge experiment per llm-judge evaluator, summarized for display.
180
+ private async loadCalibrations(
181
+ evaluators: ObsEvaluatorDocument[]
182
+ ): Promise<Map<string, EvaluatorCalibration>> {
183
+ const result = new Map<string, EvaluatorCalibration>();
184
+ const judges = evaluators.filter((row) => row.type === "llm-judge");
185
+ if (judges.length === 0) {
186
+ return result;
187
+ }
188
+ const experiments = await registerObsExperiment()
189
+ .find({
190
+ judgeEvaluatorId: {$in: judges.map((row) => row._id)},
191
+ kind: "judge",
192
+ status: "completed",
193
+ })
194
+ .sort({created: -1});
195
+ const latest = new Map<string, ObsExperimentDocument>();
196
+ for (const experiment of experiments) {
197
+ const key = String(experiment.judgeEvaluatorId);
198
+ if (!latest.has(key)) {
199
+ latest.set(key, experiment);
200
+ }
201
+ }
202
+ if (latest.size === 0) {
203
+ return result;
204
+ }
205
+ const rows = [...latest.values()];
206
+ const datasets = await registerObsDataset().find({
207
+ _id: {$in: rows.map((row) => row.datasetId)},
208
+ });
209
+ const questionSets = await registerObsEvaluator().find({
210
+ _id: {$in: rows.flatMap((row) => (row.questionSetId ? [row.questionSetId] : []))},
211
+ });
212
+ for (const judge of judges) {
213
+ const experiment = latest.get(String(judge._id));
214
+ if (!experiment) {
215
+ continue;
216
+ }
217
+ const production = judge.judgePromptName
218
+ ? (await this.promptStore.getVersionByLabel(judge.judgePromptName, "production"))?.version
219
+ : undefined;
220
+ const agreement = experiment.results?.agreement ?? {};
221
+ const best = [...experiment.versions].sort((left, right) => {
222
+ return (agreement[String(right)]?.rate ?? -1) - (agreement[String(left)]?.rate ?? -1);
223
+ })[0];
224
+ const version =
225
+ production !== undefined && experiment.versions.includes(production) ? production : best;
226
+ if (version === undefined) {
227
+ continue;
228
+ }
229
+ const questionSet = questionSets.find((row) => {
230
+ return String(row._id) === String(experiment.questionSetId);
231
+ });
232
+ const gate = experiment.results?.gates.find((row) => row.version === version);
233
+ result.set(String(judge._id), {
234
+ datasetName: datasets.find((row) => String(row._id) === String(experiment.datasetId))?.name,
235
+ experimentId: String(experiment._id),
236
+ experimentName: experiment.name,
237
+ passed: gate?.passed ?? false,
238
+ questionSetName: questionSet?.name,
239
+ rate: agreement[String(version)]?.rate,
240
+ stale:
241
+ !questionSet ||
242
+ questionSetFingerprint(questionSet.dimensions) !== experiment.questionsFingerprint,
243
+ targetAgreement: experiment.targetAgreement,
244
+ version,
245
+ });
246
+ }
247
+ return result;
154
248
  }
155
249
 
156
250
  async getByName(name: string): Promise<EvaluatorView | undefined> {
@@ -1,8 +1,8 @@
1
1
  import {afterEach, beforeEach, describe, it} from "bun:test";
2
- import {BackgroundTask} from "@terreno/api";
2
+ import {BackgroundTask, type BackgroundTaskDocument, type FindExactlyOnePlugin} from "@terreno/api";
3
3
  import {assert} from "chai";
4
4
  import mongoose from "mongoose";
5
-
5
+ import {judgeAnswering} from "../../tests/fakeJudge";
6
6
  import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
7
7
  import {LocalDatasetStore} from "./datasetStore";
8
8
  import {LocalEvaluatorStore} from "./evaluatorStore";
@@ -37,9 +37,7 @@ const createTestRunner = (params: {
37
37
  });
38
38
  runner.configureAi({
39
39
  aiService: {
40
- generateJsonObject: async () => {
41
- return {correct: true};
42
- },
40
+ generateJsonObject: judgeAnswering(() => ({correct: true})),
43
41
  generateText: params.generateText,
44
42
  modelId: "mock-model",
45
43
  },
@@ -340,9 +338,7 @@ describe("LocalExperimentRunner", () => {
340
338
  });
341
339
  defaultRunner.configureAi({
342
340
  aiService: {
343
- generateJsonObject: async () => {
344
- return {correct: true};
345
- },
341
+ generateJsonObject: judgeAnswering(() => ({correct: true})),
346
342
  generateText: async () => {
347
343
  return JSON.stringify({answer: "ok"});
348
344
  },
@@ -399,9 +395,7 @@ describe("LocalExperimentRunner", () => {
399
395
  });
400
396
  failingRunner.configureAi({
401
397
  aiService: {
402
- generateJsonObject: async () => {
403
- return {correct: true};
404
- },
398
+ generateJsonObject: judgeAnswering(() => ({correct: true})),
405
399
  generateText: async () => {
406
400
  throw new Error("generation failed");
407
401
  },
@@ -427,7 +421,10 @@ describe("LocalExperimentRunner", () => {
427
421
  });
428
422
  detail = await failingRunner.get(experiment.id);
429
423
  }
430
- const task = await BackgroundTask.findExactlyOne({_id: detail.backgroundTaskId});
424
+ // @terreno/api applies findExactlyOne to BackgroundTask, but its model type does not declare it.
425
+ const tasks = BackgroundTask as typeof BackgroundTask &
426
+ FindExactlyOnePlugin<BackgroundTaskDocument>;
427
+ const task = await tasks.findExactlyOne({_id: detail.backgroundTaskId});
431
428
  assert.equal(task.status, "failed");
432
429
  assert.include(task.error ?? "", "generation failed");
433
430
  });