@terreno/ai 58.0.0-beta.3 → 58.0.0-beta.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. package/CHANGELOG.md +99 -0
  2. package/README.md +3 -3
  3. package/dist/harness/commit.d.ts +4 -3
  4. package/dist/harness/commit.js +23 -4
  5. package/dist/index.d.ts +6 -9
  6. package/dist/index.js +14 -25
  7. package/dist/observability/adminScreens.js +5 -1
  8. package/dist/observability/evaluate.d.ts +8 -1
  9. package/dist/observability/evaluate.js +10 -2
  10. package/dist/observability/judgeAgreement.d.ts +53 -0
  11. package/dist/observability/judgeAgreement.js +123 -0
  12. package/dist/observability/local/datasetStore.d.ts +5 -0
  13. package/dist/observability/local/datasetStore.js +7 -2
  14. package/dist/observability/local/evaluatorStore.d.ts +17 -0
  15. package/dist/observability/local/evaluatorStore.js +73 -2
  16. package/dist/observability/local/experimentRunner.d.ts +40 -3
  17. package/dist/observability/local/experimentRunner.js +390 -11
  18. package/dist/observability/local/humanAnswers.d.ts +57 -0
  19. package/dist/observability/local/humanAnswers.js +174 -0
  20. package/dist/observability/local/judgeRun.d.ts +56 -0
  21. package/dist/observability/local/judgeRun.js +111 -0
  22. package/dist/observability/local/localPlugin.js +4 -0
  23. package/dist/observability/local/models/obsDatasetItem.js +4 -0
  24. package/dist/observability/local/models/obsEvaluator.js +4 -0
  25. package/dist/observability/local/models/obsExperiment.js +28 -0
  26. package/dist/observability/local/models/obsExperimentItem.js +4 -0
  27. package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
  28. package/dist/observability/local/models/obsJudgeRun.js +60 -0
  29. package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
  30. package/dist/observability/local/models/obsReviewBatch.js +48 -0
  31. package/dist/observability/local/models/obsReviewItem.js +10 -2
  32. package/dist/observability/local/models/obsTrace.js +4 -0
  33. package/dist/observability/local/promptReferences.d.ts +31 -0
  34. package/dist/observability/local/promptReferences.js +114 -0
  35. package/dist/observability/local/promptStore.d.ts +55 -19
  36. package/dist/observability/local/promptStore.js +171 -7
  37. package/dist/observability/local/reviewBatchStore.d.ts +106 -0
  38. package/dist/observability/local/reviewBatchStore.js +517 -0
  39. package/dist/observability/local/reviewStore.d.ts +24 -3
  40. package/dist/observability/local/reviewStore.js +98 -22
  41. package/dist/observability/local/traceStore.d.ts +14 -1
  42. package/dist/observability/local/traceStore.js +144 -11
  43. package/dist/observability/observabilityApp.js +10 -1
  44. package/dist/observability/questionFingerprint.d.ts +7 -0
  45. package/dist/observability/questionFingerprint.js +23 -0
  46. package/dist/observability/routes/experiments.js +12 -1
  47. package/dist/observability/routes/review.js +3 -1
  48. package/dist/observability/routes/reviewBatches.d.ts +11 -0
  49. package/dist/observability/routes/reviewBatches.js +78 -0
  50. package/dist/observability/routes/traces.d.ts +1 -1
  51. package/dist/observability/routes/traces.js +34 -2
  52. package/dist/observability/types.d.ts +10 -1
  53. package/dist/service/aiService.js +4 -1
  54. package/dist/service/chatTurn.js +38 -31
  55. package/dist/types/harness.d.ts +5 -2
  56. package/dist/types/index.d.ts +8 -3
  57. package/dist/types/observability.d.ts +74 -2
  58. package/package.json +8 -24
  59. package/src/harness/agentLoop.test.ts +0 -3
  60. package/src/harness/agentRecovery.test.ts +0 -3
  61. package/src/harness/commit.ts +24 -4
  62. package/src/harness/extensions.test.ts +0 -3
  63. package/src/harness/subagent.test.ts +27 -3
  64. package/src/index.test.ts +0 -17
  65. package/src/index.ts +23 -26
  66. package/src/isolated/gptStream.isolated.ts +0 -8
  67. package/src/observability/adminScreens.test.ts +7 -0
  68. package/src/observability/adminScreens.ts +6 -1
  69. package/src/observability/evaluate.ts +12 -2
  70. package/src/observability/judgeAgreement.test.ts +111 -0
  71. package/src/observability/judgeAgreement.ts +184 -0
  72. package/src/observability/local/datasetStore.ts +30 -11
  73. package/src/observability/local/evaluatorStore.ts +96 -2
  74. package/src/observability/local/experimentRunner.ts +495 -15
  75. package/src/observability/local/humanAnswers.ts +233 -0
  76. package/src/observability/local/judgePipeline.test.ts +444 -0
  77. package/src/observability/local/judgeRun.test.ts +173 -0
  78. package/src/observability/local/judgeRun.ts +184 -0
  79. package/src/observability/local/localPlugin.ts +4 -0
  80. package/src/observability/local/models/obsDatasetItem.ts +5 -0
  81. package/src/observability/local/models/obsEvaluator.ts +4 -0
  82. package/src/observability/local/models/obsExperiment.ts +29 -0
  83. package/src/observability/local/models/obsExperimentItem.ts +4 -0
  84. package/src/observability/local/models/obsJudgeRun.ts +61 -0
  85. package/src/observability/local/models/obsReviewBatch.ts +52 -0
  86. package/src/observability/local/models/obsReviewItem.ts +11 -2
  87. package/src/observability/local/models/obsTrace.ts +5 -0
  88. package/src/observability/local/obsModels.test.ts +13 -0
  89. package/src/observability/local/promptReferences.test.ts +128 -0
  90. package/src/observability/local/promptReferences.ts +149 -0
  91. package/src/observability/local/promptStore.test.ts +291 -0
  92. package/src/observability/local/promptStore.ts +243 -32
  93. package/src/observability/local/reviewBatchStore.test.ts +251 -0
  94. package/src/observability/local/reviewBatchStore.ts +656 -0
  95. package/src/observability/local/reviewStore.ts +122 -25
  96. package/src/observability/local/traceStore.test.ts +139 -1
  97. package/src/observability/local/traceStore.ts +162 -11
  98. package/src/observability/observabilityApp.test.ts +13 -10
  99. package/src/observability/observabilityApp.ts +10 -1
  100. package/src/observability/questionFingerprint.test.ts +33 -0
  101. package/src/observability/questionFingerprint.ts +24 -0
  102. package/src/observability/routes/experiments.test.ts +37 -0
  103. package/src/observability/routes/experiments.ts +13 -1
  104. package/src/observability/routes/review.ts +3 -1
  105. package/src/observability/routes/reviewBatches.test.ts +209 -0
  106. package/src/observability/routes/reviewBatches.ts +138 -0
  107. package/src/observability/routes/status.test.ts +4 -4
  108. package/src/observability/routes/traces.test.ts +61 -0
  109. package/src/observability/routes/traces.ts +46 -4
  110. package/src/observability/types.ts +5 -2
  111. package/src/routes/routes.test.ts +101 -76
  112. package/src/service/aiService.test.ts +21 -0
  113. package/src/service/aiService.ts +4 -1
  114. package/src/service/chatTurn.ts +42 -33
  115. package/src/tests/bunSetup.ts +0 -54
  116. package/src/types/harness.ts +5 -2
  117. package/src/types/index.ts +8 -3
  118. package/src/types/observability.ts +86 -2
  119. package/dist/langfuseApp.d.ts +0 -9
  120. package/dist/langfuseApp.js +0 -53
  121. package/dist/langfuseCache.d.ts +0 -15
  122. package/dist/langfuseCache.js +0 -54
  123. package/dist/langfuseClient.d.ts +0 -6
  124. package/dist/langfuseClient.js +0 -34
  125. package/dist/langfusePrompts.d.ts +0 -12
  126. package/dist/langfusePrompts.js +0 -89
  127. package/dist/langfuseRoutesEvaluations.d.ts +0 -3
  128. package/dist/langfuseRoutesEvaluations.js +0 -32
  129. package/dist/langfuseRoutesMiddleware.d.ts +0 -2
  130. package/dist/langfuseRoutesMiddleware.js +0 -13
  131. package/dist/langfuseRoutesPlayground.d.ts +0 -2
  132. package/dist/langfuseRoutesPlayground.js +0 -36
  133. package/dist/langfuseRoutesPrompts.d.ts +0 -2
  134. package/dist/langfuseRoutesPrompts.js +0 -56
  135. package/dist/langfuseRoutesTraces.d.ts +0 -2
  136. package/dist/langfuseRoutesTraces.js +0 -30
  137. package/dist/langfuseTracing.d.ts +0 -4
  138. package/dist/langfuseTracing.js +0 -32
  139. package/dist/langfuseTypes.d.ts +0 -96
  140. package/dist/langfuseTypes.js +0 -2
  141. package/dist/langfuseVercelAi.d.ts +0 -12
  142. package/dist/langfuseVercelAi.js +0 -49
  143. package/src/isolated/langfuseClient.isolated.ts +0 -66
  144. package/src/langfuseApp.test.ts +0 -119
  145. package/src/langfuseApp.ts +0 -62
  146. package/src/langfuseCache.test.ts +0 -75
  147. package/src/langfuseCache.ts +0 -72
  148. package/src/langfuseClient.test.ts +0 -69
  149. package/src/langfuseClient.ts +0 -34
  150. package/src/langfusePrompts.test.ts +0 -188
  151. package/src/langfusePrompts.ts +0 -121
  152. package/src/langfuseRoutes.test.ts +0 -261
  153. package/src/langfuseRoutesEvaluations.ts +0 -48
  154. package/src/langfuseRoutesMiddleware.test.ts +0 -57
  155. package/src/langfuseRoutesMiddleware.ts +0 -12
  156. package/src/langfuseRoutesPlayground.ts +0 -50
  157. package/src/langfuseRoutesPrompts.ts +0 -87
  158. package/src/langfuseRoutesTraces.ts +0 -40
  159. package/src/langfuseTracing.test.ts +0 -72
  160. package/src/langfuseTracing.ts +0 -33
  161. package/src/langfuseTypes.ts +0 -103
  162. package/src/langfuseVercelAi.test.ts +0 -82
  163. package/src/langfuseVercelAi.ts +0 -72
  164. package/src/tests/fakeLangfusePrompt.ts +0 -10
@@ -0,0 +1,233 @@
1
+ import {APIError} from "@terreno/api";
2
+ import {DateTime} from "luxon";
3
+ import mongoose from "mongoose";
4
+
5
+ import type {
6
+ ObsDatasetHumanAnswer,
7
+ ObsDatasetItemDocument,
8
+ ObsEvaluatorDocument,
9
+ } from "../../types/observability";
10
+ import {questionSetFingerprint} from "../questionFingerprint";
11
+ import {registerObsDatasetItem} from "./models/obsDatasetItem";
12
+ import {registerObsEvaluator} from "./models/obsEvaluator";
13
+ import {registerObsReviewBatch} from "./models/obsReviewBatch";
14
+ import {registerObsTrace} from "./models/obsTrace";
15
+
16
+ /** How far one question set has judged a dataset. */
17
+ export interface DatasetJudgmentSummary {
18
+ /** Items with any answer from this question set. */
19
+ answered: number;
20
+ /** Answers whose fingerprint matches the question set's questions now. */
21
+ current: number;
22
+ evaluatorId: string;
23
+ evaluatorName: string;
24
+ /** Answers given to an older version of the questions. */
25
+ stale: number;
26
+ total: number;
27
+ }
28
+
29
+ export interface DatasetHumanAnswerView {
30
+ answeredAt: string;
31
+ fingerprint: string;
32
+ reviewItemId: string;
33
+ scores: Record<string, boolean | number | string>;
34
+ userId?: string;
35
+ }
36
+
37
+ /** What reviewers (and judges) see for a dataset item: the source trace when it has one. */
38
+ export interface DatasetItemSubject {
39
+ input?: unknown;
40
+ output?: unknown;
41
+ traceId?: string;
42
+ traceName?: string;
43
+ }
44
+
45
+ export interface CurrentHumanAnswers {
46
+ fingerprint: string;
47
+ items: Array<{answer: ObsDatasetHumanAnswer; item: ObsDatasetItemDocument}>;
48
+ questionSet: ObsEvaluatorDocument;
49
+ }
50
+
51
+ const toObjectId = (value: string | mongoose.Types.ObjectId, label: string) => {
52
+ if (!mongoose.Types.ObjectId.isValid(String(value))) {
53
+ throw new APIError({status: 400, title: `${label} must be an object id`});
54
+ }
55
+ return new mongoose.Types.ObjectId(String(value));
56
+ };
57
+
58
+ export const toHumanAnswerViews = (
59
+ answers?: Record<string, ObsDatasetHumanAnswer>
60
+ ): Record<string, DatasetHumanAnswerView> | undefined => {
61
+ if (!answers) {
62
+ return undefined;
63
+ }
64
+ return Object.fromEntries(
65
+ Object.entries(answers).map(([evaluatorId, answer]) => {
66
+ return [
67
+ evaluatorId,
68
+ {
69
+ answeredAt: DateTime.fromJSDate(answer.answeredAt).toUTC().toISO() ?? "",
70
+ fingerprint: answer.fingerprint,
71
+ reviewItemId: answer.reviewItemId,
72
+ scores: answer.scores,
73
+ userId: answer.userId,
74
+ },
75
+ ];
76
+ })
77
+ );
78
+ };
79
+
80
+ /** Record (or replace) one question set's answers on a dataset item. */
81
+ export const writeHumanAnswer = async (params: {
82
+ datasetItemId: mongoose.Types.ObjectId;
83
+ evaluator: Pick<ObsEvaluatorDocument, "_id" | "dimensions">;
84
+ reviewItemId: string;
85
+ scores: Record<string, boolean | number | string>;
86
+ userId?: string;
87
+ }): Promise<void> => {
88
+ const item = await registerObsDatasetItem().findOneOrNone({_id: params.datasetItemId});
89
+ if (!item) {
90
+ return;
91
+ }
92
+ const entry: ObsDatasetHumanAnswer = {
93
+ answeredAt: DateTime.utc().toJSDate(),
94
+ fingerprint: questionSetFingerprint(params.evaluator.dimensions),
95
+ reviewItemId: params.reviewItemId,
96
+ scores: params.scores,
97
+ ...(params.userId ? {userId: params.userId} : {}),
98
+ };
99
+ item.humanAnswers = {...(item.humanAnswers ?? {}), [String(params.evaluator._id)]: entry};
100
+ item.markModified("humanAnswers");
101
+ await item.save();
102
+ };
103
+
104
+ /** Subjects for dataset items, keyed by item id: the source trace when it still exists. */
105
+ export const loadDatasetItemSubjects = async (
106
+ items: Pick<ObsDatasetItemDocument, "_id" | "expectedOutput" | "input" | "sourceTraceId">[]
107
+ ): Promise<Map<string, DatasetItemSubject>> => {
108
+ const traceIds = items.flatMap((item) => (item.sourceTraceId ? [item.sourceTraceId] : []));
109
+ const traces = traceIds.length > 0 ? await registerObsTrace().find({_id: {$in: traceIds}}) : [];
110
+ const traceById = new Map(
111
+ traces.map((trace) => {
112
+ return [String(trace._id), trace];
113
+ })
114
+ );
115
+ return new Map(
116
+ items.map((item) => {
117
+ const trace = item.sourceTraceId ? traceById.get(String(item.sourceTraceId)) : undefined;
118
+ const subject: DatasetItemSubject = trace
119
+ ? {
120
+ input: trace.input,
121
+ output: trace.output,
122
+ traceId: String(trace._id),
123
+ traceName: trace.name,
124
+ }
125
+ : {input: item.input, output: item.expectedOutput};
126
+ return [String(item._id), subject];
127
+ })
128
+ );
129
+ };
130
+
131
+ /** Dataset items whose answers from this question set match its current questions. */
132
+ export const loadCurrentHumanAnswers = async (params: {
133
+ datasetId: string | mongoose.Types.ObjectId;
134
+ questionSetId: string | mongoose.Types.ObjectId;
135
+ }): Promise<CurrentHumanAnswers> => {
136
+ const questionSet = await registerObsEvaluator().findOneOrNone({
137
+ _id: toObjectId(params.questionSetId, "questionSetId"),
138
+ });
139
+ if (!questionSet) {
140
+ throw new APIError({status: 404, title: "Unknown question set"});
141
+ }
142
+ const fingerprint = questionSetFingerprint(questionSet.dimensions);
143
+ const key = String(questionSet._id);
144
+ const rows = await registerObsDatasetItem()
145
+ .find({datasetId: toObjectId(params.datasetId, "datasetId")})
146
+ .sort({_id: 1});
147
+ const items = rows.flatMap((item) => {
148
+ const answer = item.humanAnswers?.[key];
149
+ return answer && answer.fingerprint === fingerprint ? [{answer, item}] : [];
150
+ });
151
+ return {fingerprint, items, questionSet};
152
+ };
153
+
154
+ /**
155
+ * Per-dataset judgment summaries: one row per question set that answered any item or has a
156
+ * review batch built from the dataset.
157
+ */
158
+ export const summarizeDatasetJudgments = async (
159
+ datasetIds: mongoose.Types.ObjectId[]
160
+ ): Promise<Map<string, DatasetJudgmentSummary[]>> => {
161
+ const result = new Map<string, DatasetJudgmentSummary[]>();
162
+ if (datasetIds.length === 0) {
163
+ return result;
164
+ }
165
+ const items = await registerObsDatasetItem()
166
+ .find({datasetId: {$in: datasetIds}})
167
+ .select({datasetId: 1, humanAnswers: 1});
168
+ const batches = await registerObsReviewBatch()
169
+ .find({datasetId: {$in: datasetIds}})
170
+ .select({datasetId: 1, evaluatorId: 1});
171
+ const evaluatorIdsByDataset = new Map<string, Set<string>>();
172
+ const addEvaluator = (datasetId: string, evaluatorId: string): void => {
173
+ const ids = evaluatorIdsByDataset.get(datasetId) ?? new Set<string>();
174
+ ids.add(evaluatorId);
175
+ evaluatorIdsByDataset.set(datasetId, ids);
176
+ };
177
+ for (const item of items) {
178
+ for (const evaluatorId of Object.keys(item.humanAnswers ?? {})) {
179
+ addEvaluator(String(item.datasetId), evaluatorId);
180
+ }
181
+ }
182
+ for (const batch of batches) {
183
+ addEvaluator(String(batch.datasetId), String(batch.evaluatorId));
184
+ }
185
+ const allEvaluatorIds = [
186
+ ...new Set([...evaluatorIdsByDataset.values()].flatMap((ids) => [...ids])),
187
+ ]
188
+ .filter((id) => mongoose.Types.ObjectId.isValid(id))
189
+ .map((id) => new mongoose.Types.ObjectId(id));
190
+ const evaluators = await registerObsEvaluator().find({_id: {$in: allEvaluatorIds}});
191
+ const evaluatorById = new Map(
192
+ evaluators.map((evaluator) => {
193
+ return [
194
+ String(evaluator._id),
195
+ {fingerprint: questionSetFingerprint(evaluator.dimensions), name: evaluator.name},
196
+ ];
197
+ })
198
+ );
199
+ for (const datasetId of datasetIds) {
200
+ const key = String(datasetId);
201
+ const datasetItems = items.filter((item) => String(item.datasetId) === key);
202
+ const summaries: DatasetJudgmentSummary[] = [];
203
+ for (const evaluatorId of evaluatorIdsByDataset.get(key) ?? []) {
204
+ const evaluator = evaluatorById.get(evaluatorId);
205
+ if (!evaluator) {
206
+ continue;
207
+ }
208
+ let answered = 0;
209
+ let current = 0;
210
+ for (const item of datasetItems) {
211
+ const answer = item.humanAnswers?.[evaluatorId];
212
+ if (!answer) {
213
+ continue;
214
+ }
215
+ answered += 1;
216
+ if (answer.fingerprint === evaluator.fingerprint) {
217
+ current += 1;
218
+ }
219
+ }
220
+ summaries.push({
221
+ answered,
222
+ current,
223
+ evaluatorId,
224
+ evaluatorName: evaluator.name,
225
+ stale: answered - current,
226
+ total: datasetItems.length,
227
+ });
228
+ }
229
+ summaries.sort((left, right) => left.evaluatorName.localeCompare(right.evaluatorName));
230
+ result.set(key, summaries);
231
+ }
232
+ return result;
233
+ };
@@ -0,0 +1,444 @@
1
+ import {afterEach, beforeEach, describe, expect, it} from "bun:test";
2
+ import {BackgroundTask} from "@terreno/api";
3
+ import {DateTime} from "luxon";
4
+
5
+ import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
6
+ import {questionSetFingerprint} from "../questionFingerprint";
7
+ import type {ObservabilityGenerateClient} from "../types";
8
+ import {LocalDatasetStore} from "./datasetStore";
9
+ import {LocalEvaluatorStore} from "./evaluatorStore";
10
+ import {LocalExperimentRunner} from "./experimentRunner";
11
+ import {loadCurrentHumanAnswers} from "./humanAnswers";
12
+ import {createLocalObservabilityPlugin} from "./localPlugin";
13
+ import {registerObsDataset} from "./models/obsDataset";
14
+ import {registerObsDatasetItem} from "./models/obsDatasetItem";
15
+ import {registerObsEvaluator} from "./models/obsEvaluator";
16
+ import {registerObsExperiment} from "./models/obsExperiment";
17
+ import {registerObsExperimentItem} from "./models/obsExperimentItem";
18
+ import {registerObsJudgeRun} from "./models/obsJudgeRun";
19
+ import {registerObsPrompt} from "./models/obsPrompt";
20
+ import {registerObsPromptLabel} from "./models/obsPromptLabel";
21
+ import {registerObsPromptVersion} from "./models/obsPromptVersion";
22
+ import {registerObsReviewBatch} from "./models/obsReviewBatch";
23
+ import {registerObsReviewItem} from "./models/obsReviewItem";
24
+ import {registerObsTrace} from "./models/obsTrace";
25
+ import {LocalPromptStore} from "./promptStore";
26
+ import {LocalReviewBatchStore} from "./reviewBatchStore";
27
+ import {LocalReviewStore} from "./reviewStore";
28
+ import {LocalTraceStore, MemoryScoreSink} from "./traceStore";
29
+
30
+ const SLOW_MONGO_TIMEOUT = 120_000;
31
+ const USER_ID = "64b000000000000000000001";
32
+
33
+ const QUESTIONS = [
34
+ {dataType: "boolean" as const, key: "faithful", question: "Is it faithful?", required: true},
35
+ {dataType: "boolean" as const, key: "complete", question: "Is it complete?", required: true},
36
+ ];
37
+
38
+ const judgeSchema = {
39
+ properties: {
40
+ complete: {type: "boolean"},
41
+ confidence: {type: "number"},
42
+ faithful: {type: "boolean"},
43
+ },
44
+ required: ["faithful", "complete"],
45
+ type: "object",
46
+ };
47
+
48
+ // v1 ("LENIENT") says yes to everything; v2 ("STRICT") marks outputs containing "bad" unfaithful.
49
+ const fakeClient: ObservabilityGenerateClient = {
50
+ generateJsonObject: async <OBJECT>(options: {prompt: string; systemPrompt?: string}) => {
51
+ const strict = options.systemPrompt === "STRICT";
52
+ return {
53
+ complete: true,
54
+ confidence: 0.9,
55
+ faithful: strict ? !options.prompt.includes("bad") : true,
56
+ } as OBJECT;
57
+ },
58
+ generateText: async () => "",
59
+ modelId: "fake-judge",
60
+ };
61
+
62
+ // What reviewers say for each output text.
63
+ const HUMAN: Record<string, {complete: boolean; faithful: boolean}> = {
64
+ "bad summary": {complete: true, faithful: false},
65
+ "good summary": {complete: true, faithful: true},
66
+ "manual bad": {complete: true, faithful: false},
67
+ };
68
+
69
+ const exportTrace = async (output: string, hoursAgo: number): Promise<string> => {
70
+ const startedAt = DateTime.utc().minus({hours: hoursAgo}).toISO() ?? "";
71
+ const exported = await new LocalTraceStore().exportTrace({
72
+ id: `t-${hoursAgo}`,
73
+ input: `todo list ${hoursAgo}`,
74
+ name: "summarize",
75
+ output,
76
+ prompts: [{name: "summarize", version: 1}],
77
+ sensitive: false,
78
+ spans: [{id: "s", kind: "LLM", name: "llm", startedAt, status: "ok"}],
79
+ startedAt,
80
+ status: "ok",
81
+ });
82
+ return exported.id;
83
+ };
84
+
85
+ describe("judge calibration pipeline", () => {
86
+ let promptStore: LocalPromptStore;
87
+ let datasetStore: LocalDatasetStore;
88
+ let evaluatorStore: LocalEvaluatorStore;
89
+ let batchStore: LocalReviewBatchStore;
90
+ let reviewStore: LocalReviewStore;
91
+ let runner: LocalExperimentRunner;
92
+ let questionSetId: string;
93
+ let judgeId: string;
94
+ let datasetId: string;
95
+
96
+ // Answer every pending item of a batch the way reviewers would.
97
+ const answerBatch = async (batchId: string): Promise<void> => {
98
+ const batch = await batchStore.get(batchId);
99
+ for (const item of batch.items) {
100
+ await reviewStore.submit({
101
+ id: item.reviewItemId,
102
+ scores: HUMAN[String(item.output)] ?? {complete: true, faithful: true},
103
+ sinks: [new MemoryScoreSink()],
104
+ userId: USER_ID,
105
+ });
106
+ }
107
+ };
108
+
109
+ afterEach(() => {
110
+ resetObservabilityApp();
111
+ });
112
+
113
+ beforeEach(async () => {
114
+ const plugin = createLocalObservabilityPlugin();
115
+ plugin.scoreSink = new MemoryScoreSink();
116
+ new ObservabilityApp({plugins: [plugin]});
117
+ for (const model of [
118
+ registerObsReviewItem(),
119
+ registerObsReviewBatch(),
120
+ registerObsJudgeRun(),
121
+ registerObsEvaluator(),
122
+ registerObsTrace(),
123
+ registerObsPrompt(),
124
+ registerObsPromptVersion(),
125
+ registerObsPromptLabel(),
126
+ registerObsDataset(),
127
+ registerObsDatasetItem(),
128
+ registerObsExperiment(),
129
+ registerObsExperimentItem(),
130
+ ]) {
131
+ await (model as unknown as {deleteMany: (filter: object) => Promise<unknown>}).deleteMany({});
132
+ }
133
+ await BackgroundTask.deleteMany({});
134
+ promptStore = new LocalPromptStore();
135
+ datasetStore = new LocalDatasetStore(promptStore);
136
+ evaluatorStore = new LocalEvaluatorStore(promptStore);
137
+ reviewStore = new LocalReviewStore();
138
+ batchStore = new LocalReviewBatchStore(promptStore, reviewStore);
139
+ runner = new LocalExperimentRunner({
140
+ backgroundTaskRunner: {
141
+ enqueue: async ({execute, taskType}) => {
142
+ const task = await BackgroundTask.create({status: "pending", taskType});
143
+ await execute();
144
+ return {taskId: String(task._id)};
145
+ },
146
+ },
147
+ datasetStore,
148
+ evaluatorStore,
149
+ promptStore,
150
+ });
151
+ runner.configureAi({aiService: fakeClient});
152
+
153
+ await promptStore.create({
154
+ folder: "evaluators",
155
+ name: "calib-judge",
156
+ outputSchema: judgeSchema,
157
+ system: "LENIENT",
158
+ type: "chat",
159
+ });
160
+ await promptStore.createVersion("calib-judge", {
161
+ outputSchema: judgeSchema,
162
+ system: "STRICT",
163
+ type: "chat",
164
+ });
165
+ await promptStore.moveLabel("calib-judge", {label: "production", version: 1});
166
+ questionSetId = (
167
+ await evaluatorStore.create({
168
+ dimensions: QUESTIONS,
169
+ name: "summary-questions",
170
+ target: "full trace",
171
+ type: "human",
172
+ })
173
+ ).id;
174
+ judgeId = (
175
+ await evaluatorStore.create({
176
+ dimensions: QUESTIONS.map(({question: _question, ...row}) => row),
177
+ judgePromptName: "calib-judge",
178
+ name: "summary-judge",
179
+ target: "full trace",
180
+ type: "llm-judge",
181
+ })
182
+ ).id;
183
+ datasetId = (await datasetStore.create({name: "calib-set"})).id;
184
+ await datasetStore.addTracesToDataset({
185
+ datasetId,
186
+ traceIds: [await exportTrace("good summary", 2), await exportTrace("bad summary", 1)],
187
+ });
188
+ await datasetStore.createItem(datasetId, {
189
+ expectedOutput: "manual bad",
190
+ input: {text: "manual input"},
191
+ });
192
+ });
193
+
194
+ it(
195
+ "builds review batches from a dataset and writes answers back to its items",
196
+ async () => {
197
+ await expect(
198
+ batchStore.create({datasetId, evaluatorId: questionSetId, limit: 5})
199
+ ).rejects.toThrow(/cannot be combined/);
200
+
201
+ const batch = await batchStore.create({datasetId, evaluatorId: questionSetId});
202
+ expect(batch.datasetId).toBe(datasetId);
203
+ expect(batch.datasetName).toBe("calib-set");
204
+ expect(batch.questionsFingerprint).toBe(questionSetFingerprint(QUESTIONS));
205
+ expect(batch.questionsStale).toBe(false);
206
+ expect(batch.items).toHaveLength(3);
207
+ expect(batch.items.every((item) => Boolean(item.datasetItemId))).toBe(true);
208
+ const manual = batch.items.find((item) => !item.traceId);
209
+ expect(manual?.input).toEqual({text: "manual input"});
210
+ expect(manual?.output).toBe("manual bad");
211
+ expect(manual?.traceName).toBe("Dataset item");
212
+ const traced = batch.items.find((item) => item.output === "good summary");
213
+ expect(traced?.input).toBe("todo list 2");
214
+
215
+ // Items waiting in an open batch are not queued again.
216
+ await expect(batchStore.create({datasetId, evaluatorId: questionSetId})).rejects.toThrow(
217
+ /already waiting in an open review batch/
218
+ );
219
+
220
+ const detail = await reviewStore.getDetail(manual?.reviewItemId ?? "");
221
+ expect(detail.traceId).toBeUndefined();
222
+ expect(detail.datasetItemId).toBe(manual?.datasetItemId);
223
+ expect(detail.rawOutput).toBe("manual bad");
224
+
225
+ // Submitting writes answers onto the dataset item; scores only reach sinks with a trace.
226
+ const sink = new MemoryScoreSink();
227
+ await reviewStore.submit({
228
+ id: manual?.reviewItemId ?? "",
229
+ scores: {complete: true, faithful: false},
230
+ sinks: [sink],
231
+ userId: USER_ID,
232
+ });
233
+ expect(sink.scores).toHaveLength(0);
234
+ await reviewStore.submit({
235
+ id: traced?.reviewItemId ?? "",
236
+ scores: {complete: true, faithful: true},
237
+ sinks: [sink],
238
+ });
239
+ expect(sink.scores).toHaveLength(2);
240
+ const skipped = batch.items.find((item) => item.output === "bad summary");
241
+ await reviewStore.skip(skipped?.reviewItemId ?? "");
242
+
243
+ const items = await datasetStore.listItems(datasetId);
244
+ const manualItem = items.find((item) => item.id === manual?.datasetItemId);
245
+ expect(manualItem?.humanAnswers?.[questionSetId]).toMatchObject({
246
+ fingerprint: questionSetFingerprint(QUESTIONS),
247
+ reviewItemId: manual?.reviewItemId,
248
+ scores: {complete: true, faithful: false},
249
+ userId: USER_ID,
250
+ });
251
+ expect(items.find((item) => item.id === skipped?.datasetItemId)?.humanAnswers).toBe(
252
+ undefined
253
+ );
254
+ let dataset = await datasetStore.get(datasetId);
255
+ expect(dataset.judgments).toEqual([
256
+ {
257
+ answered: 2,
258
+ current: 2,
259
+ evaluatorId: questionSetId,
260
+ evaluatorName: "summary-questions",
261
+ stale: 0,
262
+ total: 3,
263
+ },
264
+ ]);
265
+
266
+ // The skipped item is the only one left to judge.
267
+ const followUp = await batchStore.create({datasetId, evaluatorId: questionSetId});
268
+ expect(followUp.items.map((item) => item.datasetItemId)).toEqual([skipped?.datasetItemId]);
269
+ await answerBatch(followUp.id);
270
+ await expect(batchStore.create({datasetId, evaluatorId: questionSetId})).rejects.toThrow(
271
+ "Every item already has current answers"
272
+ );
273
+ expect((await loadCurrentHumanAnswers({datasetId, questionSetId})).items).toHaveLength(3);
274
+
275
+ // Editing a question makes every answer stale and the dataset judgeable again.
276
+ await evaluatorStore.update(questionSetId, {
277
+ dimensions: [{...QUESTIONS[0], question: "Is every claim backed?"}, QUESTIONS[1]],
278
+ });
279
+ dataset = (await datasetStore.list()).find((row) => row.id === datasetId) ?? dataset;
280
+ expect(dataset.judgments[0]).toMatchObject({answered: 3, current: 0, stale: 3});
281
+ expect((await loadCurrentHumanAnswers({datasetId, questionSetId})).items).toHaveLength(0);
282
+ expect((await batchStore.list()).every((row) => row.questionsStale)).toBe(true);
283
+ const rejudge = await batchStore.create({datasetId, evaluatorId: questionSetId});
284
+ expect(rejudge.items).toHaveLength(3);
285
+ },
286
+ SLOW_MONGO_TIMEOUT
287
+ );
288
+
289
+ it(
290
+ "runs LLM judges over dataset batches without a source trace",
291
+ async () => {
292
+ const batch = await batchStore.create({datasetId, evaluatorId: questionSetId});
293
+ await answerBatch(batch.id);
294
+ await promptStore.moveLabel("calib-judge", {label: "production", version: 2});
295
+ const judged = await batchStore.runJudge({
296
+ ai: {
297
+ generateJsonObject: async <OBJECT>(options: {prompt: string}) => {
298
+ return {complete: true, faithful: !options.prompt.includes("bad")} as OBJECT;
299
+ },
300
+ modelId: "fake",
301
+ },
302
+ batchId: batch.id,
303
+ evaluatorId: judgeId,
304
+ });
305
+ expect(judged.judges[0]?.agreement.rate).toBe(1);
306
+ const manual = judged.items.find((item) => !item.traceId);
307
+ expect(manual?.judgeAnswers[judgeId]?.scores).toEqual({complete: true, faithful: false});
308
+ },
309
+ SLOW_MONGO_TIMEOUT
310
+ );
311
+
312
+ it(
313
+ "ranks judge prompt versions by agreement with human answers and calibrates the judge",
314
+ async () => {
315
+ const base = {datasetId, judgeEvaluatorId: judgeId, kind: "judge" as const, name: "Calib"};
316
+ await expect(runner.create({...base, questionSetId, versions: [1, 2]})).rejects.toThrow(
317
+ "Dataset has no current human answers for this question set"
318
+ );
319
+
320
+ await answerBatch((await batchStore.create({datasetId, evaluatorId: questionSetId})).id);
321
+
322
+ await expect(runner.create({...base, questionSetId, versions: [1]})).rejects.toThrow(
323
+ /2–3 judge prompt versions/
324
+ );
325
+ await expect(runner.create({...base, questionSetId, versions: [1, 9]})).rejects.toThrow(
326
+ "Unknown judge prompt version"
327
+ );
328
+ await expect(
329
+ runner.create({...base, questionSetId: judgeId, versions: [1, 2]})
330
+ ).rejects.toThrow("questionSetId must be a human question set");
331
+ await expect(
332
+ runner.create({...base, judgeEvaluatorId: questionSetId, questionSetId, versions: [1, 2]})
333
+ ).rejects.toThrow(/must be an llm-judge evaluator/);
334
+ await expect(
335
+ runner.create({...base, questionSetId, targetAgreement: 1.5, versions: [1, 2]})
336
+ ).rejects.toThrow(/targetAgreement/);
337
+
338
+ const created = await runner.create({...base, questionSetId, versions: [1, 2]});
339
+ const experiment = await runner.get(created.id);
340
+ expect(experiment.kind).toBe("judge");
341
+ expect(experiment.status).toBe("completed");
342
+ expect(experiment.promptName).toBe("calib-judge");
343
+ expect(experiment.judgeName).toBe("summary-judge");
344
+ expect(experiment.questionSetName).toBe("summary-questions");
345
+ expect(experiment.datasetName).toBe("calib-set");
346
+ expect(experiment.targetAgreement).toBe(0.9);
347
+ expect(experiment.questionsStale).toBe(false);
348
+ expect(experiment.results?.progress).toEqual({completed: 3, total: 3});
349
+ const v1 = experiment.results?.agreement?.["1"];
350
+ const v2 = experiment.results?.agreement?.["2"];
351
+ expect(v1?.rate).toBeCloseTo(4 / 6, 5);
352
+ expect(v2?.rate).toBe(1);
353
+ expect(v1?.disagreements).toHaveLength(2);
354
+ expect(v1?.disagreements.every((row) => Boolean(row.datasetItemId))).toBe(true);
355
+ expect(experiment.results?.gates).toEqual([
356
+ {
357
+ actual: v1?.rate,
358
+ aggregate: "agreement",
359
+ dimension: "agreement",
360
+ evaluatorName: "summary-judge",
361
+ op: "gte",
362
+ passed: false,
363
+ value: 0.9,
364
+ version: 1,
365
+ },
366
+ {
367
+ actual: 1,
368
+ aggregate: "agreement",
369
+ dimension: "agreement",
370
+ evaluatorName: "summary-judge",
371
+ op: "gte",
372
+ passed: true,
373
+ value: 0.9,
374
+ version: 2,
375
+ },
376
+ ]);
377
+ expect(experiment.items).toHaveLength(3);
378
+ expect(experiment.items.filter((item) => item.failed)).toHaveLength(2);
379
+ const failed = experiment.items.find((item) => item.failed);
380
+ expect(failed?.humanScores).toEqual({complete: true, faithful: false});
381
+ expect(failed?.versionResults["1"]?.evaluatorScores[judgeId]?.scores).toEqual({
382
+ complete: true,
383
+ faithful: true,
384
+ });
385
+
386
+ expect((await runner.list({kind: "judge"})).data).toHaveLength(1);
387
+ expect((await runner.list({kind: "prompt"})).data).toHaveLength(0);
388
+
389
+ // Before promote the production judge is v1, which failed its gate.
390
+ let calibration = (await evaluatorStore.get(judgeId)).calibration;
391
+ expect(calibration).toMatchObject({passed: false, stale: false, version: 1});
392
+
393
+ await expect(runner.promote(created.id, 1)).rejects.toThrow(/Promote blocked/);
394
+ await runner.promote(created.id, 2);
395
+ expect((await promptStore.getVersionByLabel("calib-judge"))?.version).toBe(2);
396
+
397
+ calibration = (await evaluatorStore.list()).find((row) => row.id === judgeId)?.calibration;
398
+ expect(calibration).toEqual({
399
+ datasetName: "calib-set",
400
+ experimentId: created.id,
401
+ experimentName: "Calib",
402
+ passed: true,
403
+ questionSetName: "summary-questions",
404
+ rate: 1,
405
+ stale: false,
406
+ targetAgreement: 0.9,
407
+ version: 2,
408
+ });
409
+ expect((await evaluatorStore.get(questionSetId)).calibration).toBeUndefined();
410
+
411
+ await evaluatorStore.update(questionSetId, {
412
+ dimensions: [QUESTIONS[0], {...QUESTIONS[1], question: "Does it cover every todo?"}],
413
+ });
414
+ expect((await evaluatorStore.get(judgeId)).calibration?.stale).toBe(true);
415
+ expect((await runner.get(created.id)).questionsStale).toBe(true);
416
+ },
417
+ SLOW_MONGO_TIMEOUT
418
+ );
419
+
420
+ it(
421
+ "keeps prompt experiments free of agreement gates",
422
+ async () => {
423
+ await expect(
424
+ runner.create({
425
+ datasetId,
426
+ evaluatorIds: [judgeId],
427
+ name: "prompt run",
428
+ promptName: "calib-judge",
429
+ thresholds: [
430
+ {
431
+ aggregate: "agreement",
432
+ dimension: "agreement",
433
+ evaluatorName: "summary-judge",
434
+ op: "gte",
435
+ value: 0.9,
436
+ },
437
+ ],
438
+ versions: [1, 2],
439
+ })
440
+ ).rejects.toThrow(/agreement gates belong to judge experiments/);
441
+ },
442
+ SLOW_MONGO_TIMEOUT
443
+ );
444
+ });