@terreno/ai 57.21.0 → 57.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. package/CHANGELOG.md +31 -0
  2. package/README.md +3 -3
  3. package/biome.jsonc +2 -2
  4. package/dist/harness/commit.d.ts +4 -3
  5. package/dist/harness/commit.js +23 -4
  6. package/dist/index.d.ts +6 -9
  7. package/dist/index.js +14 -25
  8. package/dist/observability/adminScreens.js +5 -1
  9. package/dist/observability/evaluate.d.ts +8 -1
  10. package/dist/observability/evaluate.js +10 -2
  11. package/dist/observability/judgeAgreement.d.ts +53 -0
  12. package/dist/observability/judgeAgreement.js +123 -0
  13. package/dist/observability/local/datasetStore.d.ts +5 -0
  14. package/dist/observability/local/datasetStore.js +7 -2
  15. package/dist/observability/local/evaluatorStore.d.ts +17 -0
  16. package/dist/observability/local/evaluatorStore.js +73 -2
  17. package/dist/observability/local/experimentRunner.d.ts +40 -3
  18. package/dist/observability/local/experimentRunner.js +390 -11
  19. package/dist/observability/local/humanAnswers.d.ts +57 -0
  20. package/dist/observability/local/humanAnswers.js +174 -0
  21. package/dist/observability/local/judgeRun.d.ts +56 -0
  22. package/dist/observability/local/judgeRun.js +111 -0
  23. package/dist/observability/local/localPlugin.js +4 -0
  24. package/dist/observability/local/models/obsDatasetItem.js +4 -0
  25. package/dist/observability/local/models/obsEvaluator.js +4 -0
  26. package/dist/observability/local/models/obsExperiment.js +28 -0
  27. package/dist/observability/local/models/obsExperimentItem.js +4 -0
  28. package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
  29. package/dist/observability/local/models/obsJudgeRun.js +60 -0
  30. package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
  31. package/dist/observability/local/models/obsReviewBatch.js +48 -0
  32. package/dist/observability/local/models/obsReviewItem.js +10 -2
  33. package/dist/observability/local/models/obsTrace.js +4 -0
  34. package/dist/observability/local/promptReferences.d.ts +31 -0
  35. package/dist/observability/local/promptReferences.js +114 -0
  36. package/dist/observability/local/promptStore.d.ts +55 -19
  37. package/dist/observability/local/promptStore.js +171 -7
  38. package/dist/observability/local/reviewBatchStore.d.ts +106 -0
  39. package/dist/observability/local/reviewBatchStore.js +517 -0
  40. package/dist/observability/local/reviewStore.d.ts +24 -3
  41. package/dist/observability/local/reviewStore.js +98 -22
  42. package/dist/observability/local/traceStore.d.ts +14 -1
  43. package/dist/observability/local/traceStore.js +144 -11
  44. package/dist/observability/observabilityApp.js +10 -1
  45. package/dist/observability/questionFingerprint.d.ts +7 -0
  46. package/dist/observability/questionFingerprint.js +23 -0
  47. package/dist/observability/routes/experiments.js +12 -1
  48. package/dist/observability/routes/review.js +3 -1
  49. package/dist/observability/routes/reviewBatches.d.ts +11 -0
  50. package/dist/observability/routes/reviewBatches.js +78 -0
  51. package/dist/observability/routes/traces.d.ts +1 -1
  52. package/dist/observability/routes/traces.js +34 -2
  53. package/dist/observability/types.d.ts +10 -1
  54. package/dist/service/aiService.js +4 -1
  55. package/dist/service/chatTurn.js +38 -31
  56. package/dist/types/harness.d.ts +5 -2
  57. package/dist/types/index.d.ts +8 -3
  58. package/dist/types/observability.d.ts +74 -2
  59. package/package.json +8 -24
  60. package/src/harness/agentLoop.test.ts +0 -3
  61. package/src/harness/agentRecovery.test.ts +0 -3
  62. package/src/harness/commit.ts +24 -4
  63. package/src/harness/extensions.test.ts +0 -3
  64. package/src/harness/subagent.test.ts +27 -3
  65. package/src/index.test.ts +0 -17
  66. package/src/index.ts +23 -26
  67. package/src/isolated/gptStream.isolated.ts +0 -8
  68. package/src/observability/adminScreens.test.ts +7 -0
  69. package/src/observability/adminScreens.ts +6 -1
  70. package/src/observability/evaluate.ts +12 -2
  71. package/src/observability/judgeAgreement.test.ts +111 -0
  72. package/src/observability/judgeAgreement.ts +184 -0
  73. package/src/observability/local/datasetStore.ts +30 -11
  74. package/src/observability/local/evaluatorStore.ts +96 -2
  75. package/src/observability/local/experimentRunner.ts +495 -15
  76. package/src/observability/local/humanAnswers.ts +233 -0
  77. package/src/observability/local/judgePipeline.test.ts +444 -0
  78. package/src/observability/local/judgeRun.test.ts +173 -0
  79. package/src/observability/local/judgeRun.ts +184 -0
  80. package/src/observability/local/localPlugin.ts +4 -0
  81. package/src/observability/local/models/obsDatasetItem.ts +5 -0
  82. package/src/observability/local/models/obsEvaluator.ts +4 -0
  83. package/src/observability/local/models/obsExperiment.ts +29 -0
  84. package/src/observability/local/models/obsExperimentItem.ts +4 -0
  85. package/src/observability/local/models/obsJudgeRun.ts +61 -0
  86. package/src/observability/local/models/obsReviewBatch.ts +52 -0
  87. package/src/observability/local/models/obsReviewItem.ts +11 -2
  88. package/src/observability/local/models/obsTrace.ts +5 -0
  89. package/src/observability/local/obsModels.test.ts +13 -0
  90. package/src/observability/local/promptReferences.test.ts +128 -0
  91. package/src/observability/local/promptReferences.ts +149 -0
  92. package/src/observability/local/promptStore.test.ts +291 -0
  93. package/src/observability/local/promptStore.ts +243 -32
  94. package/src/observability/local/reviewBatchStore.test.ts +251 -0
  95. package/src/observability/local/reviewBatchStore.ts +656 -0
  96. package/src/observability/local/reviewStore.ts +122 -25
  97. package/src/observability/local/traceStore.test.ts +139 -1
  98. package/src/observability/local/traceStore.ts +162 -11
  99. package/src/observability/observabilityApp.test.ts +13 -10
  100. package/src/observability/observabilityApp.ts +10 -1
  101. package/src/observability/questionFingerprint.test.ts +33 -0
  102. package/src/observability/questionFingerprint.ts +24 -0
  103. package/src/observability/routes/experiments.test.ts +37 -0
  104. package/src/observability/routes/experiments.ts +13 -1
  105. package/src/observability/routes/review.ts +3 -1
  106. package/src/observability/routes/reviewBatches.test.ts +209 -0
  107. package/src/observability/routes/reviewBatches.ts +138 -0
  108. package/src/observability/routes/status.test.ts +4 -4
  109. package/src/observability/routes/traces.test.ts +61 -0
  110. package/src/observability/routes/traces.ts +46 -4
  111. package/src/observability/types.ts +5 -2
  112. package/src/routes/routes.test.ts +101 -76
  113. package/src/service/aiService.test.ts +21 -0
  114. package/src/service/aiService.ts +4 -1
  115. package/src/service/chatTurn.ts +42 -33
  116. package/src/tests/bunSetup.ts +0 -54
  117. package/src/types/harness.ts +5 -2
  118. package/src/types/index.ts +8 -3
  119. package/src/types/observability.ts +86 -2
  120. package/dist/langfuseApp.d.ts +0 -9
  121. package/dist/langfuseApp.js +0 -53
  122. package/dist/langfuseCache.d.ts +0 -15
  123. package/dist/langfuseCache.js +0 -54
  124. package/dist/langfuseClient.d.ts +0 -6
  125. package/dist/langfuseClient.js +0 -34
  126. package/dist/langfusePrompts.d.ts +0 -12
  127. package/dist/langfusePrompts.js +0 -89
  128. package/dist/langfuseRoutesEvaluations.d.ts +0 -3
  129. package/dist/langfuseRoutesEvaluations.js +0 -32
  130. package/dist/langfuseRoutesMiddleware.d.ts +0 -2
  131. package/dist/langfuseRoutesMiddleware.js +0 -13
  132. package/dist/langfuseRoutesPlayground.d.ts +0 -2
  133. package/dist/langfuseRoutesPlayground.js +0 -36
  134. package/dist/langfuseRoutesPrompts.d.ts +0 -2
  135. package/dist/langfuseRoutesPrompts.js +0 -56
  136. package/dist/langfuseRoutesTraces.d.ts +0 -2
  137. package/dist/langfuseRoutesTraces.js +0 -30
  138. package/dist/langfuseTracing.d.ts +0 -4
  139. package/dist/langfuseTracing.js +0 -32
  140. package/dist/langfuseTypes.d.ts +0 -96
  141. package/dist/langfuseTypes.js +0 -2
  142. package/dist/langfuseVercelAi.d.ts +0 -12
  143. package/dist/langfuseVercelAi.js +0 -49
  144. package/src/isolated/langfuseClient.isolated.ts +0 -66
  145. package/src/langfuseApp.test.ts +0 -119
  146. package/src/langfuseApp.ts +0 -62
  147. package/src/langfuseCache.test.ts +0 -75
  148. package/src/langfuseCache.ts +0 -72
  149. package/src/langfuseClient.test.ts +0 -69
  150. package/src/langfuseClient.ts +0 -34
  151. package/src/langfusePrompts.test.ts +0 -188
  152. package/src/langfusePrompts.ts +0 -121
  153. package/src/langfuseRoutes.test.ts +0 -261
  154. package/src/langfuseRoutesEvaluations.ts +0 -48
  155. package/src/langfuseRoutesMiddleware.test.ts +0 -57
  156. package/src/langfuseRoutesMiddleware.ts +0 -12
  157. package/src/langfuseRoutesPlayground.ts +0 -50
  158. package/src/langfuseRoutesPrompts.ts +0 -87
  159. package/src/langfuseRoutesTraces.ts +0 -40
  160. package/src/langfuseTracing.test.ts +0 -72
  161. package/src/langfuseTracing.ts +0 -33
  162. package/src/langfuseTypes.ts +0 -103
  163. package/src/langfuseVercelAi.test.ts +0 -82
  164. package/src/langfuseVercelAi.ts +0 -72
  165. package/src/tests/fakeLangfusePrompt.ts +0 -10
@@ -0,0 +1,517 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.LocalReviewBatchStore = exports.MAX_REVIEW_BATCH_SIZE = exports.DEFAULT_REVIEW_BATCH_SIZE = void 0;
7
+ const api_1 = require("@terreno/api");
8
+ const luxon_1 = require("luxon");
9
+ const mongoose_1 = __importDefault(require("mongoose"));
10
+ const judgeAgreement_1 = require("../judgeAgreement");
11
+ const questionFingerprint_1 = require("../questionFingerprint");
12
+ const humanAnswers_1 = require("./humanAnswers");
13
+ const judgeRun_1 = require("./judgeRun");
14
+ const obsDataset_1 = require("./models/obsDataset");
15
+ const obsDatasetItem_1 = require("./models/obsDatasetItem");
16
+ const obsEvaluator_1 = require("./models/obsEvaluator");
17
+ const obsJudgeRun_1 = require("./models/obsJudgeRun");
18
+ const obsReviewBatch_1 = require("./models/obsReviewBatch");
19
+ const obsReviewItem_1 = require("./models/obsReviewItem");
20
+ const obsTrace_1 = require("./models/obsTrace");
21
+ const promptStore_1 = require("./promptStore");
22
+ const reviewStore_1 = require("./reviewStore");
23
+ exports.DEFAULT_REVIEW_BATCH_SIZE = 10;
24
+ exports.MAX_REVIEW_BATCH_SIZE = 50;
25
+ const objectId = (value, label) => {
26
+ if (!mongoose_1.default.Types.ObjectId.isValid(value)) {
27
+ throw new api_1.APIError({ status: 400, title: `${label} must be an object id` });
28
+ }
29
+ return new mongoose_1.default.Types.ObjectId(value);
30
+ };
31
+ const toIso = (value) => {
32
+ return value ? (luxon_1.DateTime.fromJSDate(value).toUTC().toISO() ?? "") : "";
33
+ };
34
+ /**
35
+ * Review batches pair a fixed set of traces (one input and one output each) with a human question
36
+ * set. Reviewers answer every question per item; operators then run LLM judges over the same
37
+ * traces and compare judge answers with the human answers question by question.
38
+ */
39
+ class LocalReviewBatchStore {
40
+ promptStore;
41
+ reviewStore;
42
+ constructor(promptStore = new promptStore_1.LocalPromptStore(), reviewStore = new reviewStore_1.LocalReviewStore()) {
43
+ this.promptStore = promptStore;
44
+ this.reviewStore = reviewStore;
45
+ }
46
+ async create(params) {
47
+ const evaluator = await (0, obsEvaluator_1.registerObsEvaluator)().findOneOrNone({
48
+ _id: objectId(params.evaluatorId, "evaluatorId"),
49
+ });
50
+ if (!evaluator) {
51
+ throw new api_1.APIError({ status: 404, title: "Unknown evaluator" });
52
+ }
53
+ if (evaluator.type !== "human") {
54
+ throw new api_1.APIError({ status: 400, title: "Review batches require a human evaluator" });
55
+ }
56
+ if (params.datasetId) {
57
+ if (params.traceIds !== undefined || params.limit !== undefined) {
58
+ throw new api_1.APIError({
59
+ status: 400,
60
+ title: "datasetId cannot be combined with traceIds or limit",
61
+ });
62
+ }
63
+ return this.createFromDataset({ ...params, datasetId: params.datasetId, evaluator });
64
+ }
65
+ const limit = params.limit ?? exports.DEFAULT_REVIEW_BATCH_SIZE;
66
+ if (!Number.isInteger(limit) || limit < 1 || limit > exports.MAX_REVIEW_BATCH_SIZE) {
67
+ throw new api_1.APIError({
68
+ status: 400,
69
+ title: `limit must be an integer from 1 to ${exports.MAX_REVIEW_BATCH_SIZE}`,
70
+ });
71
+ }
72
+ const traceIds = params.traceIds?.length
73
+ ? params.traceIds.slice(0, exports.MAX_REVIEW_BATCH_SIZE)
74
+ : await this.recentTraceIds({
75
+ evaluatorId: evaluator._id,
76
+ limit,
77
+ promptName: params.promptName,
78
+ });
79
+ if (traceIds.length === 0) {
80
+ throw new api_1.APIError({
81
+ status: 400,
82
+ title: params.promptName
83
+ ? `No recent "${params.promptName}" traces that are not already in a batch for this question set`
84
+ : "No recent traces that are not already in a batch for this question set",
85
+ });
86
+ }
87
+ const traceObjectIds = traceIds.map((id) => objectId(id, "traceId"));
88
+ const found = await (0, obsTrace_1.registerObsTrace)()
89
+ .find({ _id: { $in: traceObjectIds } })
90
+ .select({ _id: 1 });
91
+ const foundIds = new Set(found.map((trace) => String(trace._id)));
92
+ const missing = traceIds.filter((id) => !foundIds.has(id));
93
+ if (missing.length > 0) {
94
+ throw new api_1.APIError({ detail: missing.join(", "), status: 404, title: "Unknown trace ids" });
95
+ }
96
+ const Batch = (0, obsReviewBatch_1.registerObsReviewBatch)();
97
+ const existingCount = await Batch.countDocuments({ evaluatorId: evaluator._id });
98
+ const name = params.name?.trim() || `${params.promptName ?? evaluator.name} — batch ${existingCount + 1}`;
99
+ const batch = await Batch.create({
100
+ description: params.description,
101
+ evaluatorId: evaluator._id,
102
+ name,
103
+ promptName: params.promptName,
104
+ traceIds: traceObjectIds,
105
+ });
106
+ try {
107
+ await this.reviewStore.enqueue({
108
+ batchId: String(batch._id),
109
+ evaluatorId: String(evaluator._id),
110
+ reason: "manual",
111
+ traceIds,
112
+ });
113
+ }
114
+ catch (error) {
115
+ // A half-built batch would hide its traces from later "grab recent" batches.
116
+ await (0, obsReviewItem_1.registerObsReviewItem)().deleteMany({ batchId: batch._id });
117
+ await Batch.deleteOne({ _id: batch._id });
118
+ throw error;
119
+ }
120
+ return this.get(String(batch._id));
121
+ }
122
+ /**
123
+ * One review item per dataset item that lacks a current answer from this question set and is
124
+ * not already waiting in an open review for it.
125
+ */
126
+ async createFromDataset(params) {
127
+ const { evaluator } = params;
128
+ const dataset = await (0, obsDataset_1.registerObsDataset)().findOneOrNone({
129
+ _id: objectId(params.datasetId, "datasetId"),
130
+ });
131
+ if (!dataset) {
132
+ throw new api_1.APIError({ status: 404, title: `Unknown dataset "${params.datasetId}"` });
133
+ }
134
+ const fingerprint = (0, questionFingerprint_1.questionSetFingerprint)(evaluator.dimensions);
135
+ const evaluatorKey = String(evaluator._id);
136
+ const datasetItems = await (0, obsDatasetItem_1.registerObsDatasetItem)()
137
+ .find({ datasetId: dataset._id })
138
+ .sort({ _id: 1 });
139
+ const unanswered = datasetItems.filter((item) => {
140
+ return item.humanAnswers?.[evaluatorKey]?.fingerprint !== fingerprint;
141
+ });
142
+ if (unanswered.length === 0) {
143
+ throw new api_1.APIError({
144
+ detail: datasetItems.length === 0 ? `Dataset "${dataset.name}" has no items.` : undefined,
145
+ status: 400,
146
+ title: "Every item already has current answers",
147
+ });
148
+ }
149
+ const open = await (0, obsReviewItem_1.registerObsReviewItem)()
150
+ .find({
151
+ datasetItemId: { $in: unanswered.map((item) => item._id) },
152
+ evaluatorId: evaluator._id,
153
+ status: { $in: ["in_progress", "pending"] },
154
+ })
155
+ .select({ datasetItemId: 1 });
156
+ const openIds = new Set(open.map((item) => String(item.datasetItemId)));
157
+ const toJudge = unanswered.filter((item) => !openIds.has(String(item._id)));
158
+ if (toJudge.length === 0) {
159
+ throw new api_1.APIError({
160
+ status: 400,
161
+ title: "Every unanswered item is already waiting in an open review batch",
162
+ });
163
+ }
164
+ const Batch = (0, obsReviewBatch_1.registerObsReviewBatch)();
165
+ const existingCount = await Batch.countDocuments({ datasetId: dataset._id });
166
+ const batch = await Batch.create({
167
+ datasetId: dataset._id,
168
+ description: params.description,
169
+ evaluatorId: evaluator._id,
170
+ name: params.name?.trim() || `${dataset.name} — ${evaluator.name} ${existingCount + 1}`,
171
+ questionsFingerprint: fingerprint,
172
+ traceIds: toJudge.flatMap((item) => (item.sourceTraceId ? [item.sourceTraceId] : [])),
173
+ });
174
+ try {
175
+ await this.reviewStore.enqueueDatasetItems({
176
+ batchId: String(batch._id),
177
+ evaluatorId: evaluatorKey,
178
+ items: toJudge.map((item) => {
179
+ return {
180
+ datasetItemId: String(item._id),
181
+ traceId: item.sourceTraceId ? String(item.sourceTraceId) : undefined,
182
+ };
183
+ }),
184
+ });
185
+ }
186
+ catch (error) {
187
+ await (0, obsReviewItem_1.registerObsReviewItem)().deleteMany({ batchId: batch._id });
188
+ await Batch.deleteOne({ _id: batch._id });
189
+ throw error;
190
+ }
191
+ return this.get(String(batch._id));
192
+ }
193
+ async list() {
194
+ const batches = await (0, obsReviewBatch_1.registerObsReviewBatch)().find({}).sort({ created: -1 });
195
+ if (batches.length === 0) {
196
+ return [];
197
+ }
198
+ const batchIds = batches.map((batch) => batch._id);
199
+ const reviewItems = await (0, obsReviewItem_1.registerObsReviewItem)()
200
+ .find({ batchId: { $in: batchIds } })
201
+ .select({ batchId: 1, datasetItemId: 1, scores: 1, status: 1, traceId: 1 });
202
+ const runs = await this.latestJudgeRuns(batchIds);
203
+ const evaluators = await (0, obsEvaluator_1.registerObsEvaluator)().find({
204
+ _id: {
205
+ $in: [...batches.map((batch) => batch.evaluatorId), ...runs.map((run) => run.evaluatorId)],
206
+ },
207
+ });
208
+ const evaluatorById = new Map(evaluators.map((evaluator) => {
209
+ return [String(evaluator._id), evaluator];
210
+ }));
211
+ const datasetNames = await this.datasetNames(batches);
212
+ return batches.map((batch) => {
213
+ const batchId = String(batch._id);
214
+ const evaluator = evaluatorById.get(String(batch.evaluatorId));
215
+ const items = reviewItems.filter((item) => String(item.batchId) === batchId);
216
+ const judges = this.summarizeJudges({
217
+ dimensions: evaluator?.dimensions ?? [],
218
+ evaluatorById,
219
+ reviewItems: items,
220
+ runs: runs.filter((run) => String(run.batchId) === batchId),
221
+ });
222
+ return this.toListItem({
223
+ batch,
224
+ datasetName: batch.datasetId ? datasetNames.get(String(batch.datasetId)) : undefined,
225
+ evaluator,
226
+ items,
227
+ judges,
228
+ });
229
+ });
230
+ }
231
+ async get(id) {
232
+ const batch = await (0, obsReviewBatch_1.registerObsReviewBatch)().findOneOrNone({ _id: objectId(id, "id") });
233
+ if (!batch) {
234
+ throw new api_1.APIError({ status: 404, title: `Unknown review batch "${id}"` });
235
+ }
236
+ const evaluator = await (0, obsEvaluator_1.registerObsEvaluator)().findOneOrNone({ _id: batch.evaluatorId });
237
+ const dimensions = evaluator?.dimensions ?? [];
238
+ const reviewItems = await (0, obsReviewItem_1.registerObsReviewItem)()
239
+ .find({ batchId: batch._id })
240
+ .sort({ _id: 1, enqueuedAt: 1 });
241
+ const subjects = await this.loadSubjects(reviewItems);
242
+ const runs = await this.latestJudgeRuns([batch._id]);
243
+ const judgeEvaluators = await (0, obsEvaluator_1.registerObsEvaluator)().find({
244
+ _id: { $in: runs.map((run) => run.evaluatorId) },
245
+ });
246
+ const judges = this.summarizeJudges({
247
+ dimensions,
248
+ evaluatorById: new Map(judgeEvaluators.map((judge) => {
249
+ return [String(judge._id), judge];
250
+ })),
251
+ reviewItems,
252
+ runs,
253
+ });
254
+ const items = reviewItems.map((item) => {
255
+ const subject = subjects.get(String(item._id));
256
+ const key = subject?.key ?? (0, judgeAgreement_1.agreementAnswerKey)(this.answerIds(item));
257
+ const judgeAnswers = {};
258
+ for (const run of runs) {
259
+ const result = run.results.find((entry) => {
260
+ return (0, judgeAgreement_1.agreementAnswerKey)(this.answerIds(entry)) === key;
261
+ });
262
+ if (result) {
263
+ judgeAnswers[String(run.evaluatorId)] = { error: result.error, scores: result.scores };
264
+ }
265
+ }
266
+ return {
267
+ datasetItemId: item.datasetItemId ? String(item.datasetItemId) : undefined,
268
+ input: subject?.input,
269
+ judgeAnswers,
270
+ output: subject?.output,
271
+ reviewItemId: String(item._id),
272
+ scores: item.scores,
273
+ status: item.status,
274
+ traceId: subject?.traceId ?? (item.traceId ? String(item.traceId) : undefined),
275
+ traceName: subject?.traceName ?? "Unknown trace",
276
+ };
277
+ });
278
+ const datasetNames = await this.datasetNames([batch]);
279
+ return {
280
+ ...this.toListItem({
281
+ batch,
282
+ datasetName: batch.datasetId ? datasetNames.get(String(batch.datasetId)) : undefined,
283
+ evaluator: evaluator ?? undefined,
284
+ items: reviewItems,
285
+ judges,
286
+ }),
287
+ dimensions,
288
+ instructions: evaluator?.instructions,
289
+ items,
290
+ judges,
291
+ };
292
+ }
293
+ /** Run an LLM judge over every batch trace and store its answers for comparison. */
294
+ async runJudge(params) {
295
+ const batch = await (0, obsReviewBatch_1.registerObsReviewBatch)().findOneOrNone({
296
+ _id: objectId(params.batchId, "batchId"),
297
+ });
298
+ if (!batch) {
299
+ throw new api_1.APIError({ status: 404, title: `Unknown review batch "${params.batchId}"` });
300
+ }
301
+ const judge = await (0, obsEvaluator_1.registerObsEvaluator)().findOneOrNone({
302
+ _id: objectId(params.evaluatorId, "evaluatorId"),
303
+ });
304
+ if (!judge) {
305
+ throw new api_1.APIError({ status: 404, title: "Unknown evaluator" });
306
+ }
307
+ if (judge.type !== "llm-judge" || !judge.judgePromptName) {
308
+ throw new api_1.APIError({ status: 400, title: "Only llm-judge evaluators can be compared" });
309
+ }
310
+ const questionSet = await (0, obsEvaluator_1.registerObsEvaluator)().findOneOrNone({ _id: batch.evaluatorId });
311
+ const questionKeys = new Set((questionSet?.dimensions ?? []).map((row) => row.key));
312
+ const sharedKeys = judge.dimensions.filter((row) => questionKeys.has(row.key));
313
+ if (sharedKeys.length === 0) {
314
+ throw new api_1.APIError({
315
+ detail: `Batch questions: ${[...questionKeys].join(", ")}. Judge scores: ${judge.dimensions
316
+ .map((row) => row.key)
317
+ .join(", ")}.`,
318
+ status: 400,
319
+ title: `Judge "${judge.name}" answers none of this batch's questions`,
320
+ });
321
+ }
322
+ const judgeVersion = await this.promptStore.getVersionByLabel(judge.judgePromptName);
323
+ if (!judgeVersion?.outputSchema) {
324
+ throw new api_1.APIError({
325
+ status: 400,
326
+ title: `Judge prompt "${judge.judgePromptName}" has no production output schema`,
327
+ });
328
+ }
329
+ const reviewItems = await (0, obsReviewItem_1.registerObsReviewItem)()
330
+ .find({ batchId: batch._id })
331
+ .sort({ _id: 1, enqueuedAt: 1 });
332
+ const subjectByKey = new Map();
333
+ for (const subject of (await this.loadSubjects(reviewItems)).values()) {
334
+ subjectByKey.set(subject.key, subject);
335
+ }
336
+ // Done answers pair with the judge's, exactly as summarizeJudges compares them later.
337
+ const humanAnswers = reviewItems
338
+ .filter((item) => item.status === "done")
339
+ .map((item) => {
340
+ return { ...this.answerIds(item), scores: item.scores };
341
+ });
342
+ const run = await (0, judgeRun_1.runJudgeAgainstHumans)({
343
+ ai: params.ai,
344
+ dimensions: questionSet?.dimensions ?? [],
345
+ humanAnswers,
346
+ judge,
347
+ prompt: { outputSchema: judgeVersion.outputSchema },
348
+ subjects: [...subjectByKey.values()],
349
+ });
350
+ const results = run.results.map((result) => {
351
+ return {
352
+ confidence: result.confidence,
353
+ error: result.error,
354
+ scores: result.scores,
355
+ ...(result.datasetItemId
356
+ ? { datasetItemId: new mongoose_1.default.Types.ObjectId(result.datasetItemId) }
357
+ : {}),
358
+ ...(result.traceId ? { traceId: new mongoose_1.default.Types.ObjectId(result.traceId) } : {}),
359
+ };
360
+ });
361
+ await (0, obsJudgeRun_1.registerObsJudgeRun)().create({
362
+ batchId: batch._id,
363
+ completedAt: luxon_1.DateTime.utc().toJSDate(),
364
+ evaluatorId: judge._id,
365
+ modelId: params.ai.modelId,
366
+ results,
367
+ });
368
+ return this.get(String(batch._id));
369
+ }
370
+ answerIds(row) {
371
+ return {
372
+ datasetItemId: row.datasetItemId ? String(row.datasetItemId) : undefined,
373
+ traceId: row.traceId ? String(row.traceId) : undefined,
374
+ };
375
+ }
376
+ // What reviewers saw for each review item: its trace, or the dataset item when it has none.
377
+ async loadSubjects(reviewItems) {
378
+ const traceIds = reviewItems.flatMap((item) => (item.traceId ? [item.traceId] : []));
379
+ const traces = traceIds.length ? await (0, obsTrace_1.registerObsTrace)().find({ _id: { $in: traceIds } }) : [];
380
+ const traceById = new Map(traces.map((trace) => {
381
+ return [String(trace._id), trace];
382
+ }));
383
+ const datasetItemIds = reviewItems.flatMap((item) => {
384
+ const hasTrace = item.traceId && traceById.has(String(item.traceId));
385
+ return !hasTrace && item.datasetItemId ? [item.datasetItemId] : [];
386
+ });
387
+ const datasetSubjects = datasetItemIds.length
388
+ ? await (0, humanAnswers_1.loadDatasetItemSubjects)(await (0, obsDatasetItem_1.registerObsDatasetItem)().find({ _id: { $in: datasetItemIds } }))
389
+ : new Map();
390
+ const subjects = new Map();
391
+ for (const item of reviewItems) {
392
+ const ids = this.answerIds(item);
393
+ const key = (0, judgeAgreement_1.agreementAnswerKey)(ids);
394
+ const trace = item.traceId ? traceById.get(String(item.traceId)) : undefined;
395
+ if (trace) {
396
+ subjects.set(String(item._id), {
397
+ ...ids,
398
+ input: trace.input,
399
+ key,
400
+ output: trace.output,
401
+ traceName: trace.name,
402
+ });
403
+ continue;
404
+ }
405
+ const datasetSubject = ids.datasetItemId ? datasetSubjects.get(ids.datasetItemId) : undefined;
406
+ subjects.set(String(item._id), {
407
+ ...ids,
408
+ input: datasetSubject?.input,
409
+ key,
410
+ output: datasetSubject?.output,
411
+ traceName: datasetSubject ? "Dataset item" : "Unknown trace",
412
+ });
413
+ }
414
+ return subjects;
415
+ }
416
+ async datasetNames(batches) {
417
+ const ids = batches.flatMap((batch) => (batch.datasetId ? [batch.datasetId] : []));
418
+ if (ids.length === 0) {
419
+ return new Map();
420
+ }
421
+ const datasets = await (0, obsDataset_1.registerObsDataset)().find({ _id: { $in: ids } });
422
+ return new Map(datasets.map((dataset) => {
423
+ return [String(dataset._id), dataset.name];
424
+ }));
425
+ }
426
+ async latestJudgeRuns(batchIds) {
427
+ const runs = await (0, obsJudgeRun_1.registerObsJudgeRun)()
428
+ .find({ batchId: { $in: batchIds } })
429
+ .sort({ completedAt: -1 });
430
+ const latest = new Map();
431
+ for (const run of runs) {
432
+ const key = `${String(run.batchId)}:${String(run.evaluatorId)}`;
433
+ if (!latest.has(key)) {
434
+ latest.set(key, run);
435
+ }
436
+ }
437
+ return [...latest.values()];
438
+ }
439
+ // Agreement for each judge's latest run against the done human answers, best match first.
440
+ summarizeJudges(params) {
441
+ const humanAnswers = params.reviewItems
442
+ .filter((item) => item.status === "done")
443
+ .map((item) => {
444
+ return { ...this.answerIds(item), scores: item.scores };
445
+ });
446
+ const summaries = params.runs.map((run) => {
447
+ const judge = params.evaluatorById.get(String(run.evaluatorId));
448
+ return {
449
+ agreement: (0, judgeAgreement_1.computeJudgeAgreement)({
450
+ dimensions: params.dimensions,
451
+ humanAnswers,
452
+ judgeAnswers: run.results.map((result) => {
453
+ return { ...this.answerIds(result), error: result.error, scores: result.scores };
454
+ }),
455
+ judgeKeys: (judge?.dimensions ?? []).map((dimension) => dimension.key),
456
+ }),
457
+ completedAt: toIso(run.completedAt),
458
+ evaluatorId: String(run.evaluatorId),
459
+ evaluatorName: judge?.name ?? "Deleted judge",
460
+ judgePromptName: judge?.judgePromptName,
461
+ modelId: run.modelId,
462
+ runId: String(run._id),
463
+ };
464
+ });
465
+ summaries.sort((a, b) => (b.agreement.rate ?? -1) - (a.agreement.rate ?? -1));
466
+ return summaries;
467
+ }
468
+ toListItem(params) {
469
+ const { batch, evaluator, items, judges } = params;
470
+ const questionsStale = batch.questionsFingerprint && evaluator
471
+ ? (0, questionFingerprint_1.questionSetFingerprint)(evaluator.dimensions) !== batch.questionsFingerprint
472
+ : undefined;
473
+ const best = judges[0];
474
+ return {
475
+ bestJudge: best ? { evaluatorName: best.evaluatorName, rate: best.agreement.rate } : undefined,
476
+ counts: {
477
+ done: items.filter((item) => item.status === "done").length,
478
+ pending: items.filter((item) => item.status === "pending" || item.status === "in_progress")
479
+ .length,
480
+ skipped: items.filter((item) => item.status === "skipped").length,
481
+ total: items.length,
482
+ },
483
+ created: toIso(batch.created),
484
+ datasetId: batch.datasetId ? String(batch.datasetId) : undefined,
485
+ datasetName: params.datasetName,
486
+ description: batch.description,
487
+ evaluatorId: String(batch.evaluatorId),
488
+ evaluatorName: evaluator?.name ?? "Deleted evaluator",
489
+ id: String(batch._id),
490
+ judgeCount: judges.length,
491
+ name: batch.name,
492
+ promptName: batch.promptName,
493
+ questionsFingerprint: batch.questionsFingerprint,
494
+ questionsStale,
495
+ };
496
+ }
497
+ // Newest successful traces with an output, skipping traces already batched for this question set.
498
+ async recentTraceIds(params) {
499
+ const batched = await (0, obsReviewBatch_1.registerObsReviewBatch)().find({ evaluatorId: params.evaluatorId });
500
+ const excluded = batched.flatMap((batch) => batch.traceIds);
501
+ const filter = {
502
+ _id: { $nin: excluded },
503
+ output: { $exists: true, $ne: null },
504
+ status: "ok",
505
+ };
506
+ if (params.promptName) {
507
+ filter["prompts.name"] = params.promptName;
508
+ }
509
+ const traces = await (0, obsTrace_1.registerObsTrace)()
510
+ .find(filter)
511
+ .sort({ startedAt: -1 })
512
+ .limit(params.limit)
513
+ .select({ _id: 1 });
514
+ return traces.map((trace) => String(trace._id));
515
+ }
516
+ }
517
+ exports.LocalReviewBatchStore = LocalReviewBatchStore;
@@ -4,28 +4,47 @@ import type { ScoreSink } from "../types";
4
4
  export type ReviewStatus = "done" | "in_progress" | "pending" | "skipped";
5
5
  export interface ReviewListItem {
6
6
  assigneeId?: string;
7
+ batchId?: string;
8
+ /** Dataset item under review, when the item came from a dataset review batch. */
9
+ datasetItemId?: string;
7
10
  enqueuedAt: string;
8
11
  evaluatorId: string;
9
12
  id: string;
10
13
  promptName?: string;
11
14
  reason: string;
12
15
  status: ReviewStatus;
13
- traceId: string;
16
+ /** Unset when the reviewed dataset item has no source trace. */
17
+ traceId?: string;
14
18
  traceName: string;
15
19
  }
16
20
  export declare class LocalReviewStore {
17
21
  enqueue(params: {
22
+ batchId?: string;
18
23
  evaluatorId: string;
19
24
  reason?: "dataset_candidate" | "eval" | "feedback" | "manual";
20
25
  spanId?: string;
21
26
  traceIds: string[];
22
27
  }): Promise<ReviewListItem[]>;
23
- list(status?: ReviewStatus): Promise<{
28
+ /** Queue dataset items for a human question set; items with a source trace show that trace. */
29
+ enqueueDatasetItems(params: {
30
+ batchId: string;
31
+ evaluatorId: string;
32
+ items: Array<{
33
+ datasetItemId: string;
34
+ traceId?: string;
35
+ }>;
36
+ }): Promise<ReviewListItem[]>;
37
+ list(status?: ReviewStatus, options?: {
38
+ batchId?: string;
39
+ }): Promise<{
24
40
  counts: Record<ReviewStatus, number>;
25
41
  data: ReviewListItem[];
26
42
  }>;
27
43
  getDetail(id: string): Promise<{
44
+ batchId?: string;
45
+ batchName?: string;
28
46
  comment?: string;
47
+ datasetItemId?: string;
29
48
  dimensions: EvaluatorDimension[];
30
49
  evaluatorId: string;
31
50
  id: string;
@@ -35,13 +54,15 @@ export declare class LocalReviewStore {
35
54
  rawOutput?: unknown;
36
55
  scores?: Record<string, boolean | number | string>;
37
56
  status: ReviewStatus;
38
- traceId: string;
57
+ traceId?: string;
39
58
  }>;
40
59
  submit(params: {
41
60
  comment?: string;
42
61
  id: string;
43
62
  scores: Record<string, boolean | number | string>;
44
63
  sinks: ScoreSink[];
64
+ /** Reviewer, recorded on the dataset item's human answers. */
65
+ userId?: string;
45
66
  }): Promise<void>;
46
67
  skip(id: string): Promise<void>;
47
68
  assign(id: string, assigneeId: string): Promise<void>;