@terreno/ai 58.0.0-beta.3 → 58.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. package/CHANGELOG.md +166 -0
  2. package/README.md +3 -3
  3. package/dist/harness/commit.d.ts +4 -3
  4. package/dist/harness/commit.js +23 -4
  5. package/dist/index.d.ts +6 -9
  6. package/dist/index.js +14 -25
  7. package/dist/observability/adminScreens.js +5 -1
  8. package/dist/observability/evaluate.d.ts +8 -1
  9. package/dist/observability/evaluate.js +10 -2
  10. package/dist/observability/judgeAgreement.d.ts +53 -0
  11. package/dist/observability/judgeAgreement.js +123 -0
  12. package/dist/observability/local/datasetStore.d.ts +5 -0
  13. package/dist/observability/local/datasetStore.js +7 -2
  14. package/dist/observability/local/evaluatorStore.d.ts +17 -0
  15. package/dist/observability/local/evaluatorStore.js +73 -2
  16. package/dist/observability/local/experimentRunner.d.ts +40 -3
  17. package/dist/observability/local/experimentRunner.js +390 -11
  18. package/dist/observability/local/humanAnswers.d.ts +57 -0
  19. package/dist/observability/local/humanAnswers.js +174 -0
  20. package/dist/observability/local/judgeRun.d.ts +56 -0
  21. package/dist/observability/local/judgeRun.js +111 -0
  22. package/dist/observability/local/localPlugin.js +4 -0
  23. package/dist/observability/local/models/obsDatasetItem.js +4 -0
  24. package/dist/observability/local/models/obsEvaluator.js +4 -0
  25. package/dist/observability/local/models/obsExperiment.js +28 -0
  26. package/dist/observability/local/models/obsExperimentItem.js +4 -0
  27. package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
  28. package/dist/observability/local/models/obsJudgeRun.js +60 -0
  29. package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
  30. package/dist/observability/local/models/obsReviewBatch.js +48 -0
  31. package/dist/observability/local/models/obsReviewItem.js +10 -2
  32. package/dist/observability/local/models/obsTrace.js +4 -0
  33. package/dist/observability/local/promptReferences.d.ts +31 -0
  34. package/dist/observability/local/promptReferences.js +114 -0
  35. package/dist/observability/local/promptStore.d.ts +55 -19
  36. package/dist/observability/local/promptStore.js +171 -7
  37. package/dist/observability/local/reviewBatchStore.d.ts +106 -0
  38. package/dist/observability/local/reviewBatchStore.js +517 -0
  39. package/dist/observability/local/reviewStore.d.ts +24 -3
  40. package/dist/observability/local/reviewStore.js +98 -22
  41. package/dist/observability/local/traceStore.d.ts +14 -1
  42. package/dist/observability/local/traceStore.js +144 -11
  43. package/dist/observability/observabilityApp.js +10 -1
  44. package/dist/observability/questionFingerprint.d.ts +7 -0
  45. package/dist/observability/questionFingerprint.js +23 -0
  46. package/dist/observability/routes/experiments.js +12 -1
  47. package/dist/observability/routes/review.js +3 -1
  48. package/dist/observability/routes/reviewBatches.d.ts +11 -0
  49. package/dist/observability/routes/reviewBatches.js +78 -0
  50. package/dist/observability/routes/traces.d.ts +1 -1
  51. package/dist/observability/routes/traces.js +34 -2
  52. package/dist/observability/types.d.ts +10 -1
  53. package/dist/service/aiService.js +4 -1
  54. package/dist/service/chatTurn.js +38 -31
  55. package/dist/types/harness.d.ts +5 -2
  56. package/dist/types/index.d.ts +8 -3
  57. package/dist/types/observability.d.ts +74 -2
  58. package/package.json +8 -24
  59. package/src/harness/agentLoop.test.ts +0 -3
  60. package/src/harness/agentRecovery.test.ts +0 -3
  61. package/src/harness/commit.ts +24 -4
  62. package/src/harness/extensions.test.ts +0 -3
  63. package/src/harness/subagent.test.ts +27 -3
  64. package/src/index.test.ts +0 -17
  65. package/src/index.ts +23 -26
  66. package/src/isolated/gptStream.isolated.ts +0 -8
  67. package/src/observability/adminScreens.test.ts +7 -0
  68. package/src/observability/adminScreens.ts +6 -1
  69. package/src/observability/evaluate.ts +12 -2
  70. package/src/observability/judgeAgreement.test.ts +111 -0
  71. package/src/observability/judgeAgreement.ts +184 -0
  72. package/src/observability/local/datasetStore.ts +30 -11
  73. package/src/observability/local/evaluatorStore.ts +96 -2
  74. package/src/observability/local/experimentRunner.ts +495 -15
  75. package/src/observability/local/humanAnswers.ts +233 -0
  76. package/src/observability/local/judgePipeline.test.ts +444 -0
  77. package/src/observability/local/judgeRun.test.ts +173 -0
  78. package/src/observability/local/judgeRun.ts +184 -0
  79. package/src/observability/local/localPlugin.ts +4 -0
  80. package/src/observability/local/models/obsDatasetItem.ts +5 -0
  81. package/src/observability/local/models/obsEvaluator.ts +4 -0
  82. package/src/observability/local/models/obsExperiment.ts +29 -0
  83. package/src/observability/local/models/obsExperimentItem.ts +4 -0
  84. package/src/observability/local/models/obsJudgeRun.ts +61 -0
  85. package/src/observability/local/models/obsReviewBatch.ts +52 -0
  86. package/src/observability/local/models/obsReviewItem.ts +11 -2
  87. package/src/observability/local/models/obsTrace.ts +5 -0
  88. package/src/observability/local/obsModels.test.ts +13 -0
  89. package/src/observability/local/promptReferences.test.ts +128 -0
  90. package/src/observability/local/promptReferences.ts +149 -0
  91. package/src/observability/local/promptStore.test.ts +291 -0
  92. package/src/observability/local/promptStore.ts +243 -32
  93. package/src/observability/local/reviewBatchStore.test.ts +251 -0
  94. package/src/observability/local/reviewBatchStore.ts +656 -0
  95. package/src/observability/local/reviewStore.ts +122 -25
  96. package/src/observability/local/traceStore.test.ts +139 -1
  97. package/src/observability/local/traceStore.ts +162 -11
  98. package/src/observability/observabilityApp.test.ts +13 -10
  99. package/src/observability/observabilityApp.ts +10 -1
  100. package/src/observability/questionFingerprint.test.ts +33 -0
  101. package/src/observability/questionFingerprint.ts +24 -0
  102. package/src/observability/routes/experiments.test.ts +37 -0
  103. package/src/observability/routes/experiments.ts +13 -1
  104. package/src/observability/routes/review.ts +3 -1
  105. package/src/observability/routes/reviewBatches.test.ts +209 -0
  106. package/src/observability/routes/reviewBatches.ts +138 -0
  107. package/src/observability/routes/status.test.ts +4 -4
  108. package/src/observability/routes/traces.test.ts +61 -0
  109. package/src/observability/routes/traces.ts +46 -4
  110. package/src/observability/types.ts +5 -2
  111. package/src/routes/routes.test.ts +101 -76
  112. package/src/service/aiService.test.ts +21 -0
  113. package/src/service/aiService.ts +4 -1
  114. package/src/service/chatTurn.ts +42 -33
  115. package/src/tests/bunSetup.ts +0 -54
  116. package/src/types/harness.ts +5 -2
  117. package/src/types/index.ts +8 -3
  118. package/src/types/observability.ts +86 -2
  119. package/dist/langfuseApp.d.ts +0 -9
  120. package/dist/langfuseApp.js +0 -53
  121. package/dist/langfuseCache.d.ts +0 -15
  122. package/dist/langfuseCache.js +0 -54
  123. package/dist/langfuseClient.d.ts +0 -6
  124. package/dist/langfuseClient.js +0 -34
  125. package/dist/langfusePrompts.d.ts +0 -12
  126. package/dist/langfusePrompts.js +0 -89
  127. package/dist/langfuseRoutesEvaluations.d.ts +0 -3
  128. package/dist/langfuseRoutesEvaluations.js +0 -32
  129. package/dist/langfuseRoutesMiddleware.d.ts +0 -2
  130. package/dist/langfuseRoutesMiddleware.js +0 -13
  131. package/dist/langfuseRoutesPlayground.d.ts +0 -2
  132. package/dist/langfuseRoutesPlayground.js +0 -36
  133. package/dist/langfuseRoutesPrompts.d.ts +0 -2
  134. package/dist/langfuseRoutesPrompts.js +0 -56
  135. package/dist/langfuseRoutesTraces.d.ts +0 -2
  136. package/dist/langfuseRoutesTraces.js +0 -30
  137. package/dist/langfuseTracing.d.ts +0 -4
  138. package/dist/langfuseTracing.js +0 -32
  139. package/dist/langfuseTypes.d.ts +0 -96
  140. package/dist/langfuseTypes.js +0 -2
  141. package/dist/langfuseVercelAi.d.ts +0 -12
  142. package/dist/langfuseVercelAi.js +0 -49
  143. package/src/isolated/langfuseClient.isolated.ts +0 -66
  144. package/src/langfuseApp.test.ts +0 -119
  145. package/src/langfuseApp.ts +0 -62
  146. package/src/langfuseCache.test.ts +0 -75
  147. package/src/langfuseCache.ts +0 -72
  148. package/src/langfuseClient.test.ts +0 -69
  149. package/src/langfuseClient.ts +0 -34
  150. package/src/langfusePrompts.test.ts +0 -188
  151. package/src/langfusePrompts.ts +0 -121
  152. package/src/langfuseRoutes.test.ts +0 -261
  153. package/src/langfuseRoutesEvaluations.ts +0 -48
  154. package/src/langfuseRoutesMiddleware.test.ts +0 -57
  155. package/src/langfuseRoutesMiddleware.ts +0 -12
  156. package/src/langfuseRoutesPlayground.ts +0 -50
  157. package/src/langfuseRoutesPrompts.ts +0 -87
  158. package/src/langfuseRoutesTraces.ts +0 -40
  159. package/src/langfuseTracing.test.ts +0 -72
  160. package/src/langfuseTracing.ts +0 -33
  161. package/src/langfuseTypes.ts +0 -103
  162. package/src/langfuseVercelAi.test.ts +0 -82
  163. package/src/langfuseVercelAi.ts +0 -72
  164. package/src/tests/fakeLangfusePrompt.ts +0 -10
@@ -3,15 +3,30 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
3
3
  return (mod && mod.__esModule) ? mod : { "default": mod };
4
4
  };
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.LocalExperimentRunner = exports.EXPERIMENT_LIST_DEFAULT_LIMIT = void 0;
6
+ exports.LocalExperimentRunner = exports.EXPERIMENT_LIST_DEFAULT_LIMIT = exports.DEFAULT_TARGET_AGREEMENT = void 0;
7
7
  const api_1 = require("@terreno/api");
8
8
  const luxon_1 = require("luxon");
9
9
  const mongoose_1 = __importDefault(require("mongoose"));
10
10
  const compileTemplate_1 = require("../compileTemplate");
11
11
  const evaluate_1 = require("../evaluate");
12
+ const questionFingerprint_1 = require("../questionFingerprint");
12
13
  const sopThresholds_1 = require("../sopThresholds");
14
+ const humanAnswers_1 = require("./humanAnswers");
15
+ const judgeRun_1 = require("./judgeRun");
16
+ const obsDataset_1 = require("./models/obsDataset");
17
+ const obsEvaluator_1 = require("./models/obsEvaluator");
13
18
  const obsExperiment_1 = require("./models/obsExperiment");
14
19
  const obsExperimentItem_1 = require("./models/obsExperimentItem");
20
+ exports.DEFAULT_TARGET_AGREEMENT = 0.9;
21
+ const versionBody = (version) => {
22
+ return version.system && version.system.length > 0 ? version.system : (version.template ?? "");
23
+ };
24
+ const toObjectId = (value, label) => {
25
+ if (typeof value !== "string" || !mongoose_1.default.Types.ObjectId.isValid(value)) {
26
+ throw new api_1.APIError({ status: 400, title: `${label} must be an object id` });
27
+ }
28
+ return new mongoose_1.default.Types.ObjectId(value);
29
+ };
15
30
  exports.EXPERIMENT_LIST_DEFAULT_LIMIT = 50;
16
31
  const EXPERIMENT_LIST_MAX_LIMIT = 100;
17
32
  const defaultBackgroundTaskRunner = {
@@ -114,9 +129,17 @@ class LocalExperimentRunner {
114
129
  return client;
115
130
  }
116
131
  async estimate(input) {
117
- const items = await this.deps.datasetStore.listExperimentItems(input.datasetId, input.includeUnproofread ?? false);
118
- const generations = items.length * input.versions.length;
119
- const evaluatorCount = input.evaluatorIds.length;
132
+ const itemCount = input.kind === "judge"
133
+ ? input.questionSetId
134
+ ? (await (0, humanAnswers_1.loadCurrentHumanAnswers)({
135
+ datasetId: input.datasetId,
136
+ questionSetId: input.questionSetId,
137
+ })).items.length
138
+ : 0
139
+ : (await this.deps.datasetStore.listExperimentItems(input.datasetId, input.includeUnproofread ?? false)).length;
140
+ const generations = itemCount * input.versions.length;
141
+ // Judge experiments make one judge call per item and version and no separate generation.
142
+ const evaluatorCount = input.kind === "judge" ? 0 : (input.evaluatorIds?.length ?? 0);
120
143
  const secondsPerGeneration = 2;
121
144
  const judgeSeconds = generations * evaluatorCount * 0.5;
122
145
  const modelId = (input.modelOverride
@@ -145,9 +168,22 @@ class LocalExperimentRunner {
145
168
  }
146
169
  }
147
170
  async create(input) {
148
- if (input.versions.length < 2 || input.versions.length > 3) {
171
+ const kind = input.kind ?? "prompt";
172
+ if (kind !== "prompt" && kind !== "judge") {
173
+ throw new api_1.APIError({ status: 400, title: "kind must be prompt or judge" });
174
+ }
175
+ if (input.kind === "judge") {
176
+ return this.createJudge(input);
177
+ }
178
+ if (!Array.isArray(input.versions) || input.versions.length < 2 || input.versions.length > 3) {
149
179
  throw new api_1.APIError({ status: 400, title: "experiments must compare 2–3 prompt versions" });
150
180
  }
181
+ if (input.thresholds?.some((threshold) => threshold.aggregate === "agreement")) {
182
+ throw new api_1.APIError({
183
+ status: 400,
184
+ title: "agreement gates belong to judge experiments (kind: judge)",
185
+ });
186
+ }
151
187
  await this.assertAutomaticEvaluators(input.evaluatorIds);
152
188
  this.resolveAiClient(input.modelOverride);
153
189
  await this.deps.datasetStore.get(input.datasetId);
@@ -165,6 +201,7 @@ class LocalExperimentRunner {
165
201
  return new mongoose_1.default.Types.ObjectId(id);
166
202
  }),
167
203
  includeUnproofread: input.includeUnproofread ?? false,
204
+ kind: "prompt",
168
205
  modelOverride: input.modelOverride,
169
206
  name: input.name,
170
207
  promptName: input.promptName,
@@ -172,6 +209,119 @@ class LocalExperimentRunner {
172
209
  thresholds: input.thresholds ?? sopThresholds_1.SOP_DEFAULT_THRESHOLDS,
173
210
  versions: input.versions,
174
211
  });
212
+ return this.startRun(experiment);
213
+ }
214
+ async createJudge(input) {
215
+ if (!input.name) {
216
+ throw new api_1.APIError({ status: 400, title: "name is required" });
217
+ }
218
+ if (!Array.isArray(input.versions) ||
219
+ input.versions.length < 2 ||
220
+ input.versions.length > 3 ||
221
+ !input.versions.every((version) => Number.isInteger(version))) {
222
+ throw new api_1.APIError({
223
+ status: 400,
224
+ title: "judge experiments must compare 2–3 judge prompt versions",
225
+ });
226
+ }
227
+ if (new Set(input.versions).size !== input.versions.length) {
228
+ throw new api_1.APIError({ status: 400, title: "judge prompt versions must be different" });
229
+ }
230
+ const targetAgreement = input.targetAgreement ?? exports.DEFAULT_TARGET_AGREEMENT;
231
+ if (typeof targetAgreement !== "number" || targetAgreement <= 0 || targetAgreement > 1) {
232
+ throw new api_1.APIError({
233
+ status: 400,
234
+ title: "targetAgreement must be a number above 0 and at most 1",
235
+ });
236
+ }
237
+ const questionSet = await (0, obsEvaluator_1.registerObsEvaluator)().findOneOrNone({
238
+ _id: toObjectId(input.questionSetId, "questionSetId"),
239
+ });
240
+ if (!questionSet) {
241
+ throw new api_1.APIError({ status: 404, title: "Unknown question set" });
242
+ }
243
+ if (questionSet.type !== "human") {
244
+ throw new api_1.APIError({ status: 400, title: "questionSetId must be a human question set" });
245
+ }
246
+ const judge = await (0, obsEvaluator_1.registerObsEvaluator)().findOneOrNone({
247
+ _id: toObjectId(input.judgeEvaluatorId, "judgeEvaluatorId"),
248
+ });
249
+ if (!judge) {
250
+ throw new api_1.APIError({ status: 404, title: "Unknown judge evaluator" });
251
+ }
252
+ if (judge.type !== "llm-judge" || !judge.judgePromptName) {
253
+ throw new api_1.APIError({
254
+ status: 400,
255
+ title: "judgeEvaluatorId must be an llm-judge evaluator with a judge prompt",
256
+ });
257
+ }
258
+ const questionKeys = new Set(questionSet.dimensions.map((row) => row.key));
259
+ if (!judge.dimensions.some((row) => questionKeys.has(row.key))) {
260
+ throw new api_1.APIError({
261
+ detail: `Questions: ${[...questionKeys].join(", ")}. Judge scores: ${judge.dimensions
262
+ .map((row) => row.key)
263
+ .join(", ")}.`,
264
+ status: 400,
265
+ title: "Judge answers none of the question set's questions",
266
+ });
267
+ }
268
+ const dataset = await this.deps.datasetStore.get(toObjectId(input.datasetId, "datasetId").toString());
269
+ const current = await (0, humanAnswers_1.loadCurrentHumanAnswers)({
270
+ datasetId: dataset.id,
271
+ questionSetId: String(questionSet._id),
272
+ });
273
+ if (current.items.length === 0) {
274
+ throw new api_1.APIError({
275
+ detail: `Create a review batch from dataset "${dataset.name}" with "${questionSet.name}" and answer it first.`,
276
+ status: 400,
277
+ title: "Dataset has no current human answers for this question set",
278
+ });
279
+ }
280
+ for (const version of input.versions) {
281
+ const promptVersion = await this.deps.promptStore.getVersionByNumber(judge.judgePromptName, version);
282
+ if (!promptVersion) {
283
+ throw new api_1.APIError({
284
+ detail: `Judge prompt "${judge.judgePromptName}" has no version ${version}.`,
285
+ status: 400,
286
+ title: "Unknown judge prompt version",
287
+ });
288
+ }
289
+ if (!promptVersion.outputSchema) {
290
+ throw new api_1.APIError({
291
+ detail: `Judge prompt "${judge.judgePromptName}" v${version} has no output schema.`,
292
+ status: 400,
293
+ title: "Judge prompt version has no output schema",
294
+ });
295
+ }
296
+ }
297
+ this.resolveAiClient(input.modelOverride);
298
+ const estimate = await this.estimate({
299
+ datasetId: dataset.id,
300
+ kind: "judge",
301
+ modelOverride: input.modelOverride,
302
+ questionSetId: String(questionSet._id),
303
+ versions: input.versions,
304
+ });
305
+ const experiment = await (0, obsExperiment_1.registerObsExperiment)().create({
306
+ datasetId: new mongoose_1.default.Types.ObjectId(dataset.id),
307
+ estimate,
308
+ evaluatorIds: [judge._id],
309
+ includeUnproofread: true,
310
+ judgeEvaluatorId: judge._id,
311
+ kind: "judge",
312
+ modelOverride: input.modelOverride,
313
+ name: input.name,
314
+ promptName: judge.judgePromptName,
315
+ questionSetId: questionSet._id,
316
+ questionsFingerprint: current.fingerprint,
317
+ status: "pending",
318
+ targetAgreement,
319
+ thresholds: [],
320
+ versions: input.versions,
321
+ });
322
+ return this.startRun(experiment);
323
+ }
324
+ async startRun(experiment) {
175
325
  const runner = this.deps.backgroundTaskRunner ?? defaultBackgroundTaskRunner;
176
326
  const { taskId } = await runner.enqueue({
177
327
  execute: async () => {
@@ -179,9 +329,10 @@ class LocalExperimentRunner {
179
329
  },
180
330
  taskType: `obs-experiment-${String(experiment._id)}`,
181
331
  });
182
- experiment.backgroundTaskId = new mongoose_1.default.Types.ObjectId(taskId);
183
- experiment.status = "running";
184
- await experiment.save();
332
+ const ObsExperiment = (0, obsExperiment_1.registerObsExperiment)();
333
+ await ObsExperiment.updateOne({ _id: experiment._id }, { $set: { backgroundTaskId: new mongoose_1.default.Types.ObjectId(taskId) } });
334
+ // A runner that executes inline has already finished; only a still-pending run is "running".
335
+ await ObsExperiment.updateOne({ _id: experiment._id, status: "pending" }, { $set: { status: "running" } });
185
336
  return this.get(String(experiment._id));
186
337
  }
187
338
  async list(params) {
@@ -193,15 +344,23 @@ class LocalExperimentRunner {
193
344
  if (params?.promptName) {
194
345
  filter.promptName = params.promptName;
195
346
  }
347
+ if (params?.kind === "judge") {
348
+ filter.kind = "judge";
349
+ }
350
+ else if (params?.kind === "prompt") {
351
+ // Experiments saved before `kind` existed are prompt experiments.
352
+ filter.kind = { $ne: "judge" };
353
+ }
196
354
  const ObsExperiment = (0, obsExperiment_1.registerObsExperiment)();
197
355
  const total = await ObsExperiment.countDocuments(filter);
198
356
  const rows = await ObsExperiment.find(filter)
199
357
  .sort({ created: -1 })
200
358
  .skip((page - 1) * limit)
201
359
  .limit(limit);
360
+ const names = await this.loadNames(rows);
202
361
  return {
203
362
  data: rows.map((row) => {
204
- return this.toSummaryView(row);
363
+ return this.toSummaryView(row, names.get(String(row._id)));
205
364
  }),
206
365
  meta: { limit, page, total },
207
366
  };
@@ -218,35 +377,82 @@ class LocalExperimentRunner {
218
377
  }
219
378
  return left.created.getTime() - right.created.getTime();
220
379
  });
380
+ const names = await this.loadNames([experiment]);
221
381
  return {
222
- ...this.toSummaryView(experiment),
382
+ ...this.toSummaryView(experiment, names.get(String(experiment._id))),
223
383
  items: sortedItems.map((row) => {
224
384
  return {
225
385
  datasetItemId: String(row.datasetItemId),
226
386
  failed: row.failed,
387
+ humanScores: row.humanScores,
227
388
  id: String(row._id),
228
389
  versionResults: row.versionResults,
229
390
  };
230
391
  }),
231
392
  };
232
393
  }
233
- toSummaryView(experiment) {
394
+ // Dataset, question-set, and judge names for display, plus question-set staleness.
395
+ async loadNames(experiments) {
396
+ const datasetIds = experiments.map((row) => row.datasetId);
397
+ const evaluatorIds = experiments.flatMap((row) => {
398
+ return [row.questionSetId, row.judgeEvaluatorId].filter((id) => Boolean(id));
399
+ });
400
+ const datasets = datasetIds.length
401
+ ? await (0, obsDataset_1.registerObsDataset)().find({ _id: { $in: datasetIds } })
402
+ : [];
403
+ const evaluators = evaluatorIds.length
404
+ ? await (0, obsEvaluator_1.registerObsEvaluator)().find({ _id: { $in: evaluatorIds } })
405
+ : [];
406
+ const datasetNameById = new Map(datasets.map((row) => [String(row._id), row.name]));
407
+ const evaluatorById = new Map(evaluators.map((row) => [String(row._id), row]));
408
+ return new Map(experiments.map((row) => {
409
+ const questionSet = row.questionSetId
410
+ ? evaluatorById.get(String(row.questionSetId))
411
+ : undefined;
412
+ const judge = row.judgeEvaluatorId
413
+ ? evaluatorById.get(String(row.judgeEvaluatorId))
414
+ : undefined;
415
+ return [
416
+ String(row._id),
417
+ {
418
+ datasetName: datasetNameById.get(String(row.datasetId)),
419
+ judgeName: judge?.name,
420
+ questionSetName: questionSet?.name,
421
+ questionsStale: questionSet && row.questionsFingerprint
422
+ ? (0, questionFingerprint_1.questionSetFingerprint)(questionSet.dimensions) !== row.questionsFingerprint
423
+ : undefined,
424
+ },
425
+ ];
426
+ }));
427
+ }
428
+ toSummaryView(experiment, names) {
234
429
  return {
235
430
  backgroundTaskId: experiment.backgroundTaskId
236
431
  ? String(experiment.backgroundTaskId)
237
432
  : undefined,
238
433
  created: luxon_1.DateTime.fromJSDate(experiment.created).toUTC().toISO() ?? "",
239
434
  datasetId: String(experiment.datasetId),
435
+ datasetName: names?.datasetName,
240
436
  estimate: experiment.estimate,
241
437
  evaluatorIds: experiment.evaluatorIds.map((value) => String(value)),
242
438
  id: String(experiment._id),
243
439
  includeUnproofread: experiment.includeUnproofread,
244
440
  items: [],
441
+ judgeEvaluatorId: experiment.judgeEvaluatorId
442
+ ? String(experiment.judgeEvaluatorId)
443
+ : undefined,
444
+ judgeName: names?.judgeName,
445
+ kind: experiment.kind ?? "prompt",
245
446
  modelOverride: experiment.modelOverride,
246
447
  name: experiment.name,
247
448
  promptName: experiment.promptName,
449
+ questionSetId: experiment.questionSetId ? String(experiment.questionSetId) : undefined,
450
+ questionSetName: names?.questionSetName,
451
+ questionsFingerprint: experiment.questionsFingerprint,
452
+ questionsStale: names?.questionsStale,
248
453
  results: experiment.results,
249
454
  status: experiment.status,
455
+ targetAgreement: experiment.targetAgreement,
250
456
  thresholds: experiment.thresholds,
251
457
  updated: luxon_1.DateTime.fromJSDate(experiment.updated).toUTC().toISO() ?? "",
252
458
  versions: experiment.versions,
@@ -283,6 +489,10 @@ class LocalExperimentRunner {
283
489
  throw new api_1.APIError({ status: 404, title: `Unknown experiment "${id}"` });
284
490
  }
285
491
  const aiService = this.resolveAiClient(experiment.modelOverride);
492
+ if (experiment.kind === "judge") {
493
+ await this.executeJudgeExperiment(experiment, aiService);
494
+ return;
495
+ }
286
496
  const datasetItems = await this.deps.datasetStore.listExperimentItems(String(experiment.datasetId), experiment.includeUnproofread);
287
497
  const evaluators = await Promise.all(experiment.evaluatorIds.map((evaluatorId) => {
288
498
  return this.deps.evaluatorStore.get(String(evaluatorId));
@@ -400,6 +610,175 @@ class LocalExperimentRunner {
400
610
  };
401
611
  await experiment.save();
402
612
  }
613
+ async executeJudgeExperiment(experiment, aiService) {
614
+ const judge = experiment.judgeEvaluatorId
615
+ ? await (0, obsEvaluator_1.registerObsEvaluator)().findOneOrNone({ _id: experiment.judgeEvaluatorId })
616
+ : undefined;
617
+ if (!judge?.judgePromptName || !experiment.questionSetId) {
618
+ throw new api_1.APIError({
619
+ status: 404,
620
+ title: "Judge experiment is missing its judge or question set",
621
+ });
622
+ }
623
+ const judgePromptName = judge.judgePromptName;
624
+ const current = await (0, humanAnswers_1.loadCurrentHumanAnswers)({
625
+ datasetId: experiment.datasetId,
626
+ questionSetId: experiment.questionSetId,
627
+ });
628
+ // Judges see exactly what reviewers saw: the source trace, else the item's input and expected output.
629
+ const subjects = await (0, humanAnswers_1.loadDatasetItemSubjects)(current.items.map((row) => row.item));
630
+ const versionPrompts = new Map();
631
+ for (const versionNumber of experiment.versions) {
632
+ const version = await this.deps.promptStore.getVersionByNumber(judgePromptName, versionNumber);
633
+ if (!version?.outputSchema) {
634
+ throw new api_1.APIError({
635
+ status: 404,
636
+ title: `Judge prompt "${judgePromptName}" v${versionNumber} is missing or has no output schema`,
637
+ });
638
+ }
639
+ versionPrompts.set(versionNumber, {
640
+ schema: version.outputSchema,
641
+ system: versionBody(version),
642
+ });
643
+ }
644
+ const judgeId = String(judge._id);
645
+ const subjectRows = current.items.map(({ item }) => {
646
+ const datasetItemId = String(item._id);
647
+ const subject = subjects.get(datasetItemId);
648
+ return {
649
+ datasetItemId,
650
+ input: subject?.input,
651
+ output: subject?.output,
652
+ ...(subject?.traceId ? { traceId: subject.traceId } : {}),
653
+ };
654
+ });
655
+ const humanAnswers = current.items.map(({ answer }, index) => {
656
+ const { datasetItemId, traceId } = subjectRows[index] ?? {};
657
+ return { datasetItemId, scores: answer.scores, ...(traceId ? { traceId } : {}) };
658
+ });
659
+ const total = current.items.length;
660
+ const judgeForRun = {
661
+ dimensions: judge.dimensions,
662
+ judgePromptName,
663
+ name: judge.name,
664
+ type: judge.type,
665
+ };
666
+ const versionRuns = new Map(experiment.versions.map((version) => [version, []]));
667
+ const lowConfidenceItemIds = [];
668
+ const outlierItemIds = [];
669
+ const aggregate = (completed) => {
670
+ const agreement = {};
671
+ for (const version of experiment.versions) {
672
+ agreement[String(version)] = (0, judgeRun_1.agreementForResults)({
673
+ dimensions: current.questionSet.dimensions,
674
+ humanAnswers,
675
+ judge: judgeForRun,
676
+ results: versionRuns.get(version) ?? [],
677
+ });
678
+ }
679
+ const target = experiment.targetAgreement ?? exports.DEFAULT_TARGET_AGREEMENT;
680
+ return {
681
+ agreement,
682
+ gates: experiment.versions.map((version) => {
683
+ const actual = agreement[String(version)]?.rate;
684
+ return {
685
+ actual,
686
+ aggregate: "agreement",
687
+ dimension: "agreement",
688
+ evaluatorName: judge.name,
689
+ op: "gte",
690
+ passed: actual !== undefined && actual >= target,
691
+ value: target,
692
+ version,
693
+ };
694
+ }),
695
+ lowConfidenceItemIds: [...new Set(lowConfidenceItemIds)],
696
+ outlierItemIds: [...new Set(outlierItemIds)],
697
+ progress: { completed, total },
698
+ totalCostUsd: experiment.estimate?.costUsd,
699
+ };
700
+ };
701
+ // Every version runs through the shared judge pipeline at once; an item's row is written
702
+ // (and progress saved) as soon as all versions have judged it. Writes are serialized.
703
+ const runnable = experiment.versions.filter((version) => versionPrompts.has(version));
704
+ const pending = new Map();
705
+ let completed = 0;
706
+ let writes = Promise.resolve();
707
+ const writeRow = async (index, byVersion) => {
708
+ const row = current.items[index];
709
+ if (!row) {
710
+ return;
711
+ }
712
+ const datasetItemId = String(row.item._id);
713
+ const versionResults = {};
714
+ let rowFailed = false;
715
+ for (const versionNumber of experiment.versions) {
716
+ const result = byVersion.get(versionNumber);
717
+ if (!result) {
718
+ continue;
719
+ }
720
+ versionResults[String(versionNumber)] = {
721
+ evaluatorScores: {
722
+ [judgeId]: { confidence: result.confidence, error: result.error, scores: result.scores },
723
+ },
724
+ };
725
+ versionRuns.get(versionNumber)?.push(result);
726
+ if (result.error) {
727
+ rowFailed = true;
728
+ outlierItemIds.push(datasetItemId);
729
+ }
730
+ if (result.confidence !== undefined && result.confidence < judge.confidenceAlertBelow) {
731
+ lowConfidenceItemIds.push(datasetItemId);
732
+ }
733
+ // A row fails when any version disagrees with the reviewers on any compared question.
734
+ if (result.disagrees) {
735
+ rowFailed = true;
736
+ }
737
+ }
738
+ await (0, obsExperimentItem_1.registerObsExperimentItem)().create({
739
+ datasetItemId: row.item._id,
740
+ experimentId: experiment._id,
741
+ failed: rowFailed,
742
+ humanScores: row.answer.scores,
743
+ versionResults,
744
+ });
745
+ completed += 1;
746
+ experiment.results = aggregate(completed);
747
+ await experiment.save();
748
+ };
749
+ const concurrency = Math.max(1, Math.ceil(judgeRun_1.DEFAULT_JUDGE_CONCURRENCY / Math.max(runnable.length, 1)));
750
+ await Promise.all(runnable.map(async (versionNumber) => {
751
+ const prompt = versionPrompts.get(versionNumber);
752
+ if (!prompt) {
753
+ return;
754
+ }
755
+ await (0, judgeRun_1.runJudgeAgainstHumans)({
756
+ ai: aiService,
757
+ concurrency,
758
+ dimensions: current.questionSet.dimensions,
759
+ humanAnswers,
760
+ judge: judgeForRun,
761
+ onResult: async (result) => {
762
+ const byVersion = pending.get(result.index) ?? new Map();
763
+ byVersion.set(versionNumber, result);
764
+ pending.set(result.index, byVersion);
765
+ if (byVersion.size < runnable.length) {
766
+ return;
767
+ }
768
+ pending.delete(result.index);
769
+ writes = writes.then(() => writeRow(result.index, byVersion));
770
+ await writes;
771
+ },
772
+ prompt: { outputSchema: prompt.schema, system: prompt.system },
773
+ subjects: subjectRows,
774
+ });
775
+ }));
776
+ await writes;
777
+ experiment.questionsFingerprint = current.fingerprint;
778
+ experiment.results = aggregate(total);
779
+ experiment.status = "completed";
780
+ await experiment.save();
781
+ }
403
782
  computeGates(versions, thresholds, gateBuckets) {
404
783
  return versions.flatMap((version) => {
405
784
  return thresholds.map((threshold) => {
@@ -0,0 +1,57 @@
1
+ import mongoose from "mongoose";
2
+ import type { ObsDatasetHumanAnswer, ObsDatasetItemDocument, ObsEvaluatorDocument } from "../../types/observability";
3
+ /** How far one question set has judged a dataset. */
4
+ export interface DatasetJudgmentSummary {
5
+ /** Items with any answer from this question set. */
6
+ answered: number;
7
+ /** Answers whose fingerprint matches the question set's questions now. */
8
+ current: number;
9
+ evaluatorId: string;
10
+ evaluatorName: string;
11
+ /** Answers given to an older version of the questions. */
12
+ stale: number;
13
+ total: number;
14
+ }
15
+ export interface DatasetHumanAnswerView {
16
+ answeredAt: string;
17
+ fingerprint: string;
18
+ reviewItemId: string;
19
+ scores: Record<string, boolean | number | string>;
20
+ userId?: string;
21
+ }
22
+ /** What reviewers (and judges) see for a dataset item: the source trace when it has one. */
23
+ export interface DatasetItemSubject {
24
+ input?: unknown;
25
+ output?: unknown;
26
+ traceId?: string;
27
+ traceName?: string;
28
+ }
29
+ export interface CurrentHumanAnswers {
30
+ fingerprint: string;
31
+ items: Array<{
32
+ answer: ObsDatasetHumanAnswer;
33
+ item: ObsDatasetItemDocument;
34
+ }>;
35
+ questionSet: ObsEvaluatorDocument;
36
+ }
37
+ export declare const toHumanAnswerViews: (answers?: Record<string, ObsDatasetHumanAnswer>) => Record<string, DatasetHumanAnswerView> | undefined;
38
+ /** Record (or replace) one question set's answers on a dataset item. */
39
+ export declare const writeHumanAnswer: (params: {
40
+ datasetItemId: mongoose.Types.ObjectId;
41
+ evaluator: Pick<ObsEvaluatorDocument, "_id" | "dimensions">;
42
+ reviewItemId: string;
43
+ scores: Record<string, boolean | number | string>;
44
+ userId?: string;
45
+ }) => Promise<void>;
46
+ /** Subjects for dataset items, keyed by item id: the source trace when it still exists. */
47
+ export declare const loadDatasetItemSubjects: (items: Pick<ObsDatasetItemDocument, "_id" | "expectedOutput" | "input" | "sourceTraceId">[]) => Promise<Map<string, DatasetItemSubject>>;
48
+ /** Dataset items whose answers from this question set match its current questions. */
49
+ export declare const loadCurrentHumanAnswers: (params: {
50
+ datasetId: string | mongoose.Types.ObjectId;
51
+ questionSetId: string | mongoose.Types.ObjectId;
52
+ }) => Promise<CurrentHumanAnswers>;
53
+ /**
54
+ * Per-dataset judgment summaries: one row per question set that answered any item or has a
55
+ * review batch built from the dataset.
56
+ */
57
+ export declare const summarizeDatasetJudgments: (datasetIds: mongoose.Types.ObjectId[]) => Promise<Map<string, DatasetJudgmentSummary[]>>;