@terreno/ai 57.21.0 → 57.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. package/CHANGELOG.md +31 -0
  2. package/README.md +3 -3
  3. package/biome.jsonc +2 -2
  4. package/dist/harness/commit.d.ts +4 -3
  5. package/dist/harness/commit.js +23 -4
  6. package/dist/index.d.ts +6 -9
  7. package/dist/index.js +14 -25
  8. package/dist/observability/adminScreens.js +5 -1
  9. package/dist/observability/evaluate.d.ts +8 -1
  10. package/dist/observability/evaluate.js +10 -2
  11. package/dist/observability/judgeAgreement.d.ts +53 -0
  12. package/dist/observability/judgeAgreement.js +123 -0
  13. package/dist/observability/local/datasetStore.d.ts +5 -0
  14. package/dist/observability/local/datasetStore.js +7 -2
  15. package/dist/observability/local/evaluatorStore.d.ts +17 -0
  16. package/dist/observability/local/evaluatorStore.js +73 -2
  17. package/dist/observability/local/experimentRunner.d.ts +40 -3
  18. package/dist/observability/local/experimentRunner.js +390 -11
  19. package/dist/observability/local/humanAnswers.d.ts +57 -0
  20. package/dist/observability/local/humanAnswers.js +174 -0
  21. package/dist/observability/local/judgeRun.d.ts +56 -0
  22. package/dist/observability/local/judgeRun.js +111 -0
  23. package/dist/observability/local/localPlugin.js +4 -0
  24. package/dist/observability/local/models/obsDatasetItem.js +4 -0
  25. package/dist/observability/local/models/obsEvaluator.js +4 -0
  26. package/dist/observability/local/models/obsExperiment.js +28 -0
  27. package/dist/observability/local/models/obsExperimentItem.js +4 -0
  28. package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
  29. package/dist/observability/local/models/obsJudgeRun.js +60 -0
  30. package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
  31. package/dist/observability/local/models/obsReviewBatch.js +48 -0
  32. package/dist/observability/local/models/obsReviewItem.js +10 -2
  33. package/dist/observability/local/models/obsTrace.js +4 -0
  34. package/dist/observability/local/promptReferences.d.ts +31 -0
  35. package/dist/observability/local/promptReferences.js +114 -0
  36. package/dist/observability/local/promptStore.d.ts +55 -19
  37. package/dist/observability/local/promptStore.js +171 -7
  38. package/dist/observability/local/reviewBatchStore.d.ts +106 -0
  39. package/dist/observability/local/reviewBatchStore.js +517 -0
  40. package/dist/observability/local/reviewStore.d.ts +24 -3
  41. package/dist/observability/local/reviewStore.js +98 -22
  42. package/dist/observability/local/traceStore.d.ts +14 -1
  43. package/dist/observability/local/traceStore.js +144 -11
  44. package/dist/observability/observabilityApp.js +10 -1
  45. package/dist/observability/questionFingerprint.d.ts +7 -0
  46. package/dist/observability/questionFingerprint.js +23 -0
  47. package/dist/observability/routes/experiments.js +12 -1
  48. package/dist/observability/routes/review.js +3 -1
  49. package/dist/observability/routes/reviewBatches.d.ts +11 -0
  50. package/dist/observability/routes/reviewBatches.js +78 -0
  51. package/dist/observability/routes/traces.d.ts +1 -1
  52. package/dist/observability/routes/traces.js +34 -2
  53. package/dist/observability/types.d.ts +10 -1
  54. package/dist/service/aiService.js +4 -1
  55. package/dist/service/chatTurn.js +38 -31
  56. package/dist/types/harness.d.ts +5 -2
  57. package/dist/types/index.d.ts +8 -3
  58. package/dist/types/observability.d.ts +74 -2
  59. package/package.json +8 -24
  60. package/src/harness/agentLoop.test.ts +0 -3
  61. package/src/harness/agentRecovery.test.ts +0 -3
  62. package/src/harness/commit.ts +24 -4
  63. package/src/harness/extensions.test.ts +0 -3
  64. package/src/harness/subagent.test.ts +27 -3
  65. package/src/index.test.ts +0 -17
  66. package/src/index.ts +23 -26
  67. package/src/isolated/gptStream.isolated.ts +0 -8
  68. package/src/observability/adminScreens.test.ts +7 -0
  69. package/src/observability/adminScreens.ts +6 -1
  70. package/src/observability/evaluate.ts +12 -2
  71. package/src/observability/judgeAgreement.test.ts +111 -0
  72. package/src/observability/judgeAgreement.ts +184 -0
  73. package/src/observability/local/datasetStore.ts +30 -11
  74. package/src/observability/local/evaluatorStore.ts +96 -2
  75. package/src/observability/local/experimentRunner.ts +495 -15
  76. package/src/observability/local/humanAnswers.ts +233 -0
  77. package/src/observability/local/judgePipeline.test.ts +444 -0
  78. package/src/observability/local/judgeRun.test.ts +173 -0
  79. package/src/observability/local/judgeRun.ts +184 -0
  80. package/src/observability/local/localPlugin.ts +4 -0
  81. package/src/observability/local/models/obsDatasetItem.ts +5 -0
  82. package/src/observability/local/models/obsEvaluator.ts +4 -0
  83. package/src/observability/local/models/obsExperiment.ts +29 -0
  84. package/src/observability/local/models/obsExperimentItem.ts +4 -0
  85. package/src/observability/local/models/obsJudgeRun.ts +61 -0
  86. package/src/observability/local/models/obsReviewBatch.ts +52 -0
  87. package/src/observability/local/models/obsReviewItem.ts +11 -2
  88. package/src/observability/local/models/obsTrace.ts +5 -0
  89. package/src/observability/local/obsModels.test.ts +13 -0
  90. package/src/observability/local/promptReferences.test.ts +128 -0
  91. package/src/observability/local/promptReferences.ts +149 -0
  92. package/src/observability/local/promptStore.test.ts +291 -0
  93. package/src/observability/local/promptStore.ts +243 -32
  94. package/src/observability/local/reviewBatchStore.test.ts +251 -0
  95. package/src/observability/local/reviewBatchStore.ts +656 -0
  96. package/src/observability/local/reviewStore.ts +122 -25
  97. package/src/observability/local/traceStore.test.ts +139 -1
  98. package/src/observability/local/traceStore.ts +162 -11
  99. package/src/observability/observabilityApp.test.ts +13 -10
  100. package/src/observability/observabilityApp.ts +10 -1
  101. package/src/observability/questionFingerprint.test.ts +33 -0
  102. package/src/observability/questionFingerprint.ts +24 -0
  103. package/src/observability/routes/experiments.test.ts +37 -0
  104. package/src/observability/routes/experiments.ts +13 -1
  105. package/src/observability/routes/review.ts +3 -1
  106. package/src/observability/routes/reviewBatches.test.ts +209 -0
  107. package/src/observability/routes/reviewBatches.ts +138 -0
  108. package/src/observability/routes/status.test.ts +4 -4
  109. package/src/observability/routes/traces.test.ts +61 -0
  110. package/src/observability/routes/traces.ts +46 -4
  111. package/src/observability/types.ts +5 -2
  112. package/src/routes/routes.test.ts +101 -76
  113. package/src/service/aiService.test.ts +21 -0
  114. package/src/service/aiService.ts +4 -1
  115. package/src/service/chatTurn.ts +42 -33
  116. package/src/tests/bunSetup.ts +0 -54
  117. package/src/types/harness.ts +5 -2
  118. package/src/types/index.ts +8 -3
  119. package/src/types/observability.ts +86 -2
  120. package/dist/langfuseApp.d.ts +0 -9
  121. package/dist/langfuseApp.js +0 -53
  122. package/dist/langfuseCache.d.ts +0 -15
  123. package/dist/langfuseCache.js +0 -54
  124. package/dist/langfuseClient.d.ts +0 -6
  125. package/dist/langfuseClient.js +0 -34
  126. package/dist/langfusePrompts.d.ts +0 -12
  127. package/dist/langfusePrompts.js +0 -89
  128. package/dist/langfuseRoutesEvaluations.d.ts +0 -3
  129. package/dist/langfuseRoutesEvaluations.js +0 -32
  130. package/dist/langfuseRoutesMiddleware.d.ts +0 -2
  131. package/dist/langfuseRoutesMiddleware.js +0 -13
  132. package/dist/langfuseRoutesPlayground.d.ts +0 -2
  133. package/dist/langfuseRoutesPlayground.js +0 -36
  134. package/dist/langfuseRoutesPrompts.d.ts +0 -2
  135. package/dist/langfuseRoutesPrompts.js +0 -56
  136. package/dist/langfuseRoutesTraces.d.ts +0 -2
  137. package/dist/langfuseRoutesTraces.js +0 -30
  138. package/dist/langfuseTracing.d.ts +0 -4
  139. package/dist/langfuseTracing.js +0 -32
  140. package/dist/langfuseTypes.d.ts +0 -96
  141. package/dist/langfuseTypes.js +0 -2
  142. package/dist/langfuseVercelAi.d.ts +0 -12
  143. package/dist/langfuseVercelAi.js +0 -49
  144. package/src/isolated/langfuseClient.isolated.ts +0 -66
  145. package/src/langfuseApp.test.ts +0 -119
  146. package/src/langfuseApp.ts +0 -62
  147. package/src/langfuseCache.test.ts +0 -75
  148. package/src/langfuseCache.ts +0 -72
  149. package/src/langfuseClient.test.ts +0 -69
  150. package/src/langfuseClient.ts +0 -34
  151. package/src/langfusePrompts.test.ts +0 -188
  152. package/src/langfusePrompts.ts +0 -121
  153. package/src/langfuseRoutes.test.ts +0 -261
  154. package/src/langfuseRoutesEvaluations.ts +0 -48
  155. package/src/langfuseRoutesMiddleware.test.ts +0 -57
  156. package/src/langfuseRoutesMiddleware.ts +0 -12
  157. package/src/langfuseRoutesPlayground.ts +0 -50
  158. package/src/langfuseRoutesPrompts.ts +0 -87
  159. package/src/langfuseRoutesTraces.ts +0 -40
  160. package/src/langfuseTracing.test.ts +0 -72
  161. package/src/langfuseTracing.ts +0 -33
  162. package/src/langfuseTypes.ts +0 -103
  163. package/src/langfuseVercelAi.test.ts +0 -82
  164. package/src/langfuseVercelAi.ts +0 -72
  165. package/src/tests/fakeLangfusePrompt.ts +0 -10
@@ -6,12 +6,16 @@ import type {
6
6
  ExperimentAggregates,
7
7
  ExperimentEstimate,
8
8
  ExperimentGateResult,
9
+ ExperimentKind,
9
10
  ExperimentVersionResult,
11
+ ObsEvaluatorDocument,
10
12
  ObsExperimentDocument,
11
13
  ScoreThreshold,
12
14
  } from "../../types/observability";
13
15
  import {compileTemplate} from "../compileTemplate";
14
16
  import {runEvaluator} from "../evaluate";
17
+ import type {AgreementAnswerSet, JudgeAgreement} from "../judgeAgreement";
18
+ import {questionSetFingerprint} from "../questionFingerprint";
15
19
  import {SOP_DEFAULT_THRESHOLDS} from "../sopThresholds";
16
20
  import type {
17
21
  ModelPrice,
@@ -20,6 +24,15 @@ import type {
20
24
  } from "../types";
21
25
  import type {LocalDatasetStore} from "./datasetStore";
22
26
  import type {LocalEvaluatorStore} from "./evaluatorStore";
27
+ import {loadCurrentHumanAnswers, loadDatasetItemSubjects} from "./humanAnswers";
28
+ import {
29
+ agreementForResults,
30
+ DEFAULT_JUDGE_CONCURRENCY,
31
+ type JudgeRunItemResult,
32
+ runJudgeAgainstHumans,
33
+ } from "./judgeRun";
34
+ import {registerObsDataset} from "./models/obsDataset";
35
+ import {registerObsEvaluator} from "./models/obsEvaluator";
23
36
  import {registerObsExperiment} from "./models/obsExperiment";
24
37
  import {registerObsExperimentItem} from "./models/obsExperimentItem";
25
38
  import type {LocalPromptStore, MoveLabelResult} from "./promptStore";
@@ -28,6 +41,7 @@ export interface ExperimentCreateInput {
28
41
  datasetId: string;
29
42
  evaluatorIds: string[];
30
43
  includeUnproofread?: boolean;
44
+ kind?: "prompt";
31
45
  modelOverride?: string;
32
46
  name: string;
33
47
  promptName: string;
@@ -35,20 +49,49 @@ export interface ExperimentCreateInput {
35
49
  versions: number[];
36
50
  }
37
51
 
52
+ /**
53
+ * Judge calibration: run 2–3 versions of an llm-judge's prompt over dataset items humans already
54
+ * answered with `questionSetId`, and rank versions by agreement with those answers.
55
+ */
56
+ export interface JudgeExperimentCreateInput {
57
+ datasetId: string;
58
+ judgeEvaluatorId: string;
59
+ kind: "judge";
60
+ modelOverride?: string;
61
+ name: string;
62
+ questionSetId: string;
63
+ /** Agreement rate (0–1) a version must reach to pass its gate. Defaults to 0.9. */
64
+ targetAgreement?: number;
65
+ /** Versions of the judge evaluator's `judgePromptName`. */
66
+ versions: number[];
67
+ }
68
+
69
+ export const DEFAULT_TARGET_AGREEMENT = 0.9;
70
+
38
71
  export interface ExperimentView {
39
72
  backgroundTaskId?: string;
40
73
  created: string;
41
74
  datasetId: string;
75
+ datasetName?: string;
42
76
  estimate?: ExperimentEstimate;
43
77
  evaluatorIds: string[];
44
78
  id: string;
45
79
  includeUnproofread: boolean;
46
80
  items: ExperimentItemView[];
81
+ judgeEvaluatorId?: string;
82
+ judgeName?: string;
83
+ kind: ExperimentKind;
47
84
  modelOverride?: string;
48
85
  name: string;
49
86
  promptName: string;
87
+ questionSetId?: string;
88
+ questionSetName?: string;
89
+ questionsFingerprint?: string;
90
+ /** Judge experiments: true when the question set's questions changed since the run. */
91
+ questionsStale?: boolean;
50
92
  results?: ExperimentAggregates;
51
93
  status: "completed" | "failed" | "pending" | "running";
94
+ targetAgreement?: number;
52
95
  thresholds: ScoreThreshold[];
53
96
  updated: string;
54
97
  versions: number[];
@@ -57,10 +100,30 @@ export interface ExperimentView {
57
100
  export interface ExperimentItemView {
58
101
  datasetItemId: string;
59
102
  failed: boolean;
103
+ /** Judge experiments: the human answers each judge version is compared with. */
104
+ humanScores?: Record<string, boolean | number | string>;
60
105
  id: string;
61
106
  versionResults: Record<string, ExperimentVersionResult>;
62
107
  }
63
108
 
109
+ interface ExperimentNames {
110
+ datasetName?: string;
111
+ judgeName?: string;
112
+ questionSetName?: string;
113
+ questionsStale?: boolean;
114
+ }
115
+
116
+ const versionBody = (version: {system?: string; template?: string}): string => {
117
+ return version.system && version.system.length > 0 ? version.system : (version.template ?? "");
118
+ };
119
+
120
+ const toObjectId = (value: unknown, label: string): mongoose.Types.ObjectId => {
121
+ if (typeof value !== "string" || !mongoose.Types.ObjectId.isValid(value)) {
122
+ throw new APIError({status: 400, title: `${label} must be an object id`});
123
+ }
124
+ return new mongoose.Types.ObjectId(value);
125
+ };
126
+
64
127
  export const EXPERIMENT_LIST_DEFAULT_LIMIT = 50;
65
128
  const EXPERIMENT_LIST_MAX_LIMIT = 100;
66
129
 
@@ -195,17 +258,32 @@ export class LocalExperimentRunner {
195
258
 
196
259
  async estimate(input: {
197
260
  datasetId: string;
198
- evaluatorIds: string[];
261
+ evaluatorIds?: string[];
199
262
  includeUnproofread?: boolean;
263
+ kind?: ExperimentKind;
200
264
  modelOverride?: string;
265
+ questionSetId?: string;
201
266
  versions: number[];
202
267
  }): Promise<ExperimentEstimate> {
203
- const items = await this.deps.datasetStore.listExperimentItems(
204
- input.datasetId,
205
- input.includeUnproofread ?? false
206
- );
207
- const generations = items.length * input.versions.length;
208
- const evaluatorCount = input.evaluatorIds.length;
268
+ const itemCount =
269
+ input.kind === "judge"
270
+ ? input.questionSetId
271
+ ? (
272
+ await loadCurrentHumanAnswers({
273
+ datasetId: input.datasetId,
274
+ questionSetId: input.questionSetId,
275
+ })
276
+ ).items.length
277
+ : 0
278
+ : (
279
+ await this.deps.datasetStore.listExperimentItems(
280
+ input.datasetId,
281
+ input.includeUnproofread ?? false
282
+ )
283
+ ).length;
284
+ const generations = itemCount * input.versions.length;
285
+ // Judge experiments make one judge call per item and version and no separate generation.
286
+ const evaluatorCount = input.kind === "judge" ? 0 : (input.evaluatorIds?.length ?? 0);
209
287
  const secondsPerGeneration = 2;
210
288
  const judgeSeconds = generations * evaluatorCount * 0.5;
211
289
  const modelId =
@@ -237,10 +315,23 @@ export class LocalExperimentRunner {
237
315
  }
238
316
  }
239
317
 
240
- async create(input: ExperimentCreateInput): Promise<ExperimentView> {
241
- if (input.versions.length < 2 || input.versions.length > 3) {
318
+ async create(input: ExperimentCreateInput | JudgeExperimentCreateInput): Promise<ExperimentView> {
319
+ const kind: unknown = input.kind ?? "prompt";
320
+ if (kind !== "prompt" && kind !== "judge") {
321
+ throw new APIError({status: 400, title: "kind must be prompt or judge"});
322
+ }
323
+ if (input.kind === "judge") {
324
+ return this.createJudge(input);
325
+ }
326
+ if (!Array.isArray(input.versions) || input.versions.length < 2 || input.versions.length > 3) {
242
327
  throw new APIError({status: 400, title: "experiments must compare 2–3 prompt versions"});
243
328
  }
329
+ if (input.thresholds?.some((threshold) => threshold.aggregate === "agreement")) {
330
+ throw new APIError({
331
+ status: 400,
332
+ title: "agreement gates belong to judge experiments (kind: judge)",
333
+ });
334
+ }
244
335
  await this.assertAutomaticEvaluators(input.evaluatorIds);
245
336
  this.resolveAiClient(input.modelOverride);
246
337
  await this.deps.datasetStore.get(input.datasetId);
@@ -258,6 +349,7 @@ export class LocalExperimentRunner {
258
349
  return new mongoose.Types.ObjectId(id);
259
350
  }),
260
351
  includeUnproofread: input.includeUnproofread ?? false,
352
+ kind: "prompt",
261
353
  modelOverride: input.modelOverride,
262
354
  name: input.name,
263
355
  promptName: input.promptName,
@@ -265,6 +357,128 @@ export class LocalExperimentRunner {
265
357
  thresholds: input.thresholds ?? SOP_DEFAULT_THRESHOLDS,
266
358
  versions: input.versions,
267
359
  });
360
+ return this.startRun(experiment);
361
+ }
362
+
363
+ private async createJudge(input: JudgeExperimentCreateInput): Promise<ExperimentView> {
364
+ if (!input.name) {
365
+ throw new APIError({status: 400, title: "name is required"});
366
+ }
367
+ if (
368
+ !Array.isArray(input.versions) ||
369
+ input.versions.length < 2 ||
370
+ input.versions.length > 3 ||
371
+ !input.versions.every((version) => Number.isInteger(version))
372
+ ) {
373
+ throw new APIError({
374
+ status: 400,
375
+ title: "judge experiments must compare 2–3 judge prompt versions",
376
+ });
377
+ }
378
+ if (new Set(input.versions).size !== input.versions.length) {
379
+ throw new APIError({status: 400, title: "judge prompt versions must be different"});
380
+ }
381
+ const targetAgreement = input.targetAgreement ?? DEFAULT_TARGET_AGREEMENT;
382
+ if (typeof targetAgreement !== "number" || targetAgreement <= 0 || targetAgreement > 1) {
383
+ throw new APIError({
384
+ status: 400,
385
+ title: "targetAgreement must be a number above 0 and at most 1",
386
+ });
387
+ }
388
+ const questionSet = await registerObsEvaluator().findOneOrNone({
389
+ _id: toObjectId(input.questionSetId, "questionSetId"),
390
+ });
391
+ if (!questionSet) {
392
+ throw new APIError({status: 404, title: "Unknown question set"});
393
+ }
394
+ if (questionSet.type !== "human") {
395
+ throw new APIError({status: 400, title: "questionSetId must be a human question set"});
396
+ }
397
+ const judge = await registerObsEvaluator().findOneOrNone({
398
+ _id: toObjectId(input.judgeEvaluatorId, "judgeEvaluatorId"),
399
+ });
400
+ if (!judge) {
401
+ throw new APIError({status: 404, title: "Unknown judge evaluator"});
402
+ }
403
+ if (judge.type !== "llm-judge" || !judge.judgePromptName) {
404
+ throw new APIError({
405
+ status: 400,
406
+ title: "judgeEvaluatorId must be an llm-judge evaluator with a judge prompt",
407
+ });
408
+ }
409
+ const questionKeys = new Set(questionSet.dimensions.map((row) => row.key));
410
+ if (!judge.dimensions.some((row) => questionKeys.has(row.key))) {
411
+ throw new APIError({
412
+ detail: `Questions: ${[...questionKeys].join(", ")}. Judge scores: ${judge.dimensions
413
+ .map((row) => row.key)
414
+ .join(", ")}.`,
415
+ status: 400,
416
+ title: "Judge answers none of the question set's questions",
417
+ });
418
+ }
419
+ const dataset = await this.deps.datasetStore.get(
420
+ toObjectId(input.datasetId, "datasetId").toString()
421
+ );
422
+ const current = await loadCurrentHumanAnswers({
423
+ datasetId: dataset.id,
424
+ questionSetId: String(questionSet._id),
425
+ });
426
+ if (current.items.length === 0) {
427
+ throw new APIError({
428
+ detail: `Create a review batch from dataset "${dataset.name}" with "${questionSet.name}" and answer it first.`,
429
+ status: 400,
430
+ title: "Dataset has no current human answers for this question set",
431
+ });
432
+ }
433
+ for (const version of input.versions) {
434
+ const promptVersion = await this.deps.promptStore.getVersionByNumber(
435
+ judge.judgePromptName,
436
+ version
437
+ );
438
+ if (!promptVersion) {
439
+ throw new APIError({
440
+ detail: `Judge prompt "${judge.judgePromptName}" has no version ${version}.`,
441
+ status: 400,
442
+ title: "Unknown judge prompt version",
443
+ });
444
+ }
445
+ if (!promptVersion.outputSchema) {
446
+ throw new APIError({
447
+ detail: `Judge prompt "${judge.judgePromptName}" v${version} has no output schema.`,
448
+ status: 400,
449
+ title: "Judge prompt version has no output schema",
450
+ });
451
+ }
452
+ }
453
+ this.resolveAiClient(input.modelOverride);
454
+ const estimate = await this.estimate({
455
+ datasetId: dataset.id,
456
+ kind: "judge",
457
+ modelOverride: input.modelOverride,
458
+ questionSetId: String(questionSet._id),
459
+ versions: input.versions,
460
+ });
461
+ const experiment = await registerObsExperiment().create({
462
+ datasetId: new mongoose.Types.ObjectId(dataset.id),
463
+ estimate,
464
+ evaluatorIds: [judge._id],
465
+ includeUnproofread: true,
466
+ judgeEvaluatorId: judge._id,
467
+ kind: "judge",
468
+ modelOverride: input.modelOverride,
469
+ name: input.name,
470
+ promptName: judge.judgePromptName,
471
+ questionSetId: questionSet._id,
472
+ questionsFingerprint: current.fingerprint,
473
+ status: "pending",
474
+ targetAgreement,
475
+ thresholds: [],
476
+ versions: input.versions,
477
+ });
478
+ return this.startRun(experiment);
479
+ }
480
+
481
+ private async startRun(experiment: ObsExperimentDocument): Promise<ExperimentView> {
268
482
  const runner = this.deps.backgroundTaskRunner ?? defaultBackgroundTaskRunner;
269
483
  const {taskId} = await runner.enqueue({
270
484
  execute: async () => {
@@ -272,13 +486,21 @@ export class LocalExperimentRunner {
272
486
  },
273
487
  taskType: `obs-experiment-${String(experiment._id)}`,
274
488
  });
275
- experiment.backgroundTaskId = new mongoose.Types.ObjectId(taskId);
276
- experiment.status = "running";
277
- await experiment.save();
489
+ const ObsExperiment = registerObsExperiment();
490
+ await ObsExperiment.updateOne(
491
+ {_id: experiment._id},
492
+ {$set: {backgroundTaskId: new mongoose.Types.ObjectId(taskId)}}
493
+ );
494
+ // A runner that executes inline has already finished; only a still-pending run is "running".
495
+ await ObsExperiment.updateOne(
496
+ {_id: experiment._id, status: "pending"},
497
+ {$set: {status: "running"}}
498
+ );
278
499
  return this.get(String(experiment._id));
279
500
  }
280
501
 
281
502
  async list(params?: {
503
+ kind?: ExperimentKind;
282
504
  limit?: number;
283
505
  page?: number;
284
506
  promptName?: string;
@@ -292,15 +514,22 @@ export class LocalExperimentRunner {
292
514
  if (params?.promptName) {
293
515
  filter.promptName = params.promptName;
294
516
  }
517
+ if (params?.kind === "judge") {
518
+ filter.kind = "judge";
519
+ } else if (params?.kind === "prompt") {
520
+ // Experiments saved before `kind` existed are prompt experiments.
521
+ filter.kind = {$ne: "judge"};
522
+ }
295
523
  const ObsExperiment = registerObsExperiment();
296
524
  const total = await ObsExperiment.countDocuments(filter);
297
525
  const rows = await ObsExperiment.find(filter)
298
526
  .sort({created: -1})
299
527
  .skip((page - 1) * limit)
300
528
  .limit(limit);
529
+ const names = await this.loadNames(rows);
301
530
  return {
302
531
  data: rows.map((row) => {
303
- return this.toSummaryView(row);
532
+ return this.toSummaryView(row, names.get(String(row._id)));
304
533
  }),
305
534
  meta: {limit, page, total},
306
535
  };
@@ -318,12 +547,14 @@ export class LocalExperimentRunner {
318
547
  }
319
548
  return left.created.getTime() - right.created.getTime();
320
549
  });
550
+ const names = await this.loadNames([experiment]);
321
551
  return {
322
- ...this.toSummaryView(experiment),
552
+ ...this.toSummaryView(experiment, names.get(String(experiment._id))),
323
553
  items: sortedItems.map((row) => {
324
554
  return {
325
555
  datasetItemId: String(row.datasetItemId),
326
556
  failed: row.failed,
557
+ humanScores: row.humanScores,
327
558
  id: String(row._id),
328
559
  versionResults: row.versionResults,
329
560
  };
@@ -331,23 +562,81 @@ export class LocalExperimentRunner {
331
562
  };
332
563
  }
333
564
 
334
- private toSummaryView(experiment: ObsExperimentDocument): ExperimentView {
565
+ // Dataset, question-set, and judge names for display, plus question-set staleness.
566
+ private async loadNames(
567
+ experiments: ObsExperimentDocument[]
568
+ ): Promise<Map<string, ExperimentNames>> {
569
+ const datasetIds = experiments.map((row) => row.datasetId);
570
+ const evaluatorIds = experiments.flatMap((row) => {
571
+ return [row.questionSetId, row.judgeEvaluatorId].filter((id): id is mongoose.Types.ObjectId =>
572
+ Boolean(id)
573
+ );
574
+ });
575
+ const datasets = datasetIds.length
576
+ ? await registerObsDataset().find({_id: {$in: datasetIds}})
577
+ : [];
578
+ const evaluators = evaluatorIds.length
579
+ ? await registerObsEvaluator().find({_id: {$in: evaluatorIds}})
580
+ : [];
581
+ const datasetNameById = new Map(datasets.map((row) => [String(row._id), row.name]));
582
+ const evaluatorById = new Map<string, ObsEvaluatorDocument>(
583
+ evaluators.map((row) => [String(row._id), row])
584
+ );
585
+ return new Map(
586
+ experiments.map((row) => {
587
+ const questionSet = row.questionSetId
588
+ ? evaluatorById.get(String(row.questionSetId))
589
+ : undefined;
590
+ const judge = row.judgeEvaluatorId
591
+ ? evaluatorById.get(String(row.judgeEvaluatorId))
592
+ : undefined;
593
+ return [
594
+ String(row._id),
595
+ {
596
+ datasetName: datasetNameById.get(String(row.datasetId)),
597
+ judgeName: judge?.name,
598
+ questionSetName: questionSet?.name,
599
+ questionsStale:
600
+ questionSet && row.questionsFingerprint
601
+ ? questionSetFingerprint(questionSet.dimensions) !== row.questionsFingerprint
602
+ : undefined,
603
+ },
604
+ ];
605
+ })
606
+ );
607
+ }
608
+
609
+ private toSummaryView(
610
+ experiment: ObsExperimentDocument,
611
+ names?: ExperimentNames
612
+ ): ExperimentView {
335
613
  return {
336
614
  backgroundTaskId: experiment.backgroundTaskId
337
615
  ? String(experiment.backgroundTaskId)
338
616
  : undefined,
339
617
  created: DateTime.fromJSDate(experiment.created).toUTC().toISO() ?? "",
340
618
  datasetId: String(experiment.datasetId),
619
+ datasetName: names?.datasetName,
341
620
  estimate: experiment.estimate,
342
621
  evaluatorIds: experiment.evaluatorIds.map((value) => String(value)),
343
622
  id: String(experiment._id),
344
623
  includeUnproofread: experiment.includeUnproofread,
345
624
  items: [],
625
+ judgeEvaluatorId: experiment.judgeEvaluatorId
626
+ ? String(experiment.judgeEvaluatorId)
627
+ : undefined,
628
+ judgeName: names?.judgeName,
629
+ kind: experiment.kind ?? "prompt",
346
630
  modelOverride: experiment.modelOverride,
347
631
  name: experiment.name,
348
632
  promptName: experiment.promptName,
633
+ questionSetId: experiment.questionSetId ? String(experiment.questionSetId) : undefined,
634
+ questionSetName: names?.questionSetName,
635
+ questionsFingerprint: experiment.questionsFingerprint,
636
+ questionsStale: names?.questionsStale,
349
637
  results: experiment.results,
350
638
  status: experiment.status,
639
+ targetAgreement: experiment.targetAgreement,
351
640
  thresholds: experiment.thresholds,
352
641
  updated: DateTime.fromJSDate(experiment.updated).toUTC().toISO() ?? "",
353
642
  versions: experiment.versions,
@@ -386,6 +675,10 @@ export class LocalExperimentRunner {
386
675
  throw new APIError({status: 404, title: `Unknown experiment "${id}"`});
387
676
  }
388
677
  const aiService = this.resolveAiClient(experiment.modelOverride);
678
+ if (experiment.kind === "judge") {
679
+ await this.executeJudgeExperiment(experiment, aiService);
680
+ return;
681
+ }
389
682
  const datasetItems = await this.deps.datasetStore.listExperimentItems(
390
683
  String(experiment.datasetId),
391
684
  experiment.includeUnproofread
@@ -525,6 +818,193 @@ export class LocalExperimentRunner {
525
818
  await experiment.save();
526
819
  }
527
820
 
821
+ private async executeJudgeExperiment(
822
+ experiment: ObsExperimentDocument,
823
+ aiService: ObservabilityGenerateClient
824
+ ): Promise<void> {
825
+ const judge = experiment.judgeEvaluatorId
826
+ ? await registerObsEvaluator().findOneOrNone({_id: experiment.judgeEvaluatorId})
827
+ : undefined;
828
+ if (!judge?.judgePromptName || !experiment.questionSetId) {
829
+ throw new APIError({
830
+ status: 404,
831
+ title: "Judge experiment is missing its judge or question set",
832
+ });
833
+ }
834
+ const judgePromptName = judge.judgePromptName;
835
+ const current = await loadCurrentHumanAnswers({
836
+ datasetId: experiment.datasetId,
837
+ questionSetId: experiment.questionSetId,
838
+ });
839
+ // Judges see exactly what reviewers saw: the source trace, else the item's input and expected output.
840
+ const subjects = await loadDatasetItemSubjects(current.items.map((row) => row.item));
841
+ const versionPrompts = new Map<number, {schema: Record<string, unknown>; system: string}>();
842
+ for (const versionNumber of experiment.versions) {
843
+ const version = await this.deps.promptStore.getVersionByNumber(
844
+ judgePromptName,
845
+ versionNumber
846
+ );
847
+ if (!version?.outputSchema) {
848
+ throw new APIError({
849
+ status: 404,
850
+ title: `Judge prompt "${judgePromptName}" v${versionNumber} is missing or has no output schema`,
851
+ });
852
+ }
853
+ versionPrompts.set(versionNumber, {
854
+ schema: version.outputSchema,
855
+ system: versionBody(version),
856
+ });
857
+ }
858
+ const judgeId = String(judge._id);
859
+ const subjectRows = current.items.map(({item}) => {
860
+ const datasetItemId = String(item._id);
861
+ const subject = subjects.get(datasetItemId);
862
+ return {
863
+ datasetItemId,
864
+ input: subject?.input,
865
+ output: subject?.output,
866
+ ...(subject?.traceId ? {traceId: subject.traceId} : {}),
867
+ };
868
+ });
869
+ const humanAnswers: AgreementAnswerSet[] = current.items.map(({answer}, index) => {
870
+ const {datasetItemId, traceId} = subjectRows[index] ?? {};
871
+ return {datasetItemId, scores: answer.scores, ...(traceId ? {traceId} : {})};
872
+ });
873
+ const total = current.items.length;
874
+ const judgeForRun = {
875
+ dimensions: judge.dimensions,
876
+ judgePromptName,
877
+ name: judge.name,
878
+ type: judge.type,
879
+ };
880
+ const versionRuns = new Map<number, JudgeRunItemResult[]>(
881
+ experiment.versions.map((version) => [version, []])
882
+ );
883
+ const lowConfidenceItemIds: string[] = [];
884
+ const outlierItemIds: string[] = [];
885
+ const aggregate = (completed: number): ExperimentAggregates => {
886
+ const agreement: Record<string, JudgeAgreement> = {};
887
+ for (const version of experiment.versions) {
888
+ agreement[String(version)] = agreementForResults({
889
+ dimensions: current.questionSet.dimensions,
890
+ humanAnswers,
891
+ judge: judgeForRun,
892
+ results: versionRuns.get(version) ?? [],
893
+ });
894
+ }
895
+ const target = experiment.targetAgreement ?? DEFAULT_TARGET_AGREEMENT;
896
+ return {
897
+ agreement,
898
+ gates: experiment.versions.map((version) => {
899
+ const actual = agreement[String(version)]?.rate;
900
+ return {
901
+ actual,
902
+ aggregate: "agreement" as const,
903
+ dimension: "agreement",
904
+ evaluatorName: judge.name,
905
+ op: "gte" as const,
906
+ passed: actual !== undefined && actual >= target,
907
+ value: target,
908
+ version,
909
+ };
910
+ }),
911
+ lowConfidenceItemIds: [...new Set(lowConfidenceItemIds)],
912
+ outlierItemIds: [...new Set(outlierItemIds)],
913
+ progress: {completed, total},
914
+ totalCostUsd: experiment.estimate?.costUsd,
915
+ };
916
+ };
917
+
918
+ // Every version runs through the shared judge pipeline at once; an item's row is written
919
+ // (and progress saved) as soon as all versions have judged it. Writes are serialized.
920
+ const runnable = experiment.versions.filter((version) => versionPrompts.has(version));
921
+ const pending = new Map<number, Map<number, JudgeRunItemResult>>();
922
+ let completed = 0;
923
+ let writes: Promise<void> = Promise.resolve();
924
+ const writeRow = async (
925
+ index: number,
926
+ byVersion: Map<number, JudgeRunItemResult>
927
+ ): Promise<void> => {
928
+ const row = current.items[index];
929
+ if (!row) {
930
+ return;
931
+ }
932
+ const datasetItemId = String(row.item._id);
933
+ const versionResults: Record<string, ExperimentVersionResult> = {};
934
+ let rowFailed = false;
935
+ for (const versionNumber of experiment.versions) {
936
+ const result = byVersion.get(versionNumber);
937
+ if (!result) {
938
+ continue;
939
+ }
940
+ versionResults[String(versionNumber)] = {
941
+ evaluatorScores: {
942
+ [judgeId]: {confidence: result.confidence, error: result.error, scores: result.scores},
943
+ },
944
+ };
945
+ versionRuns.get(versionNumber)?.push(result);
946
+ if (result.error) {
947
+ rowFailed = true;
948
+ outlierItemIds.push(datasetItemId);
949
+ }
950
+ if (result.confidence !== undefined && result.confidence < judge.confidenceAlertBelow) {
951
+ lowConfidenceItemIds.push(datasetItemId);
952
+ }
953
+ // A row fails when any version disagrees with the reviewers on any compared question.
954
+ if (result.disagrees) {
955
+ rowFailed = true;
956
+ }
957
+ }
958
+ await registerObsExperimentItem().create({
959
+ datasetItemId: row.item._id,
960
+ experimentId: experiment._id,
961
+ failed: rowFailed,
962
+ humanScores: row.answer.scores,
963
+ versionResults,
964
+ });
965
+ completed += 1;
966
+ experiment.results = aggregate(completed);
967
+ await experiment.save();
968
+ };
969
+ const concurrency = Math.max(
970
+ 1,
971
+ Math.ceil(DEFAULT_JUDGE_CONCURRENCY / Math.max(runnable.length, 1))
972
+ );
973
+ await Promise.all(
974
+ runnable.map(async (versionNumber) => {
975
+ const prompt = versionPrompts.get(versionNumber);
976
+ if (!prompt) {
977
+ return;
978
+ }
979
+ await runJudgeAgainstHumans({
980
+ ai: aiService,
981
+ concurrency,
982
+ dimensions: current.questionSet.dimensions,
983
+ humanAnswers,
984
+ judge: judgeForRun,
985
+ onResult: async (result) => {
986
+ const byVersion = pending.get(result.index) ?? new Map<number, JudgeRunItemResult>();
987
+ byVersion.set(versionNumber, result);
988
+ pending.set(result.index, byVersion);
989
+ if (byVersion.size < runnable.length) {
990
+ return;
991
+ }
992
+ pending.delete(result.index);
993
+ writes = writes.then(() => writeRow(result.index, byVersion));
994
+ await writes;
995
+ },
996
+ prompt: {outputSchema: prompt.schema, system: prompt.system},
997
+ subjects: subjectRows,
998
+ });
999
+ })
1000
+ );
1001
+ await writes;
1002
+ experiment.questionsFingerprint = current.fingerprint;
1003
+ experiment.results = aggregate(total);
1004
+ experiment.status = "completed";
1005
+ await experiment.save();
1006
+ }
1007
+
528
1008
  private computeGates(
529
1009
  versions: number[],
530
1010
  thresholds: ScoreThreshold[],