@terreno/ai 57.21.0 → 57.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/README.md +3 -3
- package/biome.jsonc +2 -2
- package/dist/harness/commit.d.ts +4 -3
- package/dist/harness/commit.js +23 -4
- package/dist/index.d.ts +6 -9
- package/dist/index.js +14 -25
- package/dist/observability/adminScreens.js +5 -1
- package/dist/observability/evaluate.d.ts +8 -1
- package/dist/observability/evaluate.js +10 -2
- package/dist/observability/judgeAgreement.d.ts +53 -0
- package/dist/observability/judgeAgreement.js +123 -0
- package/dist/observability/local/datasetStore.d.ts +5 -0
- package/dist/observability/local/datasetStore.js +7 -2
- package/dist/observability/local/evaluatorStore.d.ts +17 -0
- package/dist/observability/local/evaluatorStore.js +73 -2
- package/dist/observability/local/experimentRunner.d.ts +40 -3
- package/dist/observability/local/experimentRunner.js +390 -11
- package/dist/observability/local/humanAnswers.d.ts +57 -0
- package/dist/observability/local/humanAnswers.js +174 -0
- package/dist/observability/local/judgeRun.d.ts +56 -0
- package/dist/observability/local/judgeRun.js +111 -0
- package/dist/observability/local/localPlugin.js +4 -0
- package/dist/observability/local/models/obsDatasetItem.js +4 -0
- package/dist/observability/local/models/obsEvaluator.js +4 -0
- package/dist/observability/local/models/obsExperiment.js +28 -0
- package/dist/observability/local/models/obsExperimentItem.js +4 -0
- package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
- package/dist/observability/local/models/obsJudgeRun.js +60 -0
- package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
- package/dist/observability/local/models/obsReviewBatch.js +48 -0
- package/dist/observability/local/models/obsReviewItem.js +10 -2
- package/dist/observability/local/models/obsTrace.js +4 -0
- package/dist/observability/local/promptReferences.d.ts +31 -0
- package/dist/observability/local/promptReferences.js +114 -0
- package/dist/observability/local/promptStore.d.ts +55 -19
- package/dist/observability/local/promptStore.js +171 -7
- package/dist/observability/local/reviewBatchStore.d.ts +106 -0
- package/dist/observability/local/reviewBatchStore.js +517 -0
- package/dist/observability/local/reviewStore.d.ts +24 -3
- package/dist/observability/local/reviewStore.js +98 -22
- package/dist/observability/local/traceStore.d.ts +14 -1
- package/dist/observability/local/traceStore.js +144 -11
- package/dist/observability/observabilityApp.js +10 -1
- package/dist/observability/questionFingerprint.d.ts +7 -0
- package/dist/observability/questionFingerprint.js +23 -0
- package/dist/observability/routes/experiments.js +12 -1
- package/dist/observability/routes/review.js +3 -1
- package/dist/observability/routes/reviewBatches.d.ts +11 -0
- package/dist/observability/routes/reviewBatches.js +78 -0
- package/dist/observability/routes/traces.d.ts +1 -1
- package/dist/observability/routes/traces.js +34 -2
- package/dist/observability/types.d.ts +10 -1
- package/dist/service/aiService.js +4 -1
- package/dist/service/chatTurn.js +38 -31
- package/dist/types/harness.d.ts +5 -2
- package/dist/types/index.d.ts +8 -3
- package/dist/types/observability.d.ts +74 -2
- package/package.json +8 -24
- package/src/harness/agentLoop.test.ts +0 -3
- package/src/harness/agentRecovery.test.ts +0 -3
- package/src/harness/commit.ts +24 -4
- package/src/harness/extensions.test.ts +0 -3
- package/src/harness/subagent.test.ts +27 -3
- package/src/index.test.ts +0 -17
- package/src/index.ts +23 -26
- package/src/isolated/gptStream.isolated.ts +0 -8
- package/src/observability/adminScreens.test.ts +7 -0
- package/src/observability/adminScreens.ts +6 -1
- package/src/observability/evaluate.ts +12 -2
- package/src/observability/judgeAgreement.test.ts +111 -0
- package/src/observability/judgeAgreement.ts +184 -0
- package/src/observability/local/datasetStore.ts +30 -11
- package/src/observability/local/evaluatorStore.ts +96 -2
- package/src/observability/local/experimentRunner.ts +495 -15
- package/src/observability/local/humanAnswers.ts +233 -0
- package/src/observability/local/judgePipeline.test.ts +444 -0
- package/src/observability/local/judgeRun.test.ts +173 -0
- package/src/observability/local/judgeRun.ts +184 -0
- package/src/observability/local/localPlugin.ts +4 -0
- package/src/observability/local/models/obsDatasetItem.ts +5 -0
- package/src/observability/local/models/obsEvaluator.ts +4 -0
- package/src/observability/local/models/obsExperiment.ts +29 -0
- package/src/observability/local/models/obsExperimentItem.ts +4 -0
- package/src/observability/local/models/obsJudgeRun.ts +61 -0
- package/src/observability/local/models/obsReviewBatch.ts +52 -0
- package/src/observability/local/models/obsReviewItem.ts +11 -2
- package/src/observability/local/models/obsTrace.ts +5 -0
- package/src/observability/local/obsModels.test.ts +13 -0
- package/src/observability/local/promptReferences.test.ts +128 -0
- package/src/observability/local/promptReferences.ts +149 -0
- package/src/observability/local/promptStore.test.ts +291 -0
- package/src/observability/local/promptStore.ts +243 -32
- package/src/observability/local/reviewBatchStore.test.ts +251 -0
- package/src/observability/local/reviewBatchStore.ts +656 -0
- package/src/observability/local/reviewStore.ts +122 -25
- package/src/observability/local/traceStore.test.ts +139 -1
- package/src/observability/local/traceStore.ts +162 -11
- package/src/observability/observabilityApp.test.ts +13 -10
- package/src/observability/observabilityApp.ts +10 -1
- package/src/observability/questionFingerprint.test.ts +33 -0
- package/src/observability/questionFingerprint.ts +24 -0
- package/src/observability/routes/experiments.test.ts +37 -0
- package/src/observability/routes/experiments.ts +13 -1
- package/src/observability/routes/review.ts +3 -1
- package/src/observability/routes/reviewBatches.test.ts +209 -0
- package/src/observability/routes/reviewBatches.ts +138 -0
- package/src/observability/routes/status.test.ts +4 -4
- package/src/observability/routes/traces.test.ts +61 -0
- package/src/observability/routes/traces.ts +46 -4
- package/src/observability/types.ts +5 -2
- package/src/routes/routes.test.ts +101 -76
- package/src/service/aiService.test.ts +21 -0
- package/src/service/aiService.ts +4 -1
- package/src/service/chatTurn.ts +42 -33
- package/src/tests/bunSetup.ts +0 -54
- package/src/types/harness.ts +5 -2
- package/src/types/index.ts +8 -3
- package/src/types/observability.ts +86 -2
- package/dist/langfuseApp.d.ts +0 -9
- package/dist/langfuseApp.js +0 -53
- package/dist/langfuseCache.d.ts +0 -15
- package/dist/langfuseCache.js +0 -54
- package/dist/langfuseClient.d.ts +0 -6
- package/dist/langfuseClient.js +0 -34
- package/dist/langfusePrompts.d.ts +0 -12
- package/dist/langfusePrompts.js +0 -89
- package/dist/langfuseRoutesEvaluations.d.ts +0 -3
- package/dist/langfuseRoutesEvaluations.js +0 -32
- package/dist/langfuseRoutesMiddleware.d.ts +0 -2
- package/dist/langfuseRoutesMiddleware.js +0 -13
- package/dist/langfuseRoutesPlayground.d.ts +0 -2
- package/dist/langfuseRoutesPlayground.js +0 -36
- package/dist/langfuseRoutesPrompts.d.ts +0 -2
- package/dist/langfuseRoutesPrompts.js +0 -56
- package/dist/langfuseRoutesTraces.d.ts +0 -2
- package/dist/langfuseRoutesTraces.js +0 -30
- package/dist/langfuseTracing.d.ts +0 -4
- package/dist/langfuseTracing.js +0 -32
- package/dist/langfuseTypes.d.ts +0 -96
- package/dist/langfuseTypes.js +0 -2
- package/dist/langfuseVercelAi.d.ts +0 -12
- package/dist/langfuseVercelAi.js +0 -49
- package/src/isolated/langfuseClient.isolated.ts +0 -66
- package/src/langfuseApp.test.ts +0 -119
- package/src/langfuseApp.ts +0 -62
- package/src/langfuseCache.test.ts +0 -75
- package/src/langfuseCache.ts +0 -72
- package/src/langfuseClient.test.ts +0 -69
- package/src/langfuseClient.ts +0 -34
- package/src/langfusePrompts.test.ts +0 -188
- package/src/langfusePrompts.ts +0 -121
- package/src/langfuseRoutes.test.ts +0 -261
- package/src/langfuseRoutesEvaluations.ts +0 -48
- package/src/langfuseRoutesMiddleware.test.ts +0 -57
- package/src/langfuseRoutesMiddleware.ts +0 -12
- package/src/langfuseRoutesPlayground.ts +0 -50
- package/src/langfuseRoutesPrompts.ts +0 -87
- package/src/langfuseRoutesTraces.ts +0 -40
- package/src/langfuseTracing.test.ts +0 -72
- package/src/langfuseTracing.ts +0 -33
- package/src/langfuseTypes.ts +0 -103
- package/src/langfuseVercelAi.test.ts +0 -82
- package/src/langfuseVercelAi.ts +0 -72
- package/src/tests/fakeLangfusePrompt.ts +0 -10
|
@@ -6,12 +6,16 @@ import type {
|
|
|
6
6
|
ExperimentAggregates,
|
|
7
7
|
ExperimentEstimate,
|
|
8
8
|
ExperimentGateResult,
|
|
9
|
+
ExperimentKind,
|
|
9
10
|
ExperimentVersionResult,
|
|
11
|
+
ObsEvaluatorDocument,
|
|
10
12
|
ObsExperimentDocument,
|
|
11
13
|
ScoreThreshold,
|
|
12
14
|
} from "../../types/observability";
|
|
13
15
|
import {compileTemplate} from "../compileTemplate";
|
|
14
16
|
import {runEvaluator} from "../evaluate";
|
|
17
|
+
import type {AgreementAnswerSet, JudgeAgreement} from "../judgeAgreement";
|
|
18
|
+
import {questionSetFingerprint} from "../questionFingerprint";
|
|
15
19
|
import {SOP_DEFAULT_THRESHOLDS} from "../sopThresholds";
|
|
16
20
|
import type {
|
|
17
21
|
ModelPrice,
|
|
@@ -20,6 +24,15 @@ import type {
|
|
|
20
24
|
} from "../types";
|
|
21
25
|
import type {LocalDatasetStore} from "./datasetStore";
|
|
22
26
|
import type {LocalEvaluatorStore} from "./evaluatorStore";
|
|
27
|
+
import {loadCurrentHumanAnswers, loadDatasetItemSubjects} from "./humanAnswers";
|
|
28
|
+
import {
|
|
29
|
+
agreementForResults,
|
|
30
|
+
DEFAULT_JUDGE_CONCURRENCY,
|
|
31
|
+
type JudgeRunItemResult,
|
|
32
|
+
runJudgeAgainstHumans,
|
|
33
|
+
} from "./judgeRun";
|
|
34
|
+
import {registerObsDataset} from "./models/obsDataset";
|
|
35
|
+
import {registerObsEvaluator} from "./models/obsEvaluator";
|
|
23
36
|
import {registerObsExperiment} from "./models/obsExperiment";
|
|
24
37
|
import {registerObsExperimentItem} from "./models/obsExperimentItem";
|
|
25
38
|
import type {LocalPromptStore, MoveLabelResult} from "./promptStore";
|
|
@@ -28,6 +41,7 @@ export interface ExperimentCreateInput {
|
|
|
28
41
|
datasetId: string;
|
|
29
42
|
evaluatorIds: string[];
|
|
30
43
|
includeUnproofread?: boolean;
|
|
44
|
+
kind?: "prompt";
|
|
31
45
|
modelOverride?: string;
|
|
32
46
|
name: string;
|
|
33
47
|
promptName: string;
|
|
@@ -35,20 +49,49 @@ export interface ExperimentCreateInput {
|
|
|
35
49
|
versions: number[];
|
|
36
50
|
}
|
|
37
51
|
|
|
52
|
+
/**
|
|
53
|
+
* Judge calibration: run 2–3 versions of an llm-judge's prompt over dataset items humans already
|
|
54
|
+
* answered with `questionSetId`, and rank versions by agreement with those answers.
|
|
55
|
+
*/
|
|
56
|
+
export interface JudgeExperimentCreateInput {
|
|
57
|
+
datasetId: string;
|
|
58
|
+
judgeEvaluatorId: string;
|
|
59
|
+
kind: "judge";
|
|
60
|
+
modelOverride?: string;
|
|
61
|
+
name: string;
|
|
62
|
+
questionSetId: string;
|
|
63
|
+
/** Agreement rate (0–1) a version must reach to pass its gate. Defaults to 0.9. */
|
|
64
|
+
targetAgreement?: number;
|
|
65
|
+
/** Versions of the judge evaluator's `judgePromptName`. */
|
|
66
|
+
versions: number[];
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export const DEFAULT_TARGET_AGREEMENT = 0.9;
|
|
70
|
+
|
|
38
71
|
export interface ExperimentView {
|
|
39
72
|
backgroundTaskId?: string;
|
|
40
73
|
created: string;
|
|
41
74
|
datasetId: string;
|
|
75
|
+
datasetName?: string;
|
|
42
76
|
estimate?: ExperimentEstimate;
|
|
43
77
|
evaluatorIds: string[];
|
|
44
78
|
id: string;
|
|
45
79
|
includeUnproofread: boolean;
|
|
46
80
|
items: ExperimentItemView[];
|
|
81
|
+
judgeEvaluatorId?: string;
|
|
82
|
+
judgeName?: string;
|
|
83
|
+
kind: ExperimentKind;
|
|
47
84
|
modelOverride?: string;
|
|
48
85
|
name: string;
|
|
49
86
|
promptName: string;
|
|
87
|
+
questionSetId?: string;
|
|
88
|
+
questionSetName?: string;
|
|
89
|
+
questionsFingerprint?: string;
|
|
90
|
+
/** Judge experiments: true when the question set's questions changed since the run. */
|
|
91
|
+
questionsStale?: boolean;
|
|
50
92
|
results?: ExperimentAggregates;
|
|
51
93
|
status: "completed" | "failed" | "pending" | "running";
|
|
94
|
+
targetAgreement?: number;
|
|
52
95
|
thresholds: ScoreThreshold[];
|
|
53
96
|
updated: string;
|
|
54
97
|
versions: number[];
|
|
@@ -57,10 +100,30 @@ export interface ExperimentView {
|
|
|
57
100
|
export interface ExperimentItemView {
|
|
58
101
|
datasetItemId: string;
|
|
59
102
|
failed: boolean;
|
|
103
|
+
/** Judge experiments: the human answers each judge version is compared with. */
|
|
104
|
+
humanScores?: Record<string, boolean | number | string>;
|
|
60
105
|
id: string;
|
|
61
106
|
versionResults: Record<string, ExperimentVersionResult>;
|
|
62
107
|
}
|
|
63
108
|
|
|
109
|
+
interface ExperimentNames {
|
|
110
|
+
datasetName?: string;
|
|
111
|
+
judgeName?: string;
|
|
112
|
+
questionSetName?: string;
|
|
113
|
+
questionsStale?: boolean;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
const versionBody = (version: {system?: string; template?: string}): string => {
|
|
117
|
+
return version.system && version.system.length > 0 ? version.system : (version.template ?? "");
|
|
118
|
+
};
|
|
119
|
+
|
|
120
|
+
const toObjectId = (value: unknown, label: string): mongoose.Types.ObjectId => {
|
|
121
|
+
if (typeof value !== "string" || !mongoose.Types.ObjectId.isValid(value)) {
|
|
122
|
+
throw new APIError({status: 400, title: `${label} must be an object id`});
|
|
123
|
+
}
|
|
124
|
+
return new mongoose.Types.ObjectId(value);
|
|
125
|
+
};
|
|
126
|
+
|
|
64
127
|
export const EXPERIMENT_LIST_DEFAULT_LIMIT = 50;
|
|
65
128
|
const EXPERIMENT_LIST_MAX_LIMIT = 100;
|
|
66
129
|
|
|
@@ -195,17 +258,32 @@ export class LocalExperimentRunner {
|
|
|
195
258
|
|
|
196
259
|
async estimate(input: {
|
|
197
260
|
datasetId: string;
|
|
198
|
-
evaluatorIds
|
|
261
|
+
evaluatorIds?: string[];
|
|
199
262
|
includeUnproofread?: boolean;
|
|
263
|
+
kind?: ExperimentKind;
|
|
200
264
|
modelOverride?: string;
|
|
265
|
+
questionSetId?: string;
|
|
201
266
|
versions: number[];
|
|
202
267
|
}): Promise<ExperimentEstimate> {
|
|
203
|
-
const
|
|
204
|
-
input.
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
268
|
+
const itemCount =
|
|
269
|
+
input.kind === "judge"
|
|
270
|
+
? input.questionSetId
|
|
271
|
+
? (
|
|
272
|
+
await loadCurrentHumanAnswers({
|
|
273
|
+
datasetId: input.datasetId,
|
|
274
|
+
questionSetId: input.questionSetId,
|
|
275
|
+
})
|
|
276
|
+
).items.length
|
|
277
|
+
: 0
|
|
278
|
+
: (
|
|
279
|
+
await this.deps.datasetStore.listExperimentItems(
|
|
280
|
+
input.datasetId,
|
|
281
|
+
input.includeUnproofread ?? false
|
|
282
|
+
)
|
|
283
|
+
).length;
|
|
284
|
+
const generations = itemCount * input.versions.length;
|
|
285
|
+
// Judge experiments make one judge call per item and version and no separate generation.
|
|
286
|
+
const evaluatorCount = input.kind === "judge" ? 0 : (input.evaluatorIds?.length ?? 0);
|
|
209
287
|
const secondsPerGeneration = 2;
|
|
210
288
|
const judgeSeconds = generations * evaluatorCount * 0.5;
|
|
211
289
|
const modelId =
|
|
@@ -237,10 +315,23 @@ export class LocalExperimentRunner {
|
|
|
237
315
|
}
|
|
238
316
|
}
|
|
239
317
|
|
|
240
|
-
async create(input: ExperimentCreateInput): Promise<ExperimentView> {
|
|
241
|
-
|
|
318
|
+
async create(input: ExperimentCreateInput | JudgeExperimentCreateInput): Promise<ExperimentView> {
|
|
319
|
+
const kind: unknown = input.kind ?? "prompt";
|
|
320
|
+
if (kind !== "prompt" && kind !== "judge") {
|
|
321
|
+
throw new APIError({status: 400, title: "kind must be prompt or judge"});
|
|
322
|
+
}
|
|
323
|
+
if (input.kind === "judge") {
|
|
324
|
+
return this.createJudge(input);
|
|
325
|
+
}
|
|
326
|
+
if (!Array.isArray(input.versions) || input.versions.length < 2 || input.versions.length > 3) {
|
|
242
327
|
throw new APIError({status: 400, title: "experiments must compare 2–3 prompt versions"});
|
|
243
328
|
}
|
|
329
|
+
if (input.thresholds?.some((threshold) => threshold.aggregate === "agreement")) {
|
|
330
|
+
throw new APIError({
|
|
331
|
+
status: 400,
|
|
332
|
+
title: "agreement gates belong to judge experiments (kind: judge)",
|
|
333
|
+
});
|
|
334
|
+
}
|
|
244
335
|
await this.assertAutomaticEvaluators(input.evaluatorIds);
|
|
245
336
|
this.resolveAiClient(input.modelOverride);
|
|
246
337
|
await this.deps.datasetStore.get(input.datasetId);
|
|
@@ -258,6 +349,7 @@ export class LocalExperimentRunner {
|
|
|
258
349
|
return new mongoose.Types.ObjectId(id);
|
|
259
350
|
}),
|
|
260
351
|
includeUnproofread: input.includeUnproofread ?? false,
|
|
352
|
+
kind: "prompt",
|
|
261
353
|
modelOverride: input.modelOverride,
|
|
262
354
|
name: input.name,
|
|
263
355
|
promptName: input.promptName,
|
|
@@ -265,6 +357,128 @@ export class LocalExperimentRunner {
|
|
|
265
357
|
thresholds: input.thresholds ?? SOP_DEFAULT_THRESHOLDS,
|
|
266
358
|
versions: input.versions,
|
|
267
359
|
});
|
|
360
|
+
return this.startRun(experiment);
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
private async createJudge(input: JudgeExperimentCreateInput): Promise<ExperimentView> {
|
|
364
|
+
if (!input.name) {
|
|
365
|
+
throw new APIError({status: 400, title: "name is required"});
|
|
366
|
+
}
|
|
367
|
+
if (
|
|
368
|
+
!Array.isArray(input.versions) ||
|
|
369
|
+
input.versions.length < 2 ||
|
|
370
|
+
input.versions.length > 3 ||
|
|
371
|
+
!input.versions.every((version) => Number.isInteger(version))
|
|
372
|
+
) {
|
|
373
|
+
throw new APIError({
|
|
374
|
+
status: 400,
|
|
375
|
+
title: "judge experiments must compare 2–3 judge prompt versions",
|
|
376
|
+
});
|
|
377
|
+
}
|
|
378
|
+
if (new Set(input.versions).size !== input.versions.length) {
|
|
379
|
+
throw new APIError({status: 400, title: "judge prompt versions must be different"});
|
|
380
|
+
}
|
|
381
|
+
const targetAgreement = input.targetAgreement ?? DEFAULT_TARGET_AGREEMENT;
|
|
382
|
+
if (typeof targetAgreement !== "number" || targetAgreement <= 0 || targetAgreement > 1) {
|
|
383
|
+
throw new APIError({
|
|
384
|
+
status: 400,
|
|
385
|
+
title: "targetAgreement must be a number above 0 and at most 1",
|
|
386
|
+
});
|
|
387
|
+
}
|
|
388
|
+
const questionSet = await registerObsEvaluator().findOneOrNone({
|
|
389
|
+
_id: toObjectId(input.questionSetId, "questionSetId"),
|
|
390
|
+
});
|
|
391
|
+
if (!questionSet) {
|
|
392
|
+
throw new APIError({status: 404, title: "Unknown question set"});
|
|
393
|
+
}
|
|
394
|
+
if (questionSet.type !== "human") {
|
|
395
|
+
throw new APIError({status: 400, title: "questionSetId must be a human question set"});
|
|
396
|
+
}
|
|
397
|
+
const judge = await registerObsEvaluator().findOneOrNone({
|
|
398
|
+
_id: toObjectId(input.judgeEvaluatorId, "judgeEvaluatorId"),
|
|
399
|
+
});
|
|
400
|
+
if (!judge) {
|
|
401
|
+
throw new APIError({status: 404, title: "Unknown judge evaluator"});
|
|
402
|
+
}
|
|
403
|
+
if (judge.type !== "llm-judge" || !judge.judgePromptName) {
|
|
404
|
+
throw new APIError({
|
|
405
|
+
status: 400,
|
|
406
|
+
title: "judgeEvaluatorId must be an llm-judge evaluator with a judge prompt",
|
|
407
|
+
});
|
|
408
|
+
}
|
|
409
|
+
const questionKeys = new Set(questionSet.dimensions.map((row) => row.key));
|
|
410
|
+
if (!judge.dimensions.some((row) => questionKeys.has(row.key))) {
|
|
411
|
+
throw new APIError({
|
|
412
|
+
detail: `Questions: ${[...questionKeys].join(", ")}. Judge scores: ${judge.dimensions
|
|
413
|
+
.map((row) => row.key)
|
|
414
|
+
.join(", ")}.`,
|
|
415
|
+
status: 400,
|
|
416
|
+
title: "Judge answers none of the question set's questions",
|
|
417
|
+
});
|
|
418
|
+
}
|
|
419
|
+
const dataset = await this.deps.datasetStore.get(
|
|
420
|
+
toObjectId(input.datasetId, "datasetId").toString()
|
|
421
|
+
);
|
|
422
|
+
const current = await loadCurrentHumanAnswers({
|
|
423
|
+
datasetId: dataset.id,
|
|
424
|
+
questionSetId: String(questionSet._id),
|
|
425
|
+
});
|
|
426
|
+
if (current.items.length === 0) {
|
|
427
|
+
throw new APIError({
|
|
428
|
+
detail: `Create a review batch from dataset "${dataset.name}" with "${questionSet.name}" and answer it first.`,
|
|
429
|
+
status: 400,
|
|
430
|
+
title: "Dataset has no current human answers for this question set",
|
|
431
|
+
});
|
|
432
|
+
}
|
|
433
|
+
for (const version of input.versions) {
|
|
434
|
+
const promptVersion = await this.deps.promptStore.getVersionByNumber(
|
|
435
|
+
judge.judgePromptName,
|
|
436
|
+
version
|
|
437
|
+
);
|
|
438
|
+
if (!promptVersion) {
|
|
439
|
+
throw new APIError({
|
|
440
|
+
detail: `Judge prompt "${judge.judgePromptName}" has no version ${version}.`,
|
|
441
|
+
status: 400,
|
|
442
|
+
title: "Unknown judge prompt version",
|
|
443
|
+
});
|
|
444
|
+
}
|
|
445
|
+
if (!promptVersion.outputSchema) {
|
|
446
|
+
throw new APIError({
|
|
447
|
+
detail: `Judge prompt "${judge.judgePromptName}" v${version} has no output schema.`,
|
|
448
|
+
status: 400,
|
|
449
|
+
title: "Judge prompt version has no output schema",
|
|
450
|
+
});
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
this.resolveAiClient(input.modelOverride);
|
|
454
|
+
const estimate = await this.estimate({
|
|
455
|
+
datasetId: dataset.id,
|
|
456
|
+
kind: "judge",
|
|
457
|
+
modelOverride: input.modelOverride,
|
|
458
|
+
questionSetId: String(questionSet._id),
|
|
459
|
+
versions: input.versions,
|
|
460
|
+
});
|
|
461
|
+
const experiment = await registerObsExperiment().create({
|
|
462
|
+
datasetId: new mongoose.Types.ObjectId(dataset.id),
|
|
463
|
+
estimate,
|
|
464
|
+
evaluatorIds: [judge._id],
|
|
465
|
+
includeUnproofread: true,
|
|
466
|
+
judgeEvaluatorId: judge._id,
|
|
467
|
+
kind: "judge",
|
|
468
|
+
modelOverride: input.modelOverride,
|
|
469
|
+
name: input.name,
|
|
470
|
+
promptName: judge.judgePromptName,
|
|
471
|
+
questionSetId: questionSet._id,
|
|
472
|
+
questionsFingerprint: current.fingerprint,
|
|
473
|
+
status: "pending",
|
|
474
|
+
targetAgreement,
|
|
475
|
+
thresholds: [],
|
|
476
|
+
versions: input.versions,
|
|
477
|
+
});
|
|
478
|
+
return this.startRun(experiment);
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
private async startRun(experiment: ObsExperimentDocument): Promise<ExperimentView> {
|
|
268
482
|
const runner = this.deps.backgroundTaskRunner ?? defaultBackgroundTaskRunner;
|
|
269
483
|
const {taskId} = await runner.enqueue({
|
|
270
484
|
execute: async () => {
|
|
@@ -272,13 +486,21 @@ export class LocalExperimentRunner {
|
|
|
272
486
|
},
|
|
273
487
|
taskType: `obs-experiment-${String(experiment._id)}`,
|
|
274
488
|
});
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
489
|
+
const ObsExperiment = registerObsExperiment();
|
|
490
|
+
await ObsExperiment.updateOne(
|
|
491
|
+
{_id: experiment._id},
|
|
492
|
+
{$set: {backgroundTaskId: new mongoose.Types.ObjectId(taskId)}}
|
|
493
|
+
);
|
|
494
|
+
// A runner that executes inline has already finished; only a still-pending run is "running".
|
|
495
|
+
await ObsExperiment.updateOne(
|
|
496
|
+
{_id: experiment._id, status: "pending"},
|
|
497
|
+
{$set: {status: "running"}}
|
|
498
|
+
);
|
|
278
499
|
return this.get(String(experiment._id));
|
|
279
500
|
}
|
|
280
501
|
|
|
281
502
|
async list(params?: {
|
|
503
|
+
kind?: ExperimentKind;
|
|
282
504
|
limit?: number;
|
|
283
505
|
page?: number;
|
|
284
506
|
promptName?: string;
|
|
@@ -292,15 +514,22 @@ export class LocalExperimentRunner {
|
|
|
292
514
|
if (params?.promptName) {
|
|
293
515
|
filter.promptName = params.promptName;
|
|
294
516
|
}
|
|
517
|
+
if (params?.kind === "judge") {
|
|
518
|
+
filter.kind = "judge";
|
|
519
|
+
} else if (params?.kind === "prompt") {
|
|
520
|
+
// Experiments saved before `kind` existed are prompt experiments.
|
|
521
|
+
filter.kind = {$ne: "judge"};
|
|
522
|
+
}
|
|
295
523
|
const ObsExperiment = registerObsExperiment();
|
|
296
524
|
const total = await ObsExperiment.countDocuments(filter);
|
|
297
525
|
const rows = await ObsExperiment.find(filter)
|
|
298
526
|
.sort({created: -1})
|
|
299
527
|
.skip((page - 1) * limit)
|
|
300
528
|
.limit(limit);
|
|
529
|
+
const names = await this.loadNames(rows);
|
|
301
530
|
return {
|
|
302
531
|
data: rows.map((row) => {
|
|
303
|
-
return this.toSummaryView(row);
|
|
532
|
+
return this.toSummaryView(row, names.get(String(row._id)));
|
|
304
533
|
}),
|
|
305
534
|
meta: {limit, page, total},
|
|
306
535
|
};
|
|
@@ -318,12 +547,14 @@ export class LocalExperimentRunner {
|
|
|
318
547
|
}
|
|
319
548
|
return left.created.getTime() - right.created.getTime();
|
|
320
549
|
});
|
|
550
|
+
const names = await this.loadNames([experiment]);
|
|
321
551
|
return {
|
|
322
|
-
...this.toSummaryView(experiment),
|
|
552
|
+
...this.toSummaryView(experiment, names.get(String(experiment._id))),
|
|
323
553
|
items: sortedItems.map((row) => {
|
|
324
554
|
return {
|
|
325
555
|
datasetItemId: String(row.datasetItemId),
|
|
326
556
|
failed: row.failed,
|
|
557
|
+
humanScores: row.humanScores,
|
|
327
558
|
id: String(row._id),
|
|
328
559
|
versionResults: row.versionResults,
|
|
329
560
|
};
|
|
@@ -331,23 +562,81 @@ export class LocalExperimentRunner {
|
|
|
331
562
|
};
|
|
332
563
|
}
|
|
333
564
|
|
|
334
|
-
|
|
565
|
+
// Dataset, question-set, and judge names for display, plus question-set staleness.
|
|
566
|
+
private async loadNames(
|
|
567
|
+
experiments: ObsExperimentDocument[]
|
|
568
|
+
): Promise<Map<string, ExperimentNames>> {
|
|
569
|
+
const datasetIds = experiments.map((row) => row.datasetId);
|
|
570
|
+
const evaluatorIds = experiments.flatMap((row) => {
|
|
571
|
+
return [row.questionSetId, row.judgeEvaluatorId].filter((id): id is mongoose.Types.ObjectId =>
|
|
572
|
+
Boolean(id)
|
|
573
|
+
);
|
|
574
|
+
});
|
|
575
|
+
const datasets = datasetIds.length
|
|
576
|
+
? await registerObsDataset().find({_id: {$in: datasetIds}})
|
|
577
|
+
: [];
|
|
578
|
+
const evaluators = evaluatorIds.length
|
|
579
|
+
? await registerObsEvaluator().find({_id: {$in: evaluatorIds}})
|
|
580
|
+
: [];
|
|
581
|
+
const datasetNameById = new Map(datasets.map((row) => [String(row._id), row.name]));
|
|
582
|
+
const evaluatorById = new Map<string, ObsEvaluatorDocument>(
|
|
583
|
+
evaluators.map((row) => [String(row._id), row])
|
|
584
|
+
);
|
|
585
|
+
return new Map(
|
|
586
|
+
experiments.map((row) => {
|
|
587
|
+
const questionSet = row.questionSetId
|
|
588
|
+
? evaluatorById.get(String(row.questionSetId))
|
|
589
|
+
: undefined;
|
|
590
|
+
const judge = row.judgeEvaluatorId
|
|
591
|
+
? evaluatorById.get(String(row.judgeEvaluatorId))
|
|
592
|
+
: undefined;
|
|
593
|
+
return [
|
|
594
|
+
String(row._id),
|
|
595
|
+
{
|
|
596
|
+
datasetName: datasetNameById.get(String(row.datasetId)),
|
|
597
|
+
judgeName: judge?.name,
|
|
598
|
+
questionSetName: questionSet?.name,
|
|
599
|
+
questionsStale:
|
|
600
|
+
questionSet && row.questionsFingerprint
|
|
601
|
+
? questionSetFingerprint(questionSet.dimensions) !== row.questionsFingerprint
|
|
602
|
+
: undefined,
|
|
603
|
+
},
|
|
604
|
+
];
|
|
605
|
+
})
|
|
606
|
+
);
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
private toSummaryView(
|
|
610
|
+
experiment: ObsExperimentDocument,
|
|
611
|
+
names?: ExperimentNames
|
|
612
|
+
): ExperimentView {
|
|
335
613
|
return {
|
|
336
614
|
backgroundTaskId: experiment.backgroundTaskId
|
|
337
615
|
? String(experiment.backgroundTaskId)
|
|
338
616
|
: undefined,
|
|
339
617
|
created: DateTime.fromJSDate(experiment.created).toUTC().toISO() ?? "",
|
|
340
618
|
datasetId: String(experiment.datasetId),
|
|
619
|
+
datasetName: names?.datasetName,
|
|
341
620
|
estimate: experiment.estimate,
|
|
342
621
|
evaluatorIds: experiment.evaluatorIds.map((value) => String(value)),
|
|
343
622
|
id: String(experiment._id),
|
|
344
623
|
includeUnproofread: experiment.includeUnproofread,
|
|
345
624
|
items: [],
|
|
625
|
+
judgeEvaluatorId: experiment.judgeEvaluatorId
|
|
626
|
+
? String(experiment.judgeEvaluatorId)
|
|
627
|
+
: undefined,
|
|
628
|
+
judgeName: names?.judgeName,
|
|
629
|
+
kind: experiment.kind ?? "prompt",
|
|
346
630
|
modelOverride: experiment.modelOverride,
|
|
347
631
|
name: experiment.name,
|
|
348
632
|
promptName: experiment.promptName,
|
|
633
|
+
questionSetId: experiment.questionSetId ? String(experiment.questionSetId) : undefined,
|
|
634
|
+
questionSetName: names?.questionSetName,
|
|
635
|
+
questionsFingerprint: experiment.questionsFingerprint,
|
|
636
|
+
questionsStale: names?.questionsStale,
|
|
349
637
|
results: experiment.results,
|
|
350
638
|
status: experiment.status,
|
|
639
|
+
targetAgreement: experiment.targetAgreement,
|
|
351
640
|
thresholds: experiment.thresholds,
|
|
352
641
|
updated: DateTime.fromJSDate(experiment.updated).toUTC().toISO() ?? "",
|
|
353
642
|
versions: experiment.versions,
|
|
@@ -386,6 +675,10 @@ export class LocalExperimentRunner {
|
|
|
386
675
|
throw new APIError({status: 404, title: `Unknown experiment "${id}"`});
|
|
387
676
|
}
|
|
388
677
|
const aiService = this.resolveAiClient(experiment.modelOverride);
|
|
678
|
+
if (experiment.kind === "judge") {
|
|
679
|
+
await this.executeJudgeExperiment(experiment, aiService);
|
|
680
|
+
return;
|
|
681
|
+
}
|
|
389
682
|
const datasetItems = await this.deps.datasetStore.listExperimentItems(
|
|
390
683
|
String(experiment.datasetId),
|
|
391
684
|
experiment.includeUnproofread
|
|
@@ -525,6 +818,193 @@ export class LocalExperimentRunner {
|
|
|
525
818
|
await experiment.save();
|
|
526
819
|
}
|
|
527
820
|
|
|
821
|
+
private async executeJudgeExperiment(
|
|
822
|
+
experiment: ObsExperimentDocument,
|
|
823
|
+
aiService: ObservabilityGenerateClient
|
|
824
|
+
): Promise<void> {
|
|
825
|
+
const judge = experiment.judgeEvaluatorId
|
|
826
|
+
? await registerObsEvaluator().findOneOrNone({_id: experiment.judgeEvaluatorId})
|
|
827
|
+
: undefined;
|
|
828
|
+
if (!judge?.judgePromptName || !experiment.questionSetId) {
|
|
829
|
+
throw new APIError({
|
|
830
|
+
status: 404,
|
|
831
|
+
title: "Judge experiment is missing its judge or question set",
|
|
832
|
+
});
|
|
833
|
+
}
|
|
834
|
+
const judgePromptName = judge.judgePromptName;
|
|
835
|
+
const current = await loadCurrentHumanAnswers({
|
|
836
|
+
datasetId: experiment.datasetId,
|
|
837
|
+
questionSetId: experiment.questionSetId,
|
|
838
|
+
});
|
|
839
|
+
// Judges see exactly what reviewers saw: the source trace, else the item's input and expected output.
|
|
840
|
+
const subjects = await loadDatasetItemSubjects(current.items.map((row) => row.item));
|
|
841
|
+
const versionPrompts = new Map<number, {schema: Record<string, unknown>; system: string}>();
|
|
842
|
+
for (const versionNumber of experiment.versions) {
|
|
843
|
+
const version = await this.deps.promptStore.getVersionByNumber(
|
|
844
|
+
judgePromptName,
|
|
845
|
+
versionNumber
|
|
846
|
+
);
|
|
847
|
+
if (!version?.outputSchema) {
|
|
848
|
+
throw new APIError({
|
|
849
|
+
status: 404,
|
|
850
|
+
title: `Judge prompt "${judgePromptName}" v${versionNumber} is missing or has no output schema`,
|
|
851
|
+
});
|
|
852
|
+
}
|
|
853
|
+
versionPrompts.set(versionNumber, {
|
|
854
|
+
schema: version.outputSchema,
|
|
855
|
+
system: versionBody(version),
|
|
856
|
+
});
|
|
857
|
+
}
|
|
858
|
+
const judgeId = String(judge._id);
|
|
859
|
+
const subjectRows = current.items.map(({item}) => {
|
|
860
|
+
const datasetItemId = String(item._id);
|
|
861
|
+
const subject = subjects.get(datasetItemId);
|
|
862
|
+
return {
|
|
863
|
+
datasetItemId,
|
|
864
|
+
input: subject?.input,
|
|
865
|
+
output: subject?.output,
|
|
866
|
+
...(subject?.traceId ? {traceId: subject.traceId} : {}),
|
|
867
|
+
};
|
|
868
|
+
});
|
|
869
|
+
const humanAnswers: AgreementAnswerSet[] = current.items.map(({answer}, index) => {
|
|
870
|
+
const {datasetItemId, traceId} = subjectRows[index] ?? {};
|
|
871
|
+
return {datasetItemId, scores: answer.scores, ...(traceId ? {traceId} : {})};
|
|
872
|
+
});
|
|
873
|
+
const total = current.items.length;
|
|
874
|
+
const judgeForRun = {
|
|
875
|
+
dimensions: judge.dimensions,
|
|
876
|
+
judgePromptName,
|
|
877
|
+
name: judge.name,
|
|
878
|
+
type: judge.type,
|
|
879
|
+
};
|
|
880
|
+
const versionRuns = new Map<number, JudgeRunItemResult[]>(
|
|
881
|
+
experiment.versions.map((version) => [version, []])
|
|
882
|
+
);
|
|
883
|
+
const lowConfidenceItemIds: string[] = [];
|
|
884
|
+
const outlierItemIds: string[] = [];
|
|
885
|
+
const aggregate = (completed: number): ExperimentAggregates => {
|
|
886
|
+
const agreement: Record<string, JudgeAgreement> = {};
|
|
887
|
+
for (const version of experiment.versions) {
|
|
888
|
+
agreement[String(version)] = agreementForResults({
|
|
889
|
+
dimensions: current.questionSet.dimensions,
|
|
890
|
+
humanAnswers,
|
|
891
|
+
judge: judgeForRun,
|
|
892
|
+
results: versionRuns.get(version) ?? [],
|
|
893
|
+
});
|
|
894
|
+
}
|
|
895
|
+
const target = experiment.targetAgreement ?? DEFAULT_TARGET_AGREEMENT;
|
|
896
|
+
return {
|
|
897
|
+
agreement,
|
|
898
|
+
gates: experiment.versions.map((version) => {
|
|
899
|
+
const actual = agreement[String(version)]?.rate;
|
|
900
|
+
return {
|
|
901
|
+
actual,
|
|
902
|
+
aggregate: "agreement" as const,
|
|
903
|
+
dimension: "agreement",
|
|
904
|
+
evaluatorName: judge.name,
|
|
905
|
+
op: "gte" as const,
|
|
906
|
+
passed: actual !== undefined && actual >= target,
|
|
907
|
+
value: target,
|
|
908
|
+
version,
|
|
909
|
+
};
|
|
910
|
+
}),
|
|
911
|
+
lowConfidenceItemIds: [...new Set(lowConfidenceItemIds)],
|
|
912
|
+
outlierItemIds: [...new Set(outlierItemIds)],
|
|
913
|
+
progress: {completed, total},
|
|
914
|
+
totalCostUsd: experiment.estimate?.costUsd,
|
|
915
|
+
};
|
|
916
|
+
};
|
|
917
|
+
|
|
918
|
+
// Every version runs through the shared judge pipeline at once; an item's row is written
|
|
919
|
+
// (and progress saved) as soon as all versions have judged it. Writes are serialized.
|
|
920
|
+
const runnable = experiment.versions.filter((version) => versionPrompts.has(version));
|
|
921
|
+
const pending = new Map<number, Map<number, JudgeRunItemResult>>();
|
|
922
|
+
let completed = 0;
|
|
923
|
+
let writes: Promise<void> = Promise.resolve();
|
|
924
|
+
const writeRow = async (
|
|
925
|
+
index: number,
|
|
926
|
+
byVersion: Map<number, JudgeRunItemResult>
|
|
927
|
+
): Promise<void> => {
|
|
928
|
+
const row = current.items[index];
|
|
929
|
+
if (!row) {
|
|
930
|
+
return;
|
|
931
|
+
}
|
|
932
|
+
const datasetItemId = String(row.item._id);
|
|
933
|
+
const versionResults: Record<string, ExperimentVersionResult> = {};
|
|
934
|
+
let rowFailed = false;
|
|
935
|
+
for (const versionNumber of experiment.versions) {
|
|
936
|
+
const result = byVersion.get(versionNumber);
|
|
937
|
+
if (!result) {
|
|
938
|
+
continue;
|
|
939
|
+
}
|
|
940
|
+
versionResults[String(versionNumber)] = {
|
|
941
|
+
evaluatorScores: {
|
|
942
|
+
[judgeId]: {confidence: result.confidence, error: result.error, scores: result.scores},
|
|
943
|
+
},
|
|
944
|
+
};
|
|
945
|
+
versionRuns.get(versionNumber)?.push(result);
|
|
946
|
+
if (result.error) {
|
|
947
|
+
rowFailed = true;
|
|
948
|
+
outlierItemIds.push(datasetItemId);
|
|
949
|
+
}
|
|
950
|
+
if (result.confidence !== undefined && result.confidence < judge.confidenceAlertBelow) {
|
|
951
|
+
lowConfidenceItemIds.push(datasetItemId);
|
|
952
|
+
}
|
|
953
|
+
// A row fails when any version disagrees with the reviewers on any compared question.
|
|
954
|
+
if (result.disagrees) {
|
|
955
|
+
rowFailed = true;
|
|
956
|
+
}
|
|
957
|
+
}
|
|
958
|
+
await registerObsExperimentItem().create({
|
|
959
|
+
datasetItemId: row.item._id,
|
|
960
|
+
experimentId: experiment._id,
|
|
961
|
+
failed: rowFailed,
|
|
962
|
+
humanScores: row.answer.scores,
|
|
963
|
+
versionResults,
|
|
964
|
+
});
|
|
965
|
+
completed += 1;
|
|
966
|
+
experiment.results = aggregate(completed);
|
|
967
|
+
await experiment.save();
|
|
968
|
+
};
|
|
969
|
+
const concurrency = Math.max(
|
|
970
|
+
1,
|
|
971
|
+
Math.ceil(DEFAULT_JUDGE_CONCURRENCY / Math.max(runnable.length, 1))
|
|
972
|
+
);
|
|
973
|
+
await Promise.all(
|
|
974
|
+
runnable.map(async (versionNumber) => {
|
|
975
|
+
const prompt = versionPrompts.get(versionNumber);
|
|
976
|
+
if (!prompt) {
|
|
977
|
+
return;
|
|
978
|
+
}
|
|
979
|
+
await runJudgeAgainstHumans({
|
|
980
|
+
ai: aiService,
|
|
981
|
+
concurrency,
|
|
982
|
+
dimensions: current.questionSet.dimensions,
|
|
983
|
+
humanAnswers,
|
|
984
|
+
judge: judgeForRun,
|
|
985
|
+
onResult: async (result) => {
|
|
986
|
+
const byVersion = pending.get(result.index) ?? new Map<number, JudgeRunItemResult>();
|
|
987
|
+
byVersion.set(versionNumber, result);
|
|
988
|
+
pending.set(result.index, byVersion);
|
|
989
|
+
if (byVersion.size < runnable.length) {
|
|
990
|
+
return;
|
|
991
|
+
}
|
|
992
|
+
pending.delete(result.index);
|
|
993
|
+
writes = writes.then(() => writeRow(result.index, byVersion));
|
|
994
|
+
await writes;
|
|
995
|
+
},
|
|
996
|
+
prompt: {outputSchema: prompt.schema, system: prompt.system},
|
|
997
|
+
subjects: subjectRows,
|
|
998
|
+
});
|
|
999
|
+
})
|
|
1000
|
+
);
|
|
1001
|
+
await writes;
|
|
1002
|
+
experiment.questionsFingerprint = current.fingerprint;
|
|
1003
|
+
experiment.results = aggregate(total);
|
|
1004
|
+
experiment.status = "completed";
|
|
1005
|
+
await experiment.save();
|
|
1006
|
+
}
|
|
1007
|
+
|
|
528
1008
|
private computeGates(
|
|
529
1009
|
versions: number[],
|
|
530
1010
|
thresholds: ScoreThreshold[],
|