@terreno/ai 58.0.0-beta.3 → 58.0.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +99 -0
- package/README.md +3 -3
- package/dist/harness/commit.d.ts +4 -3
- package/dist/harness/commit.js +23 -4
- package/dist/index.d.ts +6 -9
- package/dist/index.js +14 -25
- package/dist/observability/adminScreens.js +5 -1
- package/dist/observability/evaluate.d.ts +8 -1
- package/dist/observability/evaluate.js +10 -2
- package/dist/observability/judgeAgreement.d.ts +53 -0
- package/dist/observability/judgeAgreement.js +123 -0
- package/dist/observability/local/datasetStore.d.ts +5 -0
- package/dist/observability/local/datasetStore.js +7 -2
- package/dist/observability/local/evaluatorStore.d.ts +17 -0
- package/dist/observability/local/evaluatorStore.js +73 -2
- package/dist/observability/local/experimentRunner.d.ts +40 -3
- package/dist/observability/local/experimentRunner.js +390 -11
- package/dist/observability/local/humanAnswers.d.ts +57 -0
- package/dist/observability/local/humanAnswers.js +174 -0
- package/dist/observability/local/judgeRun.d.ts +56 -0
- package/dist/observability/local/judgeRun.js +111 -0
- package/dist/observability/local/localPlugin.js +4 -0
- package/dist/observability/local/models/obsDatasetItem.js +4 -0
- package/dist/observability/local/models/obsEvaluator.js +4 -0
- package/dist/observability/local/models/obsExperiment.js +28 -0
- package/dist/observability/local/models/obsExperimentItem.js +4 -0
- package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
- package/dist/observability/local/models/obsJudgeRun.js +60 -0
- package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
- package/dist/observability/local/models/obsReviewBatch.js +48 -0
- package/dist/observability/local/models/obsReviewItem.js +10 -2
- package/dist/observability/local/models/obsTrace.js +4 -0
- package/dist/observability/local/promptReferences.d.ts +31 -0
- package/dist/observability/local/promptReferences.js +114 -0
- package/dist/observability/local/promptStore.d.ts +55 -19
- package/dist/observability/local/promptStore.js +171 -7
- package/dist/observability/local/reviewBatchStore.d.ts +106 -0
- package/dist/observability/local/reviewBatchStore.js +517 -0
- package/dist/observability/local/reviewStore.d.ts +24 -3
- package/dist/observability/local/reviewStore.js +98 -22
- package/dist/observability/local/traceStore.d.ts +14 -1
- package/dist/observability/local/traceStore.js +144 -11
- package/dist/observability/observabilityApp.js +10 -1
- package/dist/observability/questionFingerprint.d.ts +7 -0
- package/dist/observability/questionFingerprint.js +23 -0
- package/dist/observability/routes/experiments.js +12 -1
- package/dist/observability/routes/review.js +3 -1
- package/dist/observability/routes/reviewBatches.d.ts +11 -0
- package/dist/observability/routes/reviewBatches.js +78 -0
- package/dist/observability/routes/traces.d.ts +1 -1
- package/dist/observability/routes/traces.js +34 -2
- package/dist/observability/types.d.ts +10 -1
- package/dist/service/aiService.js +4 -1
- package/dist/service/chatTurn.js +38 -31
- package/dist/types/harness.d.ts +5 -2
- package/dist/types/index.d.ts +8 -3
- package/dist/types/observability.d.ts +74 -2
- package/package.json +8 -24
- package/src/harness/agentLoop.test.ts +0 -3
- package/src/harness/agentRecovery.test.ts +0 -3
- package/src/harness/commit.ts +24 -4
- package/src/harness/extensions.test.ts +0 -3
- package/src/harness/subagent.test.ts +27 -3
- package/src/index.test.ts +0 -17
- package/src/index.ts +23 -26
- package/src/isolated/gptStream.isolated.ts +0 -8
- package/src/observability/adminScreens.test.ts +7 -0
- package/src/observability/adminScreens.ts +6 -1
- package/src/observability/evaluate.ts +12 -2
- package/src/observability/judgeAgreement.test.ts +111 -0
- package/src/observability/judgeAgreement.ts +184 -0
- package/src/observability/local/datasetStore.ts +30 -11
- package/src/observability/local/evaluatorStore.ts +96 -2
- package/src/observability/local/experimentRunner.ts +495 -15
- package/src/observability/local/humanAnswers.ts +233 -0
- package/src/observability/local/judgePipeline.test.ts +444 -0
- package/src/observability/local/judgeRun.test.ts +173 -0
- package/src/observability/local/judgeRun.ts +184 -0
- package/src/observability/local/localPlugin.ts +4 -0
- package/src/observability/local/models/obsDatasetItem.ts +5 -0
- package/src/observability/local/models/obsEvaluator.ts +4 -0
- package/src/observability/local/models/obsExperiment.ts +29 -0
- package/src/observability/local/models/obsExperimentItem.ts +4 -0
- package/src/observability/local/models/obsJudgeRun.ts +61 -0
- package/src/observability/local/models/obsReviewBatch.ts +52 -0
- package/src/observability/local/models/obsReviewItem.ts +11 -2
- package/src/observability/local/models/obsTrace.ts +5 -0
- package/src/observability/local/obsModels.test.ts +13 -0
- package/src/observability/local/promptReferences.test.ts +128 -0
- package/src/observability/local/promptReferences.ts +149 -0
- package/src/observability/local/promptStore.test.ts +291 -0
- package/src/observability/local/promptStore.ts +243 -32
- package/src/observability/local/reviewBatchStore.test.ts +251 -0
- package/src/observability/local/reviewBatchStore.ts +656 -0
- package/src/observability/local/reviewStore.ts +122 -25
- package/src/observability/local/traceStore.test.ts +139 -1
- package/src/observability/local/traceStore.ts +162 -11
- package/src/observability/observabilityApp.test.ts +13 -10
- package/src/observability/observabilityApp.ts +10 -1
- package/src/observability/questionFingerprint.test.ts +33 -0
- package/src/observability/questionFingerprint.ts +24 -0
- package/src/observability/routes/experiments.test.ts +37 -0
- package/src/observability/routes/experiments.ts +13 -1
- package/src/observability/routes/review.ts +3 -1
- package/src/observability/routes/reviewBatches.test.ts +209 -0
- package/src/observability/routes/reviewBatches.ts +138 -0
- package/src/observability/routes/status.test.ts +4 -4
- package/src/observability/routes/traces.test.ts +61 -0
- package/src/observability/routes/traces.ts +46 -4
- package/src/observability/types.ts +5 -2
- package/src/routes/routes.test.ts +101 -76
- package/src/service/aiService.test.ts +21 -0
- package/src/service/aiService.ts +4 -1
- package/src/service/chatTurn.ts +42 -33
- package/src/tests/bunSetup.ts +0 -54
- package/src/types/harness.ts +5 -2
- package/src/types/index.ts +8 -3
- package/src/types/observability.ts +86 -2
- package/dist/langfuseApp.d.ts +0 -9
- package/dist/langfuseApp.js +0 -53
- package/dist/langfuseCache.d.ts +0 -15
- package/dist/langfuseCache.js +0 -54
- package/dist/langfuseClient.d.ts +0 -6
- package/dist/langfuseClient.js +0 -34
- package/dist/langfusePrompts.d.ts +0 -12
- package/dist/langfusePrompts.js +0 -89
- package/dist/langfuseRoutesEvaluations.d.ts +0 -3
- package/dist/langfuseRoutesEvaluations.js +0 -32
- package/dist/langfuseRoutesMiddleware.d.ts +0 -2
- package/dist/langfuseRoutesMiddleware.js +0 -13
- package/dist/langfuseRoutesPlayground.d.ts +0 -2
- package/dist/langfuseRoutesPlayground.js +0 -36
- package/dist/langfuseRoutesPrompts.d.ts +0 -2
- package/dist/langfuseRoutesPrompts.js +0 -56
- package/dist/langfuseRoutesTraces.d.ts +0 -2
- package/dist/langfuseRoutesTraces.js +0 -30
- package/dist/langfuseTracing.d.ts +0 -4
- package/dist/langfuseTracing.js +0 -32
- package/dist/langfuseTypes.d.ts +0 -96
- package/dist/langfuseTypes.js +0 -2
- package/dist/langfuseVercelAi.d.ts +0 -12
- package/dist/langfuseVercelAi.js +0 -49
- package/src/isolated/langfuseClient.isolated.ts +0 -66
- package/src/langfuseApp.test.ts +0 -119
- package/src/langfuseApp.ts +0 -62
- package/src/langfuseCache.test.ts +0 -75
- package/src/langfuseCache.ts +0 -72
- package/src/langfuseClient.test.ts +0 -69
- package/src/langfuseClient.ts +0 -34
- package/src/langfusePrompts.test.ts +0 -188
- package/src/langfusePrompts.ts +0 -121
- package/src/langfuseRoutes.test.ts +0 -261
- package/src/langfuseRoutesEvaluations.ts +0 -48
- package/src/langfuseRoutesMiddleware.test.ts +0 -57
- package/src/langfuseRoutesMiddleware.ts +0 -12
- package/src/langfuseRoutesPlayground.ts +0 -50
- package/src/langfuseRoutesPrompts.ts +0 -87
- package/src/langfuseRoutesTraces.ts +0 -40
- package/src/langfuseTracing.test.ts +0 -72
- package/src/langfuseTracing.ts +0 -33
- package/src/langfuseTypes.ts +0 -103
- package/src/langfuseVercelAi.test.ts +0 -82
- package/src/langfuseVercelAi.ts +0 -72
- package/src/tests/fakeLangfusePrompt.ts +0 -10
|
@@ -3,15 +3,30 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.LocalExperimentRunner = exports.EXPERIMENT_LIST_DEFAULT_LIMIT = void 0;
|
|
6
|
+
exports.LocalExperimentRunner = exports.EXPERIMENT_LIST_DEFAULT_LIMIT = exports.DEFAULT_TARGET_AGREEMENT = void 0;
|
|
7
7
|
const api_1 = require("@terreno/api");
|
|
8
8
|
const luxon_1 = require("luxon");
|
|
9
9
|
const mongoose_1 = __importDefault(require("mongoose"));
|
|
10
10
|
const compileTemplate_1 = require("../compileTemplate");
|
|
11
11
|
const evaluate_1 = require("../evaluate");
|
|
12
|
+
const questionFingerprint_1 = require("../questionFingerprint");
|
|
12
13
|
const sopThresholds_1 = require("../sopThresholds");
|
|
14
|
+
const humanAnswers_1 = require("./humanAnswers");
|
|
15
|
+
const judgeRun_1 = require("./judgeRun");
|
|
16
|
+
const obsDataset_1 = require("./models/obsDataset");
|
|
17
|
+
const obsEvaluator_1 = require("./models/obsEvaluator");
|
|
13
18
|
const obsExperiment_1 = require("./models/obsExperiment");
|
|
14
19
|
const obsExperimentItem_1 = require("./models/obsExperimentItem");
|
|
20
|
+
exports.DEFAULT_TARGET_AGREEMENT = 0.9;
|
|
21
|
+
const versionBody = (version) => {
|
|
22
|
+
return version.system && version.system.length > 0 ? version.system : (version.template ?? "");
|
|
23
|
+
};
|
|
24
|
+
const toObjectId = (value, label) => {
|
|
25
|
+
if (typeof value !== "string" || !mongoose_1.default.Types.ObjectId.isValid(value)) {
|
|
26
|
+
throw new api_1.APIError({ status: 400, title: `${label} must be an object id` });
|
|
27
|
+
}
|
|
28
|
+
return new mongoose_1.default.Types.ObjectId(value);
|
|
29
|
+
};
|
|
15
30
|
exports.EXPERIMENT_LIST_DEFAULT_LIMIT = 50;
|
|
16
31
|
const EXPERIMENT_LIST_MAX_LIMIT = 100;
|
|
17
32
|
const defaultBackgroundTaskRunner = {
|
|
@@ -114,9 +129,17 @@ class LocalExperimentRunner {
|
|
|
114
129
|
return client;
|
|
115
130
|
}
|
|
116
131
|
async estimate(input) {
|
|
117
|
-
const
|
|
118
|
-
|
|
119
|
-
|
|
132
|
+
const itemCount = input.kind === "judge"
|
|
133
|
+
? input.questionSetId
|
|
134
|
+
? (await (0, humanAnswers_1.loadCurrentHumanAnswers)({
|
|
135
|
+
datasetId: input.datasetId,
|
|
136
|
+
questionSetId: input.questionSetId,
|
|
137
|
+
})).items.length
|
|
138
|
+
: 0
|
|
139
|
+
: (await this.deps.datasetStore.listExperimentItems(input.datasetId, input.includeUnproofread ?? false)).length;
|
|
140
|
+
const generations = itemCount * input.versions.length;
|
|
141
|
+
// Judge experiments make one judge call per item and version and no separate generation.
|
|
142
|
+
const evaluatorCount = input.kind === "judge" ? 0 : (input.evaluatorIds?.length ?? 0);
|
|
120
143
|
const secondsPerGeneration = 2;
|
|
121
144
|
const judgeSeconds = generations * evaluatorCount * 0.5;
|
|
122
145
|
const modelId = (input.modelOverride
|
|
@@ -145,9 +168,22 @@ class LocalExperimentRunner {
|
|
|
145
168
|
}
|
|
146
169
|
}
|
|
147
170
|
async create(input) {
|
|
148
|
-
|
|
171
|
+
const kind = input.kind ?? "prompt";
|
|
172
|
+
if (kind !== "prompt" && kind !== "judge") {
|
|
173
|
+
throw new api_1.APIError({ status: 400, title: "kind must be prompt or judge" });
|
|
174
|
+
}
|
|
175
|
+
if (input.kind === "judge") {
|
|
176
|
+
return this.createJudge(input);
|
|
177
|
+
}
|
|
178
|
+
if (!Array.isArray(input.versions) || input.versions.length < 2 || input.versions.length > 3) {
|
|
149
179
|
throw new api_1.APIError({ status: 400, title: "experiments must compare 2–3 prompt versions" });
|
|
150
180
|
}
|
|
181
|
+
if (input.thresholds?.some((threshold) => threshold.aggregate === "agreement")) {
|
|
182
|
+
throw new api_1.APIError({
|
|
183
|
+
status: 400,
|
|
184
|
+
title: "agreement gates belong to judge experiments (kind: judge)",
|
|
185
|
+
});
|
|
186
|
+
}
|
|
151
187
|
await this.assertAutomaticEvaluators(input.evaluatorIds);
|
|
152
188
|
this.resolveAiClient(input.modelOverride);
|
|
153
189
|
await this.deps.datasetStore.get(input.datasetId);
|
|
@@ -165,6 +201,7 @@ class LocalExperimentRunner {
|
|
|
165
201
|
return new mongoose_1.default.Types.ObjectId(id);
|
|
166
202
|
}),
|
|
167
203
|
includeUnproofread: input.includeUnproofread ?? false,
|
|
204
|
+
kind: "prompt",
|
|
168
205
|
modelOverride: input.modelOverride,
|
|
169
206
|
name: input.name,
|
|
170
207
|
promptName: input.promptName,
|
|
@@ -172,6 +209,119 @@ class LocalExperimentRunner {
|
|
|
172
209
|
thresholds: input.thresholds ?? sopThresholds_1.SOP_DEFAULT_THRESHOLDS,
|
|
173
210
|
versions: input.versions,
|
|
174
211
|
});
|
|
212
|
+
return this.startRun(experiment);
|
|
213
|
+
}
|
|
214
|
+
async createJudge(input) {
|
|
215
|
+
if (!input.name) {
|
|
216
|
+
throw new api_1.APIError({ status: 400, title: "name is required" });
|
|
217
|
+
}
|
|
218
|
+
if (!Array.isArray(input.versions) ||
|
|
219
|
+
input.versions.length < 2 ||
|
|
220
|
+
input.versions.length > 3 ||
|
|
221
|
+
!input.versions.every((version) => Number.isInteger(version))) {
|
|
222
|
+
throw new api_1.APIError({
|
|
223
|
+
status: 400,
|
|
224
|
+
title: "judge experiments must compare 2–3 judge prompt versions",
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
if (new Set(input.versions).size !== input.versions.length) {
|
|
228
|
+
throw new api_1.APIError({ status: 400, title: "judge prompt versions must be different" });
|
|
229
|
+
}
|
|
230
|
+
const targetAgreement = input.targetAgreement ?? exports.DEFAULT_TARGET_AGREEMENT;
|
|
231
|
+
if (typeof targetAgreement !== "number" || targetAgreement <= 0 || targetAgreement > 1) {
|
|
232
|
+
throw new api_1.APIError({
|
|
233
|
+
status: 400,
|
|
234
|
+
title: "targetAgreement must be a number above 0 and at most 1",
|
|
235
|
+
});
|
|
236
|
+
}
|
|
237
|
+
const questionSet = await (0, obsEvaluator_1.registerObsEvaluator)().findOneOrNone({
|
|
238
|
+
_id: toObjectId(input.questionSetId, "questionSetId"),
|
|
239
|
+
});
|
|
240
|
+
if (!questionSet) {
|
|
241
|
+
throw new api_1.APIError({ status: 404, title: "Unknown question set" });
|
|
242
|
+
}
|
|
243
|
+
if (questionSet.type !== "human") {
|
|
244
|
+
throw new api_1.APIError({ status: 400, title: "questionSetId must be a human question set" });
|
|
245
|
+
}
|
|
246
|
+
const judge = await (0, obsEvaluator_1.registerObsEvaluator)().findOneOrNone({
|
|
247
|
+
_id: toObjectId(input.judgeEvaluatorId, "judgeEvaluatorId"),
|
|
248
|
+
});
|
|
249
|
+
if (!judge) {
|
|
250
|
+
throw new api_1.APIError({ status: 404, title: "Unknown judge evaluator" });
|
|
251
|
+
}
|
|
252
|
+
if (judge.type !== "llm-judge" || !judge.judgePromptName) {
|
|
253
|
+
throw new api_1.APIError({
|
|
254
|
+
status: 400,
|
|
255
|
+
title: "judgeEvaluatorId must be an llm-judge evaluator with a judge prompt",
|
|
256
|
+
});
|
|
257
|
+
}
|
|
258
|
+
const questionKeys = new Set(questionSet.dimensions.map((row) => row.key));
|
|
259
|
+
if (!judge.dimensions.some((row) => questionKeys.has(row.key))) {
|
|
260
|
+
throw new api_1.APIError({
|
|
261
|
+
detail: `Questions: ${[...questionKeys].join(", ")}. Judge scores: ${judge.dimensions
|
|
262
|
+
.map((row) => row.key)
|
|
263
|
+
.join(", ")}.`,
|
|
264
|
+
status: 400,
|
|
265
|
+
title: "Judge answers none of the question set's questions",
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
const dataset = await this.deps.datasetStore.get(toObjectId(input.datasetId, "datasetId").toString());
|
|
269
|
+
const current = await (0, humanAnswers_1.loadCurrentHumanAnswers)({
|
|
270
|
+
datasetId: dataset.id,
|
|
271
|
+
questionSetId: String(questionSet._id),
|
|
272
|
+
});
|
|
273
|
+
if (current.items.length === 0) {
|
|
274
|
+
throw new api_1.APIError({
|
|
275
|
+
detail: `Create a review batch from dataset "${dataset.name}" with "${questionSet.name}" and answer it first.`,
|
|
276
|
+
status: 400,
|
|
277
|
+
title: "Dataset has no current human answers for this question set",
|
|
278
|
+
});
|
|
279
|
+
}
|
|
280
|
+
for (const version of input.versions) {
|
|
281
|
+
const promptVersion = await this.deps.promptStore.getVersionByNumber(judge.judgePromptName, version);
|
|
282
|
+
if (!promptVersion) {
|
|
283
|
+
throw new api_1.APIError({
|
|
284
|
+
detail: `Judge prompt "${judge.judgePromptName}" has no version ${version}.`,
|
|
285
|
+
status: 400,
|
|
286
|
+
title: "Unknown judge prompt version",
|
|
287
|
+
});
|
|
288
|
+
}
|
|
289
|
+
if (!promptVersion.outputSchema) {
|
|
290
|
+
throw new api_1.APIError({
|
|
291
|
+
detail: `Judge prompt "${judge.judgePromptName}" v${version} has no output schema.`,
|
|
292
|
+
status: 400,
|
|
293
|
+
title: "Judge prompt version has no output schema",
|
|
294
|
+
});
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
this.resolveAiClient(input.modelOverride);
|
|
298
|
+
const estimate = await this.estimate({
|
|
299
|
+
datasetId: dataset.id,
|
|
300
|
+
kind: "judge",
|
|
301
|
+
modelOverride: input.modelOverride,
|
|
302
|
+
questionSetId: String(questionSet._id),
|
|
303
|
+
versions: input.versions,
|
|
304
|
+
});
|
|
305
|
+
const experiment = await (0, obsExperiment_1.registerObsExperiment)().create({
|
|
306
|
+
datasetId: new mongoose_1.default.Types.ObjectId(dataset.id),
|
|
307
|
+
estimate,
|
|
308
|
+
evaluatorIds: [judge._id],
|
|
309
|
+
includeUnproofread: true,
|
|
310
|
+
judgeEvaluatorId: judge._id,
|
|
311
|
+
kind: "judge",
|
|
312
|
+
modelOverride: input.modelOverride,
|
|
313
|
+
name: input.name,
|
|
314
|
+
promptName: judge.judgePromptName,
|
|
315
|
+
questionSetId: questionSet._id,
|
|
316
|
+
questionsFingerprint: current.fingerprint,
|
|
317
|
+
status: "pending",
|
|
318
|
+
targetAgreement,
|
|
319
|
+
thresholds: [],
|
|
320
|
+
versions: input.versions,
|
|
321
|
+
});
|
|
322
|
+
return this.startRun(experiment);
|
|
323
|
+
}
|
|
324
|
+
async startRun(experiment) {
|
|
175
325
|
const runner = this.deps.backgroundTaskRunner ?? defaultBackgroundTaskRunner;
|
|
176
326
|
const { taskId } = await runner.enqueue({
|
|
177
327
|
execute: async () => {
|
|
@@ -179,9 +329,10 @@ class LocalExperimentRunner {
|
|
|
179
329
|
},
|
|
180
330
|
taskType: `obs-experiment-${String(experiment._id)}`,
|
|
181
331
|
});
|
|
182
|
-
|
|
183
|
-
experiment.
|
|
184
|
-
|
|
332
|
+
const ObsExperiment = (0, obsExperiment_1.registerObsExperiment)();
|
|
333
|
+
await ObsExperiment.updateOne({ _id: experiment._id }, { $set: { backgroundTaskId: new mongoose_1.default.Types.ObjectId(taskId) } });
|
|
334
|
+
// A runner that executes inline has already finished; only a still-pending run is "running".
|
|
335
|
+
await ObsExperiment.updateOne({ _id: experiment._id, status: "pending" }, { $set: { status: "running" } });
|
|
185
336
|
return this.get(String(experiment._id));
|
|
186
337
|
}
|
|
187
338
|
async list(params) {
|
|
@@ -193,15 +344,23 @@ class LocalExperimentRunner {
|
|
|
193
344
|
if (params?.promptName) {
|
|
194
345
|
filter.promptName = params.promptName;
|
|
195
346
|
}
|
|
347
|
+
if (params?.kind === "judge") {
|
|
348
|
+
filter.kind = "judge";
|
|
349
|
+
}
|
|
350
|
+
else if (params?.kind === "prompt") {
|
|
351
|
+
// Experiments saved before `kind` existed are prompt experiments.
|
|
352
|
+
filter.kind = { $ne: "judge" };
|
|
353
|
+
}
|
|
196
354
|
const ObsExperiment = (0, obsExperiment_1.registerObsExperiment)();
|
|
197
355
|
const total = await ObsExperiment.countDocuments(filter);
|
|
198
356
|
const rows = await ObsExperiment.find(filter)
|
|
199
357
|
.sort({ created: -1 })
|
|
200
358
|
.skip((page - 1) * limit)
|
|
201
359
|
.limit(limit);
|
|
360
|
+
const names = await this.loadNames(rows);
|
|
202
361
|
return {
|
|
203
362
|
data: rows.map((row) => {
|
|
204
|
-
return this.toSummaryView(row);
|
|
363
|
+
return this.toSummaryView(row, names.get(String(row._id)));
|
|
205
364
|
}),
|
|
206
365
|
meta: { limit, page, total },
|
|
207
366
|
};
|
|
@@ -218,35 +377,82 @@ class LocalExperimentRunner {
|
|
|
218
377
|
}
|
|
219
378
|
return left.created.getTime() - right.created.getTime();
|
|
220
379
|
});
|
|
380
|
+
const names = await this.loadNames([experiment]);
|
|
221
381
|
return {
|
|
222
|
-
...this.toSummaryView(experiment),
|
|
382
|
+
...this.toSummaryView(experiment, names.get(String(experiment._id))),
|
|
223
383
|
items: sortedItems.map((row) => {
|
|
224
384
|
return {
|
|
225
385
|
datasetItemId: String(row.datasetItemId),
|
|
226
386
|
failed: row.failed,
|
|
387
|
+
humanScores: row.humanScores,
|
|
227
388
|
id: String(row._id),
|
|
228
389
|
versionResults: row.versionResults,
|
|
229
390
|
};
|
|
230
391
|
}),
|
|
231
392
|
};
|
|
232
393
|
}
|
|
233
|
-
|
|
394
|
+
// Dataset, question-set, and judge names for display, plus question-set staleness.
|
|
395
|
+
async loadNames(experiments) {
|
|
396
|
+
const datasetIds = experiments.map((row) => row.datasetId);
|
|
397
|
+
const evaluatorIds = experiments.flatMap((row) => {
|
|
398
|
+
return [row.questionSetId, row.judgeEvaluatorId].filter((id) => Boolean(id));
|
|
399
|
+
});
|
|
400
|
+
const datasets = datasetIds.length
|
|
401
|
+
? await (0, obsDataset_1.registerObsDataset)().find({ _id: { $in: datasetIds } })
|
|
402
|
+
: [];
|
|
403
|
+
const evaluators = evaluatorIds.length
|
|
404
|
+
? await (0, obsEvaluator_1.registerObsEvaluator)().find({ _id: { $in: evaluatorIds } })
|
|
405
|
+
: [];
|
|
406
|
+
const datasetNameById = new Map(datasets.map((row) => [String(row._id), row.name]));
|
|
407
|
+
const evaluatorById = new Map(evaluators.map((row) => [String(row._id), row]));
|
|
408
|
+
return new Map(experiments.map((row) => {
|
|
409
|
+
const questionSet = row.questionSetId
|
|
410
|
+
? evaluatorById.get(String(row.questionSetId))
|
|
411
|
+
: undefined;
|
|
412
|
+
const judge = row.judgeEvaluatorId
|
|
413
|
+
? evaluatorById.get(String(row.judgeEvaluatorId))
|
|
414
|
+
: undefined;
|
|
415
|
+
return [
|
|
416
|
+
String(row._id),
|
|
417
|
+
{
|
|
418
|
+
datasetName: datasetNameById.get(String(row.datasetId)),
|
|
419
|
+
judgeName: judge?.name,
|
|
420
|
+
questionSetName: questionSet?.name,
|
|
421
|
+
questionsStale: questionSet && row.questionsFingerprint
|
|
422
|
+
? (0, questionFingerprint_1.questionSetFingerprint)(questionSet.dimensions) !== row.questionsFingerprint
|
|
423
|
+
: undefined,
|
|
424
|
+
},
|
|
425
|
+
];
|
|
426
|
+
}));
|
|
427
|
+
}
|
|
428
|
+
toSummaryView(experiment, names) {
|
|
234
429
|
return {
|
|
235
430
|
backgroundTaskId: experiment.backgroundTaskId
|
|
236
431
|
? String(experiment.backgroundTaskId)
|
|
237
432
|
: undefined,
|
|
238
433
|
created: luxon_1.DateTime.fromJSDate(experiment.created).toUTC().toISO() ?? "",
|
|
239
434
|
datasetId: String(experiment.datasetId),
|
|
435
|
+
datasetName: names?.datasetName,
|
|
240
436
|
estimate: experiment.estimate,
|
|
241
437
|
evaluatorIds: experiment.evaluatorIds.map((value) => String(value)),
|
|
242
438
|
id: String(experiment._id),
|
|
243
439
|
includeUnproofread: experiment.includeUnproofread,
|
|
244
440
|
items: [],
|
|
441
|
+
judgeEvaluatorId: experiment.judgeEvaluatorId
|
|
442
|
+
? String(experiment.judgeEvaluatorId)
|
|
443
|
+
: undefined,
|
|
444
|
+
judgeName: names?.judgeName,
|
|
445
|
+
kind: experiment.kind ?? "prompt",
|
|
245
446
|
modelOverride: experiment.modelOverride,
|
|
246
447
|
name: experiment.name,
|
|
247
448
|
promptName: experiment.promptName,
|
|
449
|
+
questionSetId: experiment.questionSetId ? String(experiment.questionSetId) : undefined,
|
|
450
|
+
questionSetName: names?.questionSetName,
|
|
451
|
+
questionsFingerprint: experiment.questionsFingerprint,
|
|
452
|
+
questionsStale: names?.questionsStale,
|
|
248
453
|
results: experiment.results,
|
|
249
454
|
status: experiment.status,
|
|
455
|
+
targetAgreement: experiment.targetAgreement,
|
|
250
456
|
thresholds: experiment.thresholds,
|
|
251
457
|
updated: luxon_1.DateTime.fromJSDate(experiment.updated).toUTC().toISO() ?? "",
|
|
252
458
|
versions: experiment.versions,
|
|
@@ -283,6 +489,10 @@ class LocalExperimentRunner {
|
|
|
283
489
|
throw new api_1.APIError({ status: 404, title: `Unknown experiment "${id}"` });
|
|
284
490
|
}
|
|
285
491
|
const aiService = this.resolveAiClient(experiment.modelOverride);
|
|
492
|
+
if (experiment.kind === "judge") {
|
|
493
|
+
await this.executeJudgeExperiment(experiment, aiService);
|
|
494
|
+
return;
|
|
495
|
+
}
|
|
286
496
|
const datasetItems = await this.deps.datasetStore.listExperimentItems(String(experiment.datasetId), experiment.includeUnproofread);
|
|
287
497
|
const evaluators = await Promise.all(experiment.evaluatorIds.map((evaluatorId) => {
|
|
288
498
|
return this.deps.evaluatorStore.get(String(evaluatorId));
|
|
@@ -400,6 +610,175 @@ class LocalExperimentRunner {
|
|
|
400
610
|
};
|
|
401
611
|
await experiment.save();
|
|
402
612
|
}
|
|
613
|
+
async executeJudgeExperiment(experiment, aiService) {
|
|
614
|
+
const judge = experiment.judgeEvaluatorId
|
|
615
|
+
? await (0, obsEvaluator_1.registerObsEvaluator)().findOneOrNone({ _id: experiment.judgeEvaluatorId })
|
|
616
|
+
: undefined;
|
|
617
|
+
if (!judge?.judgePromptName || !experiment.questionSetId) {
|
|
618
|
+
throw new api_1.APIError({
|
|
619
|
+
status: 404,
|
|
620
|
+
title: "Judge experiment is missing its judge or question set",
|
|
621
|
+
});
|
|
622
|
+
}
|
|
623
|
+
const judgePromptName = judge.judgePromptName;
|
|
624
|
+
const current = await (0, humanAnswers_1.loadCurrentHumanAnswers)({
|
|
625
|
+
datasetId: experiment.datasetId,
|
|
626
|
+
questionSetId: experiment.questionSetId,
|
|
627
|
+
});
|
|
628
|
+
// Judges see exactly what reviewers saw: the source trace, else the item's input and expected output.
|
|
629
|
+
const subjects = await (0, humanAnswers_1.loadDatasetItemSubjects)(current.items.map((row) => row.item));
|
|
630
|
+
const versionPrompts = new Map();
|
|
631
|
+
for (const versionNumber of experiment.versions) {
|
|
632
|
+
const version = await this.deps.promptStore.getVersionByNumber(judgePromptName, versionNumber);
|
|
633
|
+
if (!version?.outputSchema) {
|
|
634
|
+
throw new api_1.APIError({
|
|
635
|
+
status: 404,
|
|
636
|
+
title: `Judge prompt "${judgePromptName}" v${versionNumber} is missing or has no output schema`,
|
|
637
|
+
});
|
|
638
|
+
}
|
|
639
|
+
versionPrompts.set(versionNumber, {
|
|
640
|
+
schema: version.outputSchema,
|
|
641
|
+
system: versionBody(version),
|
|
642
|
+
});
|
|
643
|
+
}
|
|
644
|
+
const judgeId = String(judge._id);
|
|
645
|
+
const subjectRows = current.items.map(({ item }) => {
|
|
646
|
+
const datasetItemId = String(item._id);
|
|
647
|
+
const subject = subjects.get(datasetItemId);
|
|
648
|
+
return {
|
|
649
|
+
datasetItemId,
|
|
650
|
+
input: subject?.input,
|
|
651
|
+
output: subject?.output,
|
|
652
|
+
...(subject?.traceId ? { traceId: subject.traceId } : {}),
|
|
653
|
+
};
|
|
654
|
+
});
|
|
655
|
+
const humanAnswers = current.items.map(({ answer }, index) => {
|
|
656
|
+
const { datasetItemId, traceId } = subjectRows[index] ?? {};
|
|
657
|
+
return { datasetItemId, scores: answer.scores, ...(traceId ? { traceId } : {}) };
|
|
658
|
+
});
|
|
659
|
+
const total = current.items.length;
|
|
660
|
+
const judgeForRun = {
|
|
661
|
+
dimensions: judge.dimensions,
|
|
662
|
+
judgePromptName,
|
|
663
|
+
name: judge.name,
|
|
664
|
+
type: judge.type,
|
|
665
|
+
};
|
|
666
|
+
const versionRuns = new Map(experiment.versions.map((version) => [version, []]));
|
|
667
|
+
const lowConfidenceItemIds = [];
|
|
668
|
+
const outlierItemIds = [];
|
|
669
|
+
const aggregate = (completed) => {
|
|
670
|
+
const agreement = {};
|
|
671
|
+
for (const version of experiment.versions) {
|
|
672
|
+
agreement[String(version)] = (0, judgeRun_1.agreementForResults)({
|
|
673
|
+
dimensions: current.questionSet.dimensions,
|
|
674
|
+
humanAnswers,
|
|
675
|
+
judge: judgeForRun,
|
|
676
|
+
results: versionRuns.get(version) ?? [],
|
|
677
|
+
});
|
|
678
|
+
}
|
|
679
|
+
const target = experiment.targetAgreement ?? exports.DEFAULT_TARGET_AGREEMENT;
|
|
680
|
+
return {
|
|
681
|
+
agreement,
|
|
682
|
+
gates: experiment.versions.map((version) => {
|
|
683
|
+
const actual = agreement[String(version)]?.rate;
|
|
684
|
+
return {
|
|
685
|
+
actual,
|
|
686
|
+
aggregate: "agreement",
|
|
687
|
+
dimension: "agreement",
|
|
688
|
+
evaluatorName: judge.name,
|
|
689
|
+
op: "gte",
|
|
690
|
+
passed: actual !== undefined && actual >= target,
|
|
691
|
+
value: target,
|
|
692
|
+
version,
|
|
693
|
+
};
|
|
694
|
+
}),
|
|
695
|
+
lowConfidenceItemIds: [...new Set(lowConfidenceItemIds)],
|
|
696
|
+
outlierItemIds: [...new Set(outlierItemIds)],
|
|
697
|
+
progress: { completed, total },
|
|
698
|
+
totalCostUsd: experiment.estimate?.costUsd,
|
|
699
|
+
};
|
|
700
|
+
};
|
|
701
|
+
// Every version runs through the shared judge pipeline at once; an item's row is written
|
|
702
|
+
// (and progress saved) as soon as all versions have judged it. Writes are serialized.
|
|
703
|
+
const runnable = experiment.versions.filter((version) => versionPrompts.has(version));
|
|
704
|
+
const pending = new Map();
|
|
705
|
+
let completed = 0;
|
|
706
|
+
let writes = Promise.resolve();
|
|
707
|
+
const writeRow = async (index, byVersion) => {
|
|
708
|
+
const row = current.items[index];
|
|
709
|
+
if (!row) {
|
|
710
|
+
return;
|
|
711
|
+
}
|
|
712
|
+
const datasetItemId = String(row.item._id);
|
|
713
|
+
const versionResults = {};
|
|
714
|
+
let rowFailed = false;
|
|
715
|
+
for (const versionNumber of experiment.versions) {
|
|
716
|
+
const result = byVersion.get(versionNumber);
|
|
717
|
+
if (!result) {
|
|
718
|
+
continue;
|
|
719
|
+
}
|
|
720
|
+
versionResults[String(versionNumber)] = {
|
|
721
|
+
evaluatorScores: {
|
|
722
|
+
[judgeId]: { confidence: result.confidence, error: result.error, scores: result.scores },
|
|
723
|
+
},
|
|
724
|
+
};
|
|
725
|
+
versionRuns.get(versionNumber)?.push(result);
|
|
726
|
+
if (result.error) {
|
|
727
|
+
rowFailed = true;
|
|
728
|
+
outlierItemIds.push(datasetItemId);
|
|
729
|
+
}
|
|
730
|
+
if (result.confidence !== undefined && result.confidence < judge.confidenceAlertBelow) {
|
|
731
|
+
lowConfidenceItemIds.push(datasetItemId);
|
|
732
|
+
}
|
|
733
|
+
// A row fails when any version disagrees with the reviewers on any compared question.
|
|
734
|
+
if (result.disagrees) {
|
|
735
|
+
rowFailed = true;
|
|
736
|
+
}
|
|
737
|
+
}
|
|
738
|
+
await (0, obsExperimentItem_1.registerObsExperimentItem)().create({
|
|
739
|
+
datasetItemId: row.item._id,
|
|
740
|
+
experimentId: experiment._id,
|
|
741
|
+
failed: rowFailed,
|
|
742
|
+
humanScores: row.answer.scores,
|
|
743
|
+
versionResults,
|
|
744
|
+
});
|
|
745
|
+
completed += 1;
|
|
746
|
+
experiment.results = aggregate(completed);
|
|
747
|
+
await experiment.save();
|
|
748
|
+
};
|
|
749
|
+
const concurrency = Math.max(1, Math.ceil(judgeRun_1.DEFAULT_JUDGE_CONCURRENCY / Math.max(runnable.length, 1)));
|
|
750
|
+
await Promise.all(runnable.map(async (versionNumber) => {
|
|
751
|
+
const prompt = versionPrompts.get(versionNumber);
|
|
752
|
+
if (!prompt) {
|
|
753
|
+
return;
|
|
754
|
+
}
|
|
755
|
+
await (0, judgeRun_1.runJudgeAgainstHumans)({
|
|
756
|
+
ai: aiService,
|
|
757
|
+
concurrency,
|
|
758
|
+
dimensions: current.questionSet.dimensions,
|
|
759
|
+
humanAnswers,
|
|
760
|
+
judge: judgeForRun,
|
|
761
|
+
onResult: async (result) => {
|
|
762
|
+
const byVersion = pending.get(result.index) ?? new Map();
|
|
763
|
+
byVersion.set(versionNumber, result);
|
|
764
|
+
pending.set(result.index, byVersion);
|
|
765
|
+
if (byVersion.size < runnable.length) {
|
|
766
|
+
return;
|
|
767
|
+
}
|
|
768
|
+
pending.delete(result.index);
|
|
769
|
+
writes = writes.then(() => writeRow(result.index, byVersion));
|
|
770
|
+
await writes;
|
|
771
|
+
},
|
|
772
|
+
prompt: { outputSchema: prompt.schema, system: prompt.system },
|
|
773
|
+
subjects: subjectRows,
|
|
774
|
+
});
|
|
775
|
+
}));
|
|
776
|
+
await writes;
|
|
777
|
+
experiment.questionsFingerprint = current.fingerprint;
|
|
778
|
+
experiment.results = aggregate(total);
|
|
779
|
+
experiment.status = "completed";
|
|
780
|
+
await experiment.save();
|
|
781
|
+
}
|
|
403
782
|
computeGates(versions, thresholds, gateBuckets) {
|
|
404
783
|
return versions.flatMap((version) => {
|
|
405
784
|
return thresholds.map((threshold) => {
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import mongoose from "mongoose";
|
|
2
|
+
import type { ObsDatasetHumanAnswer, ObsDatasetItemDocument, ObsEvaluatorDocument } from "../../types/observability";
|
|
3
|
+
/** How far one question set has judged a dataset. */
|
|
4
|
+
export interface DatasetJudgmentSummary {
|
|
5
|
+
/** Items with any answer from this question set. */
|
|
6
|
+
answered: number;
|
|
7
|
+
/** Answers whose fingerprint matches the question set's questions now. */
|
|
8
|
+
current: number;
|
|
9
|
+
evaluatorId: string;
|
|
10
|
+
evaluatorName: string;
|
|
11
|
+
/** Answers given to an older version of the questions. */
|
|
12
|
+
stale: number;
|
|
13
|
+
total: number;
|
|
14
|
+
}
|
|
15
|
+
export interface DatasetHumanAnswerView {
|
|
16
|
+
answeredAt: string;
|
|
17
|
+
fingerprint: string;
|
|
18
|
+
reviewItemId: string;
|
|
19
|
+
scores: Record<string, boolean | number | string>;
|
|
20
|
+
userId?: string;
|
|
21
|
+
}
|
|
22
|
+
/** What reviewers (and judges) see for a dataset item: the source trace when it has one. */
|
|
23
|
+
export interface DatasetItemSubject {
|
|
24
|
+
input?: unknown;
|
|
25
|
+
output?: unknown;
|
|
26
|
+
traceId?: string;
|
|
27
|
+
traceName?: string;
|
|
28
|
+
}
|
|
29
|
+
export interface CurrentHumanAnswers {
|
|
30
|
+
fingerprint: string;
|
|
31
|
+
items: Array<{
|
|
32
|
+
answer: ObsDatasetHumanAnswer;
|
|
33
|
+
item: ObsDatasetItemDocument;
|
|
34
|
+
}>;
|
|
35
|
+
questionSet: ObsEvaluatorDocument;
|
|
36
|
+
}
|
|
37
|
+
export declare const toHumanAnswerViews: (answers?: Record<string, ObsDatasetHumanAnswer>) => Record<string, DatasetHumanAnswerView> | undefined;
|
|
38
|
+
/** Record (or replace) one question set's answers on a dataset item. */
|
|
39
|
+
export declare const writeHumanAnswer: (params: {
|
|
40
|
+
datasetItemId: mongoose.Types.ObjectId;
|
|
41
|
+
evaluator: Pick<ObsEvaluatorDocument, "_id" | "dimensions">;
|
|
42
|
+
reviewItemId: string;
|
|
43
|
+
scores: Record<string, boolean | number | string>;
|
|
44
|
+
userId?: string;
|
|
45
|
+
}) => Promise<void>;
|
|
46
|
+
/** Subjects for dataset items, keyed by item id: the source trace when it still exists. */
|
|
47
|
+
export declare const loadDatasetItemSubjects: (items: Pick<ObsDatasetItemDocument, "_id" | "expectedOutput" | "input" | "sourceTraceId">[]) => Promise<Map<string, DatasetItemSubject>>;
|
|
48
|
+
/** Dataset items whose answers from this question set match its current questions. */
|
|
49
|
+
export declare const loadCurrentHumanAnswers: (params: {
|
|
50
|
+
datasetId: string | mongoose.Types.ObjectId;
|
|
51
|
+
questionSetId: string | mongoose.Types.ObjectId;
|
|
52
|
+
}) => Promise<CurrentHumanAnswers>;
|
|
53
|
+
/**
|
|
54
|
+
* Per-dataset judgment summaries: one row per question set that answered any item or has a
|
|
55
|
+
* review batch built from the dataset.
|
|
56
|
+
*/
|
|
57
|
+
export declare const summarizeDatasetJudgments: (datasetIds: mongoose.Types.ObjectId[]) => Promise<Map<string, DatasetJudgmentSummary[]>>;
|