@terreno/ai 58.0.0-beta.3 → 58.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +166 -0
- package/README.md +3 -3
- package/dist/harness/commit.d.ts +4 -3
- package/dist/harness/commit.js +23 -4
- package/dist/index.d.ts +6 -9
- package/dist/index.js +14 -25
- package/dist/observability/adminScreens.js +5 -1
- package/dist/observability/evaluate.d.ts +8 -1
- package/dist/observability/evaluate.js +10 -2
- package/dist/observability/judgeAgreement.d.ts +53 -0
- package/dist/observability/judgeAgreement.js +123 -0
- package/dist/observability/local/datasetStore.d.ts +5 -0
- package/dist/observability/local/datasetStore.js +7 -2
- package/dist/observability/local/evaluatorStore.d.ts +17 -0
- package/dist/observability/local/evaluatorStore.js +73 -2
- package/dist/observability/local/experimentRunner.d.ts +40 -3
- package/dist/observability/local/experimentRunner.js +390 -11
- package/dist/observability/local/humanAnswers.d.ts +57 -0
- package/dist/observability/local/humanAnswers.js +174 -0
- package/dist/observability/local/judgeRun.d.ts +56 -0
- package/dist/observability/local/judgeRun.js +111 -0
- package/dist/observability/local/localPlugin.js +4 -0
- package/dist/observability/local/models/obsDatasetItem.js +4 -0
- package/dist/observability/local/models/obsEvaluator.js +4 -0
- package/dist/observability/local/models/obsExperiment.js +28 -0
- package/dist/observability/local/models/obsExperimentItem.js +4 -0
- package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
- package/dist/observability/local/models/obsJudgeRun.js +60 -0
- package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
- package/dist/observability/local/models/obsReviewBatch.js +48 -0
- package/dist/observability/local/models/obsReviewItem.js +10 -2
- package/dist/observability/local/models/obsTrace.js +4 -0
- package/dist/observability/local/promptReferences.d.ts +31 -0
- package/dist/observability/local/promptReferences.js +114 -0
- package/dist/observability/local/promptStore.d.ts +55 -19
- package/dist/observability/local/promptStore.js +171 -7
- package/dist/observability/local/reviewBatchStore.d.ts +106 -0
- package/dist/observability/local/reviewBatchStore.js +517 -0
- package/dist/observability/local/reviewStore.d.ts +24 -3
- package/dist/observability/local/reviewStore.js +98 -22
- package/dist/observability/local/traceStore.d.ts +14 -1
- package/dist/observability/local/traceStore.js +144 -11
- package/dist/observability/observabilityApp.js +10 -1
- package/dist/observability/questionFingerprint.d.ts +7 -0
- package/dist/observability/questionFingerprint.js +23 -0
- package/dist/observability/routes/experiments.js +12 -1
- package/dist/observability/routes/review.js +3 -1
- package/dist/observability/routes/reviewBatches.d.ts +11 -0
- package/dist/observability/routes/reviewBatches.js +78 -0
- package/dist/observability/routes/traces.d.ts +1 -1
- package/dist/observability/routes/traces.js +34 -2
- package/dist/observability/types.d.ts +10 -1
- package/dist/service/aiService.js +4 -1
- package/dist/service/chatTurn.js +38 -31
- package/dist/types/harness.d.ts +5 -2
- package/dist/types/index.d.ts +8 -3
- package/dist/types/observability.d.ts +74 -2
- package/package.json +8 -24
- package/src/harness/agentLoop.test.ts +0 -3
- package/src/harness/agentRecovery.test.ts +0 -3
- package/src/harness/commit.ts +24 -4
- package/src/harness/extensions.test.ts +0 -3
- package/src/harness/subagent.test.ts +27 -3
- package/src/index.test.ts +0 -17
- package/src/index.ts +23 -26
- package/src/isolated/gptStream.isolated.ts +0 -8
- package/src/observability/adminScreens.test.ts +7 -0
- package/src/observability/adminScreens.ts +6 -1
- package/src/observability/evaluate.ts +12 -2
- package/src/observability/judgeAgreement.test.ts +111 -0
- package/src/observability/judgeAgreement.ts +184 -0
- package/src/observability/local/datasetStore.ts +30 -11
- package/src/observability/local/evaluatorStore.ts +96 -2
- package/src/observability/local/experimentRunner.ts +495 -15
- package/src/observability/local/humanAnswers.ts +233 -0
- package/src/observability/local/judgePipeline.test.ts +444 -0
- package/src/observability/local/judgeRun.test.ts +173 -0
- package/src/observability/local/judgeRun.ts +184 -0
- package/src/observability/local/localPlugin.ts +4 -0
- package/src/observability/local/models/obsDatasetItem.ts +5 -0
- package/src/observability/local/models/obsEvaluator.ts +4 -0
- package/src/observability/local/models/obsExperiment.ts +29 -0
- package/src/observability/local/models/obsExperimentItem.ts +4 -0
- package/src/observability/local/models/obsJudgeRun.ts +61 -0
- package/src/observability/local/models/obsReviewBatch.ts +52 -0
- package/src/observability/local/models/obsReviewItem.ts +11 -2
- package/src/observability/local/models/obsTrace.ts +5 -0
- package/src/observability/local/obsModels.test.ts +13 -0
- package/src/observability/local/promptReferences.test.ts +128 -0
- package/src/observability/local/promptReferences.ts +149 -0
- package/src/observability/local/promptStore.test.ts +291 -0
- package/src/observability/local/promptStore.ts +243 -32
- package/src/observability/local/reviewBatchStore.test.ts +251 -0
- package/src/observability/local/reviewBatchStore.ts +656 -0
- package/src/observability/local/reviewStore.ts +122 -25
- package/src/observability/local/traceStore.test.ts +139 -1
- package/src/observability/local/traceStore.ts +162 -11
- package/src/observability/observabilityApp.test.ts +13 -10
- package/src/observability/observabilityApp.ts +10 -1
- package/src/observability/questionFingerprint.test.ts +33 -0
- package/src/observability/questionFingerprint.ts +24 -0
- package/src/observability/routes/experiments.test.ts +37 -0
- package/src/observability/routes/experiments.ts +13 -1
- package/src/observability/routes/review.ts +3 -1
- package/src/observability/routes/reviewBatches.test.ts +209 -0
- package/src/observability/routes/reviewBatches.ts +138 -0
- package/src/observability/routes/status.test.ts +4 -4
- package/src/observability/routes/traces.test.ts +61 -0
- package/src/observability/routes/traces.ts +46 -4
- package/src/observability/types.ts +5 -2
- package/src/routes/routes.test.ts +101 -76
- package/src/service/aiService.test.ts +21 -0
- package/src/service/aiService.ts +4 -1
- package/src/service/chatTurn.ts +42 -33
- package/src/tests/bunSetup.ts +0 -54
- package/src/types/harness.ts +5 -2
- package/src/types/index.ts +8 -3
- package/src/types/observability.ts +86 -2
- package/dist/langfuseApp.d.ts +0 -9
- package/dist/langfuseApp.js +0 -53
- package/dist/langfuseCache.d.ts +0 -15
- package/dist/langfuseCache.js +0 -54
- package/dist/langfuseClient.d.ts +0 -6
- package/dist/langfuseClient.js +0 -34
- package/dist/langfusePrompts.d.ts +0 -12
- package/dist/langfusePrompts.js +0 -89
- package/dist/langfuseRoutesEvaluations.d.ts +0 -3
- package/dist/langfuseRoutesEvaluations.js +0 -32
- package/dist/langfuseRoutesMiddleware.d.ts +0 -2
- package/dist/langfuseRoutesMiddleware.js +0 -13
- package/dist/langfuseRoutesPlayground.d.ts +0 -2
- package/dist/langfuseRoutesPlayground.js +0 -36
- package/dist/langfuseRoutesPrompts.d.ts +0 -2
- package/dist/langfuseRoutesPrompts.js +0 -56
- package/dist/langfuseRoutesTraces.d.ts +0 -2
- package/dist/langfuseRoutesTraces.js +0 -30
- package/dist/langfuseTracing.d.ts +0 -4
- package/dist/langfuseTracing.js +0 -32
- package/dist/langfuseTypes.d.ts +0 -96
- package/dist/langfuseTypes.js +0 -2
- package/dist/langfuseVercelAi.d.ts +0 -12
- package/dist/langfuseVercelAi.js +0 -49
- package/src/isolated/langfuseClient.isolated.ts +0 -66
- package/src/langfuseApp.test.ts +0 -119
- package/src/langfuseApp.ts +0 -62
- package/src/langfuseCache.test.ts +0 -75
- package/src/langfuseCache.ts +0 -72
- package/src/langfuseClient.test.ts +0 -69
- package/src/langfuseClient.ts +0 -34
- package/src/langfusePrompts.test.ts +0 -188
- package/src/langfusePrompts.ts +0 -121
- package/src/langfuseRoutes.test.ts +0 -261
- package/src/langfuseRoutesEvaluations.ts +0 -48
- package/src/langfuseRoutesMiddleware.test.ts +0 -57
- package/src/langfuseRoutesMiddleware.ts +0 -12
- package/src/langfuseRoutesPlayground.ts +0 -50
- package/src/langfuseRoutesPrompts.ts +0 -87
- package/src/langfuseRoutesTraces.ts +0 -40
- package/src/langfuseTracing.test.ts +0 -72
- package/src/langfuseTracing.ts +0 -33
- package/src/langfuseTypes.ts +0 -103
- package/src/langfuseVercelAi.test.ts +0 -82
- package/src/langfuseVercelAi.ts +0 -72
- package/src/tests/fakeLangfusePrompt.ts +0 -10
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.computeJudgeAgreement = exports.agreementAnswerKey = exports.answersAgree = void 0;
|
|
4
|
+
// Numeric answers agree when they land within this share of the dimension's range.
|
|
5
|
+
const NUMERIC_TOLERANCE = 0.1;
|
|
6
|
+
const numericSpan = (range) => {
|
|
7
|
+
const matches = range?.match(/(-?\d+(?:\.\d+)?)\s*(?:-|\.\.)\s*(-?\d+(?:\.\d+)?)/);
|
|
8
|
+
if (!matches) {
|
|
9
|
+
return 1;
|
|
10
|
+
}
|
|
11
|
+
const span = Math.abs(Number(matches[2]) - Number(matches[1]));
|
|
12
|
+
return span > 0 ? span : 1;
|
|
13
|
+
};
|
|
14
|
+
const toBoolean = (value) => {
|
|
15
|
+
if (typeof value === "boolean") {
|
|
16
|
+
return value;
|
|
17
|
+
}
|
|
18
|
+
if (value === "true" || value === "yes") {
|
|
19
|
+
return true;
|
|
20
|
+
}
|
|
21
|
+
if (value === "false" || value === "no") {
|
|
22
|
+
return false;
|
|
23
|
+
}
|
|
24
|
+
return undefined;
|
|
25
|
+
};
|
|
26
|
+
/** True when a judge answer counts as the same answer a human gave for this dimension. */
|
|
27
|
+
const answersAgree = (dimension, human, judge) => {
|
|
28
|
+
if (judge === undefined) {
|
|
29
|
+
return false;
|
|
30
|
+
}
|
|
31
|
+
if (dimension.dataType === "boolean") {
|
|
32
|
+
const humanBool = toBoolean(human);
|
|
33
|
+
return humanBool !== undefined && humanBool === toBoolean(judge);
|
|
34
|
+
}
|
|
35
|
+
if (dimension.dataType === "numeric") {
|
|
36
|
+
const humanNumber = Number(human);
|
|
37
|
+
const judgeNumber = Number(judge);
|
|
38
|
+
if (Number.isNaN(humanNumber) || Number.isNaN(judgeNumber)) {
|
|
39
|
+
return false;
|
|
40
|
+
}
|
|
41
|
+
return (Math.abs(humanNumber - judgeNumber) <= numericSpan(dimension.range) * NUMERIC_TOLERANCE + 1e-9);
|
|
42
|
+
}
|
|
43
|
+
return String(human).trim().toLowerCase() === String(judge).trim().toLowerCase();
|
|
44
|
+
};
|
|
45
|
+
exports.answersAgree = answersAgree;
|
|
46
|
+
/** Key that pairs human and judge answers for the same item. */
|
|
47
|
+
const agreementAnswerKey = (answer) => {
|
|
48
|
+
return answer.datasetItemId ? `item:${answer.datasetItemId}` : `trace:${answer.traceId ?? ""}`;
|
|
49
|
+
};
|
|
50
|
+
exports.agreementAnswerKey = agreementAnswerKey;
|
|
51
|
+
const rateOf = (matches, compared) => {
|
|
52
|
+
return compared > 0 ? matches / compared : undefined;
|
|
53
|
+
};
|
|
54
|
+
/**
|
|
55
|
+
* Compare one judge's answers with human answers for the same traces or dataset items. Only questions the judge
|
|
56
|
+
* answers (judgeKeys) count; a judge error or a missing answer counts as a disagreement.
|
|
57
|
+
*/
|
|
58
|
+
const computeJudgeAgreement = (params) => {
|
|
59
|
+
const judgeKeys = new Set(params.judgeKeys);
|
|
60
|
+
const judgeByKey = new Map(params.judgeAnswers.map((answer) => {
|
|
61
|
+
return [(0, exports.agreementAnswerKey)(answer), answer];
|
|
62
|
+
}));
|
|
63
|
+
const questions = params.dimensions.map((dimension) => {
|
|
64
|
+
return {
|
|
65
|
+
compared: 0,
|
|
66
|
+
covered: judgeKeys.has(dimension.key),
|
|
67
|
+
key: dimension.key,
|
|
68
|
+
matches: 0,
|
|
69
|
+
question: dimension.question,
|
|
70
|
+
};
|
|
71
|
+
});
|
|
72
|
+
const disagreements = [];
|
|
73
|
+
let itemsCompared = 0;
|
|
74
|
+
let errors = 0;
|
|
75
|
+
for (const human of params.humanAnswers) {
|
|
76
|
+
const judge = judgeByKey.get((0, exports.agreementAnswerKey)(human));
|
|
77
|
+
if (!judge || !human.scores) {
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
let comparedThisItem = false;
|
|
81
|
+
if (judge.error) {
|
|
82
|
+
errors += 1;
|
|
83
|
+
}
|
|
84
|
+
params.dimensions.forEach((dimension, index) => {
|
|
85
|
+
const humanValue = human.scores?.[dimension.key];
|
|
86
|
+
const question = questions[index];
|
|
87
|
+
if (!question?.covered || humanValue === undefined) {
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
comparedThisItem = true;
|
|
91
|
+
question.compared += 1;
|
|
92
|
+
const judgeValue = judge.error ? undefined : judge.scores?.[dimension.key];
|
|
93
|
+
if ((0, exports.answersAgree)(dimension, humanValue, judgeValue)) {
|
|
94
|
+
question.matches += 1;
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
disagreements.push({
|
|
98
|
+
...(human.datasetItemId ? { datasetItemId: human.datasetItemId } : {}),
|
|
99
|
+
human: humanValue,
|
|
100
|
+
judge: judgeValue,
|
|
101
|
+
key: dimension.key,
|
|
102
|
+
...(human.traceId ? { traceId: human.traceId } : {}),
|
|
103
|
+
});
|
|
104
|
+
});
|
|
105
|
+
if (comparedThisItem) {
|
|
106
|
+
itemsCompared += 1;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
const compared = questions.reduce((total, question) => total + question.compared, 0);
|
|
110
|
+
const matches = questions.reduce((total, question) => total + question.matches, 0);
|
|
111
|
+
return {
|
|
112
|
+
compared,
|
|
113
|
+
disagreements,
|
|
114
|
+
errors,
|
|
115
|
+
itemsCompared,
|
|
116
|
+
matches,
|
|
117
|
+
questions: questions.map((question) => {
|
|
118
|
+
return { ...question, rate: rateOf(question.matches, question.compared) };
|
|
119
|
+
}),
|
|
120
|
+
rate: rateOf(matches, compared),
|
|
121
|
+
};
|
|
122
|
+
};
|
|
123
|
+
exports.computeJudgeAgreement = computeJudgeAgreement;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { ObsDatasetAnnotatedBy } from "../../types/observability";
|
|
2
2
|
import { type ParsedDatasetImportRow } from "../datasetImport";
|
|
3
|
+
import { type DatasetHumanAnswerView, type DatasetJudgmentSummary } from "./humanAnswers";
|
|
3
4
|
import { LocalPromptStore } from "./promptStore";
|
|
4
5
|
export interface DatasetWriteInput {
|
|
5
6
|
description?: string | null;
|
|
@@ -31,6 +32,8 @@ export interface DatasetView {
|
|
|
31
32
|
expectedOutputSchema?: Record<string, unknown>;
|
|
32
33
|
id: string;
|
|
33
34
|
inputSchemaPromptName?: string;
|
|
35
|
+
/** Human judging progress per question set (see `summarizeDatasetJudgments`). */
|
|
36
|
+
judgments: DatasetJudgmentSummary[];
|
|
34
37
|
name: string;
|
|
35
38
|
tags: string[];
|
|
36
39
|
updated: string;
|
|
@@ -40,6 +43,8 @@ export interface DatasetItemView {
|
|
|
40
43
|
created: string;
|
|
41
44
|
datasetId: string;
|
|
42
45
|
expectedOutput?: unknown;
|
|
46
|
+
/** Human answers keyed by question-set evaluator id. */
|
|
47
|
+
humanAnswers?: Record<string, DatasetHumanAnswerView>;
|
|
43
48
|
id: string;
|
|
44
49
|
input: unknown;
|
|
45
50
|
metadata?: Record<string, unknown>;
|
|
@@ -9,6 +9,7 @@ const luxon_1 = require("luxon");
|
|
|
9
9
|
const mongoose_1 = __importDefault(require("mongoose"));
|
|
10
10
|
const datasetImport_1 = require("../datasetImport");
|
|
11
11
|
const schemaValidation_1 = require("../schemaValidation");
|
|
12
|
+
const humanAnswers_1 = require("./humanAnswers");
|
|
12
13
|
const obsDataset_1 = require("./models/obsDataset");
|
|
13
14
|
const obsDatasetItem_1 = require("./models/obsDatasetItem");
|
|
14
15
|
const obsTrace_1 = require("./models/obsTrace");
|
|
@@ -24,8 +25,9 @@ const throwOnDatasetImportErrors = (result) => {
|
|
|
24
25
|
});
|
|
25
26
|
};
|
|
26
27
|
exports.throwOnDatasetImportErrors = throwOnDatasetImportErrors;
|
|
27
|
-
const toDatasetView = async (doc) => {
|
|
28
|
+
const toDatasetView = async (doc, judgments) => {
|
|
28
29
|
const counts = await computeDatasetCounts(String(doc._id));
|
|
30
|
+
const resolvedJudgments = judgments ?? (await (0, humanAnswers_1.summarizeDatasetJudgments)([doc._id])).get(String(doc._id)) ?? [];
|
|
29
31
|
return {
|
|
30
32
|
counts,
|
|
31
33
|
created: luxon_1.DateTime.fromJSDate(doc.created).toUTC().toISO() ?? "",
|
|
@@ -33,6 +35,7 @@ const toDatasetView = async (doc) => {
|
|
|
33
35
|
expectedOutputSchema: doc.expectedOutputSchema,
|
|
34
36
|
id: String(doc._id),
|
|
35
37
|
inputSchemaPromptName: doc.inputSchemaPromptName,
|
|
38
|
+
judgments: resolvedJudgments,
|
|
36
39
|
name: doc.name,
|
|
37
40
|
tags: doc.tags,
|
|
38
41
|
updated: luxon_1.DateTime.fromJSDate(doc.updated).toUTC().toISO() ?? "",
|
|
@@ -44,6 +47,7 @@ const toItemView = (doc) => {
|
|
|
44
47
|
created: luxon_1.DateTime.fromJSDate(doc.created).toUTC().toISO() ?? "",
|
|
45
48
|
datasetId: String(doc.datasetId),
|
|
46
49
|
expectedOutput: doc.expectedOutput,
|
|
50
|
+
humanAnswers: (0, humanAnswers_1.toHumanAnswerViews)(doc.humanAnswers),
|
|
47
51
|
id: String(doc._id),
|
|
48
52
|
input: doc.input,
|
|
49
53
|
metadata: doc.metadata,
|
|
@@ -106,8 +110,9 @@ class LocalDatasetStore {
|
|
|
106
110
|
}
|
|
107
111
|
async list() {
|
|
108
112
|
const rows = await (0, obsDataset_1.registerObsDataset)().find({}).sort({ name: 1 });
|
|
113
|
+
const judgments = await (0, humanAnswers_1.summarizeDatasetJudgments)(rows.map((row) => row._id));
|
|
109
114
|
return Promise.all(rows.map((row) => {
|
|
110
|
-
return toDatasetView(row);
|
|
115
|
+
return toDatasetView(row, judgments.get(String(row._id)) ?? []);
|
|
111
116
|
}));
|
|
112
117
|
}
|
|
113
118
|
async get(id) {
|
|
@@ -15,11 +15,27 @@ export interface EvaluatorWriteInput {
|
|
|
15
15
|
target: "dataset item" | "full trace" | "generation span";
|
|
16
16
|
type: "human" | "json-assert" | "llm-judge";
|
|
17
17
|
}
|
|
18
|
+
/** How well an llm-judge agreed with human answers in its latest judge calibration experiment. */
|
|
19
|
+
export interface EvaluatorCalibration {
|
|
20
|
+
datasetName?: string;
|
|
21
|
+
experimentId: string;
|
|
22
|
+
experimentName: string;
|
|
23
|
+
passed: boolean;
|
|
24
|
+
questionSetName?: string;
|
|
25
|
+
/** Agreement rate (0–1) of `version`; unset when nothing could be compared. */
|
|
26
|
+
rate?: number;
|
|
27
|
+
/** True when the question set's questions changed since the experiment ran. */
|
|
28
|
+
stale: boolean;
|
|
29
|
+
targetAgreement?: number;
|
|
30
|
+
/** The judge prompt's production version when it was compared, else the best version. */
|
|
31
|
+
version: number;
|
|
32
|
+
}
|
|
18
33
|
export interface EvaluatorView {
|
|
19
34
|
assertion?: {
|
|
20
35
|
constraint: string;
|
|
21
36
|
path: string;
|
|
22
37
|
};
|
|
38
|
+
calibration?: EvaluatorCalibration;
|
|
23
39
|
confidenceAlertBelow: number;
|
|
24
40
|
description?: string;
|
|
25
41
|
dimensions: EvaluatorDimension[];
|
|
@@ -37,6 +53,7 @@ export declare class LocalEvaluatorStore {
|
|
|
37
53
|
create(input: EvaluatorWriteInput): Promise<EvaluatorView>;
|
|
38
54
|
list(): Promise<EvaluatorView[]>;
|
|
39
55
|
get(id: string): Promise<EvaluatorView>;
|
|
56
|
+
private loadCalibrations;
|
|
40
57
|
getByName(name: string): Promise<EvaluatorView | undefined>;
|
|
41
58
|
update(id: string, input: Partial<EvaluatorWriteInput>): Promise<EvaluatorView>;
|
|
42
59
|
remove(id: string): Promise<void>;
|
|
@@ -4,7 +4,10 @@ exports.LocalEvaluatorStore = void 0;
|
|
|
4
4
|
const api_1 = require("@terreno/api");
|
|
5
5
|
const evaluate_1 = require("../evaluate");
|
|
6
6
|
const evaluatorTemplates_1 = require("../evaluatorTemplates");
|
|
7
|
+
const questionFingerprint_1 = require("../questionFingerprint");
|
|
8
|
+
const obsDataset_1 = require("./models/obsDataset");
|
|
7
9
|
const obsEvaluator_1 = require("./models/obsEvaluator");
|
|
10
|
+
const obsExperiment_1 = require("./models/obsExperiment");
|
|
8
11
|
const promptStore_1 = require("./promptStore");
|
|
9
12
|
const defaultRunModes = (partial) => {
|
|
10
13
|
return {
|
|
@@ -102,8 +105,9 @@ class LocalEvaluatorStore {
|
|
|
102
105
|
}
|
|
103
106
|
async list() {
|
|
104
107
|
const rows = await (0, obsEvaluator_1.registerObsEvaluator)().find({}).sort({ name: 1 });
|
|
108
|
+
const calibrations = await this.loadCalibrations(rows);
|
|
105
109
|
return rows.map((row) => {
|
|
106
|
-
return toView(row);
|
|
110
|
+
return { ...toView(row), calibration: calibrations.get(String(row._id)) };
|
|
107
111
|
});
|
|
108
112
|
}
|
|
109
113
|
async get(id) {
|
|
@@ -111,7 +115,74 @@ class LocalEvaluatorStore {
|
|
|
111
115
|
if (!doc) {
|
|
112
116
|
throw new api_1.APIError({ status: 404, title: `Unknown evaluator "${id}"` });
|
|
113
117
|
}
|
|
114
|
-
|
|
118
|
+
const calibrations = await this.loadCalibrations([doc]);
|
|
119
|
+
return { ...toView(doc), calibration: calibrations.get(String(doc._id)) };
|
|
120
|
+
}
|
|
121
|
+
// Latest completed judge experiment per llm-judge evaluator, summarized for display.
|
|
122
|
+
async loadCalibrations(evaluators) {
|
|
123
|
+
const result = new Map();
|
|
124
|
+
const judges = evaluators.filter((row) => row.type === "llm-judge");
|
|
125
|
+
if (judges.length === 0) {
|
|
126
|
+
return result;
|
|
127
|
+
}
|
|
128
|
+
const experiments = await (0, obsExperiment_1.registerObsExperiment)()
|
|
129
|
+
.find({
|
|
130
|
+
judgeEvaluatorId: { $in: judges.map((row) => row._id) },
|
|
131
|
+
kind: "judge",
|
|
132
|
+
status: "completed",
|
|
133
|
+
})
|
|
134
|
+
.sort({ created: -1 });
|
|
135
|
+
const latest = new Map();
|
|
136
|
+
for (const experiment of experiments) {
|
|
137
|
+
const key = String(experiment.judgeEvaluatorId);
|
|
138
|
+
if (!latest.has(key)) {
|
|
139
|
+
latest.set(key, experiment);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
if (latest.size === 0) {
|
|
143
|
+
return result;
|
|
144
|
+
}
|
|
145
|
+
const rows = [...latest.values()];
|
|
146
|
+
const datasets = await (0, obsDataset_1.registerObsDataset)().find({
|
|
147
|
+
_id: { $in: rows.map((row) => row.datasetId) },
|
|
148
|
+
});
|
|
149
|
+
const questionSets = await (0, obsEvaluator_1.registerObsEvaluator)().find({
|
|
150
|
+
_id: { $in: rows.flatMap((row) => (row.questionSetId ? [row.questionSetId] : [])) },
|
|
151
|
+
});
|
|
152
|
+
for (const judge of judges) {
|
|
153
|
+
const experiment = latest.get(String(judge._id));
|
|
154
|
+
if (!experiment) {
|
|
155
|
+
continue;
|
|
156
|
+
}
|
|
157
|
+
const production = judge.judgePromptName
|
|
158
|
+
? (await this.promptStore.getVersionByLabel(judge.judgePromptName, "production"))?.version
|
|
159
|
+
: undefined;
|
|
160
|
+
const agreement = experiment.results?.agreement ?? {};
|
|
161
|
+
const best = [...experiment.versions].sort((left, right) => {
|
|
162
|
+
return (agreement[String(right)]?.rate ?? -1) - (agreement[String(left)]?.rate ?? -1);
|
|
163
|
+
})[0];
|
|
164
|
+
const version = production !== undefined && experiment.versions.includes(production) ? production : best;
|
|
165
|
+
if (version === undefined) {
|
|
166
|
+
continue;
|
|
167
|
+
}
|
|
168
|
+
const questionSet = questionSets.find((row) => {
|
|
169
|
+
return String(row._id) === String(experiment.questionSetId);
|
|
170
|
+
});
|
|
171
|
+
const gate = experiment.results?.gates.find((row) => row.version === version);
|
|
172
|
+
result.set(String(judge._id), {
|
|
173
|
+
datasetName: datasets.find((row) => String(row._id) === String(experiment.datasetId))?.name,
|
|
174
|
+
experimentId: String(experiment._id),
|
|
175
|
+
experimentName: experiment.name,
|
|
176
|
+
passed: gate?.passed ?? false,
|
|
177
|
+
questionSetName: questionSet?.name,
|
|
178
|
+
rate: agreement[String(version)]?.rate,
|
|
179
|
+
stale: !questionSet ||
|
|
180
|
+
(0, questionFingerprint_1.questionSetFingerprint)(questionSet.dimensions) !== experiment.questionsFingerprint,
|
|
181
|
+
targetAgreement: experiment.targetAgreement,
|
|
182
|
+
version,
|
|
183
|
+
});
|
|
184
|
+
}
|
|
185
|
+
return result;
|
|
115
186
|
}
|
|
116
187
|
async getByName(name) {
|
|
117
188
|
const doc = await (0, obsEvaluator_1.registerObsEvaluator)().findOneOrNone({ name });
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ExperimentAggregates, ExperimentEstimate, ExperimentVersionResult, ScoreThreshold } from "../../types/observability";
|
|
1
|
+
import type { ExperimentAggregates, ExperimentEstimate, ExperimentKind, ExperimentVersionResult, ScoreThreshold } from "../../types/observability";
|
|
2
2
|
import type { ModelPrice, ObservabilityAiServiceFactory, ObservabilityGenerateClient } from "../types";
|
|
3
3
|
import type { LocalDatasetStore } from "./datasetStore";
|
|
4
4
|
import type { LocalEvaluatorStore } from "./evaluatorStore";
|
|
@@ -7,26 +7,54 @@ export interface ExperimentCreateInput {
|
|
|
7
7
|
datasetId: string;
|
|
8
8
|
evaluatorIds: string[];
|
|
9
9
|
includeUnproofread?: boolean;
|
|
10
|
+
kind?: "prompt";
|
|
10
11
|
modelOverride?: string;
|
|
11
12
|
name: string;
|
|
12
13
|
promptName: string;
|
|
13
14
|
thresholds?: ScoreThreshold[];
|
|
14
15
|
versions: number[];
|
|
15
16
|
}
|
|
17
|
+
/**
|
|
18
|
+
* Judge calibration: run 2–3 versions of an llm-judge's prompt over dataset items humans already
|
|
19
|
+
* answered with `questionSetId`, and rank versions by agreement with those answers.
|
|
20
|
+
*/
|
|
21
|
+
export interface JudgeExperimentCreateInput {
|
|
22
|
+
datasetId: string;
|
|
23
|
+
judgeEvaluatorId: string;
|
|
24
|
+
kind: "judge";
|
|
25
|
+
modelOverride?: string;
|
|
26
|
+
name: string;
|
|
27
|
+
questionSetId: string;
|
|
28
|
+
/** Agreement rate (0–1) a version must reach to pass its gate. Defaults to 0.9. */
|
|
29
|
+
targetAgreement?: number;
|
|
30
|
+
/** Versions of the judge evaluator's `judgePromptName`. */
|
|
31
|
+
versions: number[];
|
|
32
|
+
}
|
|
33
|
+
export declare const DEFAULT_TARGET_AGREEMENT = 0.9;
|
|
16
34
|
export interface ExperimentView {
|
|
17
35
|
backgroundTaskId?: string;
|
|
18
36
|
created: string;
|
|
19
37
|
datasetId: string;
|
|
38
|
+
datasetName?: string;
|
|
20
39
|
estimate?: ExperimentEstimate;
|
|
21
40
|
evaluatorIds: string[];
|
|
22
41
|
id: string;
|
|
23
42
|
includeUnproofread: boolean;
|
|
24
43
|
items: ExperimentItemView[];
|
|
44
|
+
judgeEvaluatorId?: string;
|
|
45
|
+
judgeName?: string;
|
|
46
|
+
kind: ExperimentKind;
|
|
25
47
|
modelOverride?: string;
|
|
26
48
|
name: string;
|
|
27
49
|
promptName: string;
|
|
50
|
+
questionSetId?: string;
|
|
51
|
+
questionSetName?: string;
|
|
52
|
+
questionsFingerprint?: string;
|
|
53
|
+
/** Judge experiments: true when the question set's questions changed since the run. */
|
|
54
|
+
questionsStale?: boolean;
|
|
28
55
|
results?: ExperimentAggregates;
|
|
29
56
|
status: "completed" | "failed" | "pending" | "running";
|
|
57
|
+
targetAgreement?: number;
|
|
30
58
|
thresholds: ScoreThreshold[];
|
|
31
59
|
updated: string;
|
|
32
60
|
versions: number[];
|
|
@@ -34,6 +62,8 @@ export interface ExperimentView {
|
|
|
34
62
|
export interface ExperimentItemView {
|
|
35
63
|
datasetItemId: string;
|
|
36
64
|
failed: boolean;
|
|
65
|
+
/** Judge experiments: the human answers each judge version is compared with. */
|
|
66
|
+
humanScores?: Record<string, boolean | number | string>;
|
|
37
67
|
id: string;
|
|
38
68
|
versionResults: Record<string, ExperimentVersionResult>;
|
|
39
69
|
}
|
|
@@ -73,21 +103,28 @@ export declare class LocalExperimentRunner {
|
|
|
73
103
|
private resolveAiClient;
|
|
74
104
|
estimate(input: {
|
|
75
105
|
datasetId: string;
|
|
76
|
-
evaluatorIds
|
|
106
|
+
evaluatorIds?: string[];
|
|
77
107
|
includeUnproofread?: boolean;
|
|
108
|
+
kind?: ExperimentKind;
|
|
78
109
|
modelOverride?: string;
|
|
110
|
+
questionSetId?: string;
|
|
79
111
|
versions: number[];
|
|
80
112
|
}): Promise<ExperimentEstimate>;
|
|
81
113
|
private assertAutomaticEvaluators;
|
|
82
|
-
create(input: ExperimentCreateInput): Promise<ExperimentView>;
|
|
114
|
+
create(input: ExperimentCreateInput | JudgeExperimentCreateInput): Promise<ExperimentView>;
|
|
115
|
+
private createJudge;
|
|
116
|
+
private startRun;
|
|
83
117
|
list(params?: {
|
|
118
|
+
kind?: ExperimentKind;
|
|
84
119
|
limit?: number;
|
|
85
120
|
page?: number;
|
|
86
121
|
promptName?: string;
|
|
87
122
|
}): Promise<ExperimentListResult>;
|
|
88
123
|
get(id: string): Promise<ExperimentView>;
|
|
124
|
+
private loadNames;
|
|
89
125
|
private toSummaryView;
|
|
90
126
|
promote(id: string, version: number): Promise<MoveLabelResult>;
|
|
91
127
|
executeExperiment(id: string): Promise<void>;
|
|
128
|
+
private executeJudgeExperiment;
|
|
92
129
|
private computeGates;
|
|
93
130
|
}
|