@terreno/ai 57.20.0 → 57.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +81 -0
- package/README.md +3 -3
- package/biome.jsonc +3 -2
- package/dist/harness/agentLoop.js +1 -0
- package/dist/harness/commit.d.ts +4 -3
- package/dist/harness/commit.js +23 -4
- package/dist/index.d.ts +6 -9
- package/dist/index.js +14 -25
- package/dist/observability/adminScreens.js +5 -1
- package/dist/observability/evaluate.d.ts +8 -1
- package/dist/observability/evaluate.js +10 -2
- package/dist/observability/judgeAgreement.d.ts +53 -0
- package/dist/observability/judgeAgreement.js +123 -0
- package/dist/observability/local/datasetStore.d.ts +5 -0
- package/dist/observability/local/datasetStore.js +7 -2
- package/dist/observability/local/evaluatorStore.d.ts +17 -0
- package/dist/observability/local/evaluatorStore.js +73 -2
- package/dist/observability/local/experimentRunner.d.ts +42 -8
- package/dist/observability/local/experimentRunner.js +390 -11
- package/dist/observability/local/humanAnswers.d.ts +57 -0
- package/dist/observability/local/humanAnswers.js +174 -0
- package/dist/observability/local/judgeRun.d.ts +56 -0
- package/dist/observability/local/judgeRun.js +111 -0
- package/dist/observability/local/localPlugin.js +4 -0
- package/dist/observability/local/models/obsDatasetItem.js +4 -0
- package/dist/observability/local/models/obsEvaluator.js +4 -0
- package/dist/observability/local/models/obsExperiment.js +28 -0
- package/dist/observability/local/models/obsExperimentItem.js +4 -0
- package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
- package/dist/observability/local/models/obsJudgeRun.js +60 -0
- package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
- package/dist/observability/local/models/obsReviewBatch.js +48 -0
- package/dist/observability/local/models/obsReviewItem.js +10 -2
- package/dist/observability/local/models/obsTrace.js +4 -0
- package/dist/observability/local/promptReferences.d.ts +31 -0
- package/dist/observability/local/promptReferences.js +114 -0
- package/dist/observability/local/promptStore.d.ts +55 -19
- package/dist/observability/local/promptStore.js +171 -7
- package/dist/observability/local/reviewBatchStore.d.ts +106 -0
- package/dist/observability/local/reviewBatchStore.js +517 -0
- package/dist/observability/local/reviewStore.d.ts +24 -3
- package/dist/observability/local/reviewStore.js +98 -22
- package/dist/observability/local/traceStore.d.ts +14 -1
- package/dist/observability/local/traceStore.js +144 -11
- package/dist/observability/observabilityApp.js +10 -1
- package/dist/observability/questionFingerprint.d.ts +7 -0
- package/dist/observability/questionFingerprint.js +23 -0
- package/dist/observability/routes/experiments.js +12 -1
- package/dist/observability/routes/review.js +3 -1
- package/dist/observability/routes/reviewBatches.d.ts +11 -0
- package/dist/observability/routes/reviewBatches.js +78 -0
- package/dist/observability/routes/traces.d.ts +1 -1
- package/dist/observability/routes/traces.js +34 -2
- package/dist/observability/types.d.ts +10 -1
- package/dist/service/aiService.js +4 -1
- package/dist/service/chatTurn.js +38 -31
- package/dist/types/harness.d.ts +10 -4
- package/dist/types/index.d.ts +8 -3
- package/dist/types/observability.d.ts +74 -2
- package/e2e/smoke.ts +227 -0
- package/package.json +14 -28
- package/src/aiApp.test.ts +22 -17
- package/src/harness/agentLoop.test.ts +0 -3
- package/src/harness/agentLoop.ts +2 -1
- package/src/harness/agentRecovery.test.ts +30 -30
- package/src/harness/approvals.test.ts +11 -9
- package/src/harness/commit.ts +24 -4
- package/src/harness/events.test.ts +10 -6
- package/src/harness/extensions.test.ts +6 -8
- package/src/harness/ownership.test.ts +20 -9
- package/src/harness/retries.test.ts +6 -3
- package/src/harness/routes/conversations.test.ts +2 -2
- package/src/harness/routes/tasks.test.ts +2 -2
- package/src/harness/runners/jobsRunner.test.ts +1 -1
- package/src/harness/subagent.test.ts +29 -4
- package/src/harness/waits.test.ts +6 -4
- package/src/index.test.ts +3 -20
- package/src/index.ts +23 -26
- package/src/isolated/gptStream.isolated.ts +25 -28
- package/src/isolated/lazyProjectModel.isolated.ts +2 -2
- package/src/models/gptHistory.test.ts +10 -6
- package/src/observability/adminScreens.test.ts +7 -0
- package/src/observability/adminScreens.ts +6 -1
- package/src/observability/evaluate.test.ts +12 -10
- package/src/observability/evaluate.ts +12 -2
- package/src/observability/judgeAgreement.test.ts +111 -0
- package/src/observability/judgeAgreement.ts +184 -0
- package/src/observability/local/datasetStore.ts +30 -11
- package/src/observability/local/evaluatorStore.ts +96 -2
- package/src/observability/local/experimentRunner.test.ts +9 -12
- package/src/observability/local/experimentRunner.ts +497 -17
- package/src/observability/local/humanAnswers.ts +233 -0
- package/src/observability/local/judgePipeline.test.ts +444 -0
- package/src/observability/local/judgeRun.test.ts +173 -0
- package/src/observability/local/judgeRun.ts +184 -0
- package/src/observability/local/localPlugin.ts +4 -0
- package/src/observability/local/models/obsDatasetItem.ts +5 -0
- package/src/observability/local/models/obsEvaluator.ts +4 -0
- package/src/observability/local/models/obsExperiment.ts +29 -0
- package/src/observability/local/models/obsExperimentItem.ts +4 -0
- package/src/observability/local/models/obsJudgeRun.ts +61 -0
- package/src/observability/local/models/obsReviewBatch.ts +52 -0
- package/src/observability/local/models/obsReviewItem.ts +11 -2
- package/src/observability/local/models/obsTrace.ts +5 -0
- package/src/observability/local/obsModels.test.ts +13 -0
- package/src/observability/local/promptReferences.test.ts +128 -0
- package/src/observability/local/promptReferences.ts +149 -0
- package/src/observability/local/promptStore.test.ts +295 -3
- package/src/observability/local/promptStore.ts +243 -32
- package/src/observability/local/reviewBatchStore.test.ts +251 -0
- package/src/observability/local/reviewBatchStore.ts +656 -0
- package/src/observability/local/reviewStore.ts +122 -25
- package/src/observability/local/traceStore.test.ts +139 -1
- package/src/observability/local/traceStore.ts +162 -11
- package/src/observability/observabilityApp.test.ts +13 -10
- package/src/observability/observabilityApp.ts +10 -1
- package/src/observability/questionFingerprint.test.ts +33 -0
- package/src/observability/questionFingerprint.ts +24 -0
- package/src/observability/routes/datasets.test.ts +2 -2
- package/src/observability/routes/evaluators.test.ts +2 -2
- package/src/observability/routes/experiments.test.ts +39 -2
- package/src/observability/routes/experiments.ts +13 -1
- package/src/observability/routes/prompts.test.ts +3 -3
- package/src/observability/routes/review.test.ts +2 -2
- package/src/observability/routes/review.ts +3 -1
- package/src/observability/routes/reviewBatches.test.ts +209 -0
- package/src/observability/routes/reviewBatches.ts +138 -0
- package/src/observability/routes/status.test.ts +8 -8
- package/src/observability/routes/testMultiStage.test.ts +6 -6
- package/src/observability/routes/traces.test.ts +63 -2
- package/src/observability/routes/traces.ts +46 -4
- package/src/observability/types.ts +5 -2
- package/src/routes/files.test.ts +5 -3
- package/src/routes/gpt.test.ts +21 -14
- package/src/routes/gptActions.test.ts +7 -8
- package/src/routes/gptDatasets.test.ts +1 -1
- package/src/routes/gptFiles.test.ts +1 -1
- package/src/routes/gptHistories.test.ts +15 -9
- package/src/routes/mcp.test.ts +3 -3
- package/src/routes/projects.test.ts +5 -3
- package/src/routes/routes.test.ts +156 -115
- package/src/service/aiService.test.ts +22 -1
- package/src/service/aiService.ts +4 -1
- package/src/service/asks.test.ts +14 -6
- package/src/service/chatTurn.ts +42 -33
- package/src/service/fileStorage.test.ts +5 -2
- package/src/service/getMCPTools.test.ts +9 -5
- package/src/service/mcpService.test.ts +6 -3
- package/src/tests/bunSetup.ts +0 -54
- package/src/tests/chatHarness.ts +4 -4
- package/src/tests/fakeJudge.ts +8 -0
- package/src/tests/helpers.ts +26 -4
- package/src/types/harness.ts +11 -4
- package/src/types/index.ts +8 -3
- package/src/types/observability.ts +86 -2
- package/tsconfig.typecheck.json +18 -0
- package/dist/langfuseApp.d.ts +0 -9
- package/dist/langfuseApp.js +0 -53
- package/dist/langfuseCache.d.ts +0 -15
- package/dist/langfuseCache.js +0 -54
- package/dist/langfuseClient.d.ts +0 -6
- package/dist/langfuseClient.js +0 -34
- package/dist/langfusePrompts.d.ts +0 -12
- package/dist/langfusePrompts.js +0 -89
- package/dist/langfuseRoutesEvaluations.d.ts +0 -3
- package/dist/langfuseRoutesEvaluations.js +0 -32
- package/dist/langfuseRoutesMiddleware.d.ts +0 -2
- package/dist/langfuseRoutesMiddleware.js +0 -13
- package/dist/langfuseRoutesPlayground.d.ts +0 -2
- package/dist/langfuseRoutesPlayground.js +0 -36
- package/dist/langfuseRoutesPrompts.d.ts +0 -2
- package/dist/langfuseRoutesPrompts.js +0 -56
- package/dist/langfuseRoutesTraces.d.ts +0 -2
- package/dist/langfuseRoutesTraces.js +0 -30
- package/dist/langfuseTracing.d.ts +0 -4
- package/dist/langfuseTracing.js +0 -32
- package/dist/langfuseTypes.d.ts +0 -96
- package/dist/langfuseTypes.js +0 -2
- package/dist/langfuseVercelAi.d.ts +0 -12
- package/dist/langfuseVercelAi.js +0 -49
- package/src/isolated/langfuseClient.isolated.ts +0 -60
- package/src/langfuseApp.test.ts +0 -135
- package/src/langfuseApp.ts +0 -62
- package/src/langfuseCache.test.ts +0 -75
- package/src/langfuseCache.ts +0 -72
- package/src/langfuseClient.test.ts +0 -69
- package/src/langfuseClient.ts +0 -34
- package/src/langfusePrompts.test.ts +0 -188
- package/src/langfusePrompts.ts +0 -121
- package/src/langfuseRoutes.test.ts +0 -256
- package/src/langfuseRoutesEvaluations.ts +0 -48
- package/src/langfuseRoutesMiddleware.test.ts +0 -57
- package/src/langfuseRoutesMiddleware.ts +0 -12
- package/src/langfuseRoutesPlayground.ts +0 -50
- package/src/langfuseRoutesPrompts.ts +0 -87
- package/src/langfuseRoutesTraces.ts +0 -40
- package/src/langfuseTracing.test.ts +0 -68
- package/src/langfuseTracing.ts +0 -33
- package/src/langfuseTypes.ts +0 -103
- package/src/langfuseVercelAi.test.ts +0 -79
- package/src/langfuseVercelAi.ts +0 -72
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import {describe, expect, it} from "bun:test";
|
|
2
|
+
|
|
3
|
+
import type {EvaluatorDimension} from "../types/observability";
|
|
4
|
+
import {answersAgree, computeJudgeAgreement} from "./judgeAgreement";
|
|
5
|
+
|
|
6
|
+
const QUESTIONS: EvaluatorDimension[] = [
|
|
7
|
+
{dataType: "boolean", key: "faithful", question: "Is it faithful?", required: true},
|
|
8
|
+
{dataType: "boolean", key: "complete", question: "Is it complete?", required: true},
|
|
9
|
+
{dataType: "numeric", key: "tone", range: "0-10", required: false},
|
|
10
|
+
];
|
|
11
|
+
|
|
12
|
+
describe("answersAgree", () => {
|
|
13
|
+
it("matches booleans, including yes/no strings", () => {
|
|
14
|
+
expect(answersAgree(QUESTIONS[0] as EvaluatorDimension, true, true)).toBe(true);
|
|
15
|
+
expect(answersAgree(QUESTIONS[0] as EvaluatorDimension, true, "yes")).toBe(true);
|
|
16
|
+
expect(answersAgree(QUESTIONS[0] as EvaluatorDimension, false, true)).toBe(false);
|
|
17
|
+
expect(answersAgree(QUESTIONS[0] as EvaluatorDimension, true, undefined)).toBe(false);
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
it("allows numeric answers within 10% of the range", () => {
|
|
21
|
+
const tone = QUESTIONS[2] as EvaluatorDimension;
|
|
22
|
+
expect(answersAgree(tone, 7, 8)).toBe(true);
|
|
23
|
+
expect(answersAgree(tone, 7, 8.5)).toBe(false);
|
|
24
|
+
expect(answersAgree({...tone, range: undefined}, 0.5, 0.55)).toBe(true);
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
it("matches categorical answers case-insensitively", () => {
|
|
28
|
+
const dimension: EvaluatorDimension = {dataType: "categorical", key: "c", required: true};
|
|
29
|
+
expect(answersAgree(dimension, "Warm", " warm ")).toBe(true);
|
|
30
|
+
expect(answersAgree(dimension, "warm", "cold")).toBe(false);
|
|
31
|
+
});
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
describe("computeJudgeAgreement", () => {
|
|
35
|
+
it("pairs answers by dataset item before trace", () => {
|
|
36
|
+
const agreement = computeJudgeAgreement({
|
|
37
|
+
dimensions: QUESTIONS,
|
|
38
|
+
humanAnswers: [
|
|
39
|
+
{datasetItemId: "item-1", scores: {faithful: false}, traceId: "shared-trace"},
|
|
40
|
+
{datasetItemId: "item-2", scores: {faithful: true}},
|
|
41
|
+
],
|
|
42
|
+
judgeAnswers: [
|
|
43
|
+
{datasetItemId: "item-2", scores: {faithful: true}},
|
|
44
|
+
{datasetItemId: "item-1", scores: {faithful: true}, traceId: "shared-trace"},
|
|
45
|
+
{scores: {faithful: false}, traceId: "shared-trace"},
|
|
46
|
+
],
|
|
47
|
+
judgeKeys: ["faithful"],
|
|
48
|
+
});
|
|
49
|
+
expect(agreement.rate).toBe(0.5);
|
|
50
|
+
expect(agreement.disagreements).toEqual([
|
|
51
|
+
{
|
|
52
|
+
datasetItemId: "item-1",
|
|
53
|
+
human: false,
|
|
54
|
+
judge: true,
|
|
55
|
+
key: "faithful",
|
|
56
|
+
traceId: "shared-trace",
|
|
57
|
+
},
|
|
58
|
+
]);
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
it("scores only covered questions and lists disagreements", () => {
|
|
62
|
+
const agreement = computeJudgeAgreement({
|
|
63
|
+
dimensions: QUESTIONS,
|
|
64
|
+
humanAnswers: [
|
|
65
|
+
{scores: {complete: true, faithful: true, tone: 5}, traceId: "a"},
|
|
66
|
+
{scores: {complete: false, faithful: true}, traceId: "b"},
|
|
67
|
+
{scores: {complete: true, faithful: false}, traceId: "unjudged"},
|
|
68
|
+
],
|
|
69
|
+
judgeAnswers: [
|
|
70
|
+
{scores: {complete: true, faithful: true}, traceId: "a"},
|
|
71
|
+
{scores: {complete: true, faithful: true}, traceId: "b"},
|
|
72
|
+
],
|
|
73
|
+
judgeKeys: ["faithful", "complete"],
|
|
74
|
+
});
|
|
75
|
+
expect(agreement.compared).toBe(4);
|
|
76
|
+
expect(agreement.matches).toBe(3);
|
|
77
|
+
expect(agreement.rate).toBe(0.75);
|
|
78
|
+
expect(agreement.itemsCompared).toBe(2);
|
|
79
|
+
expect(agreement.questions.map((row) => [row.key, row.covered, row.rate])).toEqual([
|
|
80
|
+
["faithful", true, 1],
|
|
81
|
+
["complete", true, 0.5],
|
|
82
|
+
["tone", false, undefined],
|
|
83
|
+
]);
|
|
84
|
+
expect(agreement.disagreements).toEqual([
|
|
85
|
+
{human: false, judge: true, key: "complete", traceId: "b"},
|
|
86
|
+
]);
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
it("counts judge errors as disagreements", () => {
|
|
90
|
+
const agreement = computeJudgeAgreement({
|
|
91
|
+
dimensions: QUESTIONS.slice(0, 1),
|
|
92
|
+
humanAnswers: [{scores: {faithful: true}, traceId: "a"}],
|
|
93
|
+
judgeAnswers: [{error: "timeout", traceId: "a"}],
|
|
94
|
+
judgeKeys: ["faithful"],
|
|
95
|
+
});
|
|
96
|
+
expect(agreement.errors).toBe(1);
|
|
97
|
+
expect(agreement.rate).toBe(0);
|
|
98
|
+
expect(agreement.disagreements[0]?.judge).toBeUndefined();
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
it("has no rate before any human answers", () => {
|
|
102
|
+
const agreement = computeJudgeAgreement({
|
|
103
|
+
dimensions: QUESTIONS,
|
|
104
|
+
humanAnswers: [],
|
|
105
|
+
judgeAnswers: [{scores: {faithful: true}, traceId: "a"}],
|
|
106
|
+
judgeKeys: ["faithful"],
|
|
107
|
+
});
|
|
108
|
+
expect(agreement.rate).toBeUndefined();
|
|
109
|
+
expect(agreement.compared).toBe(0);
|
|
110
|
+
});
|
|
111
|
+
});
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
import type {EvaluatorDimension} from "../types/observability";
|
|
2
|
+
|
|
3
|
+
export type ScoreValue = boolean | number | string;
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Answers for one reviewed item. Human and judge answers pair up by `datasetItemId` when set,
|
|
7
|
+
* otherwise by `traceId`.
|
|
8
|
+
*/
|
|
9
|
+
export interface AgreementAnswerSet {
|
|
10
|
+
datasetItemId?: string;
|
|
11
|
+
scores?: Record<string, ScoreValue>;
|
|
12
|
+
traceId?: string;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export interface JudgeAnswerSet extends AgreementAnswerSet {
|
|
16
|
+
error?: string;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface JudgeQuestionAgreement {
|
|
20
|
+
compared: number;
|
|
21
|
+
/** False when the judge does not answer this question, so it is left out of the totals. */
|
|
22
|
+
covered: boolean;
|
|
23
|
+
key: string;
|
|
24
|
+
matches: number;
|
|
25
|
+
question?: string;
|
|
26
|
+
rate?: number;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export interface JudgeDisagreement {
|
|
30
|
+
datasetItemId?: string;
|
|
31
|
+
human: ScoreValue;
|
|
32
|
+
judge?: ScoreValue;
|
|
33
|
+
key: string;
|
|
34
|
+
traceId?: string;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export interface JudgeAgreement {
|
|
38
|
+
compared: number;
|
|
39
|
+
disagreements: JudgeDisagreement[];
|
|
40
|
+
errors: number;
|
|
41
|
+
itemsCompared: number;
|
|
42
|
+
matches: number;
|
|
43
|
+
questions: JudgeQuestionAgreement[];
|
|
44
|
+
rate?: number;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// Numeric answers agree when they land within this share of the dimension's range.
|
|
48
|
+
const NUMERIC_TOLERANCE = 0.1;
|
|
49
|
+
|
|
50
|
+
const numericSpan = (range?: string): number => {
|
|
51
|
+
const matches = range?.match(/(-?\d+(?:\.\d+)?)\s*(?:-|\.\.)\s*(-?\d+(?:\.\d+)?)/);
|
|
52
|
+
if (!matches) {
|
|
53
|
+
return 1;
|
|
54
|
+
}
|
|
55
|
+
const span = Math.abs(Number(matches[2]) - Number(matches[1]));
|
|
56
|
+
return span > 0 ? span : 1;
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
const toBoolean = (value: ScoreValue): boolean | undefined => {
|
|
60
|
+
if (typeof value === "boolean") {
|
|
61
|
+
return value;
|
|
62
|
+
}
|
|
63
|
+
if (value === "true" || value === "yes") {
|
|
64
|
+
return true;
|
|
65
|
+
}
|
|
66
|
+
if (value === "false" || value === "no") {
|
|
67
|
+
return false;
|
|
68
|
+
}
|
|
69
|
+
return undefined;
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
/** True when a judge answer counts as the same answer a human gave for this dimension. */
|
|
73
|
+
export const answersAgree = (
|
|
74
|
+
dimension: EvaluatorDimension,
|
|
75
|
+
human: ScoreValue,
|
|
76
|
+
judge: ScoreValue | undefined
|
|
77
|
+
): boolean => {
|
|
78
|
+
if (judge === undefined) {
|
|
79
|
+
return false;
|
|
80
|
+
}
|
|
81
|
+
if (dimension.dataType === "boolean") {
|
|
82
|
+
const humanBool = toBoolean(human);
|
|
83
|
+
return humanBool !== undefined && humanBool === toBoolean(judge);
|
|
84
|
+
}
|
|
85
|
+
if (dimension.dataType === "numeric") {
|
|
86
|
+
const humanNumber = Number(human);
|
|
87
|
+
const judgeNumber = Number(judge);
|
|
88
|
+
if (Number.isNaN(humanNumber) || Number.isNaN(judgeNumber)) {
|
|
89
|
+
return false;
|
|
90
|
+
}
|
|
91
|
+
return (
|
|
92
|
+
Math.abs(humanNumber - judgeNumber) <= numericSpan(dimension.range) * NUMERIC_TOLERANCE + 1e-9
|
|
93
|
+
);
|
|
94
|
+
}
|
|
95
|
+
return String(human).trim().toLowerCase() === String(judge).trim().toLowerCase();
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
/** Key that pairs human and judge answers for the same item. */
|
|
99
|
+
export const agreementAnswerKey = (answer: AgreementAnswerSet): string => {
|
|
100
|
+
return answer.datasetItemId ? `item:${answer.datasetItemId}` : `trace:${answer.traceId ?? ""}`;
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
const rateOf = (matches: number, compared: number): number | undefined => {
|
|
104
|
+
return compared > 0 ? matches / compared : undefined;
|
|
105
|
+
};
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Compare one judge's answers with human answers for the same traces or dataset items. Only questions the judge
|
|
109
|
+
* answers (judgeKeys) count; a judge error or a missing answer counts as a disagreement.
|
|
110
|
+
*/
|
|
111
|
+
export const computeJudgeAgreement = (params: {
|
|
112
|
+
dimensions: EvaluatorDimension[];
|
|
113
|
+
humanAnswers: AgreementAnswerSet[];
|
|
114
|
+
judgeAnswers: JudgeAnswerSet[];
|
|
115
|
+
judgeKeys: string[];
|
|
116
|
+
}): JudgeAgreement => {
|
|
117
|
+
const judgeKeys = new Set(params.judgeKeys);
|
|
118
|
+
const judgeByKey = new Map(
|
|
119
|
+
params.judgeAnswers.map((answer) => {
|
|
120
|
+
return [agreementAnswerKey(answer), answer];
|
|
121
|
+
})
|
|
122
|
+
);
|
|
123
|
+
const questions: JudgeQuestionAgreement[] = params.dimensions.map((dimension) => {
|
|
124
|
+
return {
|
|
125
|
+
compared: 0,
|
|
126
|
+
covered: judgeKeys.has(dimension.key),
|
|
127
|
+
key: dimension.key,
|
|
128
|
+
matches: 0,
|
|
129
|
+
question: dimension.question,
|
|
130
|
+
};
|
|
131
|
+
});
|
|
132
|
+
const disagreements: JudgeDisagreement[] = [];
|
|
133
|
+
let itemsCompared = 0;
|
|
134
|
+
let errors = 0;
|
|
135
|
+
|
|
136
|
+
for (const human of params.humanAnswers) {
|
|
137
|
+
const judge = judgeByKey.get(agreementAnswerKey(human));
|
|
138
|
+
if (!judge || !human.scores) {
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
let comparedThisItem = false;
|
|
142
|
+
if (judge.error) {
|
|
143
|
+
errors += 1;
|
|
144
|
+
}
|
|
145
|
+
params.dimensions.forEach((dimension, index) => {
|
|
146
|
+
const humanValue = human.scores?.[dimension.key];
|
|
147
|
+
const question = questions[index];
|
|
148
|
+
if (!question?.covered || humanValue === undefined) {
|
|
149
|
+
return;
|
|
150
|
+
}
|
|
151
|
+
comparedThisItem = true;
|
|
152
|
+
question.compared += 1;
|
|
153
|
+
const judgeValue = judge.error ? undefined : judge.scores?.[dimension.key];
|
|
154
|
+
if (answersAgree(dimension, humanValue, judgeValue)) {
|
|
155
|
+
question.matches += 1;
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
158
|
+
disagreements.push({
|
|
159
|
+
...(human.datasetItemId ? {datasetItemId: human.datasetItemId} : {}),
|
|
160
|
+
human: humanValue,
|
|
161
|
+
judge: judgeValue,
|
|
162
|
+
key: dimension.key,
|
|
163
|
+
...(human.traceId ? {traceId: human.traceId} : {}),
|
|
164
|
+
});
|
|
165
|
+
});
|
|
166
|
+
if (comparedThisItem) {
|
|
167
|
+
itemsCompared += 1;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
const compared = questions.reduce((total, question) => total + question.compared, 0);
|
|
172
|
+
const matches = questions.reduce((total, question) => total + question.matches, 0);
|
|
173
|
+
return {
|
|
174
|
+
compared,
|
|
175
|
+
disagreements,
|
|
176
|
+
errors,
|
|
177
|
+
itemsCompared,
|
|
178
|
+
matches,
|
|
179
|
+
questions: questions.map((question) => {
|
|
180
|
+
return {...question, rate: rateOf(question.matches, question.compared)};
|
|
181
|
+
}),
|
|
182
|
+
rate: rateOf(matches, compared),
|
|
183
|
+
};
|
|
184
|
+
};
|
|
@@ -10,6 +10,12 @@ import {
|
|
|
10
10
|
parseDatasetJsonImport,
|
|
11
11
|
} from "../datasetImport";
|
|
12
12
|
import {validateAgainstSchema} from "../schemaValidation";
|
|
13
|
+
import {
|
|
14
|
+
type DatasetHumanAnswerView,
|
|
15
|
+
type DatasetJudgmentSummary,
|
|
16
|
+
summarizeDatasetJudgments,
|
|
17
|
+
toHumanAnswerViews,
|
|
18
|
+
} from "./humanAnswers";
|
|
13
19
|
import {registerObsDataset} from "./models/obsDataset";
|
|
14
20
|
import {registerObsDatasetItem} from "./models/obsDatasetItem";
|
|
15
21
|
import {registerObsTrace} from "./models/obsTrace";
|
|
@@ -48,6 +54,8 @@ export interface DatasetView {
|
|
|
48
54
|
expectedOutputSchema?: Record<string, unknown>;
|
|
49
55
|
id: string;
|
|
50
56
|
inputSchemaPromptName?: string;
|
|
57
|
+
/** Human judging progress per question set (see `summarizeDatasetJudgments`). */
|
|
58
|
+
judgments: DatasetJudgmentSummary[];
|
|
51
59
|
name: string;
|
|
52
60
|
tags: string[];
|
|
53
61
|
updated: string;
|
|
@@ -58,6 +66,8 @@ export interface DatasetItemView {
|
|
|
58
66
|
created: string;
|
|
59
67
|
datasetId: string;
|
|
60
68
|
expectedOutput?: unknown;
|
|
69
|
+
/** Human answers keyed by question-set evaluator id. */
|
|
70
|
+
humanAnswers?: Record<string, DatasetHumanAnswerView>;
|
|
61
71
|
id: string;
|
|
62
72
|
input: unknown;
|
|
63
73
|
metadata?: Record<string, unknown>;
|
|
@@ -85,17 +95,22 @@ export const throwOnDatasetImportErrors = (result: DatasetImportResult): void =>
|
|
|
85
95
|
});
|
|
86
96
|
};
|
|
87
97
|
|
|
88
|
-
const toDatasetView = async (
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
+
const toDatasetView = async (
|
|
99
|
+
doc: {
|
|
100
|
+
_id: mongoose.Types.ObjectId;
|
|
101
|
+
created: Date;
|
|
102
|
+
description?: string;
|
|
103
|
+
expectedOutputSchema?: Record<string, unknown>;
|
|
104
|
+
inputSchemaPromptName?: string;
|
|
105
|
+
name: string;
|
|
106
|
+
tags: string[];
|
|
107
|
+
updated: Date;
|
|
108
|
+
},
|
|
109
|
+
judgments?: DatasetJudgmentSummary[]
|
|
110
|
+
): Promise<DatasetView> => {
|
|
98
111
|
const counts = await computeDatasetCounts(String(doc._id));
|
|
112
|
+
const resolvedJudgments =
|
|
113
|
+
judgments ?? (await summarizeDatasetJudgments([doc._id])).get(String(doc._id)) ?? [];
|
|
99
114
|
return {
|
|
100
115
|
counts,
|
|
101
116
|
created: DateTime.fromJSDate(doc.created).toUTC().toISO() ?? "",
|
|
@@ -103,6 +118,7 @@ const toDatasetView = async (doc: {
|
|
|
103
118
|
expectedOutputSchema: doc.expectedOutputSchema,
|
|
104
119
|
id: String(doc._id),
|
|
105
120
|
inputSchemaPromptName: doc.inputSchemaPromptName,
|
|
121
|
+
judgments: resolvedJudgments,
|
|
106
122
|
name: doc.name,
|
|
107
123
|
tags: doc.tags,
|
|
108
124
|
updated: DateTime.fromJSDate(doc.updated).toUTC().toISO() ?? "",
|
|
@@ -115,6 +131,7 @@ const toItemView = (doc: {
|
|
|
115
131
|
created: Date;
|
|
116
132
|
datasetId: mongoose.Types.ObjectId;
|
|
117
133
|
expectedOutput?: unknown;
|
|
134
|
+
humanAnswers?: ObsDatasetItemDocument["humanAnswers"];
|
|
118
135
|
input: unknown;
|
|
119
136
|
metadata?: Record<string, unknown>;
|
|
120
137
|
origin: "manual" | "synthetic" | "trace";
|
|
@@ -129,6 +146,7 @@ const toItemView = (doc: {
|
|
|
129
146
|
created: DateTime.fromJSDate(doc.created).toUTC().toISO() ?? "",
|
|
130
147
|
datasetId: String(doc.datasetId),
|
|
131
148
|
expectedOutput: doc.expectedOutput,
|
|
149
|
+
humanAnswers: toHumanAnswerViews(doc.humanAnswers),
|
|
132
150
|
id: String(doc._id),
|
|
133
151
|
input: doc.input,
|
|
134
152
|
metadata: doc.metadata,
|
|
@@ -193,9 +211,10 @@ export class LocalDatasetStore {
|
|
|
193
211
|
|
|
194
212
|
async list(): Promise<DatasetView[]> {
|
|
195
213
|
const rows = await registerObsDataset().find({}).sort({name: 1});
|
|
214
|
+
const judgments = await summarizeDatasetJudgments(rows.map((row) => row._id));
|
|
196
215
|
return Promise.all(
|
|
197
216
|
rows.map((row) => {
|
|
198
|
-
return toDatasetView(row);
|
|
217
|
+
return toDatasetView(row, judgments.get(String(row._id)) ?? []);
|
|
199
218
|
})
|
|
200
219
|
);
|
|
201
220
|
}
|
|
@@ -4,10 +4,14 @@ import type {
|
|
|
4
4
|
EvaluatorDimension,
|
|
5
5
|
EvaluatorRunModes,
|
|
6
6
|
ObsEvaluatorDocument,
|
|
7
|
+
ObsExperimentDocument,
|
|
7
8
|
} from "../../types/observability";
|
|
8
9
|
import {judgeSchemaMissingDimensions} from "../evaluate";
|
|
9
10
|
import {getEvaluatorTemplate} from "../evaluatorTemplates";
|
|
11
|
+
import {questionSetFingerprint} from "../questionFingerprint";
|
|
12
|
+
import {registerObsDataset} from "./models/obsDataset";
|
|
10
13
|
import {registerObsEvaluator} from "./models/obsEvaluator";
|
|
14
|
+
import {registerObsExperiment} from "./models/obsExperiment";
|
|
11
15
|
import {LocalPromptStore} from "./promptStore";
|
|
12
16
|
|
|
13
17
|
export interface EvaluatorWriteInput {
|
|
@@ -23,8 +27,25 @@ export interface EvaluatorWriteInput {
|
|
|
23
27
|
type: "human" | "json-assert" | "llm-judge";
|
|
24
28
|
}
|
|
25
29
|
|
|
30
|
+
/** How well an llm-judge agreed with human answers in its latest judge calibration experiment. */
|
|
31
|
+
export interface EvaluatorCalibration {
|
|
32
|
+
datasetName?: string;
|
|
33
|
+
experimentId: string;
|
|
34
|
+
experimentName: string;
|
|
35
|
+
passed: boolean;
|
|
36
|
+
questionSetName?: string;
|
|
37
|
+
/** Agreement rate (0–1) of `version`; unset when nothing could be compared. */
|
|
38
|
+
rate?: number;
|
|
39
|
+
/** True when the question set's questions changed since the experiment ran. */
|
|
40
|
+
stale: boolean;
|
|
41
|
+
targetAgreement?: number;
|
|
42
|
+
/** The judge prompt's production version when it was compared, else the best version. */
|
|
43
|
+
version: number;
|
|
44
|
+
}
|
|
45
|
+
|
|
26
46
|
export interface EvaluatorView {
|
|
27
47
|
assertion?: {constraint: string; path: string};
|
|
48
|
+
calibration?: EvaluatorCalibration;
|
|
28
49
|
confidenceAlertBelow: number;
|
|
29
50
|
description?: string;
|
|
30
51
|
dimensions: EvaluatorDimension[];
|
|
@@ -140,8 +161,9 @@ export class LocalEvaluatorStore {
|
|
|
140
161
|
|
|
141
162
|
async list(): Promise<EvaluatorView[]> {
|
|
142
163
|
const rows = await registerObsEvaluator().find({}).sort({name: 1});
|
|
164
|
+
const calibrations = await this.loadCalibrations(rows);
|
|
143
165
|
return rows.map((row) => {
|
|
144
|
-
return toView(row);
|
|
166
|
+
return {...toView(row), calibration: calibrations.get(String(row._id))};
|
|
145
167
|
});
|
|
146
168
|
}
|
|
147
169
|
|
|
@@ -150,7 +172,79 @@ export class LocalEvaluatorStore {
|
|
|
150
172
|
if (!doc) {
|
|
151
173
|
throw new APIError({status: 404, title: `Unknown evaluator "${id}"`});
|
|
152
174
|
}
|
|
153
|
-
|
|
175
|
+
const calibrations = await this.loadCalibrations([doc]);
|
|
176
|
+
return {...toView(doc), calibration: calibrations.get(String(doc._id))};
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// Latest completed judge experiment per llm-judge evaluator, summarized for display.
|
|
180
|
+
private async loadCalibrations(
|
|
181
|
+
evaluators: ObsEvaluatorDocument[]
|
|
182
|
+
): Promise<Map<string, EvaluatorCalibration>> {
|
|
183
|
+
const result = new Map<string, EvaluatorCalibration>();
|
|
184
|
+
const judges = evaluators.filter((row) => row.type === "llm-judge");
|
|
185
|
+
if (judges.length === 0) {
|
|
186
|
+
return result;
|
|
187
|
+
}
|
|
188
|
+
const experiments = await registerObsExperiment()
|
|
189
|
+
.find({
|
|
190
|
+
judgeEvaluatorId: {$in: judges.map((row) => row._id)},
|
|
191
|
+
kind: "judge",
|
|
192
|
+
status: "completed",
|
|
193
|
+
})
|
|
194
|
+
.sort({created: -1});
|
|
195
|
+
const latest = new Map<string, ObsExperimentDocument>();
|
|
196
|
+
for (const experiment of experiments) {
|
|
197
|
+
const key = String(experiment.judgeEvaluatorId);
|
|
198
|
+
if (!latest.has(key)) {
|
|
199
|
+
latest.set(key, experiment);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
if (latest.size === 0) {
|
|
203
|
+
return result;
|
|
204
|
+
}
|
|
205
|
+
const rows = [...latest.values()];
|
|
206
|
+
const datasets = await registerObsDataset().find({
|
|
207
|
+
_id: {$in: rows.map((row) => row.datasetId)},
|
|
208
|
+
});
|
|
209
|
+
const questionSets = await registerObsEvaluator().find({
|
|
210
|
+
_id: {$in: rows.flatMap((row) => (row.questionSetId ? [row.questionSetId] : []))},
|
|
211
|
+
});
|
|
212
|
+
for (const judge of judges) {
|
|
213
|
+
const experiment = latest.get(String(judge._id));
|
|
214
|
+
if (!experiment) {
|
|
215
|
+
continue;
|
|
216
|
+
}
|
|
217
|
+
const production = judge.judgePromptName
|
|
218
|
+
? (await this.promptStore.getVersionByLabel(judge.judgePromptName, "production"))?.version
|
|
219
|
+
: undefined;
|
|
220
|
+
const agreement = experiment.results?.agreement ?? {};
|
|
221
|
+
const best = [...experiment.versions].sort((left, right) => {
|
|
222
|
+
return (agreement[String(right)]?.rate ?? -1) - (agreement[String(left)]?.rate ?? -1);
|
|
223
|
+
})[0];
|
|
224
|
+
const version =
|
|
225
|
+
production !== undefined && experiment.versions.includes(production) ? production : best;
|
|
226
|
+
if (version === undefined) {
|
|
227
|
+
continue;
|
|
228
|
+
}
|
|
229
|
+
const questionSet = questionSets.find((row) => {
|
|
230
|
+
return String(row._id) === String(experiment.questionSetId);
|
|
231
|
+
});
|
|
232
|
+
const gate = experiment.results?.gates.find((row) => row.version === version);
|
|
233
|
+
result.set(String(judge._id), {
|
|
234
|
+
datasetName: datasets.find((row) => String(row._id) === String(experiment.datasetId))?.name,
|
|
235
|
+
experimentId: String(experiment._id),
|
|
236
|
+
experimentName: experiment.name,
|
|
237
|
+
passed: gate?.passed ?? false,
|
|
238
|
+
questionSetName: questionSet?.name,
|
|
239
|
+
rate: agreement[String(version)]?.rate,
|
|
240
|
+
stale:
|
|
241
|
+
!questionSet ||
|
|
242
|
+
questionSetFingerprint(questionSet.dimensions) !== experiment.questionsFingerprint,
|
|
243
|
+
targetAgreement: experiment.targetAgreement,
|
|
244
|
+
version,
|
|
245
|
+
});
|
|
246
|
+
}
|
|
247
|
+
return result;
|
|
154
248
|
}
|
|
155
249
|
|
|
156
250
|
async getByName(name: string): Promise<EvaluatorView | undefined> {
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import {afterEach, beforeEach, describe, it} from "bun:test";
|
|
2
|
-
import {BackgroundTask} from "@terreno/api";
|
|
2
|
+
import {BackgroundTask, type BackgroundTaskDocument, type FindExactlyOnePlugin} from "@terreno/api";
|
|
3
3
|
import {assert} from "chai";
|
|
4
4
|
import mongoose from "mongoose";
|
|
5
|
-
|
|
5
|
+
import {judgeAnswering} from "../../tests/fakeJudge";
|
|
6
6
|
import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
|
|
7
7
|
import {LocalDatasetStore} from "./datasetStore";
|
|
8
8
|
import {LocalEvaluatorStore} from "./evaluatorStore";
|
|
@@ -37,9 +37,7 @@ const createTestRunner = (params: {
|
|
|
37
37
|
});
|
|
38
38
|
runner.configureAi({
|
|
39
39
|
aiService: {
|
|
40
|
-
generateJsonObject:
|
|
41
|
-
return {correct: true};
|
|
42
|
-
},
|
|
40
|
+
generateJsonObject: judgeAnswering(() => ({correct: true})),
|
|
43
41
|
generateText: params.generateText,
|
|
44
42
|
modelId: "mock-model",
|
|
45
43
|
},
|
|
@@ -340,9 +338,7 @@ describe("LocalExperimentRunner", () => {
|
|
|
340
338
|
});
|
|
341
339
|
defaultRunner.configureAi({
|
|
342
340
|
aiService: {
|
|
343
|
-
generateJsonObject:
|
|
344
|
-
return {correct: true};
|
|
345
|
-
},
|
|
341
|
+
generateJsonObject: judgeAnswering(() => ({correct: true})),
|
|
346
342
|
generateText: async () => {
|
|
347
343
|
return JSON.stringify({answer: "ok"});
|
|
348
344
|
},
|
|
@@ -399,9 +395,7 @@ describe("LocalExperimentRunner", () => {
|
|
|
399
395
|
});
|
|
400
396
|
failingRunner.configureAi({
|
|
401
397
|
aiService: {
|
|
402
|
-
generateJsonObject:
|
|
403
|
-
return {correct: true};
|
|
404
|
-
},
|
|
398
|
+
generateJsonObject: judgeAnswering(() => ({correct: true})),
|
|
405
399
|
generateText: async () => {
|
|
406
400
|
throw new Error("generation failed");
|
|
407
401
|
},
|
|
@@ -427,7 +421,10 @@ describe("LocalExperimentRunner", () => {
|
|
|
427
421
|
});
|
|
428
422
|
detail = await failingRunner.get(experiment.id);
|
|
429
423
|
}
|
|
430
|
-
|
|
424
|
+
// @terreno/api applies findExactlyOne to BackgroundTask, but its model type does not declare it.
|
|
425
|
+
const tasks = BackgroundTask as typeof BackgroundTask &
|
|
426
|
+
FindExactlyOnePlugin<BackgroundTaskDocument>;
|
|
427
|
+
const task = await tasks.findExactlyOne({_id: detail.backgroundTaskId});
|
|
431
428
|
assert.equal(task.status, "failed");
|
|
432
429
|
assert.include(task.error ?? "", "generation failed");
|
|
433
430
|
});
|