@terreno/ai 57.21.0 → 57.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/README.md +3 -3
- package/biome.jsonc +2 -2
- package/dist/harness/commit.d.ts +4 -3
- package/dist/harness/commit.js +23 -4
- package/dist/index.d.ts +6 -9
- package/dist/index.js +14 -25
- package/dist/observability/adminScreens.js +5 -1
- package/dist/observability/evaluate.d.ts +8 -1
- package/dist/observability/evaluate.js +10 -2
- package/dist/observability/judgeAgreement.d.ts +53 -0
- package/dist/observability/judgeAgreement.js +123 -0
- package/dist/observability/local/datasetStore.d.ts +5 -0
- package/dist/observability/local/datasetStore.js +7 -2
- package/dist/observability/local/evaluatorStore.d.ts +17 -0
- package/dist/observability/local/evaluatorStore.js +73 -2
- package/dist/observability/local/experimentRunner.d.ts +40 -3
- package/dist/observability/local/experimentRunner.js +390 -11
- package/dist/observability/local/humanAnswers.d.ts +57 -0
- package/dist/observability/local/humanAnswers.js +174 -0
- package/dist/observability/local/judgeRun.d.ts +56 -0
- package/dist/observability/local/judgeRun.js +111 -0
- package/dist/observability/local/localPlugin.js +4 -0
- package/dist/observability/local/models/obsDatasetItem.js +4 -0
- package/dist/observability/local/models/obsEvaluator.js +4 -0
- package/dist/observability/local/models/obsExperiment.js +28 -0
- package/dist/observability/local/models/obsExperimentItem.js +4 -0
- package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
- package/dist/observability/local/models/obsJudgeRun.js +60 -0
- package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
- package/dist/observability/local/models/obsReviewBatch.js +48 -0
- package/dist/observability/local/models/obsReviewItem.js +10 -2
- package/dist/observability/local/models/obsTrace.js +4 -0
- package/dist/observability/local/promptReferences.d.ts +31 -0
- package/dist/observability/local/promptReferences.js +114 -0
- package/dist/observability/local/promptStore.d.ts +55 -19
- package/dist/observability/local/promptStore.js +171 -7
- package/dist/observability/local/reviewBatchStore.d.ts +106 -0
- package/dist/observability/local/reviewBatchStore.js +517 -0
- package/dist/observability/local/reviewStore.d.ts +24 -3
- package/dist/observability/local/reviewStore.js +98 -22
- package/dist/observability/local/traceStore.d.ts +14 -1
- package/dist/observability/local/traceStore.js +144 -11
- package/dist/observability/observabilityApp.js +10 -1
- package/dist/observability/questionFingerprint.d.ts +7 -0
- package/dist/observability/questionFingerprint.js +23 -0
- package/dist/observability/routes/experiments.js +12 -1
- package/dist/observability/routes/review.js +3 -1
- package/dist/observability/routes/reviewBatches.d.ts +11 -0
- package/dist/observability/routes/reviewBatches.js +78 -0
- package/dist/observability/routes/traces.d.ts +1 -1
- package/dist/observability/routes/traces.js +34 -2
- package/dist/observability/types.d.ts +10 -1
- package/dist/service/aiService.js +4 -1
- package/dist/service/chatTurn.js +38 -31
- package/dist/types/harness.d.ts +5 -2
- package/dist/types/index.d.ts +8 -3
- package/dist/types/observability.d.ts +74 -2
- package/package.json +8 -24
- package/src/harness/agentLoop.test.ts +0 -3
- package/src/harness/agentRecovery.test.ts +0 -3
- package/src/harness/commit.ts +24 -4
- package/src/harness/extensions.test.ts +0 -3
- package/src/harness/subagent.test.ts +27 -3
- package/src/index.test.ts +0 -17
- package/src/index.ts +23 -26
- package/src/isolated/gptStream.isolated.ts +0 -8
- package/src/observability/adminScreens.test.ts +7 -0
- package/src/observability/adminScreens.ts +6 -1
- package/src/observability/evaluate.ts +12 -2
- package/src/observability/judgeAgreement.test.ts +111 -0
- package/src/observability/judgeAgreement.ts +184 -0
- package/src/observability/local/datasetStore.ts +30 -11
- package/src/observability/local/evaluatorStore.ts +96 -2
- package/src/observability/local/experimentRunner.ts +495 -15
- package/src/observability/local/humanAnswers.ts +233 -0
- package/src/observability/local/judgePipeline.test.ts +444 -0
- package/src/observability/local/judgeRun.test.ts +173 -0
- package/src/observability/local/judgeRun.ts +184 -0
- package/src/observability/local/localPlugin.ts +4 -0
- package/src/observability/local/models/obsDatasetItem.ts +5 -0
- package/src/observability/local/models/obsEvaluator.ts +4 -0
- package/src/observability/local/models/obsExperiment.ts +29 -0
- package/src/observability/local/models/obsExperimentItem.ts +4 -0
- package/src/observability/local/models/obsJudgeRun.ts +61 -0
- package/src/observability/local/models/obsReviewBatch.ts +52 -0
- package/src/observability/local/models/obsReviewItem.ts +11 -2
- package/src/observability/local/models/obsTrace.ts +5 -0
- package/src/observability/local/obsModels.test.ts +13 -0
- package/src/observability/local/promptReferences.test.ts +128 -0
- package/src/observability/local/promptReferences.ts +149 -0
- package/src/observability/local/promptStore.test.ts +291 -0
- package/src/observability/local/promptStore.ts +243 -32
- package/src/observability/local/reviewBatchStore.test.ts +251 -0
- package/src/observability/local/reviewBatchStore.ts +656 -0
- package/src/observability/local/reviewStore.ts +122 -25
- package/src/observability/local/traceStore.test.ts +139 -1
- package/src/observability/local/traceStore.ts +162 -11
- package/src/observability/observabilityApp.test.ts +13 -10
- package/src/observability/observabilityApp.ts +10 -1
- package/src/observability/questionFingerprint.test.ts +33 -0
- package/src/observability/questionFingerprint.ts +24 -0
- package/src/observability/routes/experiments.test.ts +37 -0
- package/src/observability/routes/experiments.ts +13 -1
- package/src/observability/routes/review.ts +3 -1
- package/src/observability/routes/reviewBatches.test.ts +209 -0
- package/src/observability/routes/reviewBatches.ts +138 -0
- package/src/observability/routes/status.test.ts +4 -4
- package/src/observability/routes/traces.test.ts +61 -0
- package/src/observability/routes/traces.ts +46 -4
- package/src/observability/types.ts +5 -2
- package/src/routes/routes.test.ts +101 -76
- package/src/service/aiService.test.ts +21 -0
- package/src/service/aiService.ts +4 -1
- package/src/service/chatTurn.ts +42 -33
- package/src/tests/bunSetup.ts +0 -54
- package/src/types/harness.ts +5 -2
- package/src/types/index.ts +8 -3
- package/src/types/observability.ts +86 -2
- package/dist/langfuseApp.d.ts +0 -9
- package/dist/langfuseApp.js +0 -53
- package/dist/langfuseCache.d.ts +0 -15
- package/dist/langfuseCache.js +0 -54
- package/dist/langfuseClient.d.ts +0 -6
- package/dist/langfuseClient.js +0 -34
- package/dist/langfusePrompts.d.ts +0 -12
- package/dist/langfusePrompts.js +0 -89
- package/dist/langfuseRoutesEvaluations.d.ts +0 -3
- package/dist/langfuseRoutesEvaluations.js +0 -32
- package/dist/langfuseRoutesMiddleware.d.ts +0 -2
- package/dist/langfuseRoutesMiddleware.js +0 -13
- package/dist/langfuseRoutesPlayground.d.ts +0 -2
- package/dist/langfuseRoutesPlayground.js +0 -36
- package/dist/langfuseRoutesPrompts.d.ts +0 -2
- package/dist/langfuseRoutesPrompts.js +0 -56
- package/dist/langfuseRoutesTraces.d.ts +0 -2
- package/dist/langfuseRoutesTraces.js +0 -30
- package/dist/langfuseTracing.d.ts +0 -4
- package/dist/langfuseTracing.js +0 -32
- package/dist/langfuseTypes.d.ts +0 -96
- package/dist/langfuseTypes.js +0 -2
- package/dist/langfuseVercelAi.d.ts +0 -12
- package/dist/langfuseVercelAi.js +0 -49
- package/src/isolated/langfuseClient.isolated.ts +0 -66
- package/src/langfuseApp.test.ts +0 -119
- package/src/langfuseApp.ts +0 -62
- package/src/langfuseCache.test.ts +0 -75
- package/src/langfuseCache.ts +0 -72
- package/src/langfuseClient.test.ts +0 -69
- package/src/langfuseClient.ts +0 -34
- package/src/langfusePrompts.test.ts +0 -188
- package/src/langfusePrompts.ts +0 -121
- package/src/langfuseRoutes.test.ts +0 -261
- package/src/langfuseRoutesEvaluations.ts +0 -48
- package/src/langfuseRoutesMiddleware.test.ts +0 -57
- package/src/langfuseRoutesMiddleware.ts +0 -12
- package/src/langfuseRoutesPlayground.ts +0 -50
- package/src/langfuseRoutesPrompts.ts +0 -87
- package/src/langfuseRoutesTraces.ts +0 -40
- package/src/langfuseTracing.test.ts +0 -72
- package/src/langfuseTracing.ts +0 -33
- package/src/langfuseTypes.ts +0 -103
- package/src/langfuseVercelAi.test.ts +0 -82
- package/src/langfuseVercelAi.ts +0 -72
- package/src/tests/fakeLangfusePrompt.ts +0 -10
package/src/index.test.ts
CHANGED
|
@@ -18,21 +18,10 @@ describe("@terreno/ai public exports", () => {
|
|
|
18
18
|
"addMcpRoutes",
|
|
19
19
|
"addProjectRoutes",
|
|
20
20
|
"assertVertexModelsEnabled",
|
|
21
|
-
"compilePrompt",
|
|
22
21
|
"createAskTools",
|
|
23
|
-
"createPrompt",
|
|
24
|
-
"createTelemetryConfig",
|
|
25
22
|
"createVertexProvider",
|
|
26
23
|
"findAgentBlock",
|
|
27
|
-
"getCached",
|
|
28
24
|
"getMCPTools",
|
|
29
|
-
"getLangfuseClient",
|
|
30
|
-
"getPrompt",
|
|
31
|
-
"initLangfuseClient",
|
|
32
|
-
"initTracing",
|
|
33
|
-
"invalidateCache",
|
|
34
|
-
"invalidatePromptCache",
|
|
35
|
-
"isLangfuseInitialized",
|
|
36
25
|
"isVertexModelAllowed",
|
|
37
26
|
"jsonSchema",
|
|
38
27
|
"listEnabledVertexModels",
|
|
@@ -41,15 +30,11 @@ describe("@terreno/ai public exports", () => {
|
|
|
41
30
|
"normalizeLlmJsonTextForStructuredOutput",
|
|
42
31
|
"normalizeVertexModelId",
|
|
43
32
|
"parseAiJson",
|
|
44
|
-
"preparePromptForAI",
|
|
45
33
|
"getObservabilityApp",
|
|
46
34
|
"resetObservabilityApp",
|
|
47
35
|
"resolveObservabilityControl",
|
|
48
36
|
"runBufferedChatTurn",
|
|
49
37
|
"validateObservabilityConfig",
|
|
50
|
-
"setCached",
|
|
51
|
-
"shutdownLangfuseClient",
|
|
52
|
-
"shutdownTracing",
|
|
53
38
|
"verifyVertexModelsEnabled",
|
|
54
39
|
];
|
|
55
40
|
|
|
@@ -60,8 +45,6 @@ describe("@terreno/ai public exports", () => {
|
|
|
60
45
|
"FileAttachment",
|
|
61
46
|
"FileStorageService",
|
|
62
47
|
"GptHistory",
|
|
63
|
-
"LangfuseApp",
|
|
64
|
-
"LangfuseCache",
|
|
65
48
|
"MCPService",
|
|
66
49
|
"MemoryScoreSink",
|
|
67
50
|
"MemoryTraceSink",
|
package/src/index.ts
CHANGED
|
@@ -12,44 +12,40 @@ export type {AIAdminAppOptions} from "./aiAdminApp";
|
|
|
12
12
|
export {AIAdminApp} from "./aiAdminApp";
|
|
13
13
|
export type {AiAppOptions} from "./aiApp";
|
|
14
14
|
export {AiApp} from "./aiApp";
|
|
15
|
-
export {LangfuseApp} from "./langfuseApp";
|
|
16
|
-
export {getCached, invalidateCache, LangfuseCache, setCached} from "./langfuseCache";
|
|
17
|
-
export {
|
|
18
|
-
getLangfuseClient,
|
|
19
|
-
initLangfuseClient,
|
|
20
|
-
isLangfuseInitialized,
|
|
21
|
-
shutdownLangfuseClient,
|
|
22
|
-
} from "./langfuseClient";
|
|
23
|
-
export {compilePrompt, createPrompt, getPrompt, invalidatePromptCache} from "./langfusePrompts";
|
|
24
|
-
export {initTracing, shutdownTracing} from "./langfuseTracing";
|
|
25
|
-
export type {
|
|
26
|
-
ChatMessage,
|
|
27
|
-
GetPromptOptions,
|
|
28
|
-
LangfuseAppOptions,
|
|
29
|
-
LangfuseCachedPrompt,
|
|
30
|
-
PaginatedResult,
|
|
31
|
-
PreparePromptResult,
|
|
32
|
-
PromptListItem,
|
|
33
|
-
ScoreSubmission,
|
|
34
|
-
ScoringFunction,
|
|
35
|
-
TelemetrySettings,
|
|
36
|
-
TraceListItem,
|
|
37
|
-
} from "./langfuseTypes";
|
|
38
|
-
export {createTelemetryConfig, preparePromptForAI} from "./langfuseVercelAi";
|
|
39
15
|
export {AIDataset} from "./models/aiDataset";
|
|
40
16
|
export {AIRequest} from "./models/aiRequest";
|
|
41
17
|
export {FileAttachment} from "./models/fileAttachment";
|
|
42
18
|
export {GptHistory} from "./models/gptHistory";
|
|
43
19
|
export {getProjectModel, Project} from "./models/project";
|
|
44
20
|
export {AI_OBSERVABILITY_GROUP, observabilityAdminScreens} from "./observability/adminScreens";
|
|
21
|
+
export {
|
|
22
|
+
answersAgree,
|
|
23
|
+
computeJudgeAgreement,
|
|
24
|
+
type JudgeAgreement,
|
|
25
|
+
} from "./observability/judgeAgreement";
|
|
45
26
|
export {LocalDatasetStore} from "./observability/local/datasetStore";
|
|
46
|
-
export {LocalEvaluatorStore} from "./observability/local/evaluatorStore";
|
|
47
|
-
export {
|
|
27
|
+
export {type EvaluatorCalibration, LocalEvaluatorStore} from "./observability/local/evaluatorStore";
|
|
28
|
+
export {
|
|
29
|
+
DEFAULT_TARGET_AGREEMENT,
|
|
30
|
+
type JudgeExperimentCreateInput,
|
|
31
|
+
LocalExperimentRunner,
|
|
32
|
+
} from "./observability/local/experimentRunner";
|
|
33
|
+
export {
|
|
34
|
+
type DatasetJudgmentSummary,
|
|
35
|
+
loadCurrentHumanAnswers,
|
|
36
|
+
} from "./observability/local/humanAnswers";
|
|
48
37
|
export {
|
|
49
38
|
createLocalObservabilityBundle,
|
|
50
39
|
createLocalObservabilityPlugin,
|
|
51
40
|
} from "./observability/local/localPlugin";
|
|
52
41
|
export {LocalPromptStore} from "./observability/local/promptStore";
|
|
42
|
+
export {
|
|
43
|
+
DEFAULT_REVIEW_BATCH_SIZE,
|
|
44
|
+
LocalReviewBatchStore,
|
|
45
|
+
MAX_REVIEW_BATCH_SIZE,
|
|
46
|
+
type ReviewBatchDetail,
|
|
47
|
+
type ReviewBatchListItem,
|
|
48
|
+
} from "./observability/local/reviewBatchStore";
|
|
53
49
|
export {LocalReviewStore} from "./observability/local/reviewStore";
|
|
54
50
|
export {
|
|
55
51
|
LocalScoreSink,
|
|
@@ -63,6 +59,7 @@ export {
|
|
|
63
59
|
ObservabilityApp,
|
|
64
60
|
resetObservabilityApp,
|
|
65
61
|
} from "./observability/observabilityApp";
|
|
62
|
+
export {questionSetFingerprint} from "./observability/questionFingerprint";
|
|
66
63
|
export type {
|
|
67
64
|
ObservabilityPluginStatus,
|
|
68
65
|
ObservabilityStatus,
|
|
@@ -17,14 +17,6 @@ let addGptRoutes: typeof import("../routes/gpt.js").addGptRoutes;
|
|
|
17
17
|
let AIService: typeof import("../service/aiService.js").AIService;
|
|
18
18
|
|
|
19
19
|
beforeAll(async () => {
|
|
20
|
-
// Mock langfuseVercelAi to avoid transitive langfuse SDK imports.
|
|
21
|
-
const realLangfuseVercelAi = await import("../langfuseVercelAi.js");
|
|
22
|
-
mock.module("../langfuseVercelAi", () => ({
|
|
23
|
-
...realLangfuseVercelAi,
|
|
24
|
-
createTelemetryConfig: () => ({functionId: "test", isEnabled: false}),
|
|
25
|
-
preparePromptForAI: async () => ({config: {}, prompt: "test", telemetry: {isEnabled: false}}),
|
|
26
|
-
}));
|
|
27
|
-
|
|
28
20
|
// Mock the AI SDK so we control exactly which fullStream parts the GPT prompt
|
|
29
21
|
// handler iterates over. All other exports are preserved so AIService and the
|
|
30
22
|
// route helpers keep working.
|
|
@@ -24,6 +24,7 @@ describe("observabilityAdminScreens", () => {
|
|
|
24
24
|
"ai-datasets",
|
|
25
25
|
"ai-experiments",
|
|
26
26
|
"ai-review",
|
|
27
|
+
"ai-review-batches",
|
|
27
28
|
]);
|
|
28
29
|
});
|
|
29
30
|
|
|
@@ -66,6 +67,12 @@ describe("observabilityAdminScreens", () => {
|
|
|
66
67
|
group: AI_OBSERVABILITY_GROUP,
|
|
67
68
|
name: "ai-review",
|
|
68
69
|
},
|
|
70
|
+
{
|
|
71
|
+
adminAccess: {action: "list", resource: "aiReview"},
|
|
72
|
+
displayName: "Review batches",
|
|
73
|
+
group: AI_OBSERVABILITY_GROUP,
|
|
74
|
+
name: "ai-review-batches",
|
|
75
|
+
},
|
|
69
76
|
]);
|
|
70
77
|
});
|
|
71
78
|
});
|
|
@@ -35,7 +35,12 @@ export const observabilityAdminScreens = ({localOn}: {localOn: boolean}): AdminC
|
|
|
35
35
|
];
|
|
36
36
|
if (localOn) {
|
|
37
37
|
screens.push(
|
|
38
|
-
observabilityScreen({displayName: "Review queue", name: "ai-review", resource: "aiReview"})
|
|
38
|
+
observabilityScreen({displayName: "Review queue", name: "ai-review", resource: "aiReview"}),
|
|
39
|
+
observabilityScreen({
|
|
40
|
+
displayName: "Review batches",
|
|
41
|
+
name: "ai-review-batches",
|
|
42
|
+
resource: "aiReview",
|
|
43
|
+
})
|
|
39
44
|
);
|
|
40
45
|
}
|
|
41
46
|
return screens;
|
|
@@ -170,10 +170,16 @@ export const runJsonAssertEvaluator = (
|
|
|
170
170
|
};
|
|
171
171
|
};
|
|
172
172
|
|
|
173
|
+
/**
|
|
174
|
+
* Run an llm-judge evaluator. By default the AI client resolves the judge prompt's production
|
|
175
|
+
* version by name. Pass `judgeSystemPrompt` (a specific version's resolved body) to run that
|
|
176
|
+
* version instead, as judge calibration experiments do.
|
|
177
|
+
*/
|
|
173
178
|
export const runLlmJudgeEvaluator = async (
|
|
174
179
|
context: EvaluatorExecutionContext,
|
|
175
180
|
ai: EvaluatorAiClient,
|
|
176
|
-
judgeOutputSchema: Record<string, unknown
|
|
181
|
+
judgeOutputSchema: Record<string, unknown>,
|
|
182
|
+
options: {judgeSystemPrompt?: string} = {}
|
|
177
183
|
): Promise<EvaluatorExecutionResult> => {
|
|
178
184
|
const judgePromptName = context.evaluator.judgePromptName;
|
|
179
185
|
if (!judgePromptName) {
|
|
@@ -181,9 +187,13 @@ export const runLlmJudgeEvaluator = async (
|
|
|
181
187
|
}
|
|
182
188
|
|
|
183
189
|
try {
|
|
190
|
+
const promptSource =
|
|
191
|
+
options.judgeSystemPrompt === undefined
|
|
192
|
+
? {promptName: judgePromptName}
|
|
193
|
+
: {systemPrompt: options.judgeSystemPrompt};
|
|
184
194
|
const payload = (await ai.generateJsonObject({
|
|
185
195
|
prompt: buildJudgePrompt(context),
|
|
186
|
-
|
|
196
|
+
...promptSource,
|
|
187
197
|
schema: jsonSchema(judgeOutputSchema) as FlexibleSchema<Record<string, unknown>>,
|
|
188
198
|
schemaName: `${context.evaluator.name}-judge`,
|
|
189
199
|
skipTrace: true,
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import {describe, expect, it} from "bun:test";
|
|
2
|
+
|
|
3
|
+
import type {EvaluatorDimension} from "../types/observability";
|
|
4
|
+
import {answersAgree, computeJudgeAgreement} from "./judgeAgreement";
|
|
5
|
+
|
|
6
|
+
const QUESTIONS: EvaluatorDimension[] = [
|
|
7
|
+
{dataType: "boolean", key: "faithful", question: "Is it faithful?", required: true},
|
|
8
|
+
{dataType: "boolean", key: "complete", question: "Is it complete?", required: true},
|
|
9
|
+
{dataType: "numeric", key: "tone", range: "0-10", required: false},
|
|
10
|
+
];
|
|
11
|
+
|
|
12
|
+
describe("answersAgree", () => {
|
|
13
|
+
it("matches booleans, including yes/no strings", () => {
|
|
14
|
+
expect(answersAgree(QUESTIONS[0] as EvaluatorDimension, true, true)).toBe(true);
|
|
15
|
+
expect(answersAgree(QUESTIONS[0] as EvaluatorDimension, true, "yes")).toBe(true);
|
|
16
|
+
expect(answersAgree(QUESTIONS[0] as EvaluatorDimension, false, true)).toBe(false);
|
|
17
|
+
expect(answersAgree(QUESTIONS[0] as EvaluatorDimension, true, undefined)).toBe(false);
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
it("allows numeric answers within 10% of the range", () => {
|
|
21
|
+
const tone = QUESTIONS[2] as EvaluatorDimension;
|
|
22
|
+
expect(answersAgree(tone, 7, 8)).toBe(true);
|
|
23
|
+
expect(answersAgree(tone, 7, 8.5)).toBe(false);
|
|
24
|
+
expect(answersAgree({...tone, range: undefined}, 0.5, 0.55)).toBe(true);
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
it("matches categorical answers case-insensitively", () => {
|
|
28
|
+
const dimension: EvaluatorDimension = {dataType: "categorical", key: "c", required: true};
|
|
29
|
+
expect(answersAgree(dimension, "Warm", " warm ")).toBe(true);
|
|
30
|
+
expect(answersAgree(dimension, "warm", "cold")).toBe(false);
|
|
31
|
+
});
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
describe("computeJudgeAgreement", () => {
|
|
35
|
+
it("pairs answers by dataset item before trace", () => {
|
|
36
|
+
const agreement = computeJudgeAgreement({
|
|
37
|
+
dimensions: QUESTIONS,
|
|
38
|
+
humanAnswers: [
|
|
39
|
+
{datasetItemId: "item-1", scores: {faithful: false}, traceId: "shared-trace"},
|
|
40
|
+
{datasetItemId: "item-2", scores: {faithful: true}},
|
|
41
|
+
],
|
|
42
|
+
judgeAnswers: [
|
|
43
|
+
{datasetItemId: "item-2", scores: {faithful: true}},
|
|
44
|
+
{datasetItemId: "item-1", scores: {faithful: true}, traceId: "shared-trace"},
|
|
45
|
+
{scores: {faithful: false}, traceId: "shared-trace"},
|
|
46
|
+
],
|
|
47
|
+
judgeKeys: ["faithful"],
|
|
48
|
+
});
|
|
49
|
+
expect(agreement.rate).toBe(0.5);
|
|
50
|
+
expect(agreement.disagreements).toEqual([
|
|
51
|
+
{
|
|
52
|
+
datasetItemId: "item-1",
|
|
53
|
+
human: false,
|
|
54
|
+
judge: true,
|
|
55
|
+
key: "faithful",
|
|
56
|
+
traceId: "shared-trace",
|
|
57
|
+
},
|
|
58
|
+
]);
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
it("scores only covered questions and lists disagreements", () => {
|
|
62
|
+
const agreement = computeJudgeAgreement({
|
|
63
|
+
dimensions: QUESTIONS,
|
|
64
|
+
humanAnswers: [
|
|
65
|
+
{scores: {complete: true, faithful: true, tone: 5}, traceId: "a"},
|
|
66
|
+
{scores: {complete: false, faithful: true}, traceId: "b"},
|
|
67
|
+
{scores: {complete: true, faithful: false}, traceId: "unjudged"},
|
|
68
|
+
],
|
|
69
|
+
judgeAnswers: [
|
|
70
|
+
{scores: {complete: true, faithful: true}, traceId: "a"},
|
|
71
|
+
{scores: {complete: true, faithful: true}, traceId: "b"},
|
|
72
|
+
],
|
|
73
|
+
judgeKeys: ["faithful", "complete"],
|
|
74
|
+
});
|
|
75
|
+
expect(agreement.compared).toBe(4);
|
|
76
|
+
expect(agreement.matches).toBe(3);
|
|
77
|
+
expect(agreement.rate).toBe(0.75);
|
|
78
|
+
expect(agreement.itemsCompared).toBe(2);
|
|
79
|
+
expect(agreement.questions.map((row) => [row.key, row.covered, row.rate])).toEqual([
|
|
80
|
+
["faithful", true, 1],
|
|
81
|
+
["complete", true, 0.5],
|
|
82
|
+
["tone", false, undefined],
|
|
83
|
+
]);
|
|
84
|
+
expect(agreement.disagreements).toEqual([
|
|
85
|
+
{human: false, judge: true, key: "complete", traceId: "b"},
|
|
86
|
+
]);
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
it("counts judge errors as disagreements", () => {
|
|
90
|
+
const agreement = computeJudgeAgreement({
|
|
91
|
+
dimensions: QUESTIONS.slice(0, 1),
|
|
92
|
+
humanAnswers: [{scores: {faithful: true}, traceId: "a"}],
|
|
93
|
+
judgeAnswers: [{error: "timeout", traceId: "a"}],
|
|
94
|
+
judgeKeys: ["faithful"],
|
|
95
|
+
});
|
|
96
|
+
expect(agreement.errors).toBe(1);
|
|
97
|
+
expect(agreement.rate).toBe(0);
|
|
98
|
+
expect(agreement.disagreements[0]?.judge).toBeUndefined();
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
it("has no rate before any human answers", () => {
|
|
102
|
+
const agreement = computeJudgeAgreement({
|
|
103
|
+
dimensions: QUESTIONS,
|
|
104
|
+
humanAnswers: [],
|
|
105
|
+
judgeAnswers: [{scores: {faithful: true}, traceId: "a"}],
|
|
106
|
+
judgeKeys: ["faithful"],
|
|
107
|
+
});
|
|
108
|
+
expect(agreement.rate).toBeUndefined();
|
|
109
|
+
expect(agreement.compared).toBe(0);
|
|
110
|
+
});
|
|
111
|
+
});
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
import type {EvaluatorDimension} from "../types/observability";
|
|
2
|
+
|
|
3
|
+
export type ScoreValue = boolean | number | string;
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Answers for one reviewed item. Human and judge answers pair up by `datasetItemId` when set,
|
|
7
|
+
* otherwise by `traceId`.
|
|
8
|
+
*/
|
|
9
|
+
export interface AgreementAnswerSet {
|
|
10
|
+
datasetItemId?: string;
|
|
11
|
+
scores?: Record<string, ScoreValue>;
|
|
12
|
+
traceId?: string;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export interface JudgeAnswerSet extends AgreementAnswerSet {
|
|
16
|
+
error?: string;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface JudgeQuestionAgreement {
|
|
20
|
+
compared: number;
|
|
21
|
+
/** False when the judge does not answer this question, so it is left out of the totals. */
|
|
22
|
+
covered: boolean;
|
|
23
|
+
key: string;
|
|
24
|
+
matches: number;
|
|
25
|
+
question?: string;
|
|
26
|
+
rate?: number;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export interface JudgeDisagreement {
|
|
30
|
+
datasetItemId?: string;
|
|
31
|
+
human: ScoreValue;
|
|
32
|
+
judge?: ScoreValue;
|
|
33
|
+
key: string;
|
|
34
|
+
traceId?: string;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export interface JudgeAgreement {
|
|
38
|
+
compared: number;
|
|
39
|
+
disagreements: JudgeDisagreement[];
|
|
40
|
+
errors: number;
|
|
41
|
+
itemsCompared: number;
|
|
42
|
+
matches: number;
|
|
43
|
+
questions: JudgeQuestionAgreement[];
|
|
44
|
+
rate?: number;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// Numeric answers agree when they land within this share of the dimension's range.
|
|
48
|
+
const NUMERIC_TOLERANCE = 0.1;
|
|
49
|
+
|
|
50
|
+
const numericSpan = (range?: string): number => {
|
|
51
|
+
const matches = range?.match(/(-?\d+(?:\.\d+)?)\s*(?:-|\.\.)\s*(-?\d+(?:\.\d+)?)/);
|
|
52
|
+
if (!matches) {
|
|
53
|
+
return 1;
|
|
54
|
+
}
|
|
55
|
+
const span = Math.abs(Number(matches[2]) - Number(matches[1]));
|
|
56
|
+
return span > 0 ? span : 1;
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
const toBoolean = (value: ScoreValue): boolean | undefined => {
|
|
60
|
+
if (typeof value === "boolean") {
|
|
61
|
+
return value;
|
|
62
|
+
}
|
|
63
|
+
if (value === "true" || value === "yes") {
|
|
64
|
+
return true;
|
|
65
|
+
}
|
|
66
|
+
if (value === "false" || value === "no") {
|
|
67
|
+
return false;
|
|
68
|
+
}
|
|
69
|
+
return undefined;
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
/** True when a judge answer counts as the same answer a human gave for this dimension. */
|
|
73
|
+
export const answersAgree = (
|
|
74
|
+
dimension: EvaluatorDimension,
|
|
75
|
+
human: ScoreValue,
|
|
76
|
+
judge: ScoreValue | undefined
|
|
77
|
+
): boolean => {
|
|
78
|
+
if (judge === undefined) {
|
|
79
|
+
return false;
|
|
80
|
+
}
|
|
81
|
+
if (dimension.dataType === "boolean") {
|
|
82
|
+
const humanBool = toBoolean(human);
|
|
83
|
+
return humanBool !== undefined && humanBool === toBoolean(judge);
|
|
84
|
+
}
|
|
85
|
+
if (dimension.dataType === "numeric") {
|
|
86
|
+
const humanNumber = Number(human);
|
|
87
|
+
const judgeNumber = Number(judge);
|
|
88
|
+
if (Number.isNaN(humanNumber) || Number.isNaN(judgeNumber)) {
|
|
89
|
+
return false;
|
|
90
|
+
}
|
|
91
|
+
return (
|
|
92
|
+
Math.abs(humanNumber - judgeNumber) <= numericSpan(dimension.range) * NUMERIC_TOLERANCE + 1e-9
|
|
93
|
+
);
|
|
94
|
+
}
|
|
95
|
+
return String(human).trim().toLowerCase() === String(judge).trim().toLowerCase();
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
/** Key that pairs human and judge answers for the same item. */
|
|
99
|
+
export const agreementAnswerKey = (answer: AgreementAnswerSet): string => {
|
|
100
|
+
return answer.datasetItemId ? `item:${answer.datasetItemId}` : `trace:${answer.traceId ?? ""}`;
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
const rateOf = (matches: number, compared: number): number | undefined => {
|
|
104
|
+
return compared > 0 ? matches / compared : undefined;
|
|
105
|
+
};
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Compare one judge's answers with human answers for the same traces or dataset items. Only questions the judge
|
|
109
|
+
* answers (judgeKeys) count; a judge error or a missing answer counts as a disagreement.
|
|
110
|
+
*/
|
|
111
|
+
export const computeJudgeAgreement = (params: {
|
|
112
|
+
dimensions: EvaluatorDimension[];
|
|
113
|
+
humanAnswers: AgreementAnswerSet[];
|
|
114
|
+
judgeAnswers: JudgeAnswerSet[];
|
|
115
|
+
judgeKeys: string[];
|
|
116
|
+
}): JudgeAgreement => {
|
|
117
|
+
const judgeKeys = new Set(params.judgeKeys);
|
|
118
|
+
const judgeByKey = new Map(
|
|
119
|
+
params.judgeAnswers.map((answer) => {
|
|
120
|
+
return [agreementAnswerKey(answer), answer];
|
|
121
|
+
})
|
|
122
|
+
);
|
|
123
|
+
const questions: JudgeQuestionAgreement[] = params.dimensions.map((dimension) => {
|
|
124
|
+
return {
|
|
125
|
+
compared: 0,
|
|
126
|
+
covered: judgeKeys.has(dimension.key),
|
|
127
|
+
key: dimension.key,
|
|
128
|
+
matches: 0,
|
|
129
|
+
question: dimension.question,
|
|
130
|
+
};
|
|
131
|
+
});
|
|
132
|
+
const disagreements: JudgeDisagreement[] = [];
|
|
133
|
+
let itemsCompared = 0;
|
|
134
|
+
let errors = 0;
|
|
135
|
+
|
|
136
|
+
for (const human of params.humanAnswers) {
|
|
137
|
+
const judge = judgeByKey.get(agreementAnswerKey(human));
|
|
138
|
+
if (!judge || !human.scores) {
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
let comparedThisItem = false;
|
|
142
|
+
if (judge.error) {
|
|
143
|
+
errors += 1;
|
|
144
|
+
}
|
|
145
|
+
params.dimensions.forEach((dimension, index) => {
|
|
146
|
+
const humanValue = human.scores?.[dimension.key];
|
|
147
|
+
const question = questions[index];
|
|
148
|
+
if (!question?.covered || humanValue === undefined) {
|
|
149
|
+
return;
|
|
150
|
+
}
|
|
151
|
+
comparedThisItem = true;
|
|
152
|
+
question.compared += 1;
|
|
153
|
+
const judgeValue = judge.error ? undefined : judge.scores?.[dimension.key];
|
|
154
|
+
if (answersAgree(dimension, humanValue, judgeValue)) {
|
|
155
|
+
question.matches += 1;
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
158
|
+
disagreements.push({
|
|
159
|
+
...(human.datasetItemId ? {datasetItemId: human.datasetItemId} : {}),
|
|
160
|
+
human: humanValue,
|
|
161
|
+
judge: judgeValue,
|
|
162
|
+
key: dimension.key,
|
|
163
|
+
...(human.traceId ? {traceId: human.traceId} : {}),
|
|
164
|
+
});
|
|
165
|
+
});
|
|
166
|
+
if (comparedThisItem) {
|
|
167
|
+
itemsCompared += 1;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
const compared = questions.reduce((total, question) => total + question.compared, 0);
|
|
172
|
+
const matches = questions.reduce((total, question) => total + question.matches, 0);
|
|
173
|
+
return {
|
|
174
|
+
compared,
|
|
175
|
+
disagreements,
|
|
176
|
+
errors,
|
|
177
|
+
itemsCompared,
|
|
178
|
+
matches,
|
|
179
|
+
questions: questions.map((question) => {
|
|
180
|
+
return {...question, rate: rateOf(question.matches, question.compared)};
|
|
181
|
+
}),
|
|
182
|
+
rate: rateOf(matches, compared),
|
|
183
|
+
};
|
|
184
|
+
};
|
|
@@ -10,6 +10,12 @@ import {
|
|
|
10
10
|
parseDatasetJsonImport,
|
|
11
11
|
} from "../datasetImport";
|
|
12
12
|
import {validateAgainstSchema} from "../schemaValidation";
|
|
13
|
+
import {
|
|
14
|
+
type DatasetHumanAnswerView,
|
|
15
|
+
type DatasetJudgmentSummary,
|
|
16
|
+
summarizeDatasetJudgments,
|
|
17
|
+
toHumanAnswerViews,
|
|
18
|
+
} from "./humanAnswers";
|
|
13
19
|
import {registerObsDataset} from "./models/obsDataset";
|
|
14
20
|
import {registerObsDatasetItem} from "./models/obsDatasetItem";
|
|
15
21
|
import {registerObsTrace} from "./models/obsTrace";
|
|
@@ -48,6 +54,8 @@ export interface DatasetView {
|
|
|
48
54
|
expectedOutputSchema?: Record<string, unknown>;
|
|
49
55
|
id: string;
|
|
50
56
|
inputSchemaPromptName?: string;
|
|
57
|
+
/** Human judging progress per question set (see `summarizeDatasetJudgments`). */
|
|
58
|
+
judgments: DatasetJudgmentSummary[];
|
|
51
59
|
name: string;
|
|
52
60
|
tags: string[];
|
|
53
61
|
updated: string;
|
|
@@ -58,6 +66,8 @@ export interface DatasetItemView {
|
|
|
58
66
|
created: string;
|
|
59
67
|
datasetId: string;
|
|
60
68
|
expectedOutput?: unknown;
|
|
69
|
+
/** Human answers keyed by question-set evaluator id. */
|
|
70
|
+
humanAnswers?: Record<string, DatasetHumanAnswerView>;
|
|
61
71
|
id: string;
|
|
62
72
|
input: unknown;
|
|
63
73
|
metadata?: Record<string, unknown>;
|
|
@@ -85,17 +95,22 @@ export const throwOnDatasetImportErrors = (result: DatasetImportResult): void =>
|
|
|
85
95
|
});
|
|
86
96
|
};
|
|
87
97
|
|
|
88
|
-
const toDatasetView = async (
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
+
const toDatasetView = async (
|
|
99
|
+
doc: {
|
|
100
|
+
_id: mongoose.Types.ObjectId;
|
|
101
|
+
created: Date;
|
|
102
|
+
description?: string;
|
|
103
|
+
expectedOutputSchema?: Record<string, unknown>;
|
|
104
|
+
inputSchemaPromptName?: string;
|
|
105
|
+
name: string;
|
|
106
|
+
tags: string[];
|
|
107
|
+
updated: Date;
|
|
108
|
+
},
|
|
109
|
+
judgments?: DatasetJudgmentSummary[]
|
|
110
|
+
): Promise<DatasetView> => {
|
|
98
111
|
const counts = await computeDatasetCounts(String(doc._id));
|
|
112
|
+
const resolvedJudgments =
|
|
113
|
+
judgments ?? (await summarizeDatasetJudgments([doc._id])).get(String(doc._id)) ?? [];
|
|
99
114
|
return {
|
|
100
115
|
counts,
|
|
101
116
|
created: DateTime.fromJSDate(doc.created).toUTC().toISO() ?? "",
|
|
@@ -103,6 +118,7 @@ const toDatasetView = async (doc: {
|
|
|
103
118
|
expectedOutputSchema: doc.expectedOutputSchema,
|
|
104
119
|
id: String(doc._id),
|
|
105
120
|
inputSchemaPromptName: doc.inputSchemaPromptName,
|
|
121
|
+
judgments: resolvedJudgments,
|
|
106
122
|
name: doc.name,
|
|
107
123
|
tags: doc.tags,
|
|
108
124
|
updated: DateTime.fromJSDate(doc.updated).toUTC().toISO() ?? "",
|
|
@@ -115,6 +131,7 @@ const toItemView = (doc: {
|
|
|
115
131
|
created: Date;
|
|
116
132
|
datasetId: mongoose.Types.ObjectId;
|
|
117
133
|
expectedOutput?: unknown;
|
|
134
|
+
humanAnswers?: ObsDatasetItemDocument["humanAnswers"];
|
|
118
135
|
input: unknown;
|
|
119
136
|
metadata?: Record<string, unknown>;
|
|
120
137
|
origin: "manual" | "synthetic" | "trace";
|
|
@@ -129,6 +146,7 @@ const toItemView = (doc: {
|
|
|
129
146
|
created: DateTime.fromJSDate(doc.created).toUTC().toISO() ?? "",
|
|
130
147
|
datasetId: String(doc.datasetId),
|
|
131
148
|
expectedOutput: doc.expectedOutput,
|
|
149
|
+
humanAnswers: toHumanAnswerViews(doc.humanAnswers),
|
|
132
150
|
id: String(doc._id),
|
|
133
151
|
input: doc.input,
|
|
134
152
|
metadata: doc.metadata,
|
|
@@ -193,9 +211,10 @@ export class LocalDatasetStore {
|
|
|
193
211
|
|
|
194
212
|
async list(): Promise<DatasetView[]> {
|
|
195
213
|
const rows = await registerObsDataset().find({}).sort({name: 1});
|
|
214
|
+
const judgments = await summarizeDatasetJudgments(rows.map((row) => row._id));
|
|
196
215
|
return Promise.all(
|
|
197
216
|
rows.map((row) => {
|
|
198
|
-
return toDatasetView(row);
|
|
217
|
+
return toDatasetView(row, judgments.get(String(row._id)) ?? []);
|
|
199
218
|
})
|
|
200
219
|
);
|
|
201
220
|
}
|