@terreno/ai 58.0.0-beta.3 → 58.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +166 -0
- package/README.md +3 -3
- package/dist/harness/commit.d.ts +4 -3
- package/dist/harness/commit.js +23 -4
- package/dist/index.d.ts +6 -9
- package/dist/index.js +14 -25
- package/dist/observability/adminScreens.js +5 -1
- package/dist/observability/evaluate.d.ts +8 -1
- package/dist/observability/evaluate.js +10 -2
- package/dist/observability/judgeAgreement.d.ts +53 -0
- package/dist/observability/judgeAgreement.js +123 -0
- package/dist/observability/local/datasetStore.d.ts +5 -0
- package/dist/observability/local/datasetStore.js +7 -2
- package/dist/observability/local/evaluatorStore.d.ts +17 -0
- package/dist/observability/local/evaluatorStore.js +73 -2
- package/dist/observability/local/experimentRunner.d.ts +40 -3
- package/dist/observability/local/experimentRunner.js +390 -11
- package/dist/observability/local/humanAnswers.d.ts +57 -0
- package/dist/observability/local/humanAnswers.js +174 -0
- package/dist/observability/local/judgeRun.d.ts +56 -0
- package/dist/observability/local/judgeRun.js +111 -0
- package/dist/observability/local/localPlugin.js +4 -0
- package/dist/observability/local/models/obsDatasetItem.js +4 -0
- package/dist/observability/local/models/obsEvaluator.js +4 -0
- package/dist/observability/local/models/obsExperiment.js +28 -0
- package/dist/observability/local/models/obsExperimentItem.js +4 -0
- package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
- package/dist/observability/local/models/obsJudgeRun.js +60 -0
- package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
- package/dist/observability/local/models/obsReviewBatch.js +48 -0
- package/dist/observability/local/models/obsReviewItem.js +10 -2
- package/dist/observability/local/models/obsTrace.js +4 -0
- package/dist/observability/local/promptReferences.d.ts +31 -0
- package/dist/observability/local/promptReferences.js +114 -0
- package/dist/observability/local/promptStore.d.ts +55 -19
- package/dist/observability/local/promptStore.js +171 -7
- package/dist/observability/local/reviewBatchStore.d.ts +106 -0
- package/dist/observability/local/reviewBatchStore.js +517 -0
- package/dist/observability/local/reviewStore.d.ts +24 -3
- package/dist/observability/local/reviewStore.js +98 -22
- package/dist/observability/local/traceStore.d.ts +14 -1
- package/dist/observability/local/traceStore.js +144 -11
- package/dist/observability/observabilityApp.js +10 -1
- package/dist/observability/questionFingerprint.d.ts +7 -0
- package/dist/observability/questionFingerprint.js +23 -0
- package/dist/observability/routes/experiments.js +12 -1
- package/dist/observability/routes/review.js +3 -1
- package/dist/observability/routes/reviewBatches.d.ts +11 -0
- package/dist/observability/routes/reviewBatches.js +78 -0
- package/dist/observability/routes/traces.d.ts +1 -1
- package/dist/observability/routes/traces.js +34 -2
- package/dist/observability/types.d.ts +10 -1
- package/dist/service/aiService.js +4 -1
- package/dist/service/chatTurn.js +38 -31
- package/dist/types/harness.d.ts +5 -2
- package/dist/types/index.d.ts +8 -3
- package/dist/types/observability.d.ts +74 -2
- package/package.json +8 -24
- package/src/harness/agentLoop.test.ts +0 -3
- package/src/harness/agentRecovery.test.ts +0 -3
- package/src/harness/commit.ts +24 -4
- package/src/harness/extensions.test.ts +0 -3
- package/src/harness/subagent.test.ts +27 -3
- package/src/index.test.ts +0 -17
- package/src/index.ts +23 -26
- package/src/isolated/gptStream.isolated.ts +0 -8
- package/src/observability/adminScreens.test.ts +7 -0
- package/src/observability/adminScreens.ts +6 -1
- package/src/observability/evaluate.ts +12 -2
- package/src/observability/judgeAgreement.test.ts +111 -0
- package/src/observability/judgeAgreement.ts +184 -0
- package/src/observability/local/datasetStore.ts +30 -11
- package/src/observability/local/evaluatorStore.ts +96 -2
- package/src/observability/local/experimentRunner.ts +495 -15
- package/src/observability/local/humanAnswers.ts +233 -0
- package/src/observability/local/judgePipeline.test.ts +444 -0
- package/src/observability/local/judgeRun.test.ts +173 -0
- package/src/observability/local/judgeRun.ts +184 -0
- package/src/observability/local/localPlugin.ts +4 -0
- package/src/observability/local/models/obsDatasetItem.ts +5 -0
- package/src/observability/local/models/obsEvaluator.ts +4 -0
- package/src/observability/local/models/obsExperiment.ts +29 -0
- package/src/observability/local/models/obsExperimentItem.ts +4 -0
- package/src/observability/local/models/obsJudgeRun.ts +61 -0
- package/src/observability/local/models/obsReviewBatch.ts +52 -0
- package/src/observability/local/models/obsReviewItem.ts +11 -2
- package/src/observability/local/models/obsTrace.ts +5 -0
- package/src/observability/local/obsModels.test.ts +13 -0
- package/src/observability/local/promptReferences.test.ts +128 -0
- package/src/observability/local/promptReferences.ts +149 -0
- package/src/observability/local/promptStore.test.ts +291 -0
- package/src/observability/local/promptStore.ts +243 -32
- package/src/observability/local/reviewBatchStore.test.ts +251 -0
- package/src/observability/local/reviewBatchStore.ts +656 -0
- package/src/observability/local/reviewStore.ts +122 -25
- package/src/observability/local/traceStore.test.ts +139 -1
- package/src/observability/local/traceStore.ts +162 -11
- package/src/observability/observabilityApp.test.ts +13 -10
- package/src/observability/observabilityApp.ts +10 -1
- package/src/observability/questionFingerprint.test.ts +33 -0
- package/src/observability/questionFingerprint.ts +24 -0
- package/src/observability/routes/experiments.test.ts +37 -0
- package/src/observability/routes/experiments.ts +13 -1
- package/src/observability/routes/review.ts +3 -1
- package/src/observability/routes/reviewBatches.test.ts +209 -0
- package/src/observability/routes/reviewBatches.ts +138 -0
- package/src/observability/routes/status.test.ts +4 -4
- package/src/observability/routes/traces.test.ts +61 -0
- package/src/observability/routes/traces.ts +46 -4
- package/src/observability/types.ts +5 -2
- package/src/routes/routes.test.ts +101 -76
- package/src/service/aiService.test.ts +21 -0
- package/src/service/aiService.ts +4 -1
- package/src/service/chatTurn.ts +42 -33
- package/src/tests/bunSetup.ts +0 -54
- package/src/types/harness.ts +5 -2
- package/src/types/index.ts +8 -3
- package/src/types/observability.ts +86 -2
- package/dist/langfuseApp.d.ts +0 -9
- package/dist/langfuseApp.js +0 -53
- package/dist/langfuseCache.d.ts +0 -15
- package/dist/langfuseCache.js +0 -54
- package/dist/langfuseClient.d.ts +0 -6
- package/dist/langfuseClient.js +0 -34
- package/dist/langfusePrompts.d.ts +0 -12
- package/dist/langfusePrompts.js +0 -89
- package/dist/langfuseRoutesEvaluations.d.ts +0 -3
- package/dist/langfuseRoutesEvaluations.js +0 -32
- package/dist/langfuseRoutesMiddleware.d.ts +0 -2
- package/dist/langfuseRoutesMiddleware.js +0 -13
- package/dist/langfuseRoutesPlayground.d.ts +0 -2
- package/dist/langfuseRoutesPlayground.js +0 -36
- package/dist/langfuseRoutesPrompts.d.ts +0 -2
- package/dist/langfuseRoutesPrompts.js +0 -56
- package/dist/langfuseRoutesTraces.d.ts +0 -2
- package/dist/langfuseRoutesTraces.js +0 -30
- package/dist/langfuseTracing.d.ts +0 -4
- package/dist/langfuseTracing.js +0 -32
- package/dist/langfuseTypes.d.ts +0 -96
- package/dist/langfuseTypes.js +0 -2
- package/dist/langfuseVercelAi.d.ts +0 -12
- package/dist/langfuseVercelAi.js +0 -49
- package/src/isolated/langfuseClient.isolated.ts +0 -66
- package/src/langfuseApp.test.ts +0 -119
- package/src/langfuseApp.ts +0 -62
- package/src/langfuseCache.test.ts +0 -75
- package/src/langfuseCache.ts +0 -72
- package/src/langfuseClient.test.ts +0 -69
- package/src/langfuseClient.ts +0 -34
- package/src/langfusePrompts.test.ts +0 -188
- package/src/langfusePrompts.ts +0 -121
- package/src/langfuseRoutes.test.ts +0 -261
- package/src/langfuseRoutesEvaluations.ts +0 -48
- package/src/langfuseRoutesMiddleware.test.ts +0 -57
- package/src/langfuseRoutesMiddleware.ts +0 -12
- package/src/langfuseRoutesPlayground.ts +0 -50
- package/src/langfuseRoutesPrompts.ts +0 -87
- package/src/langfuseRoutesTraces.ts +0 -40
- package/src/langfuseTracing.test.ts +0 -72
- package/src/langfuseTracing.ts +0 -33
- package/src/langfuseTypes.ts +0 -103
- package/src/langfuseVercelAi.test.ts +0 -82
- package/src/langfuseVercelAi.ts +0 -72
- package/src/tests/fakeLangfusePrompt.ts +0 -10
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import {describe, expect, it} from "bun:test";
|
|
2
|
+
|
|
3
|
+
import type {EvaluatorDimension} from "../../types/observability";
|
|
4
|
+
import type {EvaluatorAiClient} from "../evaluate";
|
|
5
|
+
import {type JudgeRunItemResult, runJudgeAgainstHumans} from "./judgeRun";
|
|
6
|
+
|
|
7
|
+
const DIMENSIONS: EvaluatorDimension[] = [
|
|
8
|
+
{dataType: "boolean", key: "faithful", question: "Is it faithful?", required: true},
|
|
9
|
+
{dataType: "boolean", key: "complete", question: "Is it complete?", required: true},
|
|
10
|
+
];
|
|
11
|
+
|
|
12
|
+
const JUDGE = {
|
|
13
|
+
dimensions: DIMENSIONS,
|
|
14
|
+
judgePromptName: "summary-judge",
|
|
15
|
+
name: "summary-judge",
|
|
16
|
+
type: "llm-judge" as const,
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
const SCHEMA = {
|
|
20
|
+
properties: {complete: {type: "boolean"}, faithful: {type: "boolean"}},
|
|
21
|
+
required: ["faithful", "complete"],
|
|
22
|
+
type: "object",
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
interface RecordedCall {
|
|
26
|
+
prompt: string;
|
|
27
|
+
promptName?: string;
|
|
28
|
+
systemPrompt?: string;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// Says "faithful" unless the output contains "bad"; throws for outputs containing "boom".
|
|
32
|
+
const makeClient = (): {
|
|
33
|
+
calls: RecordedCall[];
|
|
34
|
+
client: EvaluatorAiClient;
|
|
35
|
+
maxInFlight: () => number;
|
|
36
|
+
} => {
|
|
37
|
+
const calls: RecordedCall[] = [];
|
|
38
|
+
let inFlight = 0;
|
|
39
|
+
let peak = 0;
|
|
40
|
+
const client: EvaluatorAiClient = {
|
|
41
|
+
generateJsonObject: async <OBJECT>(options: {
|
|
42
|
+
prompt: string;
|
|
43
|
+
promptName?: string;
|
|
44
|
+
systemPrompt?: string;
|
|
45
|
+
}): Promise<OBJECT> => {
|
|
46
|
+
calls.push({
|
|
47
|
+
prompt: options.prompt,
|
|
48
|
+
promptName: options.promptName,
|
|
49
|
+
systemPrompt: options.systemPrompt,
|
|
50
|
+
});
|
|
51
|
+
inFlight += 1;
|
|
52
|
+
peak = Math.max(peak, inFlight);
|
|
53
|
+
await Bun.sleep(5);
|
|
54
|
+
inFlight -= 1;
|
|
55
|
+
if (options.prompt.includes("boom")) {
|
|
56
|
+
throw new Error("judge exploded");
|
|
57
|
+
}
|
|
58
|
+
return {complete: true, faithful: !options.prompt.includes("bad")} as OBJECT;
|
|
59
|
+
},
|
|
60
|
+
};
|
|
61
|
+
return {calls, client, maxInFlight: () => peak};
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
describe("runJudgeAgainstHumans", () => {
|
|
65
|
+
it("judges every subject with bounded concurrency and returns results in input order", async () => {
|
|
66
|
+
const {client, maxInFlight} = makeClient();
|
|
67
|
+
const subjects = Array.from({length: 7}, (_, index) => {
|
|
68
|
+
return {datasetItemId: `item-${index}`, input: "q", output: `answer ${index}`};
|
|
69
|
+
});
|
|
70
|
+
const run = await runJudgeAgainstHumans({
|
|
71
|
+
ai: client,
|
|
72
|
+
concurrency: 3,
|
|
73
|
+
dimensions: DIMENSIONS,
|
|
74
|
+
humanAnswers: [],
|
|
75
|
+
judge: JUDGE,
|
|
76
|
+
prompt: {outputSchema: SCHEMA},
|
|
77
|
+
subjects,
|
|
78
|
+
});
|
|
79
|
+
expect(maxInFlight()).toBeLessThanOrEqual(3);
|
|
80
|
+
expect(maxInFlight()).toBeGreaterThan(1);
|
|
81
|
+
expect(run.results.map((row) => row.datasetItemId)).toEqual(
|
|
82
|
+
subjects.map((row) => row.datasetItemId)
|
|
83
|
+
);
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
it("scores errors and wrong answers as disagreements in both item results and agreement", async () => {
|
|
87
|
+
const {client} = makeClient();
|
|
88
|
+
const subjects = [
|
|
89
|
+
{output: "good", traceId: "t-good"},
|
|
90
|
+
{output: "bad", traceId: "t-bad"},
|
|
91
|
+
{output: "boom", traceId: "t-boom"},
|
|
92
|
+
{output: "good", traceId: "t-unreviewed"},
|
|
93
|
+
];
|
|
94
|
+
const humanAnswers = [
|
|
95
|
+
{scores: {complete: true, faithful: true}, traceId: "t-good"},
|
|
96
|
+
{scores: {complete: true, faithful: true}, traceId: "t-bad"},
|
|
97
|
+
{scores: {complete: true, faithful: true}, traceId: "t-boom"},
|
|
98
|
+
];
|
|
99
|
+
const run = await runJudgeAgainstHumans({
|
|
100
|
+
ai: client,
|
|
101
|
+
dimensions: DIMENSIONS,
|
|
102
|
+
humanAnswers,
|
|
103
|
+
judge: JUDGE,
|
|
104
|
+
prompt: {outputSchema: SCHEMA},
|
|
105
|
+
subjects,
|
|
106
|
+
});
|
|
107
|
+
const byTrace = new Map<string, JudgeRunItemResult>(
|
|
108
|
+
run.results.map((row) => [row.traceId ?? "", row])
|
|
109
|
+
);
|
|
110
|
+
expect(byTrace.get("t-good")?.disagrees).toBe(false);
|
|
111
|
+
expect(byTrace.get("t-bad")?.disagrees).toBe(true);
|
|
112
|
+
expect(byTrace.get("t-boom")?.disagrees).toBe(true);
|
|
113
|
+
expect(byTrace.get("t-boom")?.error).toBe("judge exploded");
|
|
114
|
+
expect(byTrace.get("t-unreviewed")?.disagrees).toBe(false);
|
|
115
|
+
expect(run.agreement.errors).toBe(1);
|
|
116
|
+
expect(run.agreement.itemsCompared).toBe(3);
|
|
117
|
+
expect(run.agreement.compared).toBe(6);
|
|
118
|
+
expect(run.agreement.matches).toBe(3);
|
|
119
|
+
expect(run.agreement.disagreements.map((row) => `${row.traceId}:${row.key}`)).toEqual([
|
|
120
|
+
"t-bad:faithful",
|
|
121
|
+
"t-boom:faithful",
|
|
122
|
+
"t-boom:complete",
|
|
123
|
+
]);
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
it("uses the version body as the system prompt when given, else the prompt name", async () => {
|
|
127
|
+
const withSystem = makeClient();
|
|
128
|
+
await runJudgeAgainstHumans({
|
|
129
|
+
ai: withSystem.client,
|
|
130
|
+
dimensions: DIMENSIONS,
|
|
131
|
+
humanAnswers: [],
|
|
132
|
+
judge: JUDGE,
|
|
133
|
+
prompt: {outputSchema: SCHEMA, system: "STRICT"},
|
|
134
|
+
subjects: [{datasetItemId: "a", output: "x"}],
|
|
135
|
+
});
|
|
136
|
+
expect(withSystem.calls[0]?.systemPrompt).toBe("STRICT");
|
|
137
|
+
expect(withSystem.calls[0]?.promptName).toBeUndefined();
|
|
138
|
+
|
|
139
|
+
const byName = makeClient();
|
|
140
|
+
await runJudgeAgainstHumans({
|
|
141
|
+
ai: byName.client,
|
|
142
|
+
dimensions: DIMENSIONS,
|
|
143
|
+
humanAnswers: [],
|
|
144
|
+
judge: JUDGE,
|
|
145
|
+
prompt: {outputSchema: SCHEMA},
|
|
146
|
+
subjects: [{datasetItemId: "a", output: "x"}],
|
|
147
|
+
});
|
|
148
|
+
expect(byName.calls[0]?.promptName).toBe("summary-judge");
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
it("reports each result to onResult in input order, one at a time", async () => {
|
|
152
|
+
const {client} = makeClient();
|
|
153
|
+
const seen: string[] = [];
|
|
154
|
+
let active = 0;
|
|
155
|
+
await runJudgeAgainstHumans({
|
|
156
|
+
ai: client,
|
|
157
|
+
concurrency: 4,
|
|
158
|
+
dimensions: DIMENSIONS,
|
|
159
|
+
humanAnswers: [],
|
|
160
|
+
judge: JUDGE,
|
|
161
|
+
onResult: async (result) => {
|
|
162
|
+
active += 1;
|
|
163
|
+
expect(active).toBe(1);
|
|
164
|
+
await Bun.sleep(1);
|
|
165
|
+
seen.push(result.datasetItemId ?? "");
|
|
166
|
+
active -= 1;
|
|
167
|
+
},
|
|
168
|
+
prompt: {outputSchema: SCHEMA},
|
|
169
|
+
subjects: ["a", "b", "c", "d", "e"].map((id) => ({datasetItemId: id, output: id})),
|
|
170
|
+
});
|
|
171
|
+
expect(seen).toEqual(["a", "b", "c", "d", "e"]);
|
|
172
|
+
});
|
|
173
|
+
});
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
import type {EvaluatorDimension, ObsEvaluatorDocument} from "../../types/observability";
|
|
2
|
+
import {type EvaluatorAiClient, runLlmJudgeEvaluator} from "../evaluate";
|
|
3
|
+
import {
|
|
4
|
+
type AgreementAnswerSet,
|
|
5
|
+
agreementAnswerKey,
|
|
6
|
+
answersAgree,
|
|
7
|
+
computeJudgeAgreement,
|
|
8
|
+
type JudgeAgreement,
|
|
9
|
+
type ScoreValue,
|
|
10
|
+
} from "../judgeAgreement";
|
|
11
|
+
|
|
12
|
+
export const DEFAULT_JUDGE_CONCURRENCY = 5;
|
|
13
|
+
|
|
14
|
+
/** What a judge sees for one item, and the ids that pair it with the human answer. */
|
|
15
|
+
export interface JudgeRunSubject {
|
|
16
|
+
datasetItemId?: string;
|
|
17
|
+
input?: unknown;
|
|
18
|
+
output?: unknown;
|
|
19
|
+
traceId?: string;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** The judge prompt version to run: its output schema and, when pinned, its system text. */
|
|
23
|
+
export interface JudgeRunPrompt {
|
|
24
|
+
outputSchema: Record<string, unknown>;
|
|
25
|
+
/** Version body sent as the system prompt; unset sends the judge's prompt name instead. */
|
|
26
|
+
system?: string;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export interface JudgeRunItemResult {
|
|
30
|
+
confidence?: number;
|
|
31
|
+
datasetItemId?: string;
|
|
32
|
+
/** True when a human answered a compared question and the judge failed or answered differently. */
|
|
33
|
+
disagrees: boolean;
|
|
34
|
+
error?: string;
|
|
35
|
+
/** Index of the subject in the input list. */
|
|
36
|
+
index: number;
|
|
37
|
+
scores?: Record<string, ScoreValue>;
|
|
38
|
+
traceId?: string;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export interface JudgeRunResult {
|
|
42
|
+
agreement: JudgeAgreement;
|
|
43
|
+
results: JudgeRunItemResult[];
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** Agreement between human answers and (possibly partial) judge results from this pipeline. */
|
|
47
|
+
export const agreementForResults = (params: {
|
|
48
|
+
dimensions: EvaluatorDimension[];
|
|
49
|
+
humanAnswers: AgreementAnswerSet[];
|
|
50
|
+
judge: Pick<ObsEvaluatorDocument, "dimensions">;
|
|
51
|
+
results: JudgeRunItemResult[];
|
|
52
|
+
}): JudgeAgreement => {
|
|
53
|
+
return computeJudgeAgreement({
|
|
54
|
+
dimensions: params.dimensions,
|
|
55
|
+
humanAnswers: params.humanAnswers,
|
|
56
|
+
judgeAnswers: params.results.map((result) => {
|
|
57
|
+
return {
|
|
58
|
+
datasetItemId: result.datasetItemId,
|
|
59
|
+
error: result.error,
|
|
60
|
+
scores: result.scores,
|
|
61
|
+
traceId: result.traceId,
|
|
62
|
+
};
|
|
63
|
+
}),
|
|
64
|
+
judgeKeys: params.judge.dimensions.map((dimension) => dimension.key),
|
|
65
|
+
});
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
// Run async work with a fixed number in flight, keeping input order in the result.
|
|
69
|
+
const mapWithConcurrency = async <T, R>(
|
|
70
|
+
items: T[],
|
|
71
|
+
limit: number,
|
|
72
|
+
fn: (item: T, index: number) => Promise<R>
|
|
73
|
+
): Promise<R[]> => {
|
|
74
|
+
const results: R[] = new Array(items.length);
|
|
75
|
+
let cursor = 0;
|
|
76
|
+
const workers = Array.from({length: Math.min(Math.max(limit, 1), items.length)}, async () => {
|
|
77
|
+
while (cursor < items.length) {
|
|
78
|
+
const index = cursor;
|
|
79
|
+
cursor += 1;
|
|
80
|
+
results[index] = await fn(items[index] as T, index);
|
|
81
|
+
}
|
|
82
|
+
});
|
|
83
|
+
await Promise.all(workers);
|
|
84
|
+
return results;
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Run one judge prompt version over every subject and compare its answers with the human
|
|
89
|
+
* answers for the same items. Review-batch judge runs and judge experiments both go through
|
|
90
|
+
* here, so judge errors and disagreements are scored the same way everywhere.
|
|
91
|
+
*/
|
|
92
|
+
export const runJudgeAgainstHumans = async (params: {
|
|
93
|
+
ai: EvaluatorAiClient;
|
|
94
|
+
concurrency?: number;
|
|
95
|
+
/** Question set dimensions the humans answered. */
|
|
96
|
+
dimensions: EvaluatorDimension[];
|
|
97
|
+
humanAnswers: AgreementAnswerSet[];
|
|
98
|
+
judge: Pick<ObsEvaluatorDocument, "dimensions" | "judgePromptName" | "name" | "type">;
|
|
99
|
+
/** Called once per subject in input order, never concurrently. */
|
|
100
|
+
onResult?: (result: JudgeRunItemResult) => Promise<void> | void;
|
|
101
|
+
prompt: JudgeRunPrompt;
|
|
102
|
+
subjects: JudgeRunSubject[];
|
|
103
|
+
}): Promise<JudgeRunResult> => {
|
|
104
|
+
const judgeKeys = params.judge.dimensions.map((dimension) => dimension.key);
|
|
105
|
+
const comparedDimensions = params.dimensions.filter((dimension) => {
|
|
106
|
+
return judgeKeys.includes(dimension.key);
|
|
107
|
+
});
|
|
108
|
+
const humanByKey = new Map(
|
|
109
|
+
params.humanAnswers.map((answer) => {
|
|
110
|
+
return [agreementAnswerKey(answer), answer];
|
|
111
|
+
})
|
|
112
|
+
);
|
|
113
|
+
|
|
114
|
+
// Results are reported in input order, so buffer finished ones until their turn comes.
|
|
115
|
+
const finished: Array<JudgeRunItemResult | undefined> = new Array(params.subjects.length);
|
|
116
|
+
let nextToReport = 0;
|
|
117
|
+
let reporting: Promise<void> = Promise.resolve();
|
|
118
|
+
const report = async (): Promise<void> => {
|
|
119
|
+
while (nextToReport < finished.length && finished[nextToReport]) {
|
|
120
|
+
const result = finished[nextToReport] as JudgeRunItemResult;
|
|
121
|
+
nextToReport += 1;
|
|
122
|
+
await params.onResult?.(result);
|
|
123
|
+
}
|
|
124
|
+
};
|
|
125
|
+
|
|
126
|
+
const results = await mapWithConcurrency(
|
|
127
|
+
params.subjects,
|
|
128
|
+
params.concurrency ?? DEFAULT_JUDGE_CONCURRENCY,
|
|
129
|
+
async (subject, index) => {
|
|
130
|
+
const run = await runLlmJudgeEvaluator(
|
|
131
|
+
{
|
|
132
|
+
evaluator: {
|
|
133
|
+
dimensions: params.judge.dimensions,
|
|
134
|
+
judgePromptName: params.judge.judgePromptName,
|
|
135
|
+
name: params.judge.name,
|
|
136
|
+
type: params.judge.type,
|
|
137
|
+
},
|
|
138
|
+
input: subject.input,
|
|
139
|
+
output: subject.output,
|
|
140
|
+
},
|
|
141
|
+
params.ai,
|
|
142
|
+
params.prompt.outputSchema,
|
|
143
|
+
params.prompt.system === undefined ? {} : {judgeSystemPrompt: params.prompt.system}
|
|
144
|
+
);
|
|
145
|
+
const ids = {
|
|
146
|
+
...(subject.datasetItemId ? {datasetItemId: subject.datasetItemId} : {}),
|
|
147
|
+
...(subject.traceId ? {traceId: subject.traceId} : {}),
|
|
148
|
+
};
|
|
149
|
+
const human = humanByKey.get(agreementAnswerKey(ids));
|
|
150
|
+
// Same rule as computeJudgeAgreement: an errored judge answers nothing.
|
|
151
|
+
const disagrees = comparedDimensions.some((dimension) => {
|
|
152
|
+
const humanValue = human?.scores?.[dimension.key];
|
|
153
|
+
if (humanValue === undefined) {
|
|
154
|
+
return false;
|
|
155
|
+
}
|
|
156
|
+
const judgeValue = run.error ? undefined : run.scores?.[dimension.key];
|
|
157
|
+
return !answersAgree(dimension, humanValue, judgeValue);
|
|
158
|
+
});
|
|
159
|
+
const result: JudgeRunItemResult = {
|
|
160
|
+
...ids,
|
|
161
|
+
confidence: run.confidence,
|
|
162
|
+
disagrees,
|
|
163
|
+
error: run.error,
|
|
164
|
+
index,
|
|
165
|
+
scores: run.scores,
|
|
166
|
+
};
|
|
167
|
+
finished[index] = result;
|
|
168
|
+
reporting = reporting.then(report);
|
|
169
|
+
await reporting;
|
|
170
|
+
return result;
|
|
171
|
+
}
|
|
172
|
+
);
|
|
173
|
+
await reporting;
|
|
174
|
+
|
|
175
|
+
return {
|
|
176
|
+
agreement: agreementForResults({
|
|
177
|
+
dimensions: params.dimensions,
|
|
178
|
+
humanAnswers: params.humanAnswers,
|
|
179
|
+
judge: params.judge,
|
|
180
|
+
results,
|
|
181
|
+
}),
|
|
182
|
+
results,
|
|
183
|
+
};
|
|
184
|
+
};
|
|
@@ -7,9 +7,11 @@ import {registerObsDatasetItem} from "./models/obsDatasetItem";
|
|
|
7
7
|
import {registerObsEvaluator} from "./models/obsEvaluator";
|
|
8
8
|
import {registerObsExperiment} from "./models/obsExperiment";
|
|
9
9
|
import {registerObsExperimentItem} from "./models/obsExperimentItem";
|
|
10
|
+
import {registerObsJudgeRun} from "./models/obsJudgeRun";
|
|
10
11
|
import {registerObsPrompt} from "./models/obsPrompt";
|
|
11
12
|
import {registerObsPromptLabel} from "./models/obsPromptLabel";
|
|
12
13
|
import {registerObsPromptVersion} from "./models/obsPromptVersion";
|
|
14
|
+
import {registerObsReviewBatch} from "./models/obsReviewBatch";
|
|
13
15
|
import {registerObsReviewItem} from "./models/obsReviewItem";
|
|
14
16
|
import {registerObsScore} from "./models/obsScore";
|
|
15
17
|
import {registerObsSpan} from "./models/obsSpan";
|
|
@@ -35,6 +37,8 @@ const registerLocalObservabilityModels = (): void => {
|
|
|
35
37
|
registerObsScore();
|
|
36
38
|
registerObsEvaluator();
|
|
37
39
|
registerObsReviewItem();
|
|
40
|
+
registerObsReviewBatch();
|
|
41
|
+
registerObsJudgeRun();
|
|
38
42
|
registerObsDataset();
|
|
39
43
|
registerObsDatasetItem();
|
|
40
44
|
registerObsExperiment();
|
|
@@ -23,6 +23,11 @@ const obsDatasetItemSchema = new mongoose.Schema<ObsDatasetItemDocument, ObsData
|
|
|
23
23
|
description: "Human- or machine-authored expected output label",
|
|
24
24
|
type: mongoose.Schema.Types.Mixed,
|
|
25
25
|
},
|
|
26
|
+
humanAnswers: {
|
|
27
|
+
description:
|
|
28
|
+
"Human answers keyed by question-set evaluator id: {fingerprint, scores, reviewItemId, userId, answeredAt}. An answer is current while its fingerprint matches the question set",
|
|
29
|
+
type: mongoose.Schema.Types.Mixed,
|
|
30
|
+
},
|
|
26
31
|
input: {
|
|
27
32
|
description: "Dataset item input payload",
|
|
28
33
|
required: true,
|
|
@@ -27,6 +27,10 @@ const obsEvaluatorSchema = new mongoose.Schema<ObsEvaluatorDocument, ObsEvaluato
|
|
|
27
27
|
type: String,
|
|
28
28
|
},
|
|
29
29
|
key: {description: "Dimension key written onto scores", required: true, type: String},
|
|
30
|
+
question: {
|
|
31
|
+
description: "Question shown to human reviewers for this dimension",
|
|
32
|
+
type: String,
|
|
33
|
+
},
|
|
30
34
|
range: {
|
|
31
35
|
description: "Numeric bounds, categorical values, or a free-text hint",
|
|
32
36
|
type: String,
|
|
@@ -30,6 +30,19 @@ const obsExperimentSchema = new mongoose.Schema<ObsExperimentDocument, ObsExperi
|
|
|
30
30
|
description: "When true, synthetic/unproofread items are included",
|
|
31
31
|
type: Boolean,
|
|
32
32
|
},
|
|
33
|
+
judgeEvaluatorId: {
|
|
34
|
+
description: "Judge experiments: llm-judge evaluator whose prompt versions are compared",
|
|
35
|
+
ref: "ObsEvaluator",
|
|
36
|
+
type: mongoose.Schema.Types.ObjectId,
|
|
37
|
+
},
|
|
38
|
+
kind: {
|
|
39
|
+
default: "prompt",
|
|
40
|
+
description:
|
|
41
|
+
"prompt compares product prompt versions; judge ranks judge prompt versions by agreement with human answers",
|
|
42
|
+
enum: ["judge", "prompt"],
|
|
43
|
+
required: true,
|
|
44
|
+
type: String,
|
|
45
|
+
},
|
|
33
46
|
modelOverride: {
|
|
34
47
|
description: "Optional model id override for all generations",
|
|
35
48
|
type: String,
|
|
@@ -40,6 +53,15 @@ const obsExperimentSchema = new mongoose.Schema<ObsExperimentDocument, ObsExperi
|
|
|
40
53
|
required: true,
|
|
41
54
|
type: String,
|
|
42
55
|
},
|
|
56
|
+
questionSetId: {
|
|
57
|
+
description: "Judge experiments: human question set whose answers are the reference",
|
|
58
|
+
ref: "ObsEvaluator",
|
|
59
|
+
type: mongoose.Schema.Types.ObjectId,
|
|
60
|
+
},
|
|
61
|
+
questionsFingerprint: {
|
|
62
|
+
description: "Judge experiments: question-set fingerprint of the compared human answers",
|
|
63
|
+
type: String,
|
|
64
|
+
},
|
|
43
65
|
results: {
|
|
44
66
|
description: "Aggregated gate, outlier, and progress results",
|
|
45
67
|
type: mongoose.Schema.Types.Mixed,
|
|
@@ -51,6 +73,10 @@ const obsExperimentSchema = new mongoose.Schema<ObsExperimentDocument, ObsExperi
|
|
|
51
73
|
required: true,
|
|
52
74
|
type: String,
|
|
53
75
|
},
|
|
76
|
+
targetAgreement: {
|
|
77
|
+
description: "Judge experiments: agreement rate (0–1) a judge version must reach to pass",
|
|
78
|
+
type: Number,
|
|
79
|
+
},
|
|
54
80
|
thresholds: {
|
|
55
81
|
default: [],
|
|
56
82
|
description: "Gate thresholds evaluated after the run",
|
|
@@ -79,6 +105,9 @@ obsExperimentSchema.index({created: -1, datasetId: 1});
|
|
|
79
105
|
obsExperimentSchema.index({created: -1, status: 1});
|
|
80
106
|
// biome-ignore assist/source/useSortedKeys: promptName must be the leading key for prompt-filtered lists
|
|
81
107
|
obsExperimentSchema.index({promptName: 1, created: -1});
|
|
108
|
+
// Evaluator calibration reads the latest judge experiment per judge evaluator.
|
|
109
|
+
// biome-ignore assist/source/useSortedKeys: judgeEvaluatorId must lead for calibration lookups
|
|
110
|
+
obsExperimentSchema.index({judgeEvaluatorId: 1, created: -1});
|
|
82
111
|
|
|
83
112
|
export const registerObsExperiment = (): ObsExperimentModel => {
|
|
84
113
|
if (mongoose.models.ObsExperiment) {
|
|
@@ -25,6 +25,10 @@ const obsExperimentItemSchema = new mongoose.Schema<
|
|
|
25
25
|
description: "When true, this row failed a gate or evaluator",
|
|
26
26
|
type: Boolean,
|
|
27
27
|
},
|
|
28
|
+
humanScores: {
|
|
29
|
+
description: "Judge experiments: human answers the judge versions are compared with",
|
|
30
|
+
type: mongoose.Schema.Types.Mixed,
|
|
31
|
+
},
|
|
28
32
|
versionResults: {
|
|
29
33
|
default: {},
|
|
30
34
|
description: "Per-version outputs and evaluator score maps keyed by version number",
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import {createdUpdatedPlugin, findExactlyOne, findOneOrNone, isDeletedPlugin} from "@terreno/api";
|
|
2
|
+
import mongoose from "mongoose";
|
|
3
|
+
|
|
4
|
+
import type {ObsJudgeRunDocument, ObsJudgeRunModel} from "../../../types/observability";
|
|
5
|
+
|
|
6
|
+
const obsJudgeRunSchema = new mongoose.Schema<ObsJudgeRunDocument, ObsJudgeRunModel>(
|
|
7
|
+
{
|
|
8
|
+
batchId: {
|
|
9
|
+
description: "Review batch the judge answered",
|
|
10
|
+
ref: "ObsReviewBatch",
|
|
11
|
+
required: true,
|
|
12
|
+
type: mongoose.Schema.Types.ObjectId,
|
|
13
|
+
},
|
|
14
|
+
completedAt: {description: "When the judge finished every item", required: true, type: Date},
|
|
15
|
+
evaluatorId: {
|
|
16
|
+
description: "LLM-judge evaluator that produced the answers",
|
|
17
|
+
ref: "ObsEvaluator",
|
|
18
|
+
required: true,
|
|
19
|
+
type: mongoose.Schema.Types.ObjectId,
|
|
20
|
+
},
|
|
21
|
+
modelId: {description: "Model id of the AI client that ran the judge", type: String},
|
|
22
|
+
results: {
|
|
23
|
+
default: [],
|
|
24
|
+
description: "One judge answer set per batch trace",
|
|
25
|
+
type: [
|
|
26
|
+
{
|
|
27
|
+
confidence: {description: "Judge confidence in 0–1 when returned", type: Number},
|
|
28
|
+
datasetItemId: {
|
|
29
|
+
description: "Dataset item the judge answered, for dataset review batches",
|
|
30
|
+
ref: "ObsDatasetItem",
|
|
31
|
+
type: mongoose.Schema.Types.ObjectId,
|
|
32
|
+
},
|
|
33
|
+
error: {description: "Why the judge could not answer this item", type: String},
|
|
34
|
+
scores: {
|
|
35
|
+
description: "Dimension-keyed judge answers",
|
|
36
|
+
type: mongoose.Schema.Types.Mixed,
|
|
37
|
+
},
|
|
38
|
+
traceId: {
|
|
39
|
+
description: "Trace the judge answered, unset for dataset items without a trace",
|
|
40
|
+
ref: "ObsTrace",
|
|
41
|
+
type: mongoose.Schema.Types.ObjectId,
|
|
42
|
+
},
|
|
43
|
+
},
|
|
44
|
+
],
|
|
45
|
+
},
|
|
46
|
+
},
|
|
47
|
+
{strict: "throw", toJSON: {virtuals: true}, toObject: {virtuals: true}}
|
|
48
|
+
);
|
|
49
|
+
|
|
50
|
+
obsJudgeRunSchema.plugin(createdUpdatedPlugin);
|
|
51
|
+
obsJudgeRunSchema.plugin(isDeletedPlugin);
|
|
52
|
+
obsJudgeRunSchema.plugin(findOneOrNone);
|
|
53
|
+
obsJudgeRunSchema.plugin(findExactlyOne);
|
|
54
|
+
obsJudgeRunSchema.index({batchId: 1, completedAt: -1});
|
|
55
|
+
|
|
56
|
+
export const registerObsJudgeRun = (): ObsJudgeRunModel => {
|
|
57
|
+
if (mongoose.models.ObsJudgeRun) {
|
|
58
|
+
return mongoose.models.ObsJudgeRun as ObsJudgeRunModel;
|
|
59
|
+
}
|
|
60
|
+
return mongoose.model<ObsJudgeRunDocument, ObsJudgeRunModel>("ObsJudgeRun", obsJudgeRunSchema);
|
|
61
|
+
};
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import {createdUpdatedPlugin, findExactlyOne, findOneOrNone, isDeletedPlugin} from "@terreno/api";
|
|
2
|
+
import mongoose from "mongoose";
|
|
3
|
+
|
|
4
|
+
import type {ObsReviewBatchDocument, ObsReviewBatchModel} from "../../../types/observability";
|
|
5
|
+
|
|
6
|
+
const obsReviewBatchSchema = new mongoose.Schema<ObsReviewBatchDocument, ObsReviewBatchModel>(
|
|
7
|
+
{
|
|
8
|
+
datasetId: {
|
|
9
|
+
description: "Dataset the batch was built from, when reviewers judge dataset items",
|
|
10
|
+
ref: "ObsDataset",
|
|
11
|
+
type: mongoose.Schema.Types.ObjectId,
|
|
12
|
+
},
|
|
13
|
+
description: {description: "Operator note about why this batch exists", type: String},
|
|
14
|
+
evaluatorId: {
|
|
15
|
+
description: "Human evaluator whose questions reviewers answer for every item",
|
|
16
|
+
ref: "ObsEvaluator",
|
|
17
|
+
required: true,
|
|
18
|
+
type: mongoose.Schema.Types.ObjectId,
|
|
19
|
+
},
|
|
20
|
+
name: {description: "Operator-facing batch name", required: true, type: String},
|
|
21
|
+
promptName: {
|
|
22
|
+
description: "Prompt the batch traces were drawn from, when filtered by prompt",
|
|
23
|
+
type: String,
|
|
24
|
+
},
|
|
25
|
+
questionsFingerprint: {
|
|
26
|
+
description: "Question-set fingerprint when the batch was created",
|
|
27
|
+
type: String,
|
|
28
|
+
},
|
|
29
|
+
traceIds: {
|
|
30
|
+
default: [],
|
|
31
|
+
description: "Traces (one input and one output each) included in the batch",
|
|
32
|
+
type: [{ref: "ObsTrace", type: mongoose.Schema.Types.ObjectId}],
|
|
33
|
+
},
|
|
34
|
+
},
|
|
35
|
+
{strict: "throw", toJSON: {virtuals: true}, toObject: {virtuals: true}}
|
|
36
|
+
);
|
|
37
|
+
|
|
38
|
+
obsReviewBatchSchema.plugin(createdUpdatedPlugin);
|
|
39
|
+
obsReviewBatchSchema.plugin(isDeletedPlugin);
|
|
40
|
+
obsReviewBatchSchema.plugin(findOneOrNone);
|
|
41
|
+
obsReviewBatchSchema.plugin(findExactlyOne);
|
|
42
|
+
obsReviewBatchSchema.index({evaluatorId: 1});
|
|
43
|
+
|
|
44
|
+
export const registerObsReviewBatch = (): ObsReviewBatchModel => {
|
|
45
|
+
if (mongoose.models.ObsReviewBatch) {
|
|
46
|
+
return mongoose.models.ObsReviewBatch as ObsReviewBatchModel;
|
|
47
|
+
}
|
|
48
|
+
return mongoose.model<ObsReviewBatchDocument, ObsReviewBatchModel>(
|
|
49
|
+
"ObsReviewBatch",
|
|
50
|
+
obsReviewBatchSchema
|
|
51
|
+
);
|
|
52
|
+
};
|
|
@@ -10,6 +10,11 @@ const obsReviewItemSchema = new mongoose.Schema<ObsReviewItemDocument, ObsReview
|
|
|
10
10
|
ref: "User",
|
|
11
11
|
type: mongoose.Schema.Types.ObjectId,
|
|
12
12
|
},
|
|
13
|
+
batchId: {
|
|
14
|
+
description: "Review batch this item belongs to, when it was created as part of one",
|
|
15
|
+
ref: "ObsReviewBatch",
|
|
16
|
+
type: mongoose.Schema.Types.ObjectId,
|
|
17
|
+
},
|
|
13
18
|
comment: {description: "Reviewer comment submitted with scores", type: String},
|
|
14
19
|
datasetItemId: {
|
|
15
20
|
description: "Optional dataset item this review is bound to",
|
|
@@ -49,9 +54,9 @@ const obsReviewItemSchema = new mongoose.Schema<ObsReviewItemDocument, ObsReview
|
|
|
49
54
|
type: String,
|
|
50
55
|
},
|
|
51
56
|
traceId: {
|
|
52
|
-
description:
|
|
57
|
+
description:
|
|
58
|
+
"Trace under review. Unset when the item judges a dataset item that has no source trace",
|
|
53
59
|
ref: "ObsTrace",
|
|
54
|
-
required: true,
|
|
55
60
|
type: mongoose.Schema.Types.ObjectId,
|
|
56
61
|
},
|
|
57
62
|
},
|
|
@@ -67,6 +72,10 @@ const reviewIndexSpec: Array<[string, 1]> = [
|
|
|
67
72
|
["enqueuedAt", 1],
|
|
68
73
|
];
|
|
69
74
|
obsReviewItemSchema.index(Object.fromEntries(reviewIndexSpec));
|
|
75
|
+
// Batch list, detail, and queue reads all scope by batchId, then by status.
|
|
76
|
+
obsReviewItemSchema.index({batchId: 1, status: 1});
|
|
77
|
+
// Dataset batches skip items that already wait in an open review for the same question set.
|
|
78
|
+
obsReviewItemSchema.index({datasetItemId: 1, evaluatorId: 1});
|
|
70
79
|
|
|
71
80
|
export const registerObsReviewItem = (): ObsReviewItemModel => {
|
|
72
81
|
if (mongoose.models.ObsReviewItem) {
|
|
@@ -25,6 +25,11 @@ const obsTraceSchema = new mongoose.Schema<ObsTraceDocument, ObsTraceModel>(
|
|
|
25
25
|
// No subdocument _id, so `$addToSet` dedupes repeated refs from replayed phases.
|
|
26
26
|
new mongoose.Schema(
|
|
27
27
|
{
|
|
28
|
+
includedBy: {
|
|
29
|
+
description:
|
|
30
|
+
"Prompt whose reference pulled this version in; unset for a top-level prompt",
|
|
31
|
+
type: String,
|
|
32
|
+
},
|
|
28
33
|
label: {description: "Label resolved at call time", type: String},
|
|
29
34
|
name: {description: "Prompt name", required: true, type: String},
|
|
30
35
|
version: {description: "Prompt version number", required: true, type: Number},
|
|
@@ -6,6 +6,7 @@ import {registerObsExperiment} from "./models/obsExperiment";
|
|
|
6
6
|
import {registerObsPrompt} from "./models/obsPrompt";
|
|
7
7
|
import {registerObsPromptLabel} from "./models/obsPromptLabel";
|
|
8
8
|
import {registerObsPromptVersion} from "./models/obsPromptVersion";
|
|
9
|
+
import {registerObsReviewItem} from "./models/obsReviewItem";
|
|
9
10
|
import {registerObsScore} from "./models/obsScore";
|
|
10
11
|
import {registerObsTrace} from "./models/obsTrace";
|
|
11
12
|
|
|
@@ -111,9 +112,21 @@ describe("local observability models", () => {
|
|
|
111
112
|
expect(serialized).toContain('{"flaggedForDataset":1,"created":-1}');
|
|
112
113
|
});
|
|
113
114
|
|
|
115
|
+
it("declares an ObsReviewItem batch index for batch queues and counts", () => {
|
|
116
|
+
const ObsReviewItem = registerObsReviewItem();
|
|
117
|
+
const serialized = ObsReviewItem.schema.indexes().map(([fields]) => JSON.stringify(fields));
|
|
118
|
+
expect(serialized).toContain('{"batchId":1,"status":1}');
|
|
119
|
+
});
|
|
120
|
+
|
|
114
121
|
it("declares ObsExperiment promptName list index", () => {
|
|
115
122
|
const ObsExperiment = registerObsExperiment();
|
|
116
123
|
const serialized = ObsExperiment.schema.indexes().map(([fields]) => JSON.stringify(fields));
|
|
117
124
|
expect(serialized).toContain('{"promptName":1,"created":-1}');
|
|
118
125
|
});
|
|
126
|
+
|
|
127
|
+
it("declares an ObsReviewItem batchId index for batch reads", () => {
|
|
128
|
+
const ObsReviewItem = registerObsReviewItem();
|
|
129
|
+
const serialized = ObsReviewItem.schema.indexes().map(([fields]) => JSON.stringify(fields));
|
|
130
|
+
expect(serialized).toContain('{"batchId":1,"status":1}');
|
|
131
|
+
});
|
|
119
132
|
});
|