@terreno/ai 58.0.0-beta.2 → 58.0.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +146 -0
- package/README.md +3 -3
- package/dist/harness/commit.d.ts +4 -3
- package/dist/harness/commit.js +23 -4
- package/dist/index.d.ts +6 -9
- package/dist/index.js +14 -25
- package/dist/observability/adminScreens.js +5 -1
- package/dist/observability/evaluate.d.ts +8 -1
- package/dist/observability/evaluate.js +10 -2
- package/dist/observability/judgeAgreement.d.ts +53 -0
- package/dist/observability/judgeAgreement.js +123 -0
- package/dist/observability/local/datasetStore.d.ts +5 -0
- package/dist/observability/local/datasetStore.js +7 -2
- package/dist/observability/local/evaluatorStore.d.ts +17 -0
- package/dist/observability/local/evaluatorStore.js +73 -2
- package/dist/observability/local/experimentRunner.d.ts +40 -3
- package/dist/observability/local/experimentRunner.js +390 -11
- package/dist/observability/local/humanAnswers.d.ts +57 -0
- package/dist/observability/local/humanAnswers.js +174 -0
- package/dist/observability/local/judgeRun.d.ts +56 -0
- package/dist/observability/local/judgeRun.js +111 -0
- package/dist/observability/local/localPlugin.js +4 -0
- package/dist/observability/local/models/obsDatasetItem.js +4 -0
- package/dist/observability/local/models/obsEvaluator.js +4 -0
- package/dist/observability/local/models/obsExperiment.js +28 -0
- package/dist/observability/local/models/obsExperimentItem.js +4 -0
- package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
- package/dist/observability/local/models/obsJudgeRun.js +60 -0
- package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
- package/dist/observability/local/models/obsReviewBatch.js +48 -0
- package/dist/observability/local/models/obsReviewItem.js +10 -2
- package/dist/observability/local/models/obsTrace.js +4 -0
- package/dist/observability/local/promptReferences.d.ts +31 -0
- package/dist/observability/local/promptReferences.js +114 -0
- package/dist/observability/local/promptStore.d.ts +55 -19
- package/dist/observability/local/promptStore.js +171 -7
- package/dist/observability/local/reviewBatchStore.d.ts +106 -0
- package/dist/observability/local/reviewBatchStore.js +517 -0
- package/dist/observability/local/reviewStore.d.ts +24 -3
- package/dist/observability/local/reviewStore.js +98 -22
- package/dist/observability/local/traceStore.d.ts +14 -1
- package/dist/observability/local/traceStore.js +144 -11
- package/dist/observability/observabilityApp.js +10 -1
- package/dist/observability/questionFingerprint.d.ts +7 -0
- package/dist/observability/questionFingerprint.js +23 -0
- package/dist/observability/routes/experiments.js +12 -1
- package/dist/observability/routes/review.js +3 -1
- package/dist/observability/routes/reviewBatches.d.ts +11 -0
- package/dist/observability/routes/reviewBatches.js +78 -0
- package/dist/observability/routes/traces.d.ts +1 -1
- package/dist/observability/routes/traces.js +34 -2
- package/dist/observability/types.d.ts +10 -1
- package/dist/service/aiService.js +4 -1
- package/dist/service/chatTurn.js +38 -31
- package/dist/types/harness.d.ts +5 -2
- package/dist/types/index.d.ts +8 -3
- package/dist/types/observability.d.ts +74 -2
- package/package.json +8 -24
- package/src/harness/agentLoop.test.ts +0 -3
- package/src/harness/agentRecovery.test.ts +0 -3
- package/src/harness/commit.ts +24 -4
- package/src/harness/extensions.test.ts +0 -3
- package/src/harness/subagent.test.ts +27 -3
- package/src/index.test.ts +0 -17
- package/src/index.ts +23 -26
- package/src/isolated/gptStream.isolated.ts +0 -8
- package/src/observability/adminScreens.test.ts +7 -0
- package/src/observability/adminScreens.ts +6 -1
- package/src/observability/evaluate.ts +12 -2
- package/src/observability/judgeAgreement.test.ts +111 -0
- package/src/observability/judgeAgreement.ts +184 -0
- package/src/observability/local/datasetStore.ts +30 -11
- package/src/observability/local/evaluatorStore.ts +96 -2
- package/src/observability/local/experimentRunner.ts +495 -15
- package/src/observability/local/humanAnswers.ts +233 -0
- package/src/observability/local/judgePipeline.test.ts +444 -0
- package/src/observability/local/judgeRun.test.ts +173 -0
- package/src/observability/local/judgeRun.ts +184 -0
- package/src/observability/local/localPlugin.ts +4 -0
- package/src/observability/local/models/obsDatasetItem.ts +5 -0
- package/src/observability/local/models/obsEvaluator.ts +4 -0
- package/src/observability/local/models/obsExperiment.ts +29 -0
- package/src/observability/local/models/obsExperimentItem.ts +4 -0
- package/src/observability/local/models/obsJudgeRun.ts +61 -0
- package/src/observability/local/models/obsReviewBatch.ts +52 -0
- package/src/observability/local/models/obsReviewItem.ts +11 -2
- package/src/observability/local/models/obsTrace.ts +5 -0
- package/src/observability/local/obsModels.test.ts +13 -0
- package/src/observability/local/promptReferences.test.ts +128 -0
- package/src/observability/local/promptReferences.ts +149 -0
- package/src/observability/local/promptStore.test.ts +291 -0
- package/src/observability/local/promptStore.ts +243 -32
- package/src/observability/local/reviewBatchStore.test.ts +251 -0
- package/src/observability/local/reviewBatchStore.ts +656 -0
- package/src/observability/local/reviewStore.ts +122 -25
- package/src/observability/local/traceStore.test.ts +139 -1
- package/src/observability/local/traceStore.ts +162 -11
- package/src/observability/observabilityApp.test.ts +13 -10
- package/src/observability/observabilityApp.ts +10 -1
- package/src/observability/questionFingerprint.test.ts +33 -0
- package/src/observability/questionFingerprint.ts +24 -0
- package/src/observability/routes/experiments.test.ts +37 -0
- package/src/observability/routes/experiments.ts +13 -1
- package/src/observability/routes/review.ts +3 -1
- package/src/observability/routes/reviewBatches.test.ts +209 -0
- package/src/observability/routes/reviewBatches.ts +138 -0
- package/src/observability/routes/status.test.ts +4 -4
- package/src/observability/routes/traces.test.ts +61 -0
- package/src/observability/routes/traces.ts +46 -4
- package/src/observability/types.ts +5 -2
- package/src/routes/routes.test.ts +101 -76
- package/src/service/aiService.test.ts +21 -0
- package/src/service/aiService.ts +4 -1
- package/src/service/chatTurn.ts +42 -33
- package/src/tests/bunSetup.ts +0 -54
- package/src/types/harness.ts +5 -2
- package/src/types/index.ts +8 -3
- package/src/types/observability.ts +86 -2
- package/dist/langfuseApp.d.ts +0 -9
- package/dist/langfuseApp.js +0 -53
- package/dist/langfuseCache.d.ts +0 -15
- package/dist/langfuseCache.js +0 -54
- package/dist/langfuseClient.d.ts +0 -6
- package/dist/langfuseClient.js +0 -34
- package/dist/langfusePrompts.d.ts +0 -12
- package/dist/langfusePrompts.js +0 -89
- package/dist/langfuseRoutesEvaluations.d.ts +0 -3
- package/dist/langfuseRoutesEvaluations.js +0 -32
- package/dist/langfuseRoutesMiddleware.d.ts +0 -2
- package/dist/langfuseRoutesMiddleware.js +0 -13
- package/dist/langfuseRoutesPlayground.d.ts +0 -2
- package/dist/langfuseRoutesPlayground.js +0 -36
- package/dist/langfuseRoutesPrompts.d.ts +0 -2
- package/dist/langfuseRoutesPrompts.js +0 -56
- package/dist/langfuseRoutesTraces.d.ts +0 -2
- package/dist/langfuseRoutesTraces.js +0 -30
- package/dist/langfuseTracing.d.ts +0 -4
- package/dist/langfuseTracing.js +0 -32
- package/dist/langfuseTypes.d.ts +0 -96
- package/dist/langfuseTypes.js +0 -2
- package/dist/langfuseVercelAi.d.ts +0 -12
- package/dist/langfuseVercelAi.js +0 -49
- package/src/isolated/langfuseClient.isolated.ts +0 -66
- package/src/langfuseApp.test.ts +0 -119
- package/src/langfuseApp.ts +0 -62
- package/src/langfuseCache.test.ts +0 -75
- package/src/langfuseCache.ts +0 -72
- package/src/langfuseClient.test.ts +0 -69
- package/src/langfuseClient.ts +0 -34
- package/src/langfusePrompts.test.ts +0 -188
- package/src/langfusePrompts.ts +0 -121
- package/src/langfuseRoutes.test.ts +0 -261
- package/src/langfuseRoutesEvaluations.ts +0 -48
- package/src/langfuseRoutesMiddleware.test.ts +0 -57
- package/src/langfuseRoutesMiddleware.ts +0 -12
- package/src/langfuseRoutesPlayground.ts +0 -50
- package/src/langfuseRoutesPrompts.ts +0 -87
- package/src/langfuseRoutesTraces.ts +0 -40
- package/src/langfuseTracing.test.ts +0 -72
- package/src/langfuseTracing.ts +0 -33
- package/src/langfuseTypes.ts +0 -103
- package/src/langfuseVercelAi.test.ts +0 -82
- package/src/langfuseVercelAi.ts +0 -72
- package/src/tests/fakeLangfusePrompt.ts +0 -10
|
@@ -0,0 +1,656 @@
|
|
|
1
|
+
import {APIError} from "@terreno/api";
|
|
2
|
+
import {DateTime} from "luxon";
|
|
3
|
+
import mongoose from "mongoose";
|
|
4
|
+
|
|
5
|
+
import type {
|
|
6
|
+
EvaluatorDimension,
|
|
7
|
+
ObsEvaluatorDocument,
|
|
8
|
+
ObsJudgeRunDocument,
|
|
9
|
+
ObsReviewBatchDocument,
|
|
10
|
+
ObsReviewItemDocument,
|
|
11
|
+
} from "../../types/observability";
|
|
12
|
+
import type {EvaluatorAiClient} from "../evaluate";
|
|
13
|
+
import {
|
|
14
|
+
agreementAnswerKey,
|
|
15
|
+
computeJudgeAgreement,
|
|
16
|
+
type JudgeAgreement,
|
|
17
|
+
type ScoreValue,
|
|
18
|
+
} from "../judgeAgreement";
|
|
19
|
+
import {questionSetFingerprint} from "../questionFingerprint";
|
|
20
|
+
import {loadDatasetItemSubjects} from "./humanAnswers";
|
|
21
|
+
import {runJudgeAgainstHumans} from "./judgeRun";
|
|
22
|
+
import {registerObsDataset} from "./models/obsDataset";
|
|
23
|
+
import {registerObsDatasetItem} from "./models/obsDatasetItem";
|
|
24
|
+
import {registerObsEvaluator} from "./models/obsEvaluator";
|
|
25
|
+
import {registerObsJudgeRun} from "./models/obsJudgeRun";
|
|
26
|
+
import {registerObsReviewBatch} from "./models/obsReviewBatch";
|
|
27
|
+
import {registerObsReviewItem} from "./models/obsReviewItem";
|
|
28
|
+
import {registerObsTrace} from "./models/obsTrace";
|
|
29
|
+
import {LocalPromptStore} from "./promptStore";
|
|
30
|
+
import {LocalReviewStore, type ReviewStatus} from "./reviewStore";
|
|
31
|
+
|
|
32
|
+
export const DEFAULT_REVIEW_BATCH_SIZE = 10;
|
|
33
|
+
export const MAX_REVIEW_BATCH_SIZE = 50;
|
|
34
|
+
|
|
35
|
+
export interface ReviewBatchCounts {
|
|
36
|
+
done: number;
|
|
37
|
+
pending: number;
|
|
38
|
+
skipped: number;
|
|
39
|
+
total: number;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export interface ReviewBatchJudgeSummary {
|
|
43
|
+
agreement: JudgeAgreement;
|
|
44
|
+
completedAt: string;
|
|
45
|
+
evaluatorId: string;
|
|
46
|
+
evaluatorName: string;
|
|
47
|
+
judgePromptName?: string;
|
|
48
|
+
modelId?: string;
|
|
49
|
+
runId: string;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export interface ReviewBatchListItem {
|
|
53
|
+
bestJudge?: {evaluatorName: string; rate?: number};
|
|
54
|
+
counts: ReviewBatchCounts;
|
|
55
|
+
created: string;
|
|
56
|
+
/** Dataset the batch judges, when it was built from a dataset. */
|
|
57
|
+
datasetId?: string;
|
|
58
|
+
datasetName?: string;
|
|
59
|
+
description?: string;
|
|
60
|
+
evaluatorId: string;
|
|
61
|
+
evaluatorName: string;
|
|
62
|
+
id: string;
|
|
63
|
+
judgeCount: number;
|
|
64
|
+
name: string;
|
|
65
|
+
promptName?: string;
|
|
66
|
+
/** Question-set fingerprint when the batch was created (dataset batches). */
|
|
67
|
+
questionsFingerprint?: string;
|
|
68
|
+
/** True when the question set's questions changed since this dataset batch was created. */
|
|
69
|
+
questionsStale?: boolean;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export interface ReviewBatchItem {
|
|
73
|
+
datasetItemId?: string;
|
|
74
|
+
input?: unknown;
|
|
75
|
+
judgeAnswers: Record<string, {error?: string; scores?: Record<string, ScoreValue>}>;
|
|
76
|
+
output?: unknown;
|
|
77
|
+
reviewItemId: string;
|
|
78
|
+
scores?: Record<string, ScoreValue>;
|
|
79
|
+
status: ReviewStatus;
|
|
80
|
+
/** Unset for a dataset item without a source trace. */
|
|
81
|
+
traceId?: string;
|
|
82
|
+
traceName: string;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
interface ReviewSubject {
|
|
86
|
+
datasetItemId?: string;
|
|
87
|
+
input?: unknown;
|
|
88
|
+
key: string;
|
|
89
|
+
output?: unknown;
|
|
90
|
+
traceId?: string;
|
|
91
|
+
traceName: string;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export interface ReviewBatchDetail extends ReviewBatchListItem {
|
|
95
|
+
dimensions: EvaluatorDimension[];
|
|
96
|
+
instructions?: string;
|
|
97
|
+
items: ReviewBatchItem[];
|
|
98
|
+
judges: ReviewBatchJudgeSummary[];
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
const objectId = (value: string, label: string): mongoose.Types.ObjectId => {
|
|
102
|
+
if (!mongoose.Types.ObjectId.isValid(value)) {
|
|
103
|
+
throw new APIError({status: 400, title: `${label} must be an object id`});
|
|
104
|
+
}
|
|
105
|
+
return new mongoose.Types.ObjectId(value);
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
const toIso = (value?: Date): string => {
|
|
109
|
+
return value ? (DateTime.fromJSDate(value).toUTC().toISO() ?? "") : "";
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Review batches pair a fixed set of traces (one input and one output each) with a human question
|
|
114
|
+
* set. Reviewers answer every question per item; operators then run LLM judges over the same
|
|
115
|
+
* traces and compare judge answers with the human answers question by question.
|
|
116
|
+
*/
|
|
117
|
+
export class LocalReviewBatchStore {
|
|
118
|
+
constructor(
|
|
119
|
+
private readonly promptStore = new LocalPromptStore(),
|
|
120
|
+
private readonly reviewStore = new LocalReviewStore()
|
|
121
|
+
) {}
|
|
122
|
+
|
|
123
|
+
async create(params: {
|
|
124
|
+
datasetId?: string;
|
|
125
|
+
description?: string;
|
|
126
|
+
evaluatorId: string;
|
|
127
|
+
limit?: number;
|
|
128
|
+
name?: string;
|
|
129
|
+
promptName?: string;
|
|
130
|
+
traceIds?: string[];
|
|
131
|
+
}): Promise<ReviewBatchDetail> {
|
|
132
|
+
const evaluator = await registerObsEvaluator().findOneOrNone({
|
|
133
|
+
_id: objectId(params.evaluatorId, "evaluatorId"),
|
|
134
|
+
});
|
|
135
|
+
if (!evaluator) {
|
|
136
|
+
throw new APIError({status: 404, title: "Unknown evaluator"});
|
|
137
|
+
}
|
|
138
|
+
if (evaluator.type !== "human") {
|
|
139
|
+
throw new APIError({status: 400, title: "Review batches require a human evaluator"});
|
|
140
|
+
}
|
|
141
|
+
if (params.datasetId) {
|
|
142
|
+
if (params.traceIds !== undefined || params.limit !== undefined) {
|
|
143
|
+
throw new APIError({
|
|
144
|
+
status: 400,
|
|
145
|
+
title: "datasetId cannot be combined with traceIds or limit",
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
return this.createFromDataset({...params, datasetId: params.datasetId, evaluator});
|
|
149
|
+
}
|
|
150
|
+
const limit = params.limit ?? DEFAULT_REVIEW_BATCH_SIZE;
|
|
151
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > MAX_REVIEW_BATCH_SIZE) {
|
|
152
|
+
throw new APIError({
|
|
153
|
+
status: 400,
|
|
154
|
+
title: `limit must be an integer from 1 to ${MAX_REVIEW_BATCH_SIZE}`,
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
const traceIds = params.traceIds?.length
|
|
158
|
+
? params.traceIds.slice(0, MAX_REVIEW_BATCH_SIZE)
|
|
159
|
+
: await this.recentTraceIds({
|
|
160
|
+
evaluatorId: evaluator._id,
|
|
161
|
+
limit,
|
|
162
|
+
promptName: params.promptName,
|
|
163
|
+
});
|
|
164
|
+
if (traceIds.length === 0) {
|
|
165
|
+
throw new APIError({
|
|
166
|
+
status: 400,
|
|
167
|
+
title: params.promptName
|
|
168
|
+
? `No recent "${params.promptName}" traces that are not already in a batch for this question set`
|
|
169
|
+
: "No recent traces that are not already in a batch for this question set",
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
const traceObjectIds = traceIds.map((id) => objectId(id, "traceId"));
|
|
173
|
+
const found = await registerObsTrace()
|
|
174
|
+
.find({_id: {$in: traceObjectIds}})
|
|
175
|
+
.select({_id: 1});
|
|
176
|
+
const foundIds = new Set(found.map((trace) => String(trace._id)));
|
|
177
|
+
const missing = traceIds.filter((id) => !foundIds.has(id));
|
|
178
|
+
if (missing.length > 0) {
|
|
179
|
+
throw new APIError({detail: missing.join(", "), status: 404, title: "Unknown trace ids"});
|
|
180
|
+
}
|
|
181
|
+
const Batch = registerObsReviewBatch();
|
|
182
|
+
const existingCount = await Batch.countDocuments({evaluatorId: evaluator._id});
|
|
183
|
+
const name =
|
|
184
|
+
params.name?.trim() || `${params.promptName ?? evaluator.name} — batch ${existingCount + 1}`;
|
|
185
|
+
const batch = await Batch.create({
|
|
186
|
+
description: params.description,
|
|
187
|
+
evaluatorId: evaluator._id,
|
|
188
|
+
name,
|
|
189
|
+
promptName: params.promptName,
|
|
190
|
+
traceIds: traceObjectIds,
|
|
191
|
+
});
|
|
192
|
+
try {
|
|
193
|
+
await this.reviewStore.enqueue({
|
|
194
|
+
batchId: String(batch._id),
|
|
195
|
+
evaluatorId: String(evaluator._id),
|
|
196
|
+
reason: "manual",
|
|
197
|
+
traceIds,
|
|
198
|
+
});
|
|
199
|
+
} catch (error) {
|
|
200
|
+
// A half-built batch would hide its traces from later "grab recent" batches.
|
|
201
|
+
await registerObsReviewItem().deleteMany({batchId: batch._id});
|
|
202
|
+
await Batch.deleteOne({_id: batch._id});
|
|
203
|
+
throw error;
|
|
204
|
+
}
|
|
205
|
+
return this.get(String(batch._id));
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* One review item per dataset item that lacks a current answer from this question set and is
|
|
210
|
+
* not already waiting in an open review for it.
|
|
211
|
+
*/
|
|
212
|
+
private async createFromDataset(params: {
|
|
213
|
+
datasetId: string;
|
|
214
|
+
description?: string;
|
|
215
|
+
evaluator: ObsEvaluatorDocument;
|
|
216
|
+
name?: string;
|
|
217
|
+
}): Promise<ReviewBatchDetail> {
|
|
218
|
+
const {evaluator} = params;
|
|
219
|
+
const dataset = await registerObsDataset().findOneOrNone({
|
|
220
|
+
_id: objectId(params.datasetId, "datasetId"),
|
|
221
|
+
});
|
|
222
|
+
if (!dataset) {
|
|
223
|
+
throw new APIError({status: 404, title: `Unknown dataset "${params.datasetId}"`});
|
|
224
|
+
}
|
|
225
|
+
const fingerprint = questionSetFingerprint(evaluator.dimensions);
|
|
226
|
+
const evaluatorKey = String(evaluator._id);
|
|
227
|
+
const datasetItems = await registerObsDatasetItem()
|
|
228
|
+
.find({datasetId: dataset._id})
|
|
229
|
+
.sort({_id: 1});
|
|
230
|
+
const unanswered = datasetItems.filter((item) => {
|
|
231
|
+
return item.humanAnswers?.[evaluatorKey]?.fingerprint !== fingerprint;
|
|
232
|
+
});
|
|
233
|
+
if (unanswered.length === 0) {
|
|
234
|
+
throw new APIError({
|
|
235
|
+
detail: datasetItems.length === 0 ? `Dataset "${dataset.name}" has no items.` : undefined,
|
|
236
|
+
status: 400,
|
|
237
|
+
title: "Every item already has current answers",
|
|
238
|
+
});
|
|
239
|
+
}
|
|
240
|
+
const open = await registerObsReviewItem()
|
|
241
|
+
.find({
|
|
242
|
+
datasetItemId: {$in: unanswered.map((item) => item._id)},
|
|
243
|
+
evaluatorId: evaluator._id,
|
|
244
|
+
status: {$in: ["in_progress", "pending"]},
|
|
245
|
+
})
|
|
246
|
+
.select({datasetItemId: 1});
|
|
247
|
+
const openIds = new Set(open.map((item) => String(item.datasetItemId)));
|
|
248
|
+
const toJudge = unanswered.filter((item) => !openIds.has(String(item._id)));
|
|
249
|
+
if (toJudge.length === 0) {
|
|
250
|
+
throw new APIError({
|
|
251
|
+
status: 400,
|
|
252
|
+
title: "Every unanswered item is already waiting in an open review batch",
|
|
253
|
+
});
|
|
254
|
+
}
|
|
255
|
+
const Batch = registerObsReviewBatch();
|
|
256
|
+
const existingCount = await Batch.countDocuments({datasetId: dataset._id});
|
|
257
|
+
const batch = await Batch.create({
|
|
258
|
+
datasetId: dataset._id,
|
|
259
|
+
description: params.description,
|
|
260
|
+
evaluatorId: evaluator._id,
|
|
261
|
+
name: params.name?.trim() || `${dataset.name} — ${evaluator.name} ${existingCount + 1}`,
|
|
262
|
+
questionsFingerprint: fingerprint,
|
|
263
|
+
traceIds: toJudge.flatMap((item) => (item.sourceTraceId ? [item.sourceTraceId] : [])),
|
|
264
|
+
});
|
|
265
|
+
try {
|
|
266
|
+
await this.reviewStore.enqueueDatasetItems({
|
|
267
|
+
batchId: String(batch._id),
|
|
268
|
+
evaluatorId: evaluatorKey,
|
|
269
|
+
items: toJudge.map((item) => {
|
|
270
|
+
return {
|
|
271
|
+
datasetItemId: String(item._id),
|
|
272
|
+
traceId: item.sourceTraceId ? String(item.sourceTraceId) : undefined,
|
|
273
|
+
};
|
|
274
|
+
}),
|
|
275
|
+
});
|
|
276
|
+
} catch (error) {
|
|
277
|
+
await registerObsReviewItem().deleteMany({batchId: batch._id});
|
|
278
|
+
await Batch.deleteOne({_id: batch._id});
|
|
279
|
+
throw error;
|
|
280
|
+
}
|
|
281
|
+
return this.get(String(batch._id));
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
async list(): Promise<ReviewBatchListItem[]> {
|
|
285
|
+
const batches = await registerObsReviewBatch().find({}).sort({created: -1});
|
|
286
|
+
if (batches.length === 0) {
|
|
287
|
+
return [];
|
|
288
|
+
}
|
|
289
|
+
const batchIds = batches.map((batch) => batch._id);
|
|
290
|
+
const reviewItems = await registerObsReviewItem()
|
|
291
|
+
.find({batchId: {$in: batchIds}})
|
|
292
|
+
.select({batchId: 1, datasetItemId: 1, scores: 1, status: 1, traceId: 1});
|
|
293
|
+
const runs = await this.latestJudgeRuns(batchIds);
|
|
294
|
+
const evaluators = await registerObsEvaluator().find({
|
|
295
|
+
_id: {
|
|
296
|
+
$in: [...batches.map((batch) => batch.evaluatorId), ...runs.map((run) => run.evaluatorId)],
|
|
297
|
+
},
|
|
298
|
+
});
|
|
299
|
+
const evaluatorById = new Map(
|
|
300
|
+
evaluators.map((evaluator) => {
|
|
301
|
+
return [String(evaluator._id), evaluator];
|
|
302
|
+
})
|
|
303
|
+
);
|
|
304
|
+
const datasetNames = await this.datasetNames(batches);
|
|
305
|
+
return batches.map((batch) => {
|
|
306
|
+
const batchId = String(batch._id);
|
|
307
|
+
const evaluator = evaluatorById.get(String(batch.evaluatorId));
|
|
308
|
+
const items = reviewItems.filter((item) => String(item.batchId) === batchId);
|
|
309
|
+
const judges = this.summarizeJudges({
|
|
310
|
+
dimensions: evaluator?.dimensions ?? [],
|
|
311
|
+
evaluatorById,
|
|
312
|
+
reviewItems: items,
|
|
313
|
+
runs: runs.filter((run) => String(run.batchId) === batchId),
|
|
314
|
+
});
|
|
315
|
+
return this.toListItem({
|
|
316
|
+
batch,
|
|
317
|
+
datasetName: batch.datasetId ? datasetNames.get(String(batch.datasetId)) : undefined,
|
|
318
|
+
evaluator,
|
|
319
|
+
items,
|
|
320
|
+
judges,
|
|
321
|
+
});
|
|
322
|
+
});
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
async get(id: string): Promise<ReviewBatchDetail> {
|
|
326
|
+
const batch = await registerObsReviewBatch().findOneOrNone({_id: objectId(id, "id")});
|
|
327
|
+
if (!batch) {
|
|
328
|
+
throw new APIError({status: 404, title: `Unknown review batch "${id}"`});
|
|
329
|
+
}
|
|
330
|
+
const evaluator = await registerObsEvaluator().findOneOrNone({_id: batch.evaluatorId});
|
|
331
|
+
const dimensions = evaluator?.dimensions ?? [];
|
|
332
|
+
const reviewItems = await registerObsReviewItem()
|
|
333
|
+
.find({batchId: batch._id})
|
|
334
|
+
.sort({_id: 1, enqueuedAt: 1});
|
|
335
|
+
const subjects = await this.loadSubjects(reviewItems);
|
|
336
|
+
const runs = await this.latestJudgeRuns([batch._id]);
|
|
337
|
+
const judgeEvaluators = await registerObsEvaluator().find({
|
|
338
|
+
_id: {$in: runs.map((run) => run.evaluatorId)},
|
|
339
|
+
});
|
|
340
|
+
const judges = this.summarizeJudges({
|
|
341
|
+
dimensions,
|
|
342
|
+
evaluatorById: new Map(
|
|
343
|
+
judgeEvaluators.map((judge) => {
|
|
344
|
+
return [String(judge._id), judge];
|
|
345
|
+
})
|
|
346
|
+
),
|
|
347
|
+
reviewItems,
|
|
348
|
+
runs,
|
|
349
|
+
});
|
|
350
|
+
|
|
351
|
+
const items: ReviewBatchItem[] = reviewItems.map((item) => {
|
|
352
|
+
const subject = subjects.get(String(item._id));
|
|
353
|
+
const key = subject?.key ?? agreementAnswerKey(this.answerIds(item));
|
|
354
|
+
const judgeAnswers: ReviewBatchItem["judgeAnswers"] = {};
|
|
355
|
+
for (const run of runs) {
|
|
356
|
+
const result = run.results.find((entry) => {
|
|
357
|
+
return agreementAnswerKey(this.answerIds(entry)) === key;
|
|
358
|
+
});
|
|
359
|
+
if (result) {
|
|
360
|
+
judgeAnswers[String(run.evaluatorId)] = {error: result.error, scores: result.scores};
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
return {
|
|
364
|
+
datasetItemId: item.datasetItemId ? String(item.datasetItemId) : undefined,
|
|
365
|
+
input: subject?.input,
|
|
366
|
+
judgeAnswers,
|
|
367
|
+
output: subject?.output,
|
|
368
|
+
reviewItemId: String(item._id),
|
|
369
|
+
scores: item.scores,
|
|
370
|
+
status: item.status,
|
|
371
|
+
traceId: subject?.traceId ?? (item.traceId ? String(item.traceId) : undefined),
|
|
372
|
+
traceName: subject?.traceName ?? "Unknown trace",
|
|
373
|
+
};
|
|
374
|
+
});
|
|
375
|
+
const datasetNames = await this.datasetNames([batch]);
|
|
376
|
+
return {
|
|
377
|
+
...this.toListItem({
|
|
378
|
+
batch,
|
|
379
|
+
datasetName: batch.datasetId ? datasetNames.get(String(batch.datasetId)) : undefined,
|
|
380
|
+
evaluator: evaluator ?? undefined,
|
|
381
|
+
items: reviewItems,
|
|
382
|
+
judges,
|
|
383
|
+
}),
|
|
384
|
+
dimensions,
|
|
385
|
+
instructions: evaluator?.instructions,
|
|
386
|
+
items,
|
|
387
|
+
judges,
|
|
388
|
+
};
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
/** Run an LLM judge over every batch trace and store its answers for comparison. */
|
|
392
|
+
async runJudge(params: {
|
|
393
|
+
ai: EvaluatorAiClient & {modelId?: string};
|
|
394
|
+
batchId: string;
|
|
395
|
+
evaluatorId: string;
|
|
396
|
+
}): Promise<ReviewBatchDetail> {
|
|
397
|
+
const batch = await registerObsReviewBatch().findOneOrNone({
|
|
398
|
+
_id: objectId(params.batchId, "batchId"),
|
|
399
|
+
});
|
|
400
|
+
if (!batch) {
|
|
401
|
+
throw new APIError({status: 404, title: `Unknown review batch "${params.batchId}"`});
|
|
402
|
+
}
|
|
403
|
+
const judge = await registerObsEvaluator().findOneOrNone({
|
|
404
|
+
_id: objectId(params.evaluatorId, "evaluatorId"),
|
|
405
|
+
});
|
|
406
|
+
if (!judge) {
|
|
407
|
+
throw new APIError({status: 404, title: "Unknown evaluator"});
|
|
408
|
+
}
|
|
409
|
+
if (judge.type !== "llm-judge" || !judge.judgePromptName) {
|
|
410
|
+
throw new APIError({status: 400, title: "Only llm-judge evaluators can be compared"});
|
|
411
|
+
}
|
|
412
|
+
const questionSet = await registerObsEvaluator().findOneOrNone({_id: batch.evaluatorId});
|
|
413
|
+
const questionKeys = new Set((questionSet?.dimensions ?? []).map((row) => row.key));
|
|
414
|
+
const sharedKeys = judge.dimensions.filter((row) => questionKeys.has(row.key));
|
|
415
|
+
if (sharedKeys.length === 0) {
|
|
416
|
+
throw new APIError({
|
|
417
|
+
detail: `Batch questions: ${[...questionKeys].join(", ")}. Judge scores: ${judge.dimensions
|
|
418
|
+
.map((row) => row.key)
|
|
419
|
+
.join(", ")}.`,
|
|
420
|
+
status: 400,
|
|
421
|
+
title: `Judge "${judge.name}" answers none of this batch's questions`,
|
|
422
|
+
});
|
|
423
|
+
}
|
|
424
|
+
const judgeVersion = await this.promptStore.getVersionByLabel(judge.judgePromptName);
|
|
425
|
+
if (!judgeVersion?.outputSchema) {
|
|
426
|
+
throw new APIError({
|
|
427
|
+
status: 400,
|
|
428
|
+
title: `Judge prompt "${judge.judgePromptName}" has no production output schema`,
|
|
429
|
+
});
|
|
430
|
+
}
|
|
431
|
+
const reviewItems = await registerObsReviewItem()
|
|
432
|
+
.find({batchId: batch._id})
|
|
433
|
+
.sort({_id: 1, enqueuedAt: 1});
|
|
434
|
+
const subjectByKey = new Map<string, ReviewSubject>();
|
|
435
|
+
for (const subject of (await this.loadSubjects(reviewItems)).values()) {
|
|
436
|
+
subjectByKey.set(subject.key, subject);
|
|
437
|
+
}
|
|
438
|
+
// Done answers pair with the judge's, exactly as summarizeJudges compares them later.
|
|
439
|
+
const humanAnswers = reviewItems
|
|
440
|
+
.filter((item) => item.status === "done")
|
|
441
|
+
.map((item) => {
|
|
442
|
+
return {...this.answerIds(item), scores: item.scores};
|
|
443
|
+
});
|
|
444
|
+
const run = await runJudgeAgainstHumans({
|
|
445
|
+
ai: params.ai,
|
|
446
|
+
dimensions: questionSet?.dimensions ?? [],
|
|
447
|
+
humanAnswers,
|
|
448
|
+
judge,
|
|
449
|
+
prompt: {outputSchema: judgeVersion.outputSchema as Record<string, unknown>},
|
|
450
|
+
subjects: [...subjectByKey.values()],
|
|
451
|
+
});
|
|
452
|
+
const results = run.results.map((result) => {
|
|
453
|
+
return {
|
|
454
|
+
confidence: result.confidence,
|
|
455
|
+
error: result.error,
|
|
456
|
+
scores: result.scores,
|
|
457
|
+
...(result.datasetItemId
|
|
458
|
+
? {datasetItemId: new mongoose.Types.ObjectId(result.datasetItemId)}
|
|
459
|
+
: {}),
|
|
460
|
+
...(result.traceId ? {traceId: new mongoose.Types.ObjectId(result.traceId)} : {}),
|
|
461
|
+
};
|
|
462
|
+
});
|
|
463
|
+
await registerObsJudgeRun().create({
|
|
464
|
+
batchId: batch._id,
|
|
465
|
+
completedAt: DateTime.utc().toJSDate(),
|
|
466
|
+
evaluatorId: judge._id,
|
|
467
|
+
modelId: params.ai.modelId,
|
|
468
|
+
results,
|
|
469
|
+
});
|
|
470
|
+
return this.get(String(batch._id));
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
private answerIds(row: {
|
|
474
|
+
datasetItemId?: mongoose.Types.ObjectId;
|
|
475
|
+
traceId?: mongoose.Types.ObjectId;
|
|
476
|
+
}): {datasetItemId?: string; traceId?: string} {
|
|
477
|
+
return {
|
|
478
|
+
datasetItemId: row.datasetItemId ? String(row.datasetItemId) : undefined,
|
|
479
|
+
traceId: row.traceId ? String(row.traceId) : undefined,
|
|
480
|
+
};
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
// What reviewers saw for each review item: its trace, or the dataset item when it has none.
|
|
484
|
+
private async loadSubjects(
|
|
485
|
+
reviewItems: Pick<ObsReviewItemDocument, "_id" | "datasetItemId" | "traceId">[]
|
|
486
|
+
): Promise<Map<string, ReviewSubject>> {
|
|
487
|
+
const traceIds = reviewItems.flatMap((item) => (item.traceId ? [item.traceId] : []));
|
|
488
|
+
const traces = traceIds.length ? await registerObsTrace().find({_id: {$in: traceIds}}) : [];
|
|
489
|
+
const traceById = new Map(
|
|
490
|
+
traces.map((trace) => {
|
|
491
|
+
return [String(trace._id), trace];
|
|
492
|
+
})
|
|
493
|
+
);
|
|
494
|
+
const datasetItemIds = reviewItems.flatMap((item) => {
|
|
495
|
+
const hasTrace = item.traceId && traceById.has(String(item.traceId));
|
|
496
|
+
return !hasTrace && item.datasetItemId ? [item.datasetItemId] : [];
|
|
497
|
+
});
|
|
498
|
+
const datasetSubjects = datasetItemIds.length
|
|
499
|
+
? await loadDatasetItemSubjects(
|
|
500
|
+
await registerObsDatasetItem().find({_id: {$in: datasetItemIds}})
|
|
501
|
+
)
|
|
502
|
+
: new Map();
|
|
503
|
+
const subjects = new Map<string, ReviewSubject>();
|
|
504
|
+
for (const item of reviewItems) {
|
|
505
|
+
const ids = this.answerIds(item);
|
|
506
|
+
const key = agreementAnswerKey(ids);
|
|
507
|
+
const trace = item.traceId ? traceById.get(String(item.traceId)) : undefined;
|
|
508
|
+
if (trace) {
|
|
509
|
+
subjects.set(String(item._id), {
|
|
510
|
+
...ids,
|
|
511
|
+
input: trace.input,
|
|
512
|
+
key,
|
|
513
|
+
output: trace.output,
|
|
514
|
+
traceName: trace.name,
|
|
515
|
+
});
|
|
516
|
+
continue;
|
|
517
|
+
}
|
|
518
|
+
const datasetSubject = ids.datasetItemId ? datasetSubjects.get(ids.datasetItemId) : undefined;
|
|
519
|
+
subjects.set(String(item._id), {
|
|
520
|
+
...ids,
|
|
521
|
+
input: datasetSubject?.input,
|
|
522
|
+
key,
|
|
523
|
+
output: datasetSubject?.output,
|
|
524
|
+
traceName: datasetSubject ? "Dataset item" : "Unknown trace",
|
|
525
|
+
});
|
|
526
|
+
}
|
|
527
|
+
return subjects;
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
private async datasetNames(
|
|
531
|
+
batches: Pick<ObsReviewBatchDocument, "datasetId">[]
|
|
532
|
+
): Promise<Map<string, string>> {
|
|
533
|
+
const ids = batches.flatMap((batch) => (batch.datasetId ? [batch.datasetId] : []));
|
|
534
|
+
if (ids.length === 0) {
|
|
535
|
+
return new Map();
|
|
536
|
+
}
|
|
537
|
+
const datasets = await registerObsDataset().find({_id: {$in: ids}});
|
|
538
|
+
return new Map(
|
|
539
|
+
datasets.map((dataset) => {
|
|
540
|
+
return [String(dataset._id), dataset.name];
|
|
541
|
+
})
|
|
542
|
+
);
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
private async latestJudgeRuns(
|
|
546
|
+
batchIds: mongoose.Types.ObjectId[]
|
|
547
|
+
): Promise<ObsJudgeRunDocument[]> {
|
|
548
|
+
const runs = await registerObsJudgeRun()
|
|
549
|
+
.find({batchId: {$in: batchIds}})
|
|
550
|
+
.sort({completedAt: -1});
|
|
551
|
+
const latest = new Map<string, ObsJudgeRunDocument>();
|
|
552
|
+
for (const run of runs) {
|
|
553
|
+
const key = `${String(run.batchId)}:${String(run.evaluatorId)}`;
|
|
554
|
+
if (!latest.has(key)) {
|
|
555
|
+
latest.set(key, run);
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
return [...latest.values()];
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
// Agreement for each judge's latest run against the done human answers, best match first.
|
|
562
|
+
private summarizeJudges(params: {
|
|
563
|
+
dimensions: EvaluatorDimension[];
|
|
564
|
+
evaluatorById: Map<string, ObsEvaluatorDocument>;
|
|
565
|
+
reviewItems: Pick<ObsReviewItemDocument, "datasetItemId" | "scores" | "status" | "traceId">[];
|
|
566
|
+
runs: ObsJudgeRunDocument[];
|
|
567
|
+
}): ReviewBatchJudgeSummary[] {
|
|
568
|
+
const humanAnswers = params.reviewItems
|
|
569
|
+
.filter((item) => item.status === "done")
|
|
570
|
+
.map((item) => {
|
|
571
|
+
return {...this.answerIds(item), scores: item.scores};
|
|
572
|
+
});
|
|
573
|
+
const summaries = params.runs.map((run) => {
|
|
574
|
+
const judge = params.evaluatorById.get(String(run.evaluatorId));
|
|
575
|
+
return {
|
|
576
|
+
agreement: computeJudgeAgreement({
|
|
577
|
+
dimensions: params.dimensions,
|
|
578
|
+
humanAnswers,
|
|
579
|
+
judgeAnswers: run.results.map((result) => {
|
|
580
|
+
return {...this.answerIds(result), error: result.error, scores: result.scores};
|
|
581
|
+
}),
|
|
582
|
+
judgeKeys: (judge?.dimensions ?? []).map((dimension) => dimension.key),
|
|
583
|
+
}),
|
|
584
|
+
completedAt: toIso(run.completedAt),
|
|
585
|
+
evaluatorId: String(run.evaluatorId),
|
|
586
|
+
evaluatorName: judge?.name ?? "Deleted judge",
|
|
587
|
+
judgePromptName: judge?.judgePromptName,
|
|
588
|
+
modelId: run.modelId,
|
|
589
|
+
runId: String(run._id),
|
|
590
|
+
};
|
|
591
|
+
});
|
|
592
|
+
summaries.sort((a, b) => (b.agreement.rate ?? -1) - (a.agreement.rate ?? -1));
|
|
593
|
+
return summaries;
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
private toListItem(params: {
|
|
597
|
+
batch: ObsReviewBatchDocument;
|
|
598
|
+
datasetName?: string;
|
|
599
|
+
evaluator?: Pick<ObsEvaluatorDocument, "dimensions" | "name">;
|
|
600
|
+
items: Pick<ObsReviewItemDocument, "status">[];
|
|
601
|
+
judges: ReviewBatchJudgeSummary[];
|
|
602
|
+
}): ReviewBatchListItem {
|
|
603
|
+
const {batch, evaluator, items, judges} = params;
|
|
604
|
+
const questionsStale =
|
|
605
|
+
batch.questionsFingerprint && evaluator
|
|
606
|
+
? questionSetFingerprint(evaluator.dimensions) !== batch.questionsFingerprint
|
|
607
|
+
: undefined;
|
|
608
|
+
const best = judges[0];
|
|
609
|
+
return {
|
|
610
|
+
bestJudge: best ? {evaluatorName: best.evaluatorName, rate: best.agreement.rate} : undefined,
|
|
611
|
+
counts: {
|
|
612
|
+
done: items.filter((item) => item.status === "done").length,
|
|
613
|
+
pending: items.filter((item) => item.status === "pending" || item.status === "in_progress")
|
|
614
|
+
.length,
|
|
615
|
+
skipped: items.filter((item) => item.status === "skipped").length,
|
|
616
|
+
total: items.length,
|
|
617
|
+
},
|
|
618
|
+
created: toIso(batch.created),
|
|
619
|
+
datasetId: batch.datasetId ? String(batch.datasetId) : undefined,
|
|
620
|
+
datasetName: params.datasetName,
|
|
621
|
+
description: batch.description,
|
|
622
|
+
evaluatorId: String(batch.evaluatorId),
|
|
623
|
+
evaluatorName: evaluator?.name ?? "Deleted evaluator",
|
|
624
|
+
id: String(batch._id),
|
|
625
|
+
judgeCount: judges.length,
|
|
626
|
+
name: batch.name,
|
|
627
|
+
promptName: batch.promptName,
|
|
628
|
+
questionsFingerprint: batch.questionsFingerprint,
|
|
629
|
+
questionsStale,
|
|
630
|
+
};
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
// Newest successful traces with an output, skipping traces already batched for this question set.
|
|
634
|
+
private async recentTraceIds(params: {
|
|
635
|
+
evaluatorId: mongoose.Types.ObjectId;
|
|
636
|
+
limit: number;
|
|
637
|
+
promptName?: string;
|
|
638
|
+
}): Promise<string[]> {
|
|
639
|
+
const batched = await registerObsReviewBatch().find({evaluatorId: params.evaluatorId});
|
|
640
|
+
const excluded = batched.flatMap((batch) => batch.traceIds);
|
|
641
|
+
const filter: Record<string, unknown> = {
|
|
642
|
+
_id: {$nin: excluded},
|
|
643
|
+
output: {$exists: true, $ne: null},
|
|
644
|
+
status: "ok",
|
|
645
|
+
};
|
|
646
|
+
if (params.promptName) {
|
|
647
|
+
filter["prompts.name"] = params.promptName;
|
|
648
|
+
}
|
|
649
|
+
const traces = await registerObsTrace()
|
|
650
|
+
.find(filter)
|
|
651
|
+
.sort({startedAt: -1})
|
|
652
|
+
.limit(params.limit)
|
|
653
|
+
.select({_id: 1});
|
|
654
|
+
return traces.map((trace) => String(trace._id));
|
|
655
|
+
}
|
|
656
|
+
}
|