@terreno/ai 58.0.0-beta.3 → 58.0.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +99 -0
- package/README.md +3 -3
- package/dist/harness/commit.d.ts +4 -3
- package/dist/harness/commit.js +23 -4
- package/dist/index.d.ts +6 -9
- package/dist/index.js +14 -25
- package/dist/observability/adminScreens.js +5 -1
- package/dist/observability/evaluate.d.ts +8 -1
- package/dist/observability/evaluate.js +10 -2
- package/dist/observability/judgeAgreement.d.ts +53 -0
- package/dist/observability/judgeAgreement.js +123 -0
- package/dist/observability/local/datasetStore.d.ts +5 -0
- package/dist/observability/local/datasetStore.js +7 -2
- package/dist/observability/local/evaluatorStore.d.ts +17 -0
- package/dist/observability/local/evaluatorStore.js +73 -2
- package/dist/observability/local/experimentRunner.d.ts +40 -3
- package/dist/observability/local/experimentRunner.js +390 -11
- package/dist/observability/local/humanAnswers.d.ts +57 -0
- package/dist/observability/local/humanAnswers.js +174 -0
- package/dist/observability/local/judgeRun.d.ts +56 -0
- package/dist/observability/local/judgeRun.js +111 -0
- package/dist/observability/local/localPlugin.js +4 -0
- package/dist/observability/local/models/obsDatasetItem.js +4 -0
- package/dist/observability/local/models/obsEvaluator.js +4 -0
- package/dist/observability/local/models/obsExperiment.js +28 -0
- package/dist/observability/local/models/obsExperimentItem.js +4 -0
- package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
- package/dist/observability/local/models/obsJudgeRun.js +60 -0
- package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
- package/dist/observability/local/models/obsReviewBatch.js +48 -0
- package/dist/observability/local/models/obsReviewItem.js +10 -2
- package/dist/observability/local/models/obsTrace.js +4 -0
- package/dist/observability/local/promptReferences.d.ts +31 -0
- package/dist/observability/local/promptReferences.js +114 -0
- package/dist/observability/local/promptStore.d.ts +55 -19
- package/dist/observability/local/promptStore.js +171 -7
- package/dist/observability/local/reviewBatchStore.d.ts +106 -0
- package/dist/observability/local/reviewBatchStore.js +517 -0
- package/dist/observability/local/reviewStore.d.ts +24 -3
- package/dist/observability/local/reviewStore.js +98 -22
- package/dist/observability/local/traceStore.d.ts +14 -1
- package/dist/observability/local/traceStore.js +144 -11
- package/dist/observability/observabilityApp.js +10 -1
- package/dist/observability/questionFingerprint.d.ts +7 -0
- package/dist/observability/questionFingerprint.js +23 -0
- package/dist/observability/routes/experiments.js +12 -1
- package/dist/observability/routes/review.js +3 -1
- package/dist/observability/routes/reviewBatches.d.ts +11 -0
- package/dist/observability/routes/reviewBatches.js +78 -0
- package/dist/observability/routes/traces.d.ts +1 -1
- package/dist/observability/routes/traces.js +34 -2
- package/dist/observability/types.d.ts +10 -1
- package/dist/service/aiService.js +4 -1
- package/dist/service/chatTurn.js +38 -31
- package/dist/types/harness.d.ts +5 -2
- package/dist/types/index.d.ts +8 -3
- package/dist/types/observability.d.ts +74 -2
- package/package.json +8 -24
- package/src/harness/agentLoop.test.ts +0 -3
- package/src/harness/agentRecovery.test.ts +0 -3
- package/src/harness/commit.ts +24 -4
- package/src/harness/extensions.test.ts +0 -3
- package/src/harness/subagent.test.ts +27 -3
- package/src/index.test.ts +0 -17
- package/src/index.ts +23 -26
- package/src/isolated/gptStream.isolated.ts +0 -8
- package/src/observability/adminScreens.test.ts +7 -0
- package/src/observability/adminScreens.ts +6 -1
- package/src/observability/evaluate.ts +12 -2
- package/src/observability/judgeAgreement.test.ts +111 -0
- package/src/observability/judgeAgreement.ts +184 -0
- package/src/observability/local/datasetStore.ts +30 -11
- package/src/observability/local/evaluatorStore.ts +96 -2
- package/src/observability/local/experimentRunner.ts +495 -15
- package/src/observability/local/humanAnswers.ts +233 -0
- package/src/observability/local/judgePipeline.test.ts +444 -0
- package/src/observability/local/judgeRun.test.ts +173 -0
- package/src/observability/local/judgeRun.ts +184 -0
- package/src/observability/local/localPlugin.ts +4 -0
- package/src/observability/local/models/obsDatasetItem.ts +5 -0
- package/src/observability/local/models/obsEvaluator.ts +4 -0
- package/src/observability/local/models/obsExperiment.ts +29 -0
- package/src/observability/local/models/obsExperimentItem.ts +4 -0
- package/src/observability/local/models/obsJudgeRun.ts +61 -0
- package/src/observability/local/models/obsReviewBatch.ts +52 -0
- package/src/observability/local/models/obsReviewItem.ts +11 -2
- package/src/observability/local/models/obsTrace.ts +5 -0
- package/src/observability/local/obsModels.test.ts +13 -0
- package/src/observability/local/promptReferences.test.ts +128 -0
- package/src/observability/local/promptReferences.ts +149 -0
- package/src/observability/local/promptStore.test.ts +291 -0
- package/src/observability/local/promptStore.ts +243 -32
- package/src/observability/local/reviewBatchStore.test.ts +251 -0
- package/src/observability/local/reviewBatchStore.ts +656 -0
- package/src/observability/local/reviewStore.ts +122 -25
- package/src/observability/local/traceStore.test.ts +139 -1
- package/src/observability/local/traceStore.ts +162 -11
- package/src/observability/observabilityApp.test.ts +13 -10
- package/src/observability/observabilityApp.ts +10 -1
- package/src/observability/questionFingerprint.test.ts +33 -0
- package/src/observability/questionFingerprint.ts +24 -0
- package/src/observability/routes/experiments.test.ts +37 -0
- package/src/observability/routes/experiments.ts +13 -1
- package/src/observability/routes/review.ts +3 -1
- package/src/observability/routes/reviewBatches.test.ts +209 -0
- package/src/observability/routes/reviewBatches.ts +138 -0
- package/src/observability/routes/status.test.ts +4 -4
- package/src/observability/routes/traces.test.ts +61 -0
- package/src/observability/routes/traces.ts +46 -4
- package/src/observability/types.ts +5 -2
- package/src/routes/routes.test.ts +101 -76
- package/src/service/aiService.test.ts +21 -0
- package/src/service/aiService.ts +4 -1
- package/src/service/chatTurn.ts +42 -33
- package/src/tests/bunSetup.ts +0 -54
- package/src/types/harness.ts +5 -2
- package/src/types/index.ts +8 -3
- package/src/types/observability.ts +86 -2
- package/dist/langfuseApp.d.ts +0 -9
- package/dist/langfuseApp.js +0 -53
- package/dist/langfuseCache.d.ts +0 -15
- package/dist/langfuseCache.js +0 -54
- package/dist/langfuseClient.d.ts +0 -6
- package/dist/langfuseClient.js +0 -34
- package/dist/langfusePrompts.d.ts +0 -12
- package/dist/langfusePrompts.js +0 -89
- package/dist/langfuseRoutesEvaluations.d.ts +0 -3
- package/dist/langfuseRoutesEvaluations.js +0 -32
- package/dist/langfuseRoutesMiddleware.d.ts +0 -2
- package/dist/langfuseRoutesMiddleware.js +0 -13
- package/dist/langfuseRoutesPlayground.d.ts +0 -2
- package/dist/langfuseRoutesPlayground.js +0 -36
- package/dist/langfuseRoutesPrompts.d.ts +0 -2
- package/dist/langfuseRoutesPrompts.js +0 -56
- package/dist/langfuseRoutesTraces.d.ts +0 -2
- package/dist/langfuseRoutesTraces.js +0 -30
- package/dist/langfuseTracing.d.ts +0 -4
- package/dist/langfuseTracing.js +0 -32
- package/dist/langfuseTypes.d.ts +0 -96
- package/dist/langfuseTypes.js +0 -2
- package/dist/langfuseVercelAi.d.ts +0 -12
- package/dist/langfuseVercelAi.js +0 -49
- package/src/isolated/langfuseClient.isolated.ts +0 -66
- package/src/langfuseApp.test.ts +0 -119
- package/src/langfuseApp.ts +0 -62
- package/src/langfuseCache.test.ts +0 -75
- package/src/langfuseCache.ts +0 -72
- package/src/langfuseClient.test.ts +0 -69
- package/src/langfuseClient.ts +0 -34
- package/src/langfusePrompts.test.ts +0 -188
- package/src/langfusePrompts.ts +0 -121
- package/src/langfuseRoutes.test.ts +0 -261
- package/src/langfuseRoutesEvaluations.ts +0 -48
- package/src/langfuseRoutesMiddleware.test.ts +0 -57
- package/src/langfuseRoutesMiddleware.ts +0 -12
- package/src/langfuseRoutesPlayground.ts +0 -50
- package/src/langfuseRoutesPrompts.ts +0 -87
- package/src/langfuseRoutesTraces.ts +0 -40
- package/src/langfuseTracing.test.ts +0 -72
- package/src/langfuseTracing.ts +0 -33
- package/src/langfuseTypes.ts +0 -103
- package/src/langfuseVercelAi.test.ts +0 -82
- package/src/langfuseVercelAi.ts +0 -72
- package/src/tests/fakeLangfusePrompt.ts +0 -10
|
@@ -5,9 +5,14 @@ import mongoose from "mongoose";
|
|
|
5
5
|
import type {EvaluatorDimension} from "../../types/observability";
|
|
6
6
|
import {buildReviewPanels, type ReviewPanels} from "../reviewPanels";
|
|
7
7
|
import type {ScoreRecord, ScoreSink} from "../types";
|
|
8
|
+
import {loadDatasetItemSubjects, writeHumanAnswer} from "./humanAnswers";
|
|
9
|
+
import {registerObsDataset} from "./models/obsDataset";
|
|
10
|
+
import {registerObsDatasetItem} from "./models/obsDatasetItem";
|
|
8
11
|
import {registerObsEvaluator} from "./models/obsEvaluator";
|
|
9
12
|
import {registerObsPrompt} from "./models/obsPrompt";
|
|
13
|
+
import {registerObsPromptLabel} from "./models/obsPromptLabel";
|
|
10
14
|
import {registerObsPromptVersion} from "./models/obsPromptVersion";
|
|
15
|
+
import {registerObsReviewBatch} from "./models/obsReviewBatch";
|
|
11
16
|
import {registerObsReviewItem} from "./models/obsReviewItem";
|
|
12
17
|
import {registerObsTrace} from "./models/obsTrace";
|
|
13
18
|
import {LocalTraceStore} from "./traceStore";
|
|
@@ -16,16 +21,22 @@ export type ReviewStatus = "done" | "in_progress" | "pending" | "skipped";
|
|
|
16
21
|
|
|
17
22
|
export interface ReviewListItem {
|
|
18
23
|
assigneeId?: string;
|
|
24
|
+
batchId?: string;
|
|
25
|
+
/** Dataset item under review, when the item came from a dataset review batch. */
|
|
26
|
+
datasetItemId?: string;
|
|
19
27
|
enqueuedAt: string;
|
|
20
28
|
evaluatorId: string;
|
|
21
29
|
id: string;
|
|
22
30
|
promptName?: string;
|
|
23
31
|
reason: string;
|
|
24
32
|
status: ReviewStatus;
|
|
25
|
-
|
|
33
|
+
/** Unset when the reviewed dataset item has no source trace. */
|
|
34
|
+
traceId?: string;
|
|
26
35
|
traceName: string;
|
|
27
36
|
}
|
|
28
37
|
|
|
38
|
+
const DATASET_ITEM_NAME = "Dataset item";
|
|
39
|
+
|
|
29
40
|
const objectId = (value: string, label: string): mongoose.Types.ObjectId => {
|
|
30
41
|
if (!mongoose.Types.ObjectId.isValid(value)) {
|
|
31
42
|
throw new APIError({status: 400, title: `${label} must be an object id`});
|
|
@@ -39,6 +50,7 @@ const toIso = (value: Date): string => {
|
|
|
39
50
|
|
|
40
51
|
export class LocalReviewStore {
|
|
41
52
|
async enqueue(params: {
|
|
53
|
+
batchId?: string;
|
|
42
54
|
evaluatorId: string;
|
|
43
55
|
reason?: "dataset_candidate" | "eval" | "feedback" | "manual";
|
|
44
56
|
spanId?: string;
|
|
@@ -71,6 +83,7 @@ export class LocalReviewStore {
|
|
|
71
83
|
reason: params.reason ?? "manual",
|
|
72
84
|
status: "pending",
|
|
73
85
|
traceId: trace._id,
|
|
86
|
+
...(params.batchId ? {batchId: objectId(params.batchId, "batchId")} : {}),
|
|
74
87
|
...(params.spanId ? {spanId: objectId(params.spanId, "spanId")} : {}),
|
|
75
88
|
});
|
|
76
89
|
created.push(this.toListItem(item, trace.name, trace.prompts[0]?.name));
|
|
@@ -78,21 +91,52 @@ export class LocalReviewStore {
|
|
|
78
91
|
return created;
|
|
79
92
|
}
|
|
80
93
|
|
|
81
|
-
|
|
94
|
+
/** Queue dataset items for a human question set; items with a source trace show that trace. */
|
|
95
|
+
async enqueueDatasetItems(params: {
|
|
96
|
+
batchId: string;
|
|
97
|
+
evaluatorId: string;
|
|
98
|
+
items: Array<{datasetItemId: string; traceId?: string}>;
|
|
99
|
+
}): Promise<ReviewListItem[]> {
|
|
100
|
+
const created: ReviewListItem[] = [];
|
|
101
|
+
for (const entry of params.items) {
|
|
102
|
+
const item = await registerObsReviewItem().create({
|
|
103
|
+
batchId: objectId(params.batchId, "batchId"),
|
|
104
|
+
datasetItemId: objectId(entry.datasetItemId, "datasetItemId"),
|
|
105
|
+
enqueuedAt: DateTime.utc().toJSDate(),
|
|
106
|
+
evaluatorId: objectId(params.evaluatorId, "evaluatorId"),
|
|
107
|
+
reason: "manual",
|
|
108
|
+
status: "pending",
|
|
109
|
+
...(entry.traceId ? {traceId: objectId(entry.traceId, "traceId")} : {}),
|
|
110
|
+
});
|
|
111
|
+
created.push(this.toListItem(item, DATASET_ITEM_NAME));
|
|
112
|
+
}
|
|
113
|
+
return created;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
async list(
|
|
117
|
+
status?: ReviewStatus,
|
|
118
|
+
options: {batchId?: string} = {}
|
|
119
|
+
): Promise<{
|
|
82
120
|
counts: Record<ReviewStatus, number>;
|
|
83
121
|
data: ReviewListItem[];
|
|
84
122
|
}> {
|
|
85
123
|
const ObsReviewItem = registerObsReviewItem();
|
|
124
|
+
const scope = options.batchId ? {batchId: objectId(options.batchId, "batchId")} : {};
|
|
86
125
|
const counts = {
|
|
87
|
-
done: await ObsReviewItem.countDocuments({status: "done"}),
|
|
88
|
-
in_progress: await ObsReviewItem.countDocuments({status: "in_progress"}),
|
|
89
|
-
pending: await ObsReviewItem.countDocuments({status: "pending"}),
|
|
90
|
-
skipped: await ObsReviewItem.countDocuments({status: "skipped"}),
|
|
126
|
+
done: await ObsReviewItem.countDocuments({...scope, status: "done"}),
|
|
127
|
+
in_progress: await ObsReviewItem.countDocuments({...scope, status: "in_progress"}),
|
|
128
|
+
pending: await ObsReviewItem.countDocuments({...scope, status: "pending"}),
|
|
129
|
+
skipped: await ObsReviewItem.countDocuments({...scope, status: "skipped"}),
|
|
91
130
|
};
|
|
92
|
-
const filter = status ? {status} :
|
|
93
|
-
|
|
131
|
+
const filter = status ? {...scope, status} : scope;
|
|
132
|
+
// Batch items keep their batch order (enqueued together, newest trace first).
|
|
133
|
+
const rows = await ObsReviewItem.find(filter).sort({_id: 1, enqueuedAt: 1});
|
|
94
134
|
const data: ReviewListItem[] = [];
|
|
95
135
|
for (const row of rows) {
|
|
136
|
+
if (!row.traceId) {
|
|
137
|
+
data.push(this.toListItem(row, DATASET_ITEM_NAME));
|
|
138
|
+
continue;
|
|
139
|
+
}
|
|
96
140
|
const trace = await registerObsTrace().findOneOrNone({_id: row.traceId});
|
|
97
141
|
data.push(this.toListItem(row, trace?.name ?? "Unknown trace", trace?.prompts[0]?.name));
|
|
98
142
|
}
|
|
@@ -100,7 +144,10 @@ export class LocalReviewStore {
|
|
|
100
144
|
}
|
|
101
145
|
|
|
102
146
|
async getDetail(id: string): Promise<{
|
|
147
|
+
batchId?: string;
|
|
148
|
+
batchName?: string;
|
|
103
149
|
comment?: string;
|
|
150
|
+
datasetItemId?: string;
|
|
104
151
|
dimensions: EvaluatorDimension[];
|
|
105
152
|
evaluatorId: string;
|
|
106
153
|
id: string;
|
|
@@ -110,7 +157,7 @@ export class LocalReviewStore {
|
|
|
110
157
|
rawOutput?: unknown;
|
|
111
158
|
scores?: Record<string, boolean | number | string>;
|
|
112
159
|
status: ReviewStatus;
|
|
113
|
-
traceId
|
|
160
|
+
traceId?: string;
|
|
114
161
|
}> {
|
|
115
162
|
const item = await registerObsReviewItem().findOneOrNone({_id: objectId(id, "id")});
|
|
116
163
|
if (!item) {
|
|
@@ -120,13 +167,49 @@ export class LocalReviewStore {
|
|
|
120
167
|
if (!evaluator) {
|
|
121
168
|
throw new APIError({status: 404, title: "Unknown evaluator"});
|
|
122
169
|
}
|
|
123
|
-
const trace =
|
|
124
|
-
|
|
170
|
+
const trace = item.traceId
|
|
171
|
+
? await registerObsTrace().findOneOrNone({_id: item.traceId})
|
|
172
|
+
: undefined;
|
|
173
|
+
const batch = item.batchId
|
|
174
|
+
? await registerObsReviewBatch().findOneOrNone({_id: item.batchId})
|
|
175
|
+
: undefined;
|
|
176
|
+
let input = trace?.input;
|
|
177
|
+
let output = trace?.output;
|
|
125
178
|
let variables:
|
|
126
179
|
| {key: string; label?: string; required: boolean; reviewerNote?: string}[]
|
|
127
180
|
| undefined;
|
|
128
181
|
let outputFieldNotes: Record<string, string> | undefined;
|
|
129
182
|
let outputSchema: Record<string, unknown> | undefined;
|
|
183
|
+
// A dataset item without a (surviving) source trace shows its own input and expected output.
|
|
184
|
+
if (!trace && item.datasetItemId) {
|
|
185
|
+
const datasetItem = await registerObsDatasetItem().findOneOrNone({
|
|
186
|
+
_id: item.datasetItemId,
|
|
187
|
+
});
|
|
188
|
+
if (datasetItem) {
|
|
189
|
+
const subject = (await loadDatasetItemSubjects([datasetItem])).get(String(datasetItem._id));
|
|
190
|
+
input = subject?.input;
|
|
191
|
+
output = subject?.output;
|
|
192
|
+
const dataset = await registerObsDataset().findOneOrNone({_id: datasetItem.datasetId});
|
|
193
|
+
if (dataset?.inputSchemaPromptName) {
|
|
194
|
+
const prompt = await registerObsPrompt().findOneOrNone({
|
|
195
|
+
name: dataset.inputSchemaPromptName,
|
|
196
|
+
});
|
|
197
|
+
const label = prompt
|
|
198
|
+
? await registerObsPromptLabel().findOneOrNone({
|
|
199
|
+
label: "production",
|
|
200
|
+
promptId: prompt._id,
|
|
201
|
+
})
|
|
202
|
+
: undefined;
|
|
203
|
+
const version = label
|
|
204
|
+
? await registerObsPromptVersion().findOneOrNone({_id: label.versionId})
|
|
205
|
+
: undefined;
|
|
206
|
+
variables = version?.variables;
|
|
207
|
+
outputFieldNotes = version?.outputFieldNotes;
|
|
208
|
+
outputSchema = version?.outputSchema;
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
const promptRef = trace?.prompts[0];
|
|
130
213
|
if (promptRef) {
|
|
131
214
|
const prompt = await registerObsPrompt().findOneOrNone({name: promptRef.name});
|
|
132
215
|
if (prompt) {
|
|
@@ -140,23 +223,20 @@ export class LocalReviewStore {
|
|
|
140
223
|
}
|
|
141
224
|
}
|
|
142
225
|
return {
|
|
226
|
+
batchId: batch ? String(batch._id) : undefined,
|
|
227
|
+
batchName: batch?.name,
|
|
143
228
|
comment: item.comment,
|
|
229
|
+
datasetItemId: item.datasetItemId ? String(item.datasetItemId) : undefined,
|
|
144
230
|
dimensions: evaluator.dimensions,
|
|
145
231
|
evaluatorId: String(evaluator._id),
|
|
146
232
|
id: String(item._id),
|
|
147
233
|
instructions: evaluator.instructions,
|
|
148
|
-
panels: buildReviewPanels({
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
outputFieldNotes,
|
|
152
|
-
outputSchema,
|
|
153
|
-
variables,
|
|
154
|
-
}),
|
|
155
|
-
rawInput: trace?.input,
|
|
156
|
-
rawOutput: trace?.output,
|
|
234
|
+
panels: buildReviewPanels({input, output, outputFieldNotes, outputSchema, variables}),
|
|
235
|
+
rawInput: input,
|
|
236
|
+
rawOutput: output,
|
|
157
237
|
scores: item.scores,
|
|
158
238
|
status: item.status,
|
|
159
|
-
traceId: String(item.traceId),
|
|
239
|
+
traceId: item.traceId ? String(item.traceId) : undefined,
|
|
160
240
|
};
|
|
161
241
|
}
|
|
162
242
|
|
|
@@ -165,6 +245,8 @@ export class LocalReviewStore {
|
|
|
165
245
|
id: string;
|
|
166
246
|
scores: Record<string, boolean | number | string>;
|
|
167
247
|
sinks: ScoreSink[];
|
|
248
|
+
/** Reviewer, recorded on the dataset item's human answers. */
|
|
249
|
+
userId?: string;
|
|
168
250
|
}): Promise<void> {
|
|
169
251
|
const item = await this.requireItem(params.id);
|
|
170
252
|
const evaluator = await registerObsEvaluator().findOneOrNone({_id: item.evaluatorId});
|
|
@@ -178,7 +260,9 @@ export class LocalReviewStore {
|
|
|
178
260
|
throw new APIError({status: 400, title: `Score "${missing.key}" is required`});
|
|
179
261
|
}
|
|
180
262
|
const records: ScoreRecord[] = [];
|
|
181
|
-
|
|
263
|
+
const traceId = item.traceId;
|
|
264
|
+
// Scores attach to traces, so a dataset item without a trace only records human answers.
|
|
265
|
+
for (const dimension of traceId ? evaluator.dimensions : []) {
|
|
182
266
|
const value = params.scores[dimension.key];
|
|
183
267
|
if (value === undefined) {
|
|
184
268
|
continue;
|
|
@@ -189,7 +273,7 @@ export class LocalReviewStore {
|
|
|
189
273
|
evaluatorId: String(evaluator._id),
|
|
190
274
|
name: dimension.key,
|
|
191
275
|
source: "human" as const,
|
|
192
|
-
traceId: String(
|
|
276
|
+
traceId: String(traceId),
|
|
193
277
|
value,
|
|
194
278
|
});
|
|
195
279
|
}
|
|
@@ -213,6 +297,15 @@ export class LocalReviewStore {
|
|
|
213
297
|
item.scores = params.scores;
|
|
214
298
|
item.status = "done";
|
|
215
299
|
await item.save();
|
|
300
|
+
if (item.datasetItemId) {
|
|
301
|
+
await writeHumanAnswer({
|
|
302
|
+
datasetItemId: item.datasetItemId,
|
|
303
|
+
evaluator,
|
|
304
|
+
reviewItemId: String(item._id),
|
|
305
|
+
scores: params.scores,
|
|
306
|
+
userId: params.userId,
|
|
307
|
+
});
|
|
308
|
+
}
|
|
216
309
|
}
|
|
217
310
|
|
|
218
311
|
async skip(id: string): Promise<void> {
|
|
@@ -240,24 +333,28 @@ export class LocalReviewStore {
|
|
|
240
333
|
row: {
|
|
241
334
|
_id: mongoose.Types.ObjectId;
|
|
242
335
|
assigneeId?: mongoose.Types.ObjectId;
|
|
336
|
+
batchId?: mongoose.Types.ObjectId;
|
|
337
|
+
datasetItemId?: mongoose.Types.ObjectId;
|
|
243
338
|
enqueuedAt: Date;
|
|
244
339
|
evaluatorId: mongoose.Types.ObjectId;
|
|
245
340
|
reason: string;
|
|
246
341
|
status: ReviewStatus;
|
|
247
|
-
traceId
|
|
342
|
+
traceId?: mongoose.Types.ObjectId;
|
|
248
343
|
},
|
|
249
344
|
traceName: string,
|
|
250
345
|
promptName?: string
|
|
251
346
|
): ReviewListItem {
|
|
252
347
|
return {
|
|
253
348
|
assigneeId: row.assigneeId ? String(row.assigneeId) : undefined,
|
|
349
|
+
batchId: row.batchId ? String(row.batchId) : undefined,
|
|
350
|
+
datasetItemId: row.datasetItemId ? String(row.datasetItemId) : undefined,
|
|
254
351
|
enqueuedAt: toIso(row.enqueuedAt),
|
|
255
352
|
evaluatorId: String(row.evaluatorId),
|
|
256
353
|
id: String(row._id),
|
|
257
354
|
promptName,
|
|
258
355
|
reason: row.reason,
|
|
259
356
|
status: row.status,
|
|
260
|
-
traceId: String(row.traceId),
|
|
357
|
+
traceId: row.traceId ? String(row.traceId) : undefined,
|
|
261
358
|
traceName,
|
|
262
359
|
};
|
|
263
360
|
}
|
|
@@ -8,7 +8,13 @@ import {createLocalObservabilityPlugin} from "./localPlugin";
|
|
|
8
8
|
import {registerObsScore} from "./models/obsScore";
|
|
9
9
|
import {registerObsSpan} from "./models/obsSpan";
|
|
10
10
|
import {registerObsTrace} from "./models/obsTrace";
|
|
11
|
-
import {
|
|
11
|
+
import {
|
|
12
|
+
LocalTraceSink,
|
|
13
|
+
LocalTraceStore,
|
|
14
|
+
MemoryScoreSink,
|
|
15
|
+
MemoryTraceSink,
|
|
16
|
+
parseTraceListSort,
|
|
17
|
+
} from "./traceStore";
|
|
12
18
|
|
|
13
19
|
const iso = (msAgo: number): string => {
|
|
14
20
|
return DateTime.utc().minus({milliseconds: msAgo}).toISO() ?? "";
|
|
@@ -244,6 +250,138 @@ describe("LocalTraceStore", () => {
|
|
|
244
250
|
expect(sinkDetail.name).toBe("sink");
|
|
245
251
|
});
|
|
246
252
|
|
|
253
|
+
describe("sorting", () => {
|
|
254
|
+
const at = (base: DateTime, ms: number): string => base.plus({milliseconds: ms}).toISO() ?? "";
|
|
255
|
+
|
|
256
|
+
const span = (id: string, startedAt: string): TraceRecord["spans"][number] => {
|
|
257
|
+
return {id, kind: "LLM", name: id, startedAt, status: "ok"};
|
|
258
|
+
};
|
|
259
|
+
|
|
260
|
+
const score = (traceId: string): ScoreRecord => {
|
|
261
|
+
return {dataType: "boolean", name: "correct", source: "human", traceId, value: true};
|
|
262
|
+
};
|
|
263
|
+
|
|
264
|
+
// bravo: 15 tokens, $0.002, 300 ms, 1 span, 2 scores
|
|
265
|
+
// alpha: 150 tokens, $0.0001, 100 ms, 3 spans, 0 scores
|
|
266
|
+
// Charlie: no usage, no end time, 0 spans, 1 score
|
|
267
|
+
const seed = async (): Promise<{alpha: string; bravo: string; charlie: string}> => {
|
|
268
|
+
const base = DateTime.utc().minus({hours: 1});
|
|
269
|
+
const bravo = await store.exportTrace(
|
|
270
|
+
baseTrace({
|
|
271
|
+
endedAt: at(base, 300),
|
|
272
|
+
name: "bravo",
|
|
273
|
+
spans: [span("b1", at(base, 0))],
|
|
274
|
+
startedAt: at(base, 0),
|
|
275
|
+
usage: {costUsd: 0.002, inputTokens: 10, outputTokens: 5},
|
|
276
|
+
})
|
|
277
|
+
);
|
|
278
|
+
const alpha = await store.exportTrace(
|
|
279
|
+
baseTrace({
|
|
280
|
+
endedAt: at(base, 1100),
|
|
281
|
+
name: "alpha",
|
|
282
|
+
spans: [
|
|
283
|
+
span("a1", at(base, 1000)),
|
|
284
|
+
span("a2", at(base, 1000)),
|
|
285
|
+
span("a3", at(base, 1000)),
|
|
286
|
+
],
|
|
287
|
+
startedAt: at(base, 1000),
|
|
288
|
+
usage: {costUsd: 0.0001, inputTokens: 100, outputTokens: 50},
|
|
289
|
+
})
|
|
290
|
+
);
|
|
291
|
+
const charlie = await store.exportTrace(
|
|
292
|
+
baseTrace({
|
|
293
|
+
endedAt: undefined,
|
|
294
|
+
name: "Charlie",
|
|
295
|
+
spans: [],
|
|
296
|
+
startedAt: at(base, 2000),
|
|
297
|
+
usage: undefined,
|
|
298
|
+
})
|
|
299
|
+
);
|
|
300
|
+
await store.exportScore(score(bravo.id));
|
|
301
|
+
await store.exportScore(score(bravo.id));
|
|
302
|
+
await store.exportScore(score(charlie.id));
|
|
303
|
+
return {alpha: alpha.id, bravo: bravo.id, charlie: charlie.id};
|
|
304
|
+
};
|
|
305
|
+
|
|
306
|
+
const names = async (sort?: string): Promise<string[]> => {
|
|
307
|
+
const listed = await store.list({sort});
|
|
308
|
+
return listed.data.map((row) => row.name);
|
|
309
|
+
};
|
|
310
|
+
|
|
311
|
+
const pagedNames = async (sort: string): Promise<string[]> => {
|
|
312
|
+
const first = await store.list({limit: 2, page: 1, sort});
|
|
313
|
+
const second = await store.list({limit: 2, page: 2, sort});
|
|
314
|
+
expect(first.meta.total).toBe(3);
|
|
315
|
+
return [...first.data, ...second.data].map((row) => row.name);
|
|
316
|
+
};
|
|
317
|
+
|
|
318
|
+
const cases: Array<[string, string[]]> = [
|
|
319
|
+
["startedAt", ["bravo", "alpha", "Charlie"]],
|
|
320
|
+
["-startedAt", ["Charlie", "alpha", "bravo"]],
|
|
321
|
+
["name", ["alpha", "bravo", "Charlie"]],
|
|
322
|
+
["-name", ["Charlie", "bravo", "alpha"]],
|
|
323
|
+
["tokens", ["bravo", "alpha", "Charlie"]],
|
|
324
|
+
["-tokens", ["alpha", "bravo", "Charlie"]],
|
|
325
|
+
["cost", ["alpha", "bravo", "Charlie"]],
|
|
326
|
+
["-cost", ["bravo", "alpha", "Charlie"]],
|
|
327
|
+
["latency", ["alpha", "bravo", "Charlie"]],
|
|
328
|
+
["-latency", ["bravo", "alpha", "Charlie"]],
|
|
329
|
+
["spans", ["Charlie", "bravo", "alpha"]],
|
|
330
|
+
["-spans", ["alpha", "bravo", "Charlie"]],
|
|
331
|
+
["scores", ["alpha", "Charlie", "bravo"]],
|
|
332
|
+
["-scores", ["bravo", "Charlie", "alpha"]],
|
|
333
|
+
];
|
|
334
|
+
|
|
335
|
+
for (const [sort, expected] of cases) {
|
|
336
|
+
it(`orders every page by ${sort}`, async () => {
|
|
337
|
+
await seed();
|
|
338
|
+
expect(await names(sort)).toEqual(expected);
|
|
339
|
+
expect(await pagedNames(sort)).toEqual(expected);
|
|
340
|
+
});
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
it("defaults to newest first and keeps counts on sorted rows", async () => {
|
|
344
|
+
const ids = await seed();
|
|
345
|
+
expect(await names()).toEqual(["Charlie", "alpha", "bravo"]);
|
|
346
|
+
const bySpans = await store.list({sort: "-spans"});
|
|
347
|
+
expect(bySpans.data[0]).toMatchObject({id: ids.alpha, scoreCount: 0, spanCount: 3});
|
|
348
|
+
});
|
|
349
|
+
|
|
350
|
+
it("orders by status in both directions", async () => {
|
|
351
|
+
await seed();
|
|
352
|
+
await store.exportTrace(baseTrace({name: "zulu", status: "error"}));
|
|
353
|
+
const asc = await store.list({sort: "status"});
|
|
354
|
+
expect(asc.data.map((row) => row.status)).toEqual(["error", "ok", "ok", "ok"]);
|
|
355
|
+
const desc = await store.list({limit: 3, page: 2, sort: "-status"});
|
|
356
|
+
expect(desc.data.map((row) => row.name)).toEqual(["zulu"]);
|
|
357
|
+
});
|
|
358
|
+
|
|
359
|
+
it("sorts inside the filtered set", async () => {
|
|
360
|
+
await seed();
|
|
361
|
+
await store.exportTrace(baseTrace({name: "zulu", status: "error"}));
|
|
362
|
+
const okOnly = await store.list({sort: "-tokens", status: "ok"});
|
|
363
|
+
expect(okOnly.data.map((row) => row.name)).toEqual(["alpha", "bravo", "Charlie"]);
|
|
364
|
+
});
|
|
365
|
+
|
|
366
|
+
it("rejects unknown sort keys with a 400", async () => {
|
|
367
|
+
await expect(store.list({sort: "created"})).rejects.toMatchObject({status: 400});
|
|
368
|
+
await expect(store.list({sort: "-"})).rejects.toMatchObject({status: 400});
|
|
369
|
+
expect(parseTraceListSort(undefined)).toEqual({direction: -1, key: "startedAt"});
|
|
370
|
+
expect(parseTraceListSort("cost")).toEqual({direction: 1, key: "cost"});
|
|
371
|
+
});
|
|
372
|
+
});
|
|
373
|
+
|
|
374
|
+
it("filters by any of several prompt names", async () => {
|
|
375
|
+
await store.exportTrace(baseTrace({name: "one", prompts: [{name: "summarize", version: 1}]}));
|
|
376
|
+
await store.exportTrace(baseTrace({name: "two", prompts: [{name: "title", version: 1}]}));
|
|
377
|
+
await store.exportTrace(baseTrace({name: "three", prompts: [{name: "other", version: 1}]}));
|
|
378
|
+
const listed = await store.list({prompt: ["summarize", "title"], sort: "name"});
|
|
379
|
+
expect(listed.data.map((row) => row.name)).toEqual(["one", "two"]);
|
|
380
|
+
await expect(store.list({prompt: ["summarize", "title"], promptVersion: 1})).rejects.toThrow(
|
|
381
|
+
/exactly one prompt/
|
|
382
|
+
);
|
|
383
|
+
});
|
|
384
|
+
|
|
247
385
|
it("throws when loading an unknown trace", async () => {
|
|
248
386
|
await expect(store.getDetail(new mongoose.Types.ObjectId().toString())).rejects.toThrow(
|
|
249
387
|
/Unknown trace/
|
|
@@ -7,17 +7,42 @@ import {registerObsScore} from "./models/obsScore";
|
|
|
7
7
|
import {registerObsSpan} from "./models/obsSpan";
|
|
8
8
|
import {registerObsTrace} from "./models/obsTrace";
|
|
9
9
|
|
|
10
|
+
/** Columns the trace list can sort by. Prefix with `-` for descending. */
|
|
11
|
+
export const TRACE_LIST_SORT_KEYS = [
|
|
12
|
+
"cost",
|
|
13
|
+
"latency",
|
|
14
|
+
"name",
|
|
15
|
+
"scores",
|
|
16
|
+
"spans",
|
|
17
|
+
"startedAt",
|
|
18
|
+
"status",
|
|
19
|
+
"tokens",
|
|
20
|
+
] as const;
|
|
21
|
+
|
|
22
|
+
export type TraceListSortKey = (typeof TRACE_LIST_SORT_KEYS)[number];
|
|
23
|
+
|
|
24
|
+
export interface TraceListSort {
|
|
25
|
+
direction: 1 | -1;
|
|
26
|
+
key: TraceListSortKey;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** Newest first, matching the admin Traces screen default. */
|
|
30
|
+
export const DEFAULT_TRACE_LIST_SORT = "-startedAt";
|
|
31
|
+
|
|
10
32
|
export interface TraceListQuery {
|
|
11
33
|
flaggedForDataset?: boolean;
|
|
12
34
|
from?: string;
|
|
13
35
|
hasScore?: boolean;
|
|
14
36
|
limit?: number;
|
|
15
37
|
page?: number;
|
|
16
|
-
prompt
|
|
38
|
+
/** One prompt name, or several to match traces that used any of them. */
|
|
39
|
+
prompt?: string | string[];
|
|
17
40
|
promptVersion?: number;
|
|
18
41
|
scope?: string;
|
|
19
42
|
sensitive?: boolean;
|
|
20
43
|
sessionId?: string;
|
|
44
|
+
/** One of `TRACE_LIST_SORT_KEYS`, optionally prefixed with `-` for descending. */
|
|
45
|
+
sort?: string;
|
|
21
46
|
status?: "error" | "ok";
|
|
22
47
|
to?: string;
|
|
23
48
|
userId?: string;
|
|
@@ -129,6 +154,112 @@ const usageWithoutUndefinedCost = (
|
|
|
129
154
|
return usage;
|
|
130
155
|
};
|
|
131
156
|
|
|
157
|
+
const isTraceListSortKey = (value: string): value is TraceListSortKey => {
|
|
158
|
+
return (TRACE_LIST_SORT_KEYS as readonly string[]).includes(value);
|
|
159
|
+
};
|
|
160
|
+
|
|
161
|
+
export const parseTraceListSort = (value?: string): TraceListSort => {
|
|
162
|
+
const raw = value?.trim() || DEFAULT_TRACE_LIST_SORT;
|
|
163
|
+
const descending = raw.startsWith("-");
|
|
164
|
+
const key = descending ? raw.slice(1) : raw;
|
|
165
|
+
if (!isTraceListSortKey(key)) {
|
|
166
|
+
throw new APIError({
|
|
167
|
+
detail: `Use one of ${TRACE_LIST_SORT_KEYS.join(", ")}, optionally prefixed with "-".`,
|
|
168
|
+
status: 400,
|
|
169
|
+
title: "Unknown trace sort key",
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
return {direction: descending ? -1 : 1, key};
|
|
173
|
+
};
|
|
174
|
+
|
|
175
|
+
const promptNames = (prompt?: string | string[]): string[] => {
|
|
176
|
+
const values = Array.isArray(prompt) ? prompt : prompt === undefined ? [] : [prompt];
|
|
177
|
+
return values.map((entry) => entry.trim()).filter((entry) => entry !== "");
|
|
178
|
+
};
|
|
179
|
+
|
|
180
|
+
/** Count rows in `from` whose traceId matches the trace, for span and score sorts. */
|
|
181
|
+
const countLookupStages = (from: string): mongoose.PipelineStage[] => {
|
|
182
|
+
return [
|
|
183
|
+
{
|
|
184
|
+
$lookup: {
|
|
185
|
+
as: "_sortCounts",
|
|
186
|
+
from,
|
|
187
|
+
let: {traceId: "$_id"},
|
|
188
|
+
pipeline: [{$match: {$expr: {$eq: ["$traceId", "$$traceId"]}}}, {$count: "n"}],
|
|
189
|
+
},
|
|
190
|
+
},
|
|
191
|
+
{$addFields: {_sortValue: {$ifNull: [{$arrayElemAt: ["$_sortCounts.n", 0]}, 0]}}},
|
|
192
|
+
];
|
|
193
|
+
};
|
|
194
|
+
|
|
195
|
+
const missingAware = (expression: unknown): mongoose.PipelineStage[] => {
|
|
196
|
+
return [
|
|
197
|
+
{$addFields: {_sortValue: expression}},
|
|
198
|
+
{$addFields: {_sortMissing: {$cond: [{$eq: ["$_sortValue", null]}, 1, 0]}}},
|
|
199
|
+
];
|
|
200
|
+
};
|
|
201
|
+
|
|
202
|
+
const isMissing = (path: string): Record<string, unknown> => {
|
|
203
|
+
return {$eq: [{$type: path}, "missing"]};
|
|
204
|
+
};
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* Stages that compute `_sortValue` (and `_sortMissing` for usage-derived values, so traces
|
|
208
|
+
* without usage or an end time always list last) before the sort runs.
|
|
209
|
+
*/
|
|
210
|
+
const sortValueStages = (key: TraceListSortKey): mongoose.PipelineStage[] => {
|
|
211
|
+
if (key === "cost") {
|
|
212
|
+
return missingAware({$ifNull: ["$usage.costUsd", null]});
|
|
213
|
+
}
|
|
214
|
+
if (key === "tokens") {
|
|
215
|
+
return missingAware({
|
|
216
|
+
$cond: [
|
|
217
|
+
{$and: [isMissing("$usage.inputTokens"), isMissing("$usage.outputTokens")]},
|
|
218
|
+
null,
|
|
219
|
+
{$add: [{$ifNull: ["$usage.inputTokens", 0]}, {$ifNull: ["$usage.outputTokens", 0]}]},
|
|
220
|
+
],
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
if (key === "latency") {
|
|
224
|
+
return missingAware({
|
|
225
|
+
$cond: [
|
|
226
|
+
{$or: [{$eq: [{$ifNull: ["$endedAt", null]}, null]}, {$eq: ["$startedAt", null]}]},
|
|
227
|
+
null,
|
|
228
|
+
{$subtract: ["$endedAt", "$startedAt"]},
|
|
229
|
+
],
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
if (key === "spans") {
|
|
233
|
+
return countLookupStages(registerObsSpan().collection.name);
|
|
234
|
+
}
|
|
235
|
+
if (key === "scores") {
|
|
236
|
+
return countLookupStages(registerObsScore().collection.name);
|
|
237
|
+
}
|
|
238
|
+
return [];
|
|
239
|
+
};
|
|
240
|
+
|
|
241
|
+
/** `$sort` keys are order-significant, so the spec is built from an ordered entry list. */
|
|
242
|
+
const sortSpec = (sort: TraceListSort): Record<string, 1 | -1> => {
|
|
243
|
+
const dir = sort.direction;
|
|
244
|
+
if (sort.key === "startedAt" || sort.key === "name" || sort.key === "status") {
|
|
245
|
+
return Object.fromEntries([
|
|
246
|
+
[sort.key, dir],
|
|
247
|
+
["_id", dir],
|
|
248
|
+
]);
|
|
249
|
+
}
|
|
250
|
+
if (sort.key === "spans" || sort.key === "scores") {
|
|
251
|
+
return Object.fromEntries([
|
|
252
|
+
["_sortValue", dir],
|
|
253
|
+
["_id", dir],
|
|
254
|
+
]);
|
|
255
|
+
}
|
|
256
|
+
return Object.fromEntries([
|
|
257
|
+
["_sortMissing", 1],
|
|
258
|
+
["_sortValue", dir],
|
|
259
|
+
["_id", dir],
|
|
260
|
+
]);
|
|
261
|
+
};
|
|
262
|
+
|
|
132
263
|
const countByTraceId = async (
|
|
133
264
|
model: mongoose.Model<mongoose.Document>,
|
|
134
265
|
ids: mongoose.Types.ObjectId[]
|
|
@@ -229,15 +360,22 @@ export class LocalTraceStore {
|
|
|
229
360
|
if (query.scope) {
|
|
230
361
|
filter.scope = query.scope;
|
|
231
362
|
}
|
|
232
|
-
|
|
363
|
+
const sort = parseTraceListSort(query.sort);
|
|
364
|
+
const prompts = promptNames(query.prompt);
|
|
365
|
+
if (query.promptVersion !== undefined && prompts.length === 0) {
|
|
233
366
|
throw new APIError({status: 400, title: "prompt is required when promptVersion is set"});
|
|
234
367
|
}
|
|
235
|
-
if (query.
|
|
368
|
+
if (query.promptVersion !== undefined && prompts.length > 1) {
|
|
369
|
+
throw new APIError({status: 400, title: "promptVersion needs exactly one prompt"});
|
|
370
|
+
}
|
|
371
|
+
if (prompts.length === 1 && query.promptVersion !== undefined) {
|
|
236
372
|
filter.prompts = {
|
|
237
|
-
$elemMatch: {name:
|
|
373
|
+
$elemMatch: {name: prompts[0], version: query.promptVersion},
|
|
238
374
|
};
|
|
239
|
-
} else if (
|
|
240
|
-
filter["prompts.name"] =
|
|
375
|
+
} else if (prompts.length === 1) {
|
|
376
|
+
filter["prompts.name"] = prompts[0];
|
|
377
|
+
} else if (prompts.length > 1) {
|
|
378
|
+
filter["prompts.name"] = {$in: prompts};
|
|
241
379
|
}
|
|
242
380
|
if (query.sensitive !== undefined) {
|
|
243
381
|
filter.sensitive = query.sensitive;
|
|
@@ -270,11 +408,24 @@ export class LocalTraceStore {
|
|
|
270
408
|
}
|
|
271
409
|
const ObsTrace = registerObsTrace();
|
|
272
410
|
const total = await ObsTrace.countDocuments(filter);
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
.
|
|
277
|
-
|
|
411
|
+
// Sort in Mongo before skip/limit so derived columns (latency, counts) order every page.
|
|
412
|
+
const aggregate = ObsTrace.aggregate<{_id: mongoose.Types.ObjectId}>([
|
|
413
|
+
{$match: {deleted: {$ne: true}, ...filter}},
|
|
414
|
+
...sortValueStages(sort.key),
|
|
415
|
+
{$sort: sortSpec(sort)},
|
|
416
|
+
{$skip: (page - 1) * limit},
|
|
417
|
+
{$limit: limit},
|
|
418
|
+
{$project: {_id: 1}},
|
|
419
|
+
]);
|
|
420
|
+
if (sort.key === "name") {
|
|
421
|
+
aggregate.collation({locale: "en"});
|
|
422
|
+
}
|
|
423
|
+
const ordered = await aggregate.exec();
|
|
424
|
+
const ids = ordered.map((row) => row._id);
|
|
425
|
+
const position = new Map(ids.map((id, index) => [String(id), index]));
|
|
426
|
+
const rows = (await ObsTrace.find({_id: {$in: ids}})).sort((a, b) => {
|
|
427
|
+
return (position.get(String(a._id)) ?? 0) - (position.get(String(b._id)) ?? 0);
|
|
428
|
+
});
|
|
278
429
|
const spanCountById = await countByTraceId(registerObsSpan(), ids);
|
|
279
430
|
const scoreCountById = await countByTraceId(registerObsScore(), ids);
|
|
280
431
|
return {
|