@terreno/ai 58.0.0-beta.2 → 58.0.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +146 -0
- package/README.md +3 -3
- package/dist/harness/commit.d.ts +4 -3
- package/dist/harness/commit.js +23 -4
- package/dist/index.d.ts +6 -9
- package/dist/index.js +14 -25
- package/dist/observability/adminScreens.js +5 -1
- package/dist/observability/evaluate.d.ts +8 -1
- package/dist/observability/evaluate.js +10 -2
- package/dist/observability/judgeAgreement.d.ts +53 -0
- package/dist/observability/judgeAgreement.js +123 -0
- package/dist/observability/local/datasetStore.d.ts +5 -0
- package/dist/observability/local/datasetStore.js +7 -2
- package/dist/observability/local/evaluatorStore.d.ts +17 -0
- package/dist/observability/local/evaluatorStore.js +73 -2
- package/dist/observability/local/experimentRunner.d.ts +40 -3
- package/dist/observability/local/experimentRunner.js +390 -11
- package/dist/observability/local/humanAnswers.d.ts +57 -0
- package/dist/observability/local/humanAnswers.js +174 -0
- package/dist/observability/local/judgeRun.d.ts +56 -0
- package/dist/observability/local/judgeRun.js +111 -0
- package/dist/observability/local/localPlugin.js +4 -0
- package/dist/observability/local/models/obsDatasetItem.js +4 -0
- package/dist/observability/local/models/obsEvaluator.js +4 -0
- package/dist/observability/local/models/obsExperiment.js +28 -0
- package/dist/observability/local/models/obsExperimentItem.js +4 -0
- package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
- package/dist/observability/local/models/obsJudgeRun.js +60 -0
- package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
- package/dist/observability/local/models/obsReviewBatch.js +48 -0
- package/dist/observability/local/models/obsReviewItem.js +10 -2
- package/dist/observability/local/models/obsTrace.js +4 -0
- package/dist/observability/local/promptReferences.d.ts +31 -0
- package/dist/observability/local/promptReferences.js +114 -0
- package/dist/observability/local/promptStore.d.ts +55 -19
- package/dist/observability/local/promptStore.js +171 -7
- package/dist/observability/local/reviewBatchStore.d.ts +106 -0
- package/dist/observability/local/reviewBatchStore.js +517 -0
- package/dist/observability/local/reviewStore.d.ts +24 -3
- package/dist/observability/local/reviewStore.js +98 -22
- package/dist/observability/local/traceStore.d.ts +14 -1
- package/dist/observability/local/traceStore.js +144 -11
- package/dist/observability/observabilityApp.js +10 -1
- package/dist/observability/questionFingerprint.d.ts +7 -0
- package/dist/observability/questionFingerprint.js +23 -0
- package/dist/observability/routes/experiments.js +12 -1
- package/dist/observability/routes/review.js +3 -1
- package/dist/observability/routes/reviewBatches.d.ts +11 -0
- package/dist/observability/routes/reviewBatches.js +78 -0
- package/dist/observability/routes/traces.d.ts +1 -1
- package/dist/observability/routes/traces.js +34 -2
- package/dist/observability/types.d.ts +10 -1
- package/dist/service/aiService.js +4 -1
- package/dist/service/chatTurn.js +38 -31
- package/dist/types/harness.d.ts +5 -2
- package/dist/types/index.d.ts +8 -3
- package/dist/types/observability.d.ts +74 -2
- package/package.json +8 -24
- package/src/harness/agentLoop.test.ts +0 -3
- package/src/harness/agentRecovery.test.ts +0 -3
- package/src/harness/commit.ts +24 -4
- package/src/harness/extensions.test.ts +0 -3
- package/src/harness/subagent.test.ts +27 -3
- package/src/index.test.ts +0 -17
- package/src/index.ts +23 -26
- package/src/isolated/gptStream.isolated.ts +0 -8
- package/src/observability/adminScreens.test.ts +7 -0
- package/src/observability/adminScreens.ts +6 -1
- package/src/observability/evaluate.ts +12 -2
- package/src/observability/judgeAgreement.test.ts +111 -0
- package/src/observability/judgeAgreement.ts +184 -0
- package/src/observability/local/datasetStore.ts +30 -11
- package/src/observability/local/evaluatorStore.ts +96 -2
- package/src/observability/local/experimentRunner.ts +495 -15
- package/src/observability/local/humanAnswers.ts +233 -0
- package/src/observability/local/judgePipeline.test.ts +444 -0
- package/src/observability/local/judgeRun.test.ts +173 -0
- package/src/observability/local/judgeRun.ts +184 -0
- package/src/observability/local/localPlugin.ts +4 -0
- package/src/observability/local/models/obsDatasetItem.ts +5 -0
- package/src/observability/local/models/obsEvaluator.ts +4 -0
- package/src/observability/local/models/obsExperiment.ts +29 -0
- package/src/observability/local/models/obsExperimentItem.ts +4 -0
- package/src/observability/local/models/obsJudgeRun.ts +61 -0
- package/src/observability/local/models/obsReviewBatch.ts +52 -0
- package/src/observability/local/models/obsReviewItem.ts +11 -2
- package/src/observability/local/models/obsTrace.ts +5 -0
- package/src/observability/local/obsModels.test.ts +13 -0
- package/src/observability/local/promptReferences.test.ts +128 -0
- package/src/observability/local/promptReferences.ts +149 -0
- package/src/observability/local/promptStore.test.ts +291 -0
- package/src/observability/local/promptStore.ts +243 -32
- package/src/observability/local/reviewBatchStore.test.ts +251 -0
- package/src/observability/local/reviewBatchStore.ts +656 -0
- package/src/observability/local/reviewStore.ts +122 -25
- package/src/observability/local/traceStore.test.ts +139 -1
- package/src/observability/local/traceStore.ts +162 -11
- package/src/observability/observabilityApp.test.ts +13 -10
- package/src/observability/observabilityApp.ts +10 -1
- package/src/observability/questionFingerprint.test.ts +33 -0
- package/src/observability/questionFingerprint.ts +24 -0
- package/src/observability/routes/experiments.test.ts +37 -0
- package/src/observability/routes/experiments.ts +13 -1
- package/src/observability/routes/review.ts +3 -1
- package/src/observability/routes/reviewBatches.test.ts +209 -0
- package/src/observability/routes/reviewBatches.ts +138 -0
- package/src/observability/routes/status.test.ts +4 -4
- package/src/observability/routes/traces.test.ts +61 -0
- package/src/observability/routes/traces.ts +46 -4
- package/src/observability/types.ts +5 -2
- package/src/routes/routes.test.ts +101 -76
- package/src/service/aiService.test.ts +21 -0
- package/src/service/aiService.ts +4 -1
- package/src/service/chatTurn.ts +42 -33
- package/src/tests/bunSetup.ts +0 -54
- package/src/types/harness.ts +5 -2
- package/src/types/index.ts +8 -3
- package/src/types/observability.ts +86 -2
- package/dist/langfuseApp.d.ts +0 -9
- package/dist/langfuseApp.js +0 -53
- package/dist/langfuseCache.d.ts +0 -15
- package/dist/langfuseCache.js +0 -54
- package/dist/langfuseClient.d.ts +0 -6
- package/dist/langfuseClient.js +0 -34
- package/dist/langfusePrompts.d.ts +0 -12
- package/dist/langfusePrompts.js +0 -89
- package/dist/langfuseRoutesEvaluations.d.ts +0 -3
- package/dist/langfuseRoutesEvaluations.js +0 -32
- package/dist/langfuseRoutesMiddleware.d.ts +0 -2
- package/dist/langfuseRoutesMiddleware.js +0 -13
- package/dist/langfuseRoutesPlayground.d.ts +0 -2
- package/dist/langfuseRoutesPlayground.js +0 -36
- package/dist/langfuseRoutesPrompts.d.ts +0 -2
- package/dist/langfuseRoutesPrompts.js +0 -56
- package/dist/langfuseRoutesTraces.d.ts +0 -2
- package/dist/langfuseRoutesTraces.js +0 -30
- package/dist/langfuseTracing.d.ts +0 -4
- package/dist/langfuseTracing.js +0 -32
- package/dist/langfuseTypes.d.ts +0 -96
- package/dist/langfuseTypes.js +0 -2
- package/dist/langfuseVercelAi.d.ts +0 -12
- package/dist/langfuseVercelAi.js +0 -49
- package/src/isolated/langfuseClient.isolated.ts +0 -66
- package/src/langfuseApp.test.ts +0 -119
- package/src/langfuseApp.ts +0 -62
- package/src/langfuseCache.test.ts +0 -75
- package/src/langfuseCache.ts +0 -72
- package/src/langfuseClient.test.ts +0 -69
- package/src/langfuseClient.ts +0 -34
- package/src/langfusePrompts.test.ts +0 -188
- package/src/langfusePrompts.ts +0 -121
- package/src/langfuseRoutes.test.ts +0 -261
- package/src/langfuseRoutesEvaluations.ts +0 -48
- package/src/langfuseRoutesMiddleware.test.ts +0 -57
- package/src/langfuseRoutesMiddleware.ts +0 -12
- package/src/langfuseRoutesPlayground.ts +0 -50
- package/src/langfuseRoutesPrompts.ts +0 -87
- package/src/langfuseRoutesTraces.ts +0 -40
- package/src/langfuseTracing.test.ts +0 -72
- package/src/langfuseTracing.ts +0 -33
- package/src/langfuseTypes.ts +0 -103
- package/src/langfuseVercelAi.test.ts +0 -82
- package/src/langfuseVercelAi.ts +0 -72
- package/src/tests/fakeLangfusePrompt.ts +0 -10
|
@@ -8,12 +8,18 @@ const api_1 = require("@terreno/api");
|
|
|
8
8
|
const luxon_1 = require("luxon");
|
|
9
9
|
const mongoose_1 = __importDefault(require("mongoose"));
|
|
10
10
|
const reviewPanels_1 = require("../reviewPanels");
|
|
11
|
+
const humanAnswers_1 = require("./humanAnswers");
|
|
12
|
+
const obsDataset_1 = require("./models/obsDataset");
|
|
13
|
+
const obsDatasetItem_1 = require("./models/obsDatasetItem");
|
|
11
14
|
const obsEvaluator_1 = require("./models/obsEvaluator");
|
|
12
15
|
const obsPrompt_1 = require("./models/obsPrompt");
|
|
16
|
+
const obsPromptLabel_1 = require("./models/obsPromptLabel");
|
|
13
17
|
const obsPromptVersion_1 = require("./models/obsPromptVersion");
|
|
18
|
+
const obsReviewBatch_1 = require("./models/obsReviewBatch");
|
|
14
19
|
const obsReviewItem_1 = require("./models/obsReviewItem");
|
|
15
20
|
const obsTrace_1 = require("./models/obsTrace");
|
|
16
21
|
const traceStore_1 = require("./traceStore");
|
|
22
|
+
const DATASET_ITEM_NAME = "Dataset item";
|
|
17
23
|
const objectId = (value, label) => {
|
|
18
24
|
if (!mongoose_1.default.Types.ObjectId.isValid(value)) {
|
|
19
25
|
throw new api_1.APIError({ status: 400, title: `${label} must be an object id` });
|
|
@@ -52,24 +58,48 @@ class LocalReviewStore {
|
|
|
52
58
|
reason: params.reason ?? "manual",
|
|
53
59
|
status: "pending",
|
|
54
60
|
traceId: trace._id,
|
|
61
|
+
...(params.batchId ? { batchId: objectId(params.batchId, "batchId") } : {}),
|
|
55
62
|
...(params.spanId ? { spanId: objectId(params.spanId, "spanId") } : {}),
|
|
56
63
|
});
|
|
57
64
|
created.push(this.toListItem(item, trace.name, trace.prompts[0]?.name));
|
|
58
65
|
}
|
|
59
66
|
return created;
|
|
60
67
|
}
|
|
61
|
-
|
|
68
|
+
/** Queue dataset items for a human question set; items with a source trace show that trace. */
|
|
69
|
+
async enqueueDatasetItems(params) {
|
|
70
|
+
const created = [];
|
|
71
|
+
for (const entry of params.items) {
|
|
72
|
+
const item = await (0, obsReviewItem_1.registerObsReviewItem)().create({
|
|
73
|
+
batchId: objectId(params.batchId, "batchId"),
|
|
74
|
+
datasetItemId: objectId(entry.datasetItemId, "datasetItemId"),
|
|
75
|
+
enqueuedAt: luxon_1.DateTime.utc().toJSDate(),
|
|
76
|
+
evaluatorId: objectId(params.evaluatorId, "evaluatorId"),
|
|
77
|
+
reason: "manual",
|
|
78
|
+
status: "pending",
|
|
79
|
+
...(entry.traceId ? { traceId: objectId(entry.traceId, "traceId") } : {}),
|
|
80
|
+
});
|
|
81
|
+
created.push(this.toListItem(item, DATASET_ITEM_NAME));
|
|
82
|
+
}
|
|
83
|
+
return created;
|
|
84
|
+
}
|
|
85
|
+
async list(status, options = {}) {
|
|
62
86
|
const ObsReviewItem = (0, obsReviewItem_1.registerObsReviewItem)();
|
|
87
|
+
const scope = options.batchId ? { batchId: objectId(options.batchId, "batchId") } : {};
|
|
63
88
|
const counts = {
|
|
64
|
-
done: await ObsReviewItem.countDocuments({ status: "done" }),
|
|
65
|
-
in_progress: await ObsReviewItem.countDocuments({ status: "in_progress" }),
|
|
66
|
-
pending: await ObsReviewItem.countDocuments({ status: "pending" }),
|
|
67
|
-
skipped: await ObsReviewItem.countDocuments({ status: "skipped" }),
|
|
89
|
+
done: await ObsReviewItem.countDocuments({ ...scope, status: "done" }),
|
|
90
|
+
in_progress: await ObsReviewItem.countDocuments({ ...scope, status: "in_progress" }),
|
|
91
|
+
pending: await ObsReviewItem.countDocuments({ ...scope, status: "pending" }),
|
|
92
|
+
skipped: await ObsReviewItem.countDocuments({ ...scope, status: "skipped" }),
|
|
68
93
|
};
|
|
69
|
-
const filter = status ? { status } :
|
|
70
|
-
|
|
94
|
+
const filter = status ? { ...scope, status } : scope;
|
|
95
|
+
// Batch items keep their batch order (enqueued together, newest trace first).
|
|
96
|
+
const rows = await ObsReviewItem.find(filter).sort({ _id: 1, enqueuedAt: 1 });
|
|
71
97
|
const data = [];
|
|
72
98
|
for (const row of rows) {
|
|
99
|
+
if (!row.traceId) {
|
|
100
|
+
data.push(this.toListItem(row, DATASET_ITEM_NAME));
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
73
103
|
const trace = await (0, obsTrace_1.registerObsTrace)().findOneOrNone({ _id: row.traceId });
|
|
74
104
|
data.push(this.toListItem(row, trace?.name ?? "Unknown trace", trace?.prompts[0]?.name));
|
|
75
105
|
}
|
|
@@ -84,11 +114,47 @@ class LocalReviewStore {
|
|
|
84
114
|
if (!evaluator) {
|
|
85
115
|
throw new api_1.APIError({ status: 404, title: "Unknown evaluator" });
|
|
86
116
|
}
|
|
87
|
-
const trace =
|
|
88
|
-
|
|
117
|
+
const trace = item.traceId
|
|
118
|
+
? await (0, obsTrace_1.registerObsTrace)().findOneOrNone({ _id: item.traceId })
|
|
119
|
+
: undefined;
|
|
120
|
+
const batch = item.batchId
|
|
121
|
+
? await (0, obsReviewBatch_1.registerObsReviewBatch)().findOneOrNone({ _id: item.batchId })
|
|
122
|
+
: undefined;
|
|
123
|
+
let input = trace?.input;
|
|
124
|
+
let output = trace?.output;
|
|
89
125
|
let variables;
|
|
90
126
|
let outputFieldNotes;
|
|
91
127
|
let outputSchema;
|
|
128
|
+
// A dataset item without a (surviving) source trace shows its own input and expected output.
|
|
129
|
+
if (!trace && item.datasetItemId) {
|
|
130
|
+
const datasetItem = await (0, obsDatasetItem_1.registerObsDatasetItem)().findOneOrNone({
|
|
131
|
+
_id: item.datasetItemId,
|
|
132
|
+
});
|
|
133
|
+
if (datasetItem) {
|
|
134
|
+
const subject = (await (0, humanAnswers_1.loadDatasetItemSubjects)([datasetItem])).get(String(datasetItem._id));
|
|
135
|
+
input = subject?.input;
|
|
136
|
+
output = subject?.output;
|
|
137
|
+
const dataset = await (0, obsDataset_1.registerObsDataset)().findOneOrNone({ _id: datasetItem.datasetId });
|
|
138
|
+
if (dataset?.inputSchemaPromptName) {
|
|
139
|
+
const prompt = await (0, obsPrompt_1.registerObsPrompt)().findOneOrNone({
|
|
140
|
+
name: dataset.inputSchemaPromptName,
|
|
141
|
+
});
|
|
142
|
+
const label = prompt
|
|
143
|
+
? await (0, obsPromptLabel_1.registerObsPromptLabel)().findOneOrNone({
|
|
144
|
+
label: "production",
|
|
145
|
+
promptId: prompt._id,
|
|
146
|
+
})
|
|
147
|
+
: undefined;
|
|
148
|
+
const version = label
|
|
149
|
+
? await (0, obsPromptVersion_1.registerObsPromptVersion)().findOneOrNone({ _id: label.versionId })
|
|
150
|
+
: undefined;
|
|
151
|
+
variables = version?.variables;
|
|
152
|
+
outputFieldNotes = version?.outputFieldNotes;
|
|
153
|
+
outputSchema = version?.outputSchema;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
const promptRef = trace?.prompts[0];
|
|
92
158
|
if (promptRef) {
|
|
93
159
|
const prompt = await (0, obsPrompt_1.registerObsPrompt)().findOneOrNone({ name: promptRef.name });
|
|
94
160
|
if (prompt) {
|
|
@@ -102,23 +168,20 @@ class LocalReviewStore {
|
|
|
102
168
|
}
|
|
103
169
|
}
|
|
104
170
|
return {
|
|
171
|
+
batchId: batch ? String(batch._id) : undefined,
|
|
172
|
+
batchName: batch?.name,
|
|
105
173
|
comment: item.comment,
|
|
174
|
+
datasetItemId: item.datasetItemId ? String(item.datasetItemId) : undefined,
|
|
106
175
|
dimensions: evaluator.dimensions,
|
|
107
176
|
evaluatorId: String(evaluator._id),
|
|
108
177
|
id: String(item._id),
|
|
109
178
|
instructions: evaluator.instructions,
|
|
110
|
-
panels: (0, reviewPanels_1.buildReviewPanels)({
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
outputFieldNotes,
|
|
114
|
-
outputSchema,
|
|
115
|
-
variables,
|
|
116
|
-
}),
|
|
117
|
-
rawInput: trace?.input,
|
|
118
|
-
rawOutput: trace?.output,
|
|
179
|
+
panels: (0, reviewPanels_1.buildReviewPanels)({ input, output, outputFieldNotes, outputSchema, variables }),
|
|
180
|
+
rawInput: input,
|
|
181
|
+
rawOutput: output,
|
|
119
182
|
scores: item.scores,
|
|
120
183
|
status: item.status,
|
|
121
|
-
traceId: String(item.traceId),
|
|
184
|
+
traceId: item.traceId ? String(item.traceId) : undefined,
|
|
122
185
|
};
|
|
123
186
|
}
|
|
124
187
|
async submit(params) {
|
|
@@ -134,7 +197,9 @@ class LocalReviewStore {
|
|
|
134
197
|
throw new api_1.APIError({ status: 400, title: `Score "${missing.key}" is required` });
|
|
135
198
|
}
|
|
136
199
|
const records = [];
|
|
137
|
-
|
|
200
|
+
const traceId = item.traceId;
|
|
201
|
+
// Scores attach to traces, so a dataset item without a trace only records human answers.
|
|
202
|
+
for (const dimension of traceId ? evaluator.dimensions : []) {
|
|
138
203
|
const value = params.scores[dimension.key];
|
|
139
204
|
if (value === undefined) {
|
|
140
205
|
continue;
|
|
@@ -145,7 +210,7 @@ class LocalReviewStore {
|
|
|
145
210
|
evaluatorId: String(evaluator._id),
|
|
146
211
|
name: dimension.key,
|
|
147
212
|
source: "human",
|
|
148
|
-
traceId: String(
|
|
213
|
+
traceId: String(traceId),
|
|
149
214
|
value,
|
|
150
215
|
});
|
|
151
216
|
}
|
|
@@ -167,6 +232,15 @@ class LocalReviewStore {
|
|
|
167
232
|
item.scores = params.scores;
|
|
168
233
|
item.status = "done";
|
|
169
234
|
await item.save();
|
|
235
|
+
if (item.datasetItemId) {
|
|
236
|
+
await (0, humanAnswers_1.writeHumanAnswer)({
|
|
237
|
+
datasetItemId: item.datasetItemId,
|
|
238
|
+
evaluator,
|
|
239
|
+
reviewItemId: String(item._id),
|
|
240
|
+
scores: params.scores,
|
|
241
|
+
userId: params.userId,
|
|
242
|
+
});
|
|
243
|
+
}
|
|
170
244
|
}
|
|
171
245
|
async skip(id) {
|
|
172
246
|
const item = await this.requireItem(id);
|
|
@@ -189,13 +263,15 @@ class LocalReviewStore {
|
|
|
189
263
|
toListItem(row, traceName, promptName) {
|
|
190
264
|
return {
|
|
191
265
|
assigneeId: row.assigneeId ? String(row.assigneeId) : undefined,
|
|
266
|
+
batchId: row.batchId ? String(row.batchId) : undefined,
|
|
267
|
+
datasetItemId: row.datasetItemId ? String(row.datasetItemId) : undefined,
|
|
192
268
|
enqueuedAt: toIso(row.enqueuedAt),
|
|
193
269
|
evaluatorId: String(row.evaluatorId),
|
|
194
270
|
id: String(row._id),
|
|
195
271
|
promptName,
|
|
196
272
|
reason: row.reason,
|
|
197
273
|
status: row.status,
|
|
198
|
-
traceId: String(row.traceId),
|
|
274
|
+
traceId: row.traceId ? String(row.traceId) : undefined,
|
|
199
275
|
traceName,
|
|
200
276
|
};
|
|
201
277
|
}
|
|
@@ -1,15 +1,27 @@
|
|
|
1
1
|
import type { ScoreRecord, ScoreSink, TraceRecord, TraceSink } from "../types";
|
|
2
|
+
/** Columns the trace list can sort by. Prefix with `-` for descending. */
|
|
3
|
+
export declare const TRACE_LIST_SORT_KEYS: readonly ["cost", "latency", "name", "scores", "spans", "startedAt", "status", "tokens"];
|
|
4
|
+
export type TraceListSortKey = (typeof TRACE_LIST_SORT_KEYS)[number];
|
|
5
|
+
export interface TraceListSort {
|
|
6
|
+
direction: 1 | -1;
|
|
7
|
+
key: TraceListSortKey;
|
|
8
|
+
}
|
|
9
|
+
/** Newest first, matching the admin Traces screen default. */
|
|
10
|
+
export declare const DEFAULT_TRACE_LIST_SORT = "-startedAt";
|
|
2
11
|
export interface TraceListQuery {
|
|
3
12
|
flaggedForDataset?: boolean;
|
|
4
13
|
from?: string;
|
|
5
14
|
hasScore?: boolean;
|
|
6
15
|
limit?: number;
|
|
7
16
|
page?: number;
|
|
8
|
-
prompt
|
|
17
|
+
/** One prompt name, or several to match traces that used any of them. */
|
|
18
|
+
prompt?: string | string[];
|
|
9
19
|
promptVersion?: number;
|
|
10
20
|
scope?: string;
|
|
11
21
|
sensitive?: boolean;
|
|
12
22
|
sessionId?: string;
|
|
23
|
+
/** One of `TRACE_LIST_SORT_KEYS`, optionally prefixed with `-` for descending. */
|
|
24
|
+
sort?: string;
|
|
13
25
|
status?: "error" | "ok";
|
|
14
26
|
to?: string;
|
|
15
27
|
userId?: string;
|
|
@@ -69,6 +81,7 @@ export interface TraceListItem {
|
|
|
69
81
|
usage?: TraceRecord["usage"];
|
|
70
82
|
userId?: string;
|
|
71
83
|
}
|
|
84
|
+
export declare const parseTraceListSort: (value?: string) => TraceListSort;
|
|
72
85
|
export declare class LocalTraceStore {
|
|
73
86
|
exportTrace(trace: TraceRecord): Promise<{
|
|
74
87
|
id: string;
|
|
@@ -3,13 +3,26 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.MemoryScoreSink = exports.MemoryTraceSink = exports.LocalScoreSink = exports.LocalTraceSink = exports.LocalTraceStore = void 0;
|
|
6
|
+
exports.MemoryScoreSink = exports.MemoryTraceSink = exports.LocalScoreSink = exports.LocalTraceSink = exports.LocalTraceStore = exports.parseTraceListSort = exports.DEFAULT_TRACE_LIST_SORT = exports.TRACE_LIST_SORT_KEYS = void 0;
|
|
7
7
|
const api_1 = require("@terreno/api");
|
|
8
8
|
const luxon_1 = require("luxon");
|
|
9
9
|
const mongoose_1 = __importDefault(require("mongoose"));
|
|
10
10
|
const obsScore_1 = require("./models/obsScore");
|
|
11
11
|
const obsSpan_1 = require("./models/obsSpan");
|
|
12
12
|
const obsTrace_1 = require("./models/obsTrace");
|
|
13
|
+
/** Columns the trace list can sort by. Prefix with `-` for descending. */
|
|
14
|
+
exports.TRACE_LIST_SORT_KEYS = [
|
|
15
|
+
"cost",
|
|
16
|
+
"latency",
|
|
17
|
+
"name",
|
|
18
|
+
"scores",
|
|
19
|
+
"spans",
|
|
20
|
+
"startedAt",
|
|
21
|
+
"status",
|
|
22
|
+
"tokens",
|
|
23
|
+
];
|
|
24
|
+
/** Newest first, matching the admin Traces screen default. */
|
|
25
|
+
exports.DEFAULT_TRACE_LIST_SORT = "-startedAt";
|
|
13
26
|
const toIso = (value) => {
|
|
14
27
|
if (!value) {
|
|
15
28
|
return undefined;
|
|
@@ -51,6 +64,105 @@ const usageWithoutUndefinedCost = (usage) => {
|
|
|
51
64
|
}
|
|
52
65
|
return usage;
|
|
53
66
|
};
|
|
67
|
+
const isTraceListSortKey = (value) => {
|
|
68
|
+
return exports.TRACE_LIST_SORT_KEYS.includes(value);
|
|
69
|
+
};
|
|
70
|
+
const parseTraceListSort = (value) => {
|
|
71
|
+
const raw = value?.trim() || exports.DEFAULT_TRACE_LIST_SORT;
|
|
72
|
+
const descending = raw.startsWith("-");
|
|
73
|
+
const key = descending ? raw.slice(1) : raw;
|
|
74
|
+
if (!isTraceListSortKey(key)) {
|
|
75
|
+
throw new api_1.APIError({
|
|
76
|
+
detail: `Use one of ${exports.TRACE_LIST_SORT_KEYS.join(", ")}, optionally prefixed with "-".`,
|
|
77
|
+
status: 400,
|
|
78
|
+
title: "Unknown trace sort key",
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
return { direction: descending ? -1 : 1, key };
|
|
82
|
+
};
|
|
83
|
+
exports.parseTraceListSort = parseTraceListSort;
|
|
84
|
+
const promptNames = (prompt) => {
|
|
85
|
+
const values = Array.isArray(prompt) ? prompt : prompt === undefined ? [] : [prompt];
|
|
86
|
+
return values.map((entry) => entry.trim()).filter((entry) => entry !== "");
|
|
87
|
+
};
|
|
88
|
+
/** Count rows in `from` whose traceId matches the trace, for span and score sorts. */
|
|
89
|
+
const countLookupStages = (from) => {
|
|
90
|
+
return [
|
|
91
|
+
{
|
|
92
|
+
$lookup: {
|
|
93
|
+
as: "_sortCounts",
|
|
94
|
+
from,
|
|
95
|
+
let: { traceId: "$_id" },
|
|
96
|
+
pipeline: [{ $match: { $expr: { $eq: ["$traceId", "$$traceId"] } } }, { $count: "n" }],
|
|
97
|
+
},
|
|
98
|
+
},
|
|
99
|
+
{ $addFields: { _sortValue: { $ifNull: [{ $arrayElemAt: ["$_sortCounts.n", 0] }, 0] } } },
|
|
100
|
+
];
|
|
101
|
+
};
|
|
102
|
+
const missingAware = (expression) => {
|
|
103
|
+
return [
|
|
104
|
+
{ $addFields: { _sortValue: expression } },
|
|
105
|
+
{ $addFields: { _sortMissing: { $cond: [{ $eq: ["$_sortValue", null] }, 1, 0] } } },
|
|
106
|
+
];
|
|
107
|
+
};
|
|
108
|
+
const isMissing = (path) => {
|
|
109
|
+
return { $eq: [{ $type: path }, "missing"] };
|
|
110
|
+
};
|
|
111
|
+
/**
|
|
112
|
+
* Stages that compute `_sortValue` (and `_sortMissing` for usage-derived values, so traces
|
|
113
|
+
* without usage or an end time always list last) before the sort runs.
|
|
114
|
+
*/
|
|
115
|
+
const sortValueStages = (key) => {
|
|
116
|
+
if (key === "cost") {
|
|
117
|
+
return missingAware({ $ifNull: ["$usage.costUsd", null] });
|
|
118
|
+
}
|
|
119
|
+
if (key === "tokens") {
|
|
120
|
+
return missingAware({
|
|
121
|
+
$cond: [
|
|
122
|
+
{ $and: [isMissing("$usage.inputTokens"), isMissing("$usage.outputTokens")] },
|
|
123
|
+
null,
|
|
124
|
+
{ $add: [{ $ifNull: ["$usage.inputTokens", 0] }, { $ifNull: ["$usage.outputTokens", 0] }] },
|
|
125
|
+
],
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
if (key === "latency") {
|
|
129
|
+
return missingAware({
|
|
130
|
+
$cond: [
|
|
131
|
+
{ $or: [{ $eq: [{ $ifNull: ["$endedAt", null] }, null] }, { $eq: ["$startedAt", null] }] },
|
|
132
|
+
null,
|
|
133
|
+
{ $subtract: ["$endedAt", "$startedAt"] },
|
|
134
|
+
],
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
if (key === "spans") {
|
|
138
|
+
return countLookupStages((0, obsSpan_1.registerObsSpan)().collection.name);
|
|
139
|
+
}
|
|
140
|
+
if (key === "scores") {
|
|
141
|
+
return countLookupStages((0, obsScore_1.registerObsScore)().collection.name);
|
|
142
|
+
}
|
|
143
|
+
return [];
|
|
144
|
+
};
|
|
145
|
+
/** `$sort` keys are order-significant, so the spec is built from an ordered entry list. */
|
|
146
|
+
const sortSpec = (sort) => {
|
|
147
|
+
const dir = sort.direction;
|
|
148
|
+
if (sort.key === "startedAt" || sort.key === "name" || sort.key === "status") {
|
|
149
|
+
return Object.fromEntries([
|
|
150
|
+
[sort.key, dir],
|
|
151
|
+
["_id", dir],
|
|
152
|
+
]);
|
|
153
|
+
}
|
|
154
|
+
if (sort.key === "spans" || sort.key === "scores") {
|
|
155
|
+
return Object.fromEntries([
|
|
156
|
+
["_sortValue", dir],
|
|
157
|
+
["_id", dir],
|
|
158
|
+
]);
|
|
159
|
+
}
|
|
160
|
+
return Object.fromEntries([
|
|
161
|
+
["_sortMissing", 1],
|
|
162
|
+
["_sortValue", dir],
|
|
163
|
+
["_id", dir],
|
|
164
|
+
]);
|
|
165
|
+
};
|
|
54
166
|
const countByTraceId = async (model, ids) => {
|
|
55
167
|
const counts = new Map();
|
|
56
168
|
if (ids.length === 0) {
|
|
@@ -143,16 +255,24 @@ class LocalTraceStore {
|
|
|
143
255
|
if (query.scope) {
|
|
144
256
|
filter.scope = query.scope;
|
|
145
257
|
}
|
|
146
|
-
|
|
258
|
+
const sort = (0, exports.parseTraceListSort)(query.sort);
|
|
259
|
+
const prompts = promptNames(query.prompt);
|
|
260
|
+
if (query.promptVersion !== undefined && prompts.length === 0) {
|
|
147
261
|
throw new api_1.APIError({ status: 400, title: "prompt is required when promptVersion is set" });
|
|
148
262
|
}
|
|
149
|
-
if (query.
|
|
263
|
+
if (query.promptVersion !== undefined && prompts.length > 1) {
|
|
264
|
+
throw new api_1.APIError({ status: 400, title: "promptVersion needs exactly one prompt" });
|
|
265
|
+
}
|
|
266
|
+
if (prompts.length === 1 && query.promptVersion !== undefined) {
|
|
150
267
|
filter.prompts = {
|
|
151
|
-
$elemMatch: { name:
|
|
268
|
+
$elemMatch: { name: prompts[0], version: query.promptVersion },
|
|
152
269
|
};
|
|
153
270
|
}
|
|
154
|
-
else if (
|
|
155
|
-
filter["prompts.name"] =
|
|
271
|
+
else if (prompts.length === 1) {
|
|
272
|
+
filter["prompts.name"] = prompts[0];
|
|
273
|
+
}
|
|
274
|
+
else if (prompts.length > 1) {
|
|
275
|
+
filter["prompts.name"] = { $in: prompts };
|
|
156
276
|
}
|
|
157
277
|
if (query.sensitive !== undefined) {
|
|
158
278
|
filter.sensitive = query.sensitive;
|
|
@@ -186,11 +306,24 @@ class LocalTraceStore {
|
|
|
186
306
|
}
|
|
187
307
|
const ObsTrace = (0, obsTrace_1.registerObsTrace)();
|
|
188
308
|
const total = await ObsTrace.countDocuments(filter);
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
.
|
|
193
|
-
|
|
309
|
+
// Sort in Mongo before skip/limit so derived columns (latency, counts) order every page.
|
|
310
|
+
const aggregate = ObsTrace.aggregate([
|
|
311
|
+
{ $match: { deleted: { $ne: true }, ...filter } },
|
|
312
|
+
...sortValueStages(sort.key),
|
|
313
|
+
{ $sort: sortSpec(sort) },
|
|
314
|
+
{ $skip: (page - 1) * limit },
|
|
315
|
+
{ $limit: limit },
|
|
316
|
+
{ $project: { _id: 1 } },
|
|
317
|
+
]);
|
|
318
|
+
if (sort.key === "name") {
|
|
319
|
+
aggregate.collation({ locale: "en" });
|
|
320
|
+
}
|
|
321
|
+
const ordered = await aggregate.exec();
|
|
322
|
+
const ids = ordered.map((row) => row._id);
|
|
323
|
+
const position = new Map(ids.map((id, index) => [String(id), index]));
|
|
324
|
+
const rows = (await ObsTrace.find({ _id: { $in: ids } })).sort((a, b) => {
|
|
325
|
+
return (position.get(String(a._id)) ?? 0) - (position.get(String(b._id)) ?? 0);
|
|
326
|
+
});
|
|
194
327
|
const spanCountById = await countByTraceId((0, obsSpan_1.registerObsSpan)(), ids);
|
|
195
328
|
const scoreCountById = await countByTraceId((0, obsScore_1.registerObsScore)(), ids);
|
|
196
329
|
return {
|
|
@@ -7,6 +7,7 @@ const datasetStore_1 = require("./local/datasetStore");
|
|
|
7
7
|
const evaluatorStore_1 = require("./local/evaluatorStore");
|
|
8
8
|
const experimentRunner_1 = require("./local/experimentRunner");
|
|
9
9
|
const promptStore_1 = require("./local/promptStore");
|
|
10
|
+
const reviewBatchStore_1 = require("./local/reviewBatchStore");
|
|
10
11
|
const reviewStore_1 = require("./local/reviewStore");
|
|
11
12
|
const traceStore_1 = require("./local/traceStore");
|
|
12
13
|
const observabilityAppRegistry_1 = require("./observabilityAppRegistry");
|
|
@@ -17,6 +18,7 @@ const evaluators_1 = require("./routes/evaluators");
|
|
|
17
18
|
const experiments_1 = require("./routes/experiments");
|
|
18
19
|
const prompts_1 = require("./routes/prompts");
|
|
19
20
|
const review_1 = require("./routes/review");
|
|
21
|
+
const reviewBatches_1 = require("./routes/reviewBatches");
|
|
20
22
|
const status_1 = require("./routes/status");
|
|
21
23
|
const testMultiStage_1 = require("./routes/testMultiStage");
|
|
22
24
|
const traces_1 = require("./routes/traces");
|
|
@@ -120,9 +122,16 @@ class ObservabilityApp {
|
|
|
120
122
|
...routeAccess,
|
|
121
123
|
store: evaluatorStore,
|
|
122
124
|
});
|
|
125
|
+
const reviewStore = new reviewStore_1.LocalReviewStore();
|
|
126
|
+
(0, reviewBatches_1.addObservabilityReviewBatchRoutes)(app, {
|
|
127
|
+
...routeAccess,
|
|
128
|
+
aiService: this.aiService,
|
|
129
|
+
requestAiServiceFactory: this.requestAiServiceFactory,
|
|
130
|
+
store: new reviewBatchStore_1.LocalReviewBatchStore(promptStore ?? new promptStore_1.LocalPromptStore(), reviewStore),
|
|
131
|
+
});
|
|
123
132
|
(0, review_1.addObservabilityReviewRoutes)(app, {
|
|
124
133
|
...routeAccess,
|
|
125
|
-
store:
|
|
134
|
+
store: reviewStore,
|
|
126
135
|
});
|
|
127
136
|
if (localPlugin.datasetStore instanceof datasetStore_1.LocalDatasetStore) {
|
|
128
137
|
(0, datasets_1.addObservabilityDatasetRoutes)(app, {
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { EvaluatorDimension } from "../types/observability";
|
|
2
|
+
/**
|
|
3
|
+
* Stable fingerprint of a question set's questions. Human answers stay reusable while the
|
|
4
|
+
* fingerprint matches; editing a question's key, type, wording, or range makes earlier answers
|
|
5
|
+
* stale. Dimension order and the `required` flag do not change the fingerprint.
|
|
6
|
+
*/
|
|
7
|
+
export declare const questionSetFingerprint: (dimensions: Pick<EvaluatorDimension, "dataType" | "key" | "question" | "range">[]) => string;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.questionSetFingerprint = void 0;
|
|
4
|
+
const node_crypto_1 = require("node:crypto");
|
|
5
|
+
/**
|
|
6
|
+
* Stable fingerprint of a question set's questions. Human answers stay reusable while the
|
|
7
|
+
* fingerprint matches; editing a question's key, type, wording, or range makes earlier answers
|
|
8
|
+
* stale. Dimension order and the `required` flag do not change the fingerprint.
|
|
9
|
+
*/
|
|
10
|
+
const questionSetFingerprint = (dimensions) => {
|
|
11
|
+
const canonical = [...dimensions]
|
|
12
|
+
.map((dimension) => {
|
|
13
|
+
return {
|
|
14
|
+
dataType: dimension.dataType,
|
|
15
|
+
key: dimension.key,
|
|
16
|
+
question: dimension.question?.trim() ?? null,
|
|
17
|
+
range: dimension.range?.trim() ?? null,
|
|
18
|
+
};
|
|
19
|
+
})
|
|
20
|
+
.sort((left, right) => left.key.localeCompare(right.key));
|
|
21
|
+
return (0, node_crypto_1.createHash)("sha256").update(JSON.stringify(canonical)).digest("hex");
|
|
22
|
+
};
|
|
23
|
+
exports.questionSetFingerprint = questionSetFingerprint;
|
|
@@ -18,6 +18,15 @@ const parseOptionalPromptName = (value) => {
|
|
|
18
18
|
}
|
|
19
19
|
return trimmed;
|
|
20
20
|
};
|
|
21
|
+
const parseKind = (value) => {
|
|
22
|
+
if (value === undefined || value === "") {
|
|
23
|
+
return undefined;
|
|
24
|
+
}
|
|
25
|
+
if (value !== "judge" && value !== "prompt") {
|
|
26
|
+
throw new api_1.APIError({ status: 400, title: "kind must be prompt or judge" });
|
|
27
|
+
}
|
|
28
|
+
return value;
|
|
29
|
+
};
|
|
21
30
|
const parsePositiveInt = (value, fallback) => {
|
|
22
31
|
if (value === undefined) {
|
|
23
32
|
return fallback;
|
|
@@ -45,11 +54,13 @@ const addObservabilityExperimentRoutes = (router, options) => {
|
|
|
45
54
|
.withTags(["observability"])
|
|
46
55
|
.withSummary("List experiments")
|
|
47
56
|
.withQueryParameter("promptName", { type: "string" }, { required: false })
|
|
57
|
+
.withQueryParameter("kind", { enum: ["judge", "prompt"], type: "string" }, { required: false })
|
|
48
58
|
.withQueryParameter("page", { type: "number" }, { required: false })
|
|
49
59
|
.withQueryParameter("limit", { type: "number" }, { required: false })
|
|
50
60
|
.withResponse(200, {})
|
|
51
61
|
.build()), (0, api_1.asyncHandler)(async (req, res) => {
|
|
52
62
|
const listed = await options.runner.list({
|
|
63
|
+
kind: parseKind(req.query.kind),
|
|
53
64
|
limit: parsePositiveInt(req.query.limit, experimentRunner_1.EXPERIMENT_LIST_DEFAULT_LIMIT),
|
|
54
65
|
page: parsePositiveInt(req.query.page, 1),
|
|
55
66
|
promptName: parseOptionalPromptName(req.query.promptName),
|
|
@@ -65,7 +76,7 @@ const addObservabilityExperimentRoutes = (router, options) => {
|
|
|
65
76
|
}));
|
|
66
77
|
router.post(`${BASE_PATH}/experiments`, (0, observabilityRouteAccess_1.observabilityRouteMiddleware)(options.accessControl, { action: "create", resource: "aiExperiment" }, builder()
|
|
67
78
|
.withTags(["observability"])
|
|
68
|
-
.withSummary("Create
|
|
79
|
+
.withSummary("Create a prompt comparison or judge calibration experiment")
|
|
69
80
|
.withResponse(201, {})
|
|
70
81
|
.build()), (0, api_1.asyncHandler)(async (req, res) => {
|
|
71
82
|
const data = await options.runner.create(req.body);
|
|
@@ -32,7 +32,8 @@ const addObservabilityReviewRoutes = (router, options) => {
|
|
|
32
32
|
.withResponse(200, { data: { type: "array" } })
|
|
33
33
|
.build()), (0, api_1.asyncHandler)(async (req, res) => {
|
|
34
34
|
const status = req.query.status;
|
|
35
|
-
|
|
35
|
+
const batchId = typeof req.query.batchId === "string" ? req.query.batchId : undefined;
|
|
36
|
+
return res.json({ ...(await options.store.list(status, { batchId })), more: false });
|
|
36
37
|
}));
|
|
37
38
|
router.get(`${BASE_PATH}/review/:id`, (0, observabilityRouteAccess_1.observabilityRouteMiddleware)(options.accessControl, { action: "read", resource: "aiReview" }, builder()
|
|
38
39
|
.withTags(["observability"])
|
|
@@ -59,6 +60,7 @@ const addObservabilityReviewRoutes = (router, options) => {
|
|
|
59
60
|
id: req.params.id,
|
|
60
61
|
scores: body.scores ?? {},
|
|
61
62
|
sinks: (0, observabilityAppRegistry_1.getObservabilityApp)()?.scoreSinks ?? [],
|
|
63
|
+
userId: req.user?._id ? String(req.user._id) : undefined,
|
|
62
64
|
});
|
|
63
65
|
}
|
|
64
66
|
else if (body.action === "skip") {
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type express from "express";
|
|
2
|
+
import type { LocalReviewBatchStore } from "../local/reviewBatchStore";
|
|
3
|
+
import type { ObservabilityGenerateClient, ObservabilityRequestAiServiceFactory } from "../types";
|
|
4
|
+
import { type ObservabilityRouteAccessOptions } from "./observabilityRouteAccess";
|
|
5
|
+
export interface ObservabilityReviewBatchRouteOptions extends ObservabilityRouteAccessOptions {
|
|
6
|
+
aiService?: ObservabilityGenerateClient;
|
|
7
|
+
requestAiServiceFactory?: ObservabilityRequestAiServiceFactory;
|
|
8
|
+
store: LocalReviewBatchStore;
|
|
9
|
+
}
|
|
10
|
+
/** Register before the review item routes so `/review/batches` is not read as an item id. */
|
|
11
|
+
export declare const addObservabilityReviewBatchRoutes: (router: express.Application, options: ObservabilityReviewBatchRouteOptions) => void;
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.addObservabilityReviewBatchRoutes = void 0;
|
|
4
|
+
const api_1 = require("@terreno/api");
|
|
5
|
+
const observabilityRouteAccess_1 = require("./observabilityRouteAccess");
|
|
6
|
+
const requestAiService_1 = require("./requestAiService");
|
|
7
|
+
const BASE_PATH = "/ai/observability";
|
|
8
|
+
/** Register before the review item routes so `/review/batches` is not read as an item id. */
|
|
9
|
+
const addObservabilityReviewBatchRoutes = (router, options) => {
|
|
10
|
+
const openApiOptions = options.openApi ? { openApi: options.openApi } : {};
|
|
11
|
+
const builder = () => {
|
|
12
|
+
return (0, api_1.createOpenApiBuilder)(openApiOptions);
|
|
13
|
+
};
|
|
14
|
+
router.get(`${BASE_PATH}/review/batches`, (0, observabilityRouteAccess_1.observabilityRouteMiddleware)(options.accessControl, { action: "list", resource: "aiReview" }, builder()
|
|
15
|
+
.withTags(["observability"])
|
|
16
|
+
.withSummary("List human review batches with judge agreement")
|
|
17
|
+
.withResponse(200, { data: { type: "array" } })
|
|
18
|
+
.build()), (0, api_1.asyncHandler)(async (_req, res) => {
|
|
19
|
+
return res.json({ data: await options.store.list(), more: false });
|
|
20
|
+
}));
|
|
21
|
+
router.post(`${BASE_PATH}/review/batches`, (0, observabilityRouteAccess_1.observabilityRouteMiddleware)(options.accessControl, { action: "assign", resource: "aiReview" }, builder()
|
|
22
|
+
.withTags(["observability"])
|
|
23
|
+
.withSummary("Create a review batch from traces, the most recent traces, or a dataset's unanswered items")
|
|
24
|
+
.withResponse(201, { data: { type: "object" } })
|
|
25
|
+
.build()), (0, api_1.asyncHandler)(async (req, res) => {
|
|
26
|
+
const body = req.body;
|
|
27
|
+
if (!body.evaluatorId) {
|
|
28
|
+
throw new api_1.APIError({ status: 400, title: "evaluatorId is required" });
|
|
29
|
+
}
|
|
30
|
+
if (body.traceIds !== undefined && !Array.isArray(body.traceIds)) {
|
|
31
|
+
throw new api_1.APIError({ status: 400, title: "traceIds must be an array" });
|
|
32
|
+
}
|
|
33
|
+
if (body.datasetId !== undefined && typeof body.datasetId !== "string") {
|
|
34
|
+
throw new api_1.APIError({ status: 400, title: "datasetId must be a string" });
|
|
35
|
+
}
|
|
36
|
+
const data = await options.store.create({
|
|
37
|
+
datasetId: body.datasetId || undefined,
|
|
38
|
+
description: body.description,
|
|
39
|
+
evaluatorId: body.evaluatorId,
|
|
40
|
+
limit: body.limit,
|
|
41
|
+
name: body.name,
|
|
42
|
+
promptName: body.promptName || undefined,
|
|
43
|
+
traceIds: body.traceIds,
|
|
44
|
+
});
|
|
45
|
+
return res.status(201).json({ data });
|
|
46
|
+
}));
|
|
47
|
+
router.get(`${BASE_PATH}/review/batches/:id`, (0, observabilityRouteAccess_1.observabilityRouteMiddleware)(options.accessControl, { action: "read", resource: "aiReview" }, builder()
|
|
48
|
+
.withTags(["observability"])
|
|
49
|
+
.withSummary("Get a review batch with human answers and judge agreement")
|
|
50
|
+
.withPathParameter("id", { type: "string" })
|
|
51
|
+
.withResponse(200, { data: { type: "object" } })
|
|
52
|
+
.build()), (0, api_1.asyncHandler)(async (req, res) => {
|
|
53
|
+
return res.json({ data: await options.store.get(req.params.id) });
|
|
54
|
+
}));
|
|
55
|
+
router.post(`${BASE_PATH}/review/batches/:id/judgeRuns`, (0, observabilityRouteAccess_1.observabilityRouteMiddleware)(options.accessControl, { action: "assign", resource: "aiReview" }, builder()
|
|
56
|
+
.withTags(["observability"])
|
|
57
|
+
.withSummary("Run an LLM judge over a review batch")
|
|
58
|
+
.withPathParameter("id", { type: "string" })
|
|
59
|
+
.withResponse(201, { data: { type: "object" } })
|
|
60
|
+
.build()), (0, api_1.asyncHandler)(async (req, res) => {
|
|
61
|
+
const body = req.body;
|
|
62
|
+
if (!body.evaluatorId) {
|
|
63
|
+
throw new api_1.APIError({ status: 400, title: "evaluatorId is required" });
|
|
64
|
+
}
|
|
65
|
+
const ai = (0, requestAiService_1.resolveRequestAiService)({
|
|
66
|
+
aiService: options.aiService,
|
|
67
|
+
req,
|
|
68
|
+
requestAiServiceFactory: options.requestAiServiceFactory,
|
|
69
|
+
});
|
|
70
|
+
const data = await options.store.runJudge({
|
|
71
|
+
ai,
|
|
72
|
+
batchId: req.params.id,
|
|
73
|
+
evaluatorId: body.evaluatorId,
|
|
74
|
+
});
|
|
75
|
+
return res.status(201).json({ data });
|
|
76
|
+
}));
|
|
77
|
+
};
|
|
78
|
+
exports.addObservabilityReviewBatchRoutes = addObservabilityReviewBatchRoutes;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type express from "express";
|
|
2
|
-
import type
|
|
2
|
+
import { type LocalTraceStore } from "../local/traceStore";
|
|
3
3
|
import { type ObservabilityRouteAccessOptions } from "./observabilityRouteAccess";
|
|
4
4
|
export interface ObservabilityTraceRouteOptions extends ObservabilityRouteAccessOptions {
|
|
5
5
|
store: LocalTraceStore;
|