@terreno/ai 58.0.5 → 58.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +58 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +6 -2
- package/dist/models/gptHistory.js +4 -0
- package/dist/observability/feedback.d.ts +31 -0
- package/dist/observability/feedback.js +218 -0
- package/dist/observability/local/datasetStore.d.ts +7 -0
- package/dist/observability/local/datasetStore.js +48 -2
- package/dist/observability/local/models/obsReviewItem.js +6 -0
- package/dist/observability/local/models/obsScore.js +12 -1
- package/dist/observability/local/promptStore.d.ts +1 -0
- package/dist/observability/local/promptStore.js +1 -0
- package/dist/observability/local/reviewStore.d.ts +5 -0
- package/dist/observability/local/reviewStore.js +17 -0
- package/dist/observability/local/traceStore.d.ts +23 -0
- package/dist/observability/local/traceStore.js +121 -10
- package/dist/observability/observabilityApp.d.ts +1 -0
- package/dist/observability/observabilityApp.js +2 -0
- package/dist/observability/observabilityAppRegistry.d.ts +1 -0
- package/dist/observability/routes/traces.js +72 -0
- package/dist/observability/types.d.ts +8 -1
- package/dist/routes/gpt.js +42 -2
- package/dist/routes/gptHistories.js +42 -2
- package/dist/service/aiService.d.ts +4 -1
- package/dist/service/aiService.js +17 -2
- package/dist/service/chatTurn.js +10 -1
- package/dist/types/index.d.ts +4 -0
- package/dist/types/observability.d.ts +1 -1
- package/package.json +9 -9
- package/src/index.ts +8 -1
- package/src/models/gptHistory.ts +5 -0
- package/src/observability/feedback.test.ts +688 -0
- package/src/observability/feedback.ts +263 -0
- package/src/observability/local/datasetStore.ts +57 -2
- package/src/observability/local/models/obsReviewItem.ts +9 -0
- package/src/observability/local/models/obsScore.ts +18 -1
- package/src/observability/local/obsModels.test.ts +10 -2
- package/src/observability/local/promptStore.test.ts +2 -0
- package/src/observability/local/promptStore.ts +2 -0
- package/src/observability/local/reviewStore.test.ts +55 -0
- package/src/observability/local/reviewStore.ts +21 -0
- package/src/observability/local/traceStore.test.ts +239 -1
- package/src/observability/local/traceStore.ts +140 -9
- package/src/observability/observabilityApp.ts +2 -0
- package/src/observability/observabilityAppRegistry.ts +1 -0
- package/src/observability/routes/datasets.test.ts +253 -0
- package/src/observability/routes/prompts.test.ts +1 -0
- package/src/observability/routes/traces.test.ts +581 -3
- package/src/observability/routes/traces.ts +87 -1
- package/src/observability/types.ts +6 -1
- package/src/routes/gpt.test.ts +263 -1
- package/src/routes/gpt.ts +43 -2
- package/src/routes/gptHistories.ts +55 -2
- package/src/service/aiService.observability.test.ts +244 -0
- package/src/service/aiService.ts +15 -2
- package/src/service/chatTurn.ts +11 -2
- package/src/types/index.ts +4 -0
- package/src/types/observability.ts +1 -1
|
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.MemoryScoreSink = exports.MemoryTraceSink = exports.LocalScoreSink = exports.LocalTraceSink = exports.LocalTraceStore = exports.parseTraceListSort = exports.DEFAULT_TRACE_LIST_SORT = exports.TRACE_LIST_SORT_KEYS = void 0;
|
|
6
|
+
exports.MemoryScoreSink = exports.MemoryTraceSink = exports.LocalScoreSink = exports.LocalTraceSink = exports.LocalTraceStore = exports.resetObsScoreIndexBuild = exports.parseTraceListSort = exports.DEFAULT_TRACE_LIST_SORT = exports.TRACE_LIST_SORT_KEYS = void 0;
|
|
7
7
|
const api_1 = require("@terreno/api");
|
|
8
8
|
const luxon_1 = require("luxon");
|
|
9
9
|
const mongoose_1 = __importDefault(require("mongoose"));
|
|
@@ -113,7 +113,11 @@ const countLookupStages = (from) => {
|
|
|
113
113
|
as: "_sortCounts",
|
|
114
114
|
from,
|
|
115
115
|
let: { traceId: "$_id" },
|
|
116
|
-
pipeline: [
|
|
116
|
+
pipeline: [
|
|
117
|
+
// Soft-deleted rows (e.g. a cleared thumbs vote) do not count.
|
|
118
|
+
{ $match: { $expr: { $eq: ["$traceId", "$$traceId"] }, deleted: { $ne: true } } },
|
|
119
|
+
{ $count: "n" },
|
|
120
|
+
],
|
|
117
121
|
},
|
|
118
122
|
},
|
|
119
123
|
{ $addFields: { _sortValue: { $ifNull: [{ $arrayElemAt: ["$_sortCounts.n", 0] }, 0] } } },
|
|
@@ -189,7 +193,7 @@ const countByTraceId = async (model, ids) => {
|
|
|
189
193
|
return counts;
|
|
190
194
|
}
|
|
191
195
|
const grouped = (await model.aggregate([
|
|
192
|
-
{ $match: { traceId: { $in: ids } } },
|
|
196
|
+
{ $match: { deleted: { $ne: true }, traceId: { $in: ids } } },
|
|
193
197
|
{ $group: { _id: "$traceId", n: { $sum: 1 } } },
|
|
194
198
|
]));
|
|
195
199
|
for (const row of grouped) {
|
|
@@ -197,12 +201,45 @@ const countByTraceId = async (model, ids) => {
|
|
|
197
201
|
}
|
|
198
202
|
return counts;
|
|
199
203
|
};
|
|
204
|
+
/** Score sources written by end users; the local store keeps one live score per name. */
|
|
205
|
+
const USER_SCORE_SOURCES = new Set(["user-feedback", "user-signal"]);
|
|
206
|
+
let obsScoreIndexes;
|
|
207
|
+
/** Builds ObsScore indexes once, so the unique user-score index exists before the first upsert. */
|
|
208
|
+
const ensureObsScoreIndexes = () => {
|
|
209
|
+
// A database that already holds duplicate live end-user scores (written through the admin
|
|
210
|
+
// scores route before these indexes existed) cannot build them. Feedback must keep working:
|
|
211
|
+
// log once and upsert without the uniqueness guarantee until the duplicates are removed.
|
|
212
|
+
obsScoreIndexes ??= (0, obsScore_1.registerObsScore)()
|
|
213
|
+
.init()
|
|
214
|
+
.catch((error) => {
|
|
215
|
+
// Only existing duplicates are tolerated; any other failure retries on the next write.
|
|
216
|
+
if (error.code !== 11000) {
|
|
217
|
+
obsScoreIndexes = undefined;
|
|
218
|
+
throw error;
|
|
219
|
+
}
|
|
220
|
+
api_1.logger.error("ObsScore indexes failed to build; remove duplicate live user-feedback/user-signal scores per trace and name", { error });
|
|
221
|
+
});
|
|
222
|
+
return obsScoreIndexes;
|
|
223
|
+
};
|
|
224
|
+
/**
|
|
225
|
+
* Forgets the index build so a test can simulate a failed build.
|
|
226
|
+
* @internal Test seam. Production builds the indexes once per process.
|
|
227
|
+
*/
|
|
228
|
+
const resetObsScoreIndexBuild = () => {
|
|
229
|
+
obsScoreIndexes = undefined;
|
|
230
|
+
};
|
|
231
|
+
exports.resetObsScoreIndexBuild = resetObsScoreIndexBuild;
|
|
200
232
|
class LocalTraceStore {
|
|
201
233
|
async exportTrace(trace) {
|
|
202
234
|
const ObsTrace = (0, obsTrace_1.registerObsTrace)();
|
|
203
235
|
const ObsSpan = (0, obsSpan_1.registerObsSpan)();
|
|
204
236
|
const userId = objectIdOrUndefined(trace.userId);
|
|
237
|
+
// Only a 24-hex id is a caller-chosen trace id; other ids (uuids from older callers) are ignored.
|
|
238
|
+
const traceId = /^[0-9a-f]{24}$/i.test(trace.id ?? "")
|
|
239
|
+
? new mongoose_1.default.Types.ObjectId(trace.id)
|
|
240
|
+
: undefined;
|
|
205
241
|
const created = await ObsTrace.create({
|
|
242
|
+
...(traceId ? { _id: traceId } : {}),
|
|
206
243
|
endedAt: parseDate(trace.endedAt),
|
|
207
244
|
errorSummary: firstErrorSummary(trace),
|
|
208
245
|
flaggedForDataset: trace.flaggedForDataset ?? false,
|
|
@@ -250,6 +287,10 @@ class LocalTraceStore {
|
|
|
250
287
|
}
|
|
251
288
|
const spanId = objectIdOrUndefined(score.spanId);
|
|
252
289
|
const evaluatorId = objectIdOrUndefined(score.evaluatorId);
|
|
290
|
+
if (USER_SCORE_SOURCES.has(score.source)) {
|
|
291
|
+
await this.upsertUserScore({ ...score, traceId: String(traceId) });
|
|
292
|
+
return;
|
|
293
|
+
}
|
|
253
294
|
await (0, obsScore_1.registerObsScore)().create({
|
|
254
295
|
comment: score.comment,
|
|
255
296
|
confidence: score.confidence,
|
|
@@ -262,6 +303,57 @@ class LocalTraceStore {
|
|
|
262
303
|
...(evaluatorId ? { evaluatorId } : {}),
|
|
263
304
|
});
|
|
264
305
|
}
|
|
306
|
+
/**
|
|
307
|
+
* Keeps one live score per (trace, source, name) for end-user sources, so a changed vote
|
|
308
|
+
* replaces the old one. One atomic upsert backed by a partial unique index, so concurrent
|
|
309
|
+
* taps cannot leave two live scores.
|
|
310
|
+
*/
|
|
311
|
+
async upsertUserScore(score) {
|
|
312
|
+
const ObsScore = (0, obsScore_1.registerObsScore)();
|
|
313
|
+
await ensureObsScoreIndexes();
|
|
314
|
+
const filter = {
|
|
315
|
+
deleted: false,
|
|
316
|
+
name: score.name,
|
|
317
|
+
source: score.source,
|
|
318
|
+
traceId: new mongoose_1.default.Types.ObjectId(score.traceId),
|
|
319
|
+
};
|
|
320
|
+
const set = {
|
|
321
|
+
dataType: score.dataType,
|
|
322
|
+
updated: luxon_1.DateTime.utc().toJSDate(),
|
|
323
|
+
value: score.value,
|
|
324
|
+
};
|
|
325
|
+
const unset = {};
|
|
326
|
+
if (score.comment) {
|
|
327
|
+
set.comment = score.comment;
|
|
328
|
+
}
|
|
329
|
+
else {
|
|
330
|
+
unset.comment = "";
|
|
331
|
+
}
|
|
332
|
+
const update = {
|
|
333
|
+
$set: set,
|
|
334
|
+
$setOnInsert: { created: luxon_1.DateTime.utc().toJSDate() },
|
|
335
|
+
...(Object.keys(unset).length > 0 ? { $unset: unset } : {}),
|
|
336
|
+
};
|
|
337
|
+
try {
|
|
338
|
+
await ObsScore.collection.updateOne(filter, update, { upsert: true });
|
|
339
|
+
}
|
|
340
|
+
catch (error) {
|
|
341
|
+
// A concurrent upsert may insert first; the retry then matches and updates that row.
|
|
342
|
+
if (error.code !== 11000) {
|
|
343
|
+
throw error;
|
|
344
|
+
}
|
|
345
|
+
await ObsScore.collection.updateOne(filter, update, { upsert: true });
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
/** Soft-deletes the live end-user score for (trace, source, name), e.g. a cleared thumbs vote. */
|
|
349
|
+
async deleteFeedbackScore(params) {
|
|
350
|
+
await (0, obsScore_1.registerObsScore)().collection.updateMany({
|
|
351
|
+
deleted: false,
|
|
352
|
+
name: params.name,
|
|
353
|
+
source: params.source,
|
|
354
|
+
traceId: new mongoose_1.default.Types.ObjectId(params.traceId),
|
|
355
|
+
}, { $set: { deleted: true, updated: luxon_1.DateTime.utc().toJSDate() } });
|
|
356
|
+
}
|
|
265
357
|
async list(query) {
|
|
266
358
|
const page = query.page && query.page > 0 ? query.page : 1;
|
|
267
359
|
const limit = query.limit && query.limit > 0 ? Math.min(query.limit, 100) : 20;
|
|
@@ -323,14 +415,33 @@ class LocalTraceStore {
|
|
|
323
415
|
...(to ? { $lte: to } : {}),
|
|
324
416
|
};
|
|
325
417
|
}
|
|
418
|
+
// Score-based filters each narrow the trace ids; `$and` lets several apply at once.
|
|
419
|
+
const idConstraints = [];
|
|
420
|
+
const scoredTraceIds = (scoreFilter) => {
|
|
421
|
+
return (0, obsScore_1.registerObsScore)().distinct("traceId", { deleted: { $ne: true }, ...scoreFilter });
|
|
422
|
+
};
|
|
326
423
|
if (query.hasScore !== undefined) {
|
|
327
|
-
const scoredIds = await (
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
424
|
+
const scoredIds = await scoredTraceIds({});
|
|
425
|
+
idConstraints.push({ _id: query.hasScore ? { $in: scoredIds } : { $nin: scoredIds } });
|
|
426
|
+
}
|
|
427
|
+
if (query.thumbs) {
|
|
428
|
+
const ids = await scoredTraceIds({
|
|
429
|
+
name: "thumbs_up",
|
|
430
|
+
source: "user-feedback",
|
|
431
|
+
value: query.thumbs === "up",
|
|
432
|
+
});
|
|
433
|
+
idConstraints.push({ _id: { $in: ids } });
|
|
434
|
+
}
|
|
435
|
+
if (query.signal) {
|
|
436
|
+
const ids = await scoredTraceIds({ name: query.signal, source: "user-signal" });
|
|
437
|
+
idConstraints.push({ _id: { $in: ids } });
|
|
438
|
+
}
|
|
439
|
+
if (query.hasComment !== undefined) {
|
|
440
|
+
const ids = await scoredTraceIds({ comment: { $nin: [null, ""] }, source: "user-feedback" });
|
|
441
|
+
idConstraints.push({ _id: query.hasComment ? { $in: ids } : { $nin: ids } });
|
|
442
|
+
}
|
|
443
|
+
if (idConstraints.length > 0) {
|
|
444
|
+
filter.$and = [...(filter.$and ?? []), ...idConstraints];
|
|
334
445
|
}
|
|
335
446
|
const ObsTrace = (0, obsTrace_1.registerObsTrace)();
|
|
336
447
|
const total = await ObsTrace.countDocuments(filter);
|
|
@@ -13,6 +13,7 @@ export declare class ObservabilityApp implements TerrenoPlugin {
|
|
|
13
13
|
readonly priceMap: ObservabilityAppOptions["priceMap"];
|
|
14
14
|
readonly requestAiServiceFactory?: ObservabilityRequestAiServiceFactory;
|
|
15
15
|
readonly sampleRate: number;
|
|
16
|
+
readonly feedbackReview?: ObservabilityAppOptions["feedbackReview"];
|
|
16
17
|
constructor(options: ObservabilityAppOptions);
|
|
17
18
|
get promptRegistry(): PromptRegistry | undefined;
|
|
18
19
|
get scoreSinks(): ScoreSink[];
|
|
@@ -33,6 +33,7 @@ class ObservabilityApp {
|
|
|
33
33
|
priceMap;
|
|
34
34
|
requestAiServiceFactory;
|
|
35
35
|
sampleRate;
|
|
36
|
+
feedbackReview;
|
|
36
37
|
constructor(options) {
|
|
37
38
|
this.accessControl = options.accessControl;
|
|
38
39
|
this.aiService = options.aiService;
|
|
@@ -42,6 +43,7 @@ class ObservabilityApp {
|
|
|
42
43
|
this.priceMap = options.priceMap;
|
|
43
44
|
this.requestAiServiceFactory = options.requestAiServiceFactory;
|
|
44
45
|
this.sampleRate = options.sampleRate ?? 0;
|
|
46
|
+
this.feedbackReview = options.feedbackReview;
|
|
45
47
|
(0, observabilityAppRegistry_1.registerObservabilityApp)(this);
|
|
46
48
|
}
|
|
47
49
|
get promptRegistry() {
|
|
@@ -2,6 +2,7 @@ import type { ObservabilityAppOptions, ObservabilityControlConfig, Observability
|
|
|
2
2
|
export interface RegisteredObservabilityApp {
|
|
3
3
|
aiService?: ObservabilityGenerateClient;
|
|
4
4
|
control: ObservabilityControlConfig;
|
|
5
|
+
feedbackReview?: ObservabilityAppOptions["feedbackReview"];
|
|
5
6
|
plugins: ReadonlyArray<ObservabilityPlugin>;
|
|
6
7
|
priceMap?: ObservabilityAppOptions["priceMap"];
|
|
7
8
|
promptRegistry?: PromptRegistry;
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.addObservabilityTraceRoutes = void 0;
|
|
4
4
|
const api_1 = require("@terreno/api");
|
|
5
|
+
const feedback_1 = require("../feedback");
|
|
5
6
|
const traceStore_1 = require("../local/traceStore");
|
|
6
7
|
const observabilityAppRegistry_1 = require("../observabilityAppRegistry");
|
|
7
8
|
const observabilityRouteAccess_1 = require("./observabilityRouteAccess");
|
|
@@ -54,6 +55,27 @@ const parseSort = (value) => {
|
|
|
54
55
|
(0, traceStore_1.parseTraceListSort)(value);
|
|
55
56
|
return value;
|
|
56
57
|
};
|
|
58
|
+
const parseThumbs = (value) => {
|
|
59
|
+
if (value === undefined) {
|
|
60
|
+
return undefined;
|
|
61
|
+
}
|
|
62
|
+
if (value === "up" || value === "down") {
|
|
63
|
+
return value;
|
|
64
|
+
}
|
|
65
|
+
throw new api_1.APIError({ status: 400, title: "thumbs must be up or down" });
|
|
66
|
+
};
|
|
67
|
+
const parseSignal = (value) => {
|
|
68
|
+
if (value === undefined) {
|
|
69
|
+
return undefined;
|
|
70
|
+
}
|
|
71
|
+
if (typeof value === "string" && feedback_1.FEEDBACK_SIGNALS.includes(value)) {
|
|
72
|
+
return value;
|
|
73
|
+
}
|
|
74
|
+
throw new api_1.APIError({
|
|
75
|
+
status: 400,
|
|
76
|
+
title: `signal must be one of ${feedback_1.FEEDBACK_SIGNALS.join(", ")}`,
|
|
77
|
+
});
|
|
78
|
+
};
|
|
57
79
|
const parsePositiveInt = (value, fallback) => {
|
|
58
80
|
if (value === undefined) {
|
|
59
81
|
return fallback;
|
|
@@ -75,6 +97,9 @@ const addObservabilityTraceRoutes = (router, options) => {
|
|
|
75
97
|
.withQueryParameter("prompt", { type: "string" }, { required: false })
|
|
76
98
|
.withQueryParameter("promptVersion", { type: "number" }, { required: false })
|
|
77
99
|
.withQueryParameter("scope", { type: "string" }, { required: false })
|
|
100
|
+
.withQueryParameter("thumbs", { enum: ["up", "down"], type: "string" }, { required: false })
|
|
101
|
+
.withQueryParameter("signal", { enum: [...feedback_1.FEEDBACK_SIGNALS], type: "string" }, { required: false })
|
|
102
|
+
.withQueryParameter("hasComment", { type: "boolean" }, { required: false })
|
|
78
103
|
.withQueryParameter("sort", {
|
|
79
104
|
enum: traceStore_1.TRACE_LIST_SORT_KEYS.flatMap((key) => [key, `-${key}`]),
|
|
80
105
|
type: "string",
|
|
@@ -88,6 +113,7 @@ const addObservabilityTraceRoutes = (router, options) => {
|
|
|
88
113
|
const listed = await options.store.list({
|
|
89
114
|
flaggedForDataset: parseBoolean(req.query.flaggedForDataset),
|
|
90
115
|
from: typeof req.query.from === "string" ? req.query.from : undefined,
|
|
116
|
+
hasComment: parseBoolean(req.query.hasComment),
|
|
91
117
|
hasScore: parseBoolean(req.query.hasScore),
|
|
92
118
|
limit: parsePositiveInt(req.query.limit, 20),
|
|
93
119
|
page: parsePositiveInt(req.query.page, 1),
|
|
@@ -96,8 +122,10 @@ const addObservabilityTraceRoutes = (router, options) => {
|
|
|
96
122
|
scope: typeof req.query.scope === "string" ? req.query.scope : undefined,
|
|
97
123
|
sensitive: parseBoolean(req.query.sensitive),
|
|
98
124
|
sessionId: typeof req.query.sessionId === "string" ? req.query.sessionId : undefined,
|
|
125
|
+
signal: parseSignal(req.query.signal),
|
|
99
126
|
sort: parseSort(req.query.sort),
|
|
100
127
|
status: req.query.status === "ok" || req.query.status === "error" ? req.query.status : undefined,
|
|
128
|
+
thumbs: parseThumbs(req.query.thumbs),
|
|
101
129
|
to: typeof req.query.to === "string" ? req.query.to : undefined,
|
|
102
130
|
userId: typeof req.query.userId === "string" ? req.query.userId : undefined,
|
|
103
131
|
});
|
|
@@ -119,6 +147,50 @@ const addObservabilityTraceRoutes = (router, options) => {
|
|
|
119
147
|
const data = await options.store.getDetail(req.params.id);
|
|
120
148
|
return res.json({ data });
|
|
121
149
|
}));
|
|
150
|
+
// End-user feedback: authenticated and owner-only, so it skips the admin observability
|
|
151
|
+
// middleware every other route here uses.
|
|
152
|
+
router.post(`${BASE_PATH}/traces/:id/feedback`, (0, api_1.authenticateMiddleware)(), builder()
|
|
153
|
+
.withTags(["observability"])
|
|
154
|
+
.withSummary("Record the trace owner's feedback on an AI output")
|
|
155
|
+
.withDescription("Thumbs (with an optional comment), outcome class, a passive signal, or flag-for-dataset. " +
|
|
156
|
+
"Only the trace's own user may call it.")
|
|
157
|
+
.withPathParameter("id", { type: "string" })
|
|
158
|
+
.withRequestBody({
|
|
159
|
+
comment: {
|
|
160
|
+
description: "Free-text comment attached to the thumbs or outcome score (≤ 2000 chars)",
|
|
161
|
+
type: "string",
|
|
162
|
+
},
|
|
163
|
+
flagDataset: { description: "Flag the trace as a dataset candidate", type: "boolean" },
|
|
164
|
+
outcome: { enum: ["tp", "fp", "tn", "fn"], type: "string" },
|
|
165
|
+
signal: { enum: [...feedback_1.FEEDBACK_SIGNALS], type: "string" },
|
|
166
|
+
thumbs: {
|
|
167
|
+
// `null` clears the vote. The request validator checks enum membership even for a
|
|
168
|
+
// nullable field, so null must be listed (the builder's types only allow strings).
|
|
169
|
+
enum: ["up", "down", null],
|
|
170
|
+
nullable: true,
|
|
171
|
+
type: "string",
|
|
172
|
+
},
|
|
173
|
+
})
|
|
174
|
+
.withResponse(200, {
|
|
175
|
+
data: {
|
|
176
|
+
properties: {
|
|
177
|
+
flaggedForDataset: { type: "boolean" },
|
|
178
|
+
reviewItemId: { type: "string" },
|
|
179
|
+
scores: { items: { type: "object" }, type: "array" },
|
|
180
|
+
traceId: { type: "string" },
|
|
181
|
+
},
|
|
182
|
+
type: "object",
|
|
183
|
+
},
|
|
184
|
+
})
|
|
185
|
+
.build(), (0, api_1.asyncHandler)(async (req, res) => {
|
|
186
|
+
const user = req.user;
|
|
187
|
+
const data = await (0, feedback_1.recordTraceFeedback)({
|
|
188
|
+
input: req.body,
|
|
189
|
+
traceId: req.params.id,
|
|
190
|
+
user: { admin: user?.admin, id: String(user?._id) },
|
|
191
|
+
});
|
|
192
|
+
return res.json({ data });
|
|
193
|
+
}));
|
|
122
194
|
router.post(`${BASE_PATH}/traces/:id/scores`, (0, observabilityRouteAccess_1.observabilityRouteMiddleware)(options.accessControl, { action: "score", resource: "aiReview" }, builder()
|
|
123
195
|
.withTags(["observability"])
|
|
124
196
|
.withSummary("Add a score to a trace")
|
|
@@ -63,7 +63,7 @@ export interface ScoreRecord {
|
|
|
63
63
|
dataType: "boolean" | "categorical" | "numeric";
|
|
64
64
|
evaluatorId?: string;
|
|
65
65
|
name: string;
|
|
66
|
-
source: "code" | "human" | "llm-judge" | "user-feedback";
|
|
66
|
+
source: "code" | "human" | "llm-judge" | "user-feedback" | "user-signal";
|
|
67
67
|
spanId?: string;
|
|
68
68
|
traceId: string;
|
|
69
69
|
value: boolean | number | string;
|
|
@@ -152,6 +152,13 @@ export interface ObservabilityAppOptions {
|
|
|
152
152
|
accessControl?: AnyTerrenoAccess;
|
|
153
153
|
aiService?: ObservabilityGenerateClient;
|
|
154
154
|
aiServiceFactory?: ObservabilityAiServiceFactory;
|
|
155
|
+
/**
|
|
156
|
+
* When set, an end user's thumbs down queues the trace for review (`reason: "feedback"`) against
|
|
157
|
+
* this human evaluator, unless the trace already has an open item for it. Unset: no queueing.
|
|
158
|
+
*/
|
|
159
|
+
feedbackReview?: {
|
|
160
|
+
evaluatorName: string;
|
|
161
|
+
};
|
|
155
162
|
control?: {
|
|
156
163
|
datasets?: ControlPrimary;
|
|
157
164
|
experiments?: ControlPrimary;
|
package/dist/routes/gpt.js
CHANGED
|
@@ -5,6 +5,8 @@ const api_1 = require("@terreno/api");
|
|
|
5
5
|
const blocks_1 = require("@terreno/blocks");
|
|
6
6
|
const luxon_1 = require("luxon");
|
|
7
7
|
const gptHistory_1 = require("../models/gptHistory");
|
|
8
|
+
const feedback_1 = require("../observability/feedback");
|
|
9
|
+
const obsTrace_1 = require("../observability/local/models/obsTrace");
|
|
8
10
|
const aiDatasets_1 = require("../service/aiDatasets");
|
|
9
11
|
const asks_1 = require("../service/asks");
|
|
10
12
|
const chatTurn_1 = require("../service/chatTurn");
|
|
@@ -228,14 +230,23 @@ const addGptRoutes = (router, options) => {
|
|
|
228
230
|
.withSummary("Rate a prompt in a GPT history")
|
|
229
231
|
.withPathParameter("id", { type: "string" })
|
|
230
232
|
.withRequestBody({
|
|
233
|
+
comment: {
|
|
234
|
+
description: "Optional comment on the rating (≤ 2000 chars), recorded on the reply's trace",
|
|
235
|
+
type: "string",
|
|
236
|
+
},
|
|
231
237
|
promptIndex: { type: "number" },
|
|
232
|
-
rating: {
|
|
238
|
+
rating: {
|
|
239
|
+
// `null` clears the rating; listed so the request validator accepts it.
|
|
240
|
+
enum: ["up", "down", null],
|
|
241
|
+
nullable: true,
|
|
242
|
+
type: "string",
|
|
243
|
+
},
|
|
233
244
|
})
|
|
234
245
|
.withResponse(200, { data: { type: "object" } })
|
|
235
246
|
.build(),
|
|
236
247
|
], (0, api_1.asyncHandler)(async (req, res) => {
|
|
237
248
|
const { id } = req.params;
|
|
238
|
-
const { promptIndex, rating } = req.body;
|
|
249
|
+
const { comment, promptIndex, rating } = req.body;
|
|
239
250
|
const userId = req.user?._id;
|
|
240
251
|
if (typeof promptIndex !== "number" || !Number.isInteger(promptIndex) || promptIndex < 0) {
|
|
241
252
|
throw new api_1.APIError({ status: 400, title: "promptIndex must be a non-negative number" });
|
|
@@ -243,6 +254,15 @@ const addGptRoutes = (router, options) => {
|
|
|
243
254
|
if (rating !== null && rating !== "up" && rating !== "down") {
|
|
244
255
|
throw new api_1.APIError({ status: 400, title: "rating must be 'up', 'down', or null" });
|
|
245
256
|
}
|
|
257
|
+
if (comment !== undefined && typeof comment !== "string") {
|
|
258
|
+
throw new api_1.APIError({ status: 400, title: "comment must be a string" });
|
|
259
|
+
}
|
|
260
|
+
if (typeof comment === "string" && comment.trim().length > feedback_1.FEEDBACK_COMMENT_MAX_LENGTH) {
|
|
261
|
+
throw new api_1.APIError({
|
|
262
|
+
status: 400,
|
|
263
|
+
title: `comment must be at most ${feedback_1.FEEDBACK_COMMENT_MAX_LENGTH} characters`,
|
|
264
|
+
});
|
|
265
|
+
}
|
|
246
266
|
const history = await gptHistory_1.GptHistory.findById(id);
|
|
247
267
|
if (!history) {
|
|
248
268
|
throw new api_1.APIError({ status: 404, title: "History not found" });
|
|
@@ -256,6 +276,26 @@ const addGptRoutes = (router, options) => {
|
|
|
256
276
|
// One field update, so rows a running turn appends meanwhile are kept.
|
|
257
277
|
const ratingPath = `prompts.${promptIndex}.rating`;
|
|
258
278
|
await gptHistory_1.GptHistory.updateOne({ _id: history._id }, rating === null ? { $unset: { [ratingPath]: "" } } : { $set: { [ratingPath]: rating } });
|
|
279
|
+
// The rating is also the reply's thumbs feedback when its trace exists; a missing trace
|
|
280
|
+
// (old history, no local trace sink) never fails the rating.
|
|
281
|
+
const traceId = history.prompts[promptIndex]?.traceId;
|
|
282
|
+
if (traceId && userId) {
|
|
283
|
+
const trimmed = typeof comment === "string" ? comment.trim() : "";
|
|
284
|
+
try {
|
|
285
|
+
if (!(await (0, obsTrace_1.registerObsTrace)().exists({ _id: traceId }))) {
|
|
286
|
+
return res.json({ data: { promptIndex, rating } });
|
|
287
|
+
}
|
|
288
|
+
await (0, feedback_1.recordTraceFeedback)({
|
|
289
|
+
input: { thumbs: rating, ...(trimmed && rating !== null ? { comment: trimmed } : {}) },
|
|
290
|
+
traceId,
|
|
291
|
+
user: { id: userId.toString() },
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
catch (error) {
|
|
295
|
+
// The rating is already saved; the trace score is best-effort.
|
|
296
|
+
api_1.logger.error("Failed to record chat rating as trace feedback", { error, traceId });
|
|
297
|
+
}
|
|
298
|
+
}
|
|
259
299
|
return res.json({ data: { promptIndex, rating } });
|
|
260
300
|
}));
|
|
261
301
|
router.post("/gpt/remix", [
|
|
@@ -93,6 +93,46 @@ const withoutClientGcsKeys = async (body, historyId) => {
|
|
|
93
93
|
return kept === undefined ? [] : [[key, kept]];
|
|
94
94
|
}));
|
|
95
95
|
};
|
|
96
|
+
/** The trace ids the chat turn stored on a history's assistant rows. */
|
|
97
|
+
const storedTraceIds = async (historyId) => {
|
|
98
|
+
if (typeof historyId !== "string") {
|
|
99
|
+
return new Set();
|
|
100
|
+
}
|
|
101
|
+
const history = await gptHistory_1.GptHistory.findById(historyId).select({ "prompts.traceId": 1 }).lean();
|
|
102
|
+
return new Set((history?.prompts ?? []).flatMap((prompt) => (prompt.traceId ? [prompt.traceId] : [])));
|
|
103
|
+
};
|
|
104
|
+
const rowWithKnownTraceId = (row, known) => {
|
|
105
|
+
if (!isFragment(row) || row.traceId === undefined || known.has(String(row.traceId))) {
|
|
106
|
+
return row;
|
|
107
|
+
}
|
|
108
|
+
const { traceId: _dropped, ...rest } = row;
|
|
109
|
+
return rest;
|
|
110
|
+
};
|
|
111
|
+
/**
|
|
112
|
+
* Only the chat turn writes a row's `traceId` (feedback is recorded against it), so a client may
|
|
113
|
+
* round-trip ids the server stored but never plant a new one, including through dotted paths.
|
|
114
|
+
*/
|
|
115
|
+
const withoutClientTraceIds = async (body, historyId) => {
|
|
116
|
+
const entries = Object.entries(body);
|
|
117
|
+
if (!entries.some(([key]) => isPromptsKey(key))) {
|
|
118
|
+
return body;
|
|
119
|
+
}
|
|
120
|
+
const known = await storedTraceIds(historyId);
|
|
121
|
+
return Object.fromEntries(entries.flatMap(([key, value]) => {
|
|
122
|
+
if (key === "prompts") {
|
|
123
|
+
return [
|
|
124
|
+
[key, Array.isArray(value) ? value.map((row) => rowWithKnownTraceId(row, known)) : value],
|
|
125
|
+
];
|
|
126
|
+
}
|
|
127
|
+
if (/^prompts\.\d+\.traceId$/.test(key)) {
|
|
128
|
+
return [];
|
|
129
|
+
}
|
|
130
|
+
if (/^prompts\.\d+$/.test(key)) {
|
|
131
|
+
return [[key, rowWithKnownTraceId(value, known)]];
|
|
132
|
+
}
|
|
133
|
+
return [[key, value]];
|
|
134
|
+
}));
|
|
135
|
+
};
|
|
96
136
|
/** Server-only fields of `pendingAsk`: the paused turn's messages, replayed only to the model. */
|
|
97
137
|
const SERVER_ONLY_PENDING_ASK_FIELDS = ["promptIndex", "responseMessages"];
|
|
98
138
|
const withoutServerOnlyFields = (pendingAsk) => Object.fromEntries(Object.entries(pendingAsk).filter(([key]) => !SERVER_ONLY_PENDING_ASK_FIELDS.includes(key)));
|
|
@@ -263,12 +303,12 @@ const addGptHistoryRoutes = (router, options) => {
|
|
|
263
303
|
},
|
|
264
304
|
preCreate: async (body, req) => {
|
|
265
305
|
return {
|
|
266
|
-
...(await withoutClientGcsKeys(withoutPendingAsk(body))),
|
|
306
|
+
...(await withoutClientTraceIds(await withoutClientGcsKeys(withoutPendingAsk(body)))),
|
|
267
307
|
userId: req.user?._id,
|
|
268
308
|
};
|
|
269
309
|
},
|
|
270
310
|
preUpdate: async (body, req) => {
|
|
271
|
-
const update = await withoutClientGcsKeys(withoutPendingAsk(body), req.params.id);
|
|
311
|
+
const update = await withoutClientTraceIds(await withoutClientGcsKeys(withoutPendingAsk(body), req.params.id), req.params.id);
|
|
272
312
|
// A host hook that returns null still refuses the update; modelRouter checks the value.
|
|
273
313
|
return (routerOptions.preUpdate ? await routerOptions.preUpdate(update, req) : update);
|
|
274
314
|
},
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type BlocksDocument } from "@terreno/blocks";
|
|
2
2
|
import type { JSONValue, LanguageModel, ModelMessage } from "ai";
|
|
3
|
-
import
|
|
3
|
+
import mongoose from "mongoose";
|
|
4
4
|
import type { ModelPrice, PromptVersionRef, SpanRecord } from "../observability/types";
|
|
5
5
|
import type { AIRequestType, AIServiceOptions, GenerateChatStreamOptions, GenerateJsonArrayOptions, GenerateJsonObjectOptions, GenerateJsonValueOptions, GenerateObservabilityOptions, GenerateStreamOptions, GenerateTextOptions, GptHistoryPrompt, RemixOptions, SummaryOptions, TranslateOptions } from "../types";
|
|
6
6
|
export declare const TemperaturePresets: {
|
|
@@ -18,7 +18,10 @@ interface ResolvedObservability {
|
|
|
18
18
|
sessionId?: string;
|
|
19
19
|
skipTrace: boolean;
|
|
20
20
|
systemPrompt?: string;
|
|
21
|
+
traceId?: string;
|
|
21
22
|
}
|
|
23
|
+
/** Mints an id a caller can pass as `traceId` before a generate call and return to its client. */
|
|
24
|
+
export declare const createTraceId: () => string;
|
|
22
25
|
export declare class AIService {
|
|
23
26
|
readonly model: LanguageModel;
|
|
24
27
|
readonly defaultTemperature: number;
|
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
2
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.AIService = exports.TemperaturePresets = void 0;
|
|
6
|
+
exports.AIService = exports.createTraceId = exports.TemperaturePresets = void 0;
|
|
4
7
|
const node_crypto_1 = require("node:crypto");
|
|
5
8
|
const api_1 = require("@terreno/api");
|
|
6
9
|
const blocks_1 = require("@terreno/blocks");
|
|
7
10
|
const ai_1 = require("ai");
|
|
8
11
|
const luxon_1 = require("luxon");
|
|
12
|
+
const mongoose_1 = __importDefault(require("mongoose"));
|
|
9
13
|
const aiRequest_1 = require("../models/aiRequest");
|
|
10
14
|
const observabilityAppRegistry_1 = require("../observability/observabilityAppRegistry");
|
|
11
15
|
const jsonFenceModel_1 = require("./jsonFenceModel");
|
|
@@ -61,6 +65,12 @@ const computeCostUsd = (params) => {
|
|
|
61
65
|
return ((price.inputPerMTok * params.inputTokens + price.outputPerMTok * params.outputTokens) /
|
|
62
66
|
1_000_000);
|
|
63
67
|
};
|
|
68
|
+
const TRACE_ID_PATTERN = /^[0-9a-f]{24}$/i;
|
|
69
|
+
/** Mints an id a caller can pass as `traceId` before a generate call and return to its client. */
|
|
70
|
+
const createTraceId = () => {
|
|
71
|
+
return new mongoose_1.default.Types.ObjectId().toHexString();
|
|
72
|
+
};
|
|
73
|
+
exports.createTraceId = createTraceId;
|
|
64
74
|
class AIService {
|
|
65
75
|
model;
|
|
66
76
|
defaultTemperature;
|
|
@@ -153,6 +163,9 @@ class AIService {
|
|
|
153
163
|
}
|
|
154
164
|
}
|
|
155
165
|
async resolveObservability(options) {
|
|
166
|
+
if (options.traceId !== undefined && !TRACE_ID_PATTERN.test(options.traceId)) {
|
|
167
|
+
throw new api_1.APIError({ status: 400, title: "traceId must be an object id" });
|
|
168
|
+
}
|
|
156
169
|
let promptRef;
|
|
157
170
|
let systemPrompt = options.systemPrompt;
|
|
158
171
|
let sensitive = options.sensitive ?? false;
|
|
@@ -182,6 +195,7 @@ class AIService {
|
|
|
182
195
|
sessionId: options.sessionId,
|
|
183
196
|
skipTrace: options.skipTrace === true,
|
|
184
197
|
systemPrompt,
|
|
198
|
+
traceId: options.traceId,
|
|
185
199
|
};
|
|
186
200
|
}
|
|
187
201
|
observabilityFields(options) {
|
|
@@ -192,6 +206,7 @@ class AIService {
|
|
|
192
206
|
sensitive: options.sensitive,
|
|
193
207
|
sessionId: options.sessionId,
|
|
194
208
|
skipTrace: options.skipTrace,
|
|
209
|
+
traceId: options.traceId,
|
|
195
210
|
};
|
|
196
211
|
}
|
|
197
212
|
buildTraceRecord(params) {
|
|
@@ -262,7 +277,7 @@ class AIService {
|
|
|
262
277
|
const promptRef = params.observability.promptRef;
|
|
263
278
|
return {
|
|
264
279
|
endedAt,
|
|
265
|
-
id: (0, node_crypto_1.randomUUID)(),
|
|
280
|
+
id: params.observability.traceId ?? (0, node_crypto_1.randomUUID)(),
|
|
266
281
|
input: params.prompt,
|
|
267
282
|
name: spanName,
|
|
268
283
|
output: params.response,
|
package/dist/service/chatTurn.js
CHANGED
|
@@ -1244,12 +1244,15 @@ const runChatTurn = async ({ body, options, req, sink, }) => {
|
|
|
1244
1244
|
systemPrompt,
|
|
1245
1245
|
userId,
|
|
1246
1246
|
});
|
|
1247
|
+
// Minted up front so the assistant row carries it and feedback on the reply can find the trace.
|
|
1248
|
+
const traceId = (0, observabilityAppRegistry_1.getObservabilityApp)() ? (0, aiService_1.createTraceId)() : undefined;
|
|
1247
1249
|
const resolvedObservability = await aiService.resolveGenerateObservability({
|
|
1248
1250
|
promptLabel,
|
|
1249
1251
|
promptName,
|
|
1250
1252
|
sensitive: body.sensitive === true ? true : undefined,
|
|
1251
1253
|
sessionId,
|
|
1252
1254
|
systemPrompt: effectiveSystemPrompt,
|
|
1255
|
+
traceId,
|
|
1253
1256
|
});
|
|
1254
1257
|
const observability = registryPrompt
|
|
1255
1258
|
? {
|
|
@@ -1360,6 +1363,7 @@ const runChatTurn = async ({ body, options, req, sink, }) => {
|
|
|
1360
1363
|
status: "streaming",
|
|
1361
1364
|
streamId,
|
|
1362
1365
|
text: "",
|
|
1366
|
+
...(traceId ? { traceId } : {}),
|
|
1363
1367
|
type: "assistant",
|
|
1364
1368
|
},
|
|
1365
1369
|
]);
|
|
@@ -1581,7 +1585,12 @@ const runChatTurn = async ({ body, options, req, sink, }) => {
|
|
|
1581
1585
|
fullResponse: storedResponse,
|
|
1582
1586
|
generatedImages,
|
|
1583
1587
|
modelId: aiService.modelId,
|
|
1584
|
-
}).map((row) => ({
|
|
1588
|
+
}).map((row) => ({
|
|
1589
|
+
...row,
|
|
1590
|
+
status: "complete",
|
|
1591
|
+
streamId,
|
|
1592
|
+
...(traceId && row.type === "assistant" ? { traceId } : {}),
|
|
1593
|
+
})),
|
|
1585
1594
|
...(ask
|
|
1586
1595
|
? [
|
|
1587
1596
|
askCallRow(ask, "pending"),
|
package/dist/types/index.d.ts
CHANGED
|
@@ -84,6 +84,8 @@ export type GptHistoryPromptStatus = "streaming" | "complete" | "error";
|
|
|
84
84
|
export interface GptHistoryPrompt {
|
|
85
85
|
model?: string;
|
|
86
86
|
rating?: "up" | "down";
|
|
87
|
+
/** Observability trace id of an assistant reply (written by the chat turn). */
|
|
88
|
+
traceId?: string;
|
|
87
89
|
/** Set on assistant replies produced by /gpt/prompt. "streaming" while partial text is persisted. */
|
|
88
90
|
status?: GptHistoryPromptStatus;
|
|
89
91
|
/** Identifies one /gpt/prompt reply so resume clients can follow it. */
|
|
@@ -165,6 +167,8 @@ export interface GenerateObservabilityOptions {
|
|
|
165
167
|
sensitive?: boolean;
|
|
166
168
|
sessionId?: string;
|
|
167
169
|
skipTrace?: boolean;
|
|
170
|
+
/** Id to store the trace under (from `createTraceId()`), so the caller can return it for feedback. */
|
|
171
|
+
traceId?: string;
|
|
168
172
|
}
|
|
169
173
|
export interface GenerateTextOptions extends GenerateObservabilityOptions {
|
|
170
174
|
maxOutputTokens?: number;
|
|
@@ -121,7 +121,7 @@ export interface ObsScoreDocument extends mongoose.Document<mongoose.Types.Objec
|
|
|
121
121
|
deleted: boolean;
|
|
122
122
|
evaluatorId?: mongoose.Types.ObjectId;
|
|
123
123
|
name: string;
|
|
124
|
-
source: "code" | "human" | "llm-judge" | "user-feedback";
|
|
124
|
+
source: "code" | "human" | "llm-judge" | "user-feedback" | "user-signal";
|
|
125
125
|
spanId?: mongoose.Types.ObjectId;
|
|
126
126
|
traceId: mongoose.Types.ObjectId;
|
|
127
127
|
updated: Date;
|