@terreno/ai 58.0.6 → 58.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +6 -2
- package/dist/models/gptHistory.js +4 -0
- package/dist/observability/feedback.d.ts +31 -0
- package/dist/observability/feedback.js +218 -0
- package/dist/observability/local/datasetStore.d.ts +7 -0
- package/dist/observability/local/datasetStore.js +48 -2
- package/dist/observability/local/models/obsReviewItem.js +6 -0
- package/dist/observability/local/models/obsScore.js +12 -1
- package/dist/observability/local/promptStore.d.ts +1 -0
- package/dist/observability/local/promptStore.js +1 -0
- package/dist/observability/local/reviewStore.d.ts +5 -0
- package/dist/observability/local/reviewStore.js +17 -0
- package/dist/observability/local/traceStore.d.ts +23 -0
- package/dist/observability/local/traceStore.js +121 -10
- package/dist/observability/observabilityApp.d.ts +1 -0
- package/dist/observability/observabilityApp.js +2 -0
- package/dist/observability/observabilityAppRegistry.d.ts +1 -0
- package/dist/observability/routes/traces.js +72 -0
- package/dist/observability/types.d.ts +8 -1
- package/dist/routes/gpt.js +42 -2
- package/dist/routes/gptHistories.js +42 -2
- package/dist/service/aiService.d.ts +4 -1
- package/dist/service/aiService.js +17 -2
- package/dist/service/chatTurn.js +10 -1
- package/dist/types/index.d.ts +4 -0
- package/dist/types/observability.d.ts +1 -1
- package/package.json +8 -8
- package/src/index.ts +8 -1
- package/src/models/gptHistory.ts +5 -0
- package/src/observability/feedback.test.ts +688 -0
- package/src/observability/feedback.ts +263 -0
- package/src/observability/local/datasetStore.ts +57 -2
- package/src/observability/local/models/obsReviewItem.ts +9 -0
- package/src/observability/local/models/obsScore.ts +18 -1
- package/src/observability/local/obsModels.test.ts +10 -2
- package/src/observability/local/promptStore.test.ts +2 -0
- package/src/observability/local/promptStore.ts +2 -0
- package/src/observability/local/reviewStore.test.ts +55 -0
- package/src/observability/local/reviewStore.ts +21 -0
- package/src/observability/local/traceStore.test.ts +239 -1
- package/src/observability/local/traceStore.ts +140 -9
- package/src/observability/observabilityApp.ts +2 -0
- package/src/observability/observabilityAppRegistry.ts +1 -0
- package/src/observability/routes/datasets.test.ts +253 -0
- package/src/observability/routes/prompts.test.ts +1 -0
- package/src/observability/routes/traces.test.ts +581 -3
- package/src/observability/routes/traces.ts +87 -1
- package/src/observability/types.ts +6 -1
- package/src/routes/gpt.test.ts +263 -1
- package/src/routes/gpt.ts +43 -2
- package/src/routes/gptHistories.ts +55 -2
- package/src/service/aiService.observability.test.ts +244 -0
- package/src/service/aiService.ts +15 -2
- package/src/service/chatTurn.ts +11 -2
- package/src/types/index.ts +4 -0
- package/src/types/observability.ts +1 -1
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import {APIError, asyncHandler, createOpenApiBuilder} from "@terreno/api";
|
|
1
|
+
import {APIError, asyncHandler, authenticateMiddleware, createOpenApiBuilder} from "@terreno/api";
|
|
2
2
|
import type express from "express";
|
|
3
|
+
import {FEEDBACK_SIGNALS, recordTraceFeedback, type TraceFeedbackInput} from "../feedback";
|
|
3
4
|
import {
|
|
4
5
|
DEFAULT_TRACE_LIST_SORT,
|
|
5
6
|
type LocalTraceStore,
|
|
@@ -71,6 +72,29 @@ const parseSort = (value: unknown): string | undefined => {
|
|
|
71
72
|
return value;
|
|
72
73
|
};
|
|
73
74
|
|
|
75
|
+
const parseThumbs = (value: unknown): "down" | "up" | undefined => {
|
|
76
|
+
if (value === undefined) {
|
|
77
|
+
return undefined;
|
|
78
|
+
}
|
|
79
|
+
if (value === "up" || value === "down") {
|
|
80
|
+
return value;
|
|
81
|
+
}
|
|
82
|
+
throw new APIError({status: 400, title: "thumbs must be up or down"});
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
const parseSignal = (value: unknown): string | undefined => {
|
|
86
|
+
if (value === undefined) {
|
|
87
|
+
return undefined;
|
|
88
|
+
}
|
|
89
|
+
if (typeof value === "string" && (FEEDBACK_SIGNALS as readonly string[]).includes(value)) {
|
|
90
|
+
return value;
|
|
91
|
+
}
|
|
92
|
+
throw new APIError({
|
|
93
|
+
status: 400,
|
|
94
|
+
title: `signal must be one of ${FEEDBACK_SIGNALS.join(", ")}`,
|
|
95
|
+
});
|
|
96
|
+
};
|
|
97
|
+
|
|
74
98
|
const parsePositiveInt = (value: unknown, fallback: number): number => {
|
|
75
99
|
if (value === undefined) {
|
|
76
100
|
return fallback;
|
|
@@ -102,6 +126,13 @@ export const addObservabilityTraceRoutes = (
|
|
|
102
126
|
.withQueryParameter("prompt", {type: "string"}, {required: false})
|
|
103
127
|
.withQueryParameter("promptVersion", {type: "number"}, {required: false})
|
|
104
128
|
.withQueryParameter("scope", {type: "string"}, {required: false})
|
|
129
|
+
.withQueryParameter("thumbs", {enum: ["up", "down"], type: "string"}, {required: false})
|
|
130
|
+
.withQueryParameter(
|
|
131
|
+
"signal",
|
|
132
|
+
{enum: [...FEEDBACK_SIGNALS], type: "string"},
|
|
133
|
+
{required: false}
|
|
134
|
+
)
|
|
135
|
+
.withQueryParameter("hasComment", {type: "boolean"}, {required: false})
|
|
105
136
|
.withQueryParameter(
|
|
106
137
|
"sort",
|
|
107
138
|
{
|
|
@@ -121,6 +152,7 @@ export const addObservabilityTraceRoutes = (
|
|
|
121
152
|
const listed = await options.store.list({
|
|
122
153
|
flaggedForDataset: parseBoolean(req.query.flaggedForDataset),
|
|
123
154
|
from: typeof req.query.from === "string" ? req.query.from : undefined,
|
|
155
|
+
hasComment: parseBoolean(req.query.hasComment),
|
|
124
156
|
hasScore: parseBoolean(req.query.hasScore),
|
|
125
157
|
limit: parsePositiveInt(req.query.limit, 20),
|
|
126
158
|
page: parsePositiveInt(req.query.page, 1),
|
|
@@ -129,9 +161,11 @@ export const addObservabilityTraceRoutes = (
|
|
|
129
161
|
scope: typeof req.query.scope === "string" ? req.query.scope : undefined,
|
|
130
162
|
sensitive: parseBoolean(req.query.sensitive),
|
|
131
163
|
sessionId: typeof req.query.sessionId === "string" ? req.query.sessionId : undefined,
|
|
164
|
+
signal: parseSignal(req.query.signal),
|
|
132
165
|
sort: parseSort(req.query.sort),
|
|
133
166
|
status:
|
|
134
167
|
req.query.status === "ok" || req.query.status === "error" ? req.query.status : undefined,
|
|
168
|
+
thumbs: parseThumbs(req.query.thumbs),
|
|
135
169
|
to: typeof req.query.to === "string" ? req.query.to : undefined,
|
|
136
170
|
userId: typeof req.query.userId === "string" ? req.query.userId : undefined,
|
|
137
171
|
});
|
|
@@ -164,6 +198,58 @@ export const addObservabilityTraceRoutes = (
|
|
|
164
198
|
})
|
|
165
199
|
);
|
|
166
200
|
|
|
201
|
+
// End-user feedback: authenticated and owner-only, so it skips the admin observability
|
|
202
|
+
// middleware every other route here uses.
|
|
203
|
+
router.post(
|
|
204
|
+
`${BASE_PATH}/traces/:id/feedback`,
|
|
205
|
+
authenticateMiddleware(),
|
|
206
|
+
builder()
|
|
207
|
+
.withTags(["observability"])
|
|
208
|
+
.withSummary("Record the trace owner's feedback on an AI output")
|
|
209
|
+
.withDescription(
|
|
210
|
+
"Thumbs (with an optional comment), outcome class, a passive signal, or flag-for-dataset. " +
|
|
211
|
+
"Only the trace's own user may call it."
|
|
212
|
+
)
|
|
213
|
+
.withPathParameter("id", {type: "string"})
|
|
214
|
+
.withRequestBody({
|
|
215
|
+
comment: {
|
|
216
|
+
description: "Free-text comment attached to the thumbs or outcome score (≤ 2000 chars)",
|
|
217
|
+
type: "string",
|
|
218
|
+
},
|
|
219
|
+
flagDataset: {description: "Flag the trace as a dataset candidate", type: "boolean"},
|
|
220
|
+
outcome: {enum: ["tp", "fp", "tn", "fn"], type: "string"},
|
|
221
|
+
signal: {enum: [...FEEDBACK_SIGNALS], type: "string"},
|
|
222
|
+
thumbs: {
|
|
223
|
+
// `null` clears the vote. The request validator checks enum membership even for a
|
|
224
|
+
// nullable field, so null must be listed (the builder's types only allow strings).
|
|
225
|
+
enum: ["up", "down", null] as unknown as string[],
|
|
226
|
+
nullable: true,
|
|
227
|
+
type: "string",
|
|
228
|
+
},
|
|
229
|
+
})
|
|
230
|
+
.withResponse(200, {
|
|
231
|
+
data: {
|
|
232
|
+
properties: {
|
|
233
|
+
flaggedForDataset: {type: "boolean"},
|
|
234
|
+
reviewItemId: {type: "string"},
|
|
235
|
+
scores: {items: {type: "object"}, type: "array"},
|
|
236
|
+
traceId: {type: "string"},
|
|
237
|
+
},
|
|
238
|
+
type: "object",
|
|
239
|
+
},
|
|
240
|
+
})
|
|
241
|
+
.build(),
|
|
242
|
+
asyncHandler(async (req, res) => {
|
|
243
|
+
const user = req.user as {_id?: unknown; admin?: boolean} | undefined;
|
|
244
|
+
const data = await recordTraceFeedback({
|
|
245
|
+
input: req.body as TraceFeedbackInput,
|
|
246
|
+
traceId: req.params.id,
|
|
247
|
+
user: {admin: user?.admin, id: String(user?._id)},
|
|
248
|
+
});
|
|
249
|
+
return res.json({data});
|
|
250
|
+
})
|
|
251
|
+
);
|
|
252
|
+
|
|
167
253
|
router.post(
|
|
168
254
|
`${BASE_PATH}/traces/:id/scores`,
|
|
169
255
|
observabilityRouteMiddleware(
|
|
@@ -71,7 +71,7 @@ export interface ScoreRecord {
|
|
|
71
71
|
dataType: "boolean" | "categorical" | "numeric";
|
|
72
72
|
evaluatorId?: string;
|
|
73
73
|
name: string;
|
|
74
|
-
source: "code" | "human" | "llm-judge" | "user-feedback";
|
|
74
|
+
source: "code" | "human" | "llm-judge" | "user-feedback" | "user-signal";
|
|
75
75
|
spanId?: string;
|
|
76
76
|
traceId: string;
|
|
77
77
|
value: boolean | number | string;
|
|
@@ -168,6 +168,11 @@ export interface ObservabilityAppOptions {
|
|
|
168
168
|
accessControl?: AnyTerrenoAccess;
|
|
169
169
|
aiService?: ObservabilityGenerateClient;
|
|
170
170
|
aiServiceFactory?: ObservabilityAiServiceFactory;
|
|
171
|
+
/**
|
|
172
|
+
* When set, an end user's thumbs down queues the trace for review (`reason: "feedback"`) against
|
|
173
|
+
* this human evaluator, unless the trace already has an open item for it. Unset: no queueing.
|
|
174
|
+
*/
|
|
175
|
+
feedbackReview?: {evaluatorName: string};
|
|
171
176
|
control?: {
|
|
172
177
|
datasets?: ControlPrimary;
|
|
173
178
|
experiments?: ControlPrimary;
|
package/src/routes/gpt.test.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {afterEach, beforeAll, describe, expect, it, spyOn} from "bun:test";
|
|
1
|
+
import {afterEach, beforeAll, beforeEach, describe, expect, it, spyOn} from "bun:test";
|
|
2
2
|
import {TerrenoApp} from "@terreno/api";
|
|
3
3
|
import {askPromptSection, blocksPromptSection} from "@terreno/blocks";
|
|
4
4
|
import {jsonSchema, type LanguageModel, type Tool, tool} from "ai";
|
|
@@ -6,6 +6,13 @@ import express from "express";
|
|
|
6
6
|
|
|
7
7
|
import {AIRequest} from "../models/aiRequest";
|
|
8
8
|
import {GptHistory} from "../models/gptHistory";
|
|
9
|
+
import {LocalEvaluatorStore} from "../observability/local/evaluatorStore";
|
|
10
|
+
import {createLocalObservabilityPlugin} from "../observability/local/localPlugin";
|
|
11
|
+
import {registerObsEvaluator} from "../observability/local/models/obsEvaluator";
|
|
12
|
+
import {registerObsReviewItem} from "../observability/local/models/obsReviewItem";
|
|
13
|
+
import {registerObsScore} from "../observability/local/models/obsScore";
|
|
14
|
+
import {registerObsTrace} from "../observability/local/models/obsTrace";
|
|
15
|
+
import {ObservabilityApp, resetObservabilityApp} from "../observability/observabilityApp";
|
|
9
16
|
import {AIService} from "../service/aiService";
|
|
10
17
|
import type {MCPService} from "../service/mcpService";
|
|
11
18
|
import {
|
|
@@ -3415,3 +3422,258 @@ blocks:
|
|
|
3415
3422
|
expect(blocksEventOf(events)).toEqual({errors: [], ok: true, warnings: []});
|
|
3416
3423
|
});
|
|
3417
3424
|
});
|
|
3425
|
+
|
|
3426
|
+
describe("chat trace ids and ratings", () => {
|
|
3427
|
+
const REVIEW_EVALUATOR = "chat-feedback-human";
|
|
3428
|
+
let ownerId: string;
|
|
3429
|
+
|
|
3430
|
+
const startObservability = (feedbackReview?: {evaluatorName: string}): void => {
|
|
3431
|
+
resetObservabilityApp();
|
|
3432
|
+
new ObservabilityApp({feedbackReview, plugins: [createLocalObservabilityPlugin()]});
|
|
3433
|
+
};
|
|
3434
|
+
|
|
3435
|
+
// Runs one chat turn and returns the history id and the assistant row's trace id.
|
|
3436
|
+
const chatTurn = async (agent: Agent): Promise<{historyId: string; traceId: string}> => {
|
|
3437
|
+
const {status} = await streamPrompt(agent, {prompt: "Hi"});
|
|
3438
|
+
expect(status).toBe(200);
|
|
3439
|
+
const historyId = await onlyHistoryId();
|
|
3440
|
+
const rows = (await loadHistory(historyId)).prompts as unknown as {
|
|
3441
|
+
traceId?: string;
|
|
3442
|
+
type: string;
|
|
3443
|
+
}[];
|
|
3444
|
+
const traceId = rows.find((row) => row.type === "assistant")?.traceId;
|
|
3445
|
+
// Every test below needs the minted id, so a missing one fails here with a clear reason.
|
|
3446
|
+
expect(traceId).toBeString();
|
|
3447
|
+
return {historyId, traceId: traceId as string};
|
|
3448
|
+
};
|
|
3449
|
+
|
|
3450
|
+
const rate = (
|
|
3451
|
+
agent: Agent,
|
|
3452
|
+
historyId: string,
|
|
3453
|
+
body: Record<string, unknown>
|
|
3454
|
+
): ReturnType<Agent["patch"]> =>
|
|
3455
|
+
agent.patch(`/gpt/histories/${historyId}/rating`).send({promptIndex: 1, ...body});
|
|
3456
|
+
|
|
3457
|
+
const liveScores = async (traceId: string): Promise<Record<string, unknown>[]> =>
|
|
3458
|
+
(await registerObsScore().find({deleted: false, traceId})).map(
|
|
3459
|
+
(doc) => doc.toObject() as unknown as Record<string, unknown>
|
|
3460
|
+
);
|
|
3461
|
+
|
|
3462
|
+
beforeAll(async () => {
|
|
3463
|
+
const users = await ensureTestUsers();
|
|
3464
|
+
ownerId = String(users[1]?._id);
|
|
3465
|
+
});
|
|
3466
|
+
|
|
3467
|
+
beforeEach(async () => {
|
|
3468
|
+
await registerObsTrace().deleteMany({});
|
|
3469
|
+
await registerObsScore().collection.deleteMany({});
|
|
3470
|
+
await registerObsReviewItem().deleteMany({});
|
|
3471
|
+
await registerObsEvaluator().deleteMany({});
|
|
3472
|
+
startObservability();
|
|
3473
|
+
});
|
|
3474
|
+
|
|
3475
|
+
afterEach(async () => {
|
|
3476
|
+
resetObservabilityApp();
|
|
3477
|
+
await AIRequest.deleteMany({});
|
|
3478
|
+
await GptHistory.collection.deleteMany({});
|
|
3479
|
+
});
|
|
3480
|
+
|
|
3481
|
+
const chatAgent = async (): Promise<Agent> =>
|
|
3482
|
+
authAsUser(buildApp({model: createScriptedModel({steps: [textStep("Hello!")]})}), "notAdmin");
|
|
3483
|
+
|
|
3484
|
+
it("a chat turn writes a trace id on the assistant row, and that trace exists for the user", async () => {
|
|
3485
|
+
const agent = await chatAgent();
|
|
3486
|
+
|
|
3487
|
+
const {historyId, traceId} = await chatTurn(agent);
|
|
3488
|
+
|
|
3489
|
+
expect(traceId).toMatch(/^[0-9a-f]{24}$/);
|
|
3490
|
+
const trace = await registerObsTrace().findById(traceId);
|
|
3491
|
+
expect(trace).not.toBeNull();
|
|
3492
|
+
expect(String(trace?.userId)).toBe(ownerId);
|
|
3493
|
+
const rows = (await loadHistory(historyId)).prompts as unknown as {traceId?: string}[];
|
|
3494
|
+
expect(rows[0]?.traceId).toBeUndefined();
|
|
3495
|
+
});
|
|
3496
|
+
|
|
3497
|
+
it("two turns get two different trace ids", async () => {
|
|
3498
|
+
const model = createScriptedModel({steps: [textStep("One"), textStep("Two")]});
|
|
3499
|
+
const agent = await authAsUser(buildApp({model}), "notAdmin");
|
|
3500
|
+
const first = await chatTurn(agent);
|
|
3501
|
+
await streamPrompt(agent, {historyId: first.historyId, prompt: "Again"});
|
|
3502
|
+
|
|
3503
|
+
const rows = (await loadHistory(first.historyId)).prompts as unknown as {
|
|
3504
|
+
traceId?: string;
|
|
3505
|
+
type: string;
|
|
3506
|
+
}[];
|
|
3507
|
+
const ids = rows.filter((row) => row.type === "assistant").map((row) => row.traceId);
|
|
3508
|
+
expect(ids).toHaveLength(2);
|
|
3509
|
+
expect(ids[0]).toBeDefined();
|
|
3510
|
+
expect(ids[1]).toBeDefined();
|
|
3511
|
+
expect(ids[0]).not.toBe(ids[1]);
|
|
3512
|
+
});
|
|
3513
|
+
|
|
3514
|
+
it("GptHistory accepts traceId on an assistant row", async () => {
|
|
3515
|
+
const created = await GptHistory.create({
|
|
3516
|
+
prompts: [
|
|
3517
|
+
{text: "hi", type: "user"},
|
|
3518
|
+
{text: "yo", traceId: "64b64b64b64b64b64b64b64b", type: "assistant"},
|
|
3519
|
+
],
|
|
3520
|
+
userId: ownerId,
|
|
3521
|
+
});
|
|
3522
|
+
const reloaded = await GptHistory.findById(created._id);
|
|
3523
|
+
const rows = reloaded?.prompts as unknown as {traceId?: string}[];
|
|
3524
|
+
expect(rows[1]?.traceId).toBe("64b64b64b64b64b64b64b64b");
|
|
3525
|
+
});
|
|
3526
|
+
|
|
3527
|
+
describe("rating", () => {
|
|
3528
|
+
it("down with a comment sets the row rating and a thumbs_up false score with the comment", async () => {
|
|
3529
|
+
const agent = await chatAgent();
|
|
3530
|
+
const {historyId, traceId} = await chatTurn(agent);
|
|
3531
|
+
|
|
3532
|
+
const res = await rate(agent, historyId, {comment: "Too vague", rating: "down"});
|
|
3533
|
+
|
|
3534
|
+
expect(res.status).toBe(200);
|
|
3535
|
+
expect(res.body.data).toEqual({promptIndex: 1, rating: "down"});
|
|
3536
|
+
expect(rowsOf(await loadHistory(historyId))[1]).toMatchObject({rating: "down"});
|
|
3537
|
+
const scores = await liveScores(traceId);
|
|
3538
|
+
expect(scores).toHaveLength(1);
|
|
3539
|
+
expect(scores[0]).toMatchObject({
|
|
3540
|
+
comment: "Too vague",
|
|
3541
|
+
name: "thumbs_up",
|
|
3542
|
+
source: "user-feedback",
|
|
3543
|
+
value: false,
|
|
3544
|
+
});
|
|
3545
|
+
});
|
|
3546
|
+
|
|
3547
|
+
it("up records thumbs_up true", async () => {
|
|
3548
|
+
const agent = await chatAgent();
|
|
3549
|
+
const {historyId, traceId} = await chatTurn(agent);
|
|
3550
|
+
|
|
3551
|
+
const res = await rate(agent, historyId, {rating: "up"});
|
|
3552
|
+
|
|
3553
|
+
expect(res.status).toBe(200);
|
|
3554
|
+
const scores = await liveScores(traceId);
|
|
3555
|
+
expect(scores).toHaveLength(1);
|
|
3556
|
+
expect(scores[0]).toMatchObject({name: "thumbs_up", value: true});
|
|
3557
|
+
});
|
|
3558
|
+
|
|
3559
|
+
it("null clears the history rating and soft-deletes the score", async () => {
|
|
3560
|
+
const agent = await chatAgent();
|
|
3561
|
+
const {historyId, traceId} = await chatTurn(agent);
|
|
3562
|
+
await rate(agent, historyId, {rating: "down"});
|
|
3563
|
+
|
|
3564
|
+
const res = await rate(agent, historyId, {rating: null});
|
|
3565
|
+
|
|
3566
|
+
expect(res.status).toBe(200);
|
|
3567
|
+
expect(res.body.data).toEqual({promptIndex: 1, rating: null});
|
|
3568
|
+
expect(rowsOf(await loadHistory(historyId))[1]?.rating).toBeUndefined();
|
|
3569
|
+
expect(await liveScores(traceId)).toHaveLength(0);
|
|
3570
|
+
expect(await registerObsScore().countDocuments({deleted: true, traceId})).toBe(1);
|
|
3571
|
+
});
|
|
3572
|
+
|
|
3573
|
+
it("with feedbackReview set, down creates one pending feedback review item", async () => {
|
|
3574
|
+
await new LocalEvaluatorStore().create({
|
|
3575
|
+
dimensions: [{dataType: "boolean", key: "correct", required: true}],
|
|
3576
|
+
name: REVIEW_EVALUATOR,
|
|
3577
|
+
target: "full trace",
|
|
3578
|
+
type: "human",
|
|
3579
|
+
});
|
|
3580
|
+
startObservability({evaluatorName: REVIEW_EVALUATOR});
|
|
3581
|
+
const agent = await chatAgent();
|
|
3582
|
+
const {historyId, traceId} = await chatTurn(agent);
|
|
3583
|
+
|
|
3584
|
+
const res = await rate(agent, historyId, {rating: "down"});
|
|
3585
|
+
|
|
3586
|
+
expect(res.status).toBe(200);
|
|
3587
|
+
const items = await registerObsReviewItem().find({});
|
|
3588
|
+
expect(items).toHaveLength(1);
|
|
3589
|
+
expect(items[0]).toMatchObject({reason: "feedback", status: "pending"});
|
|
3590
|
+
expect(String(items[0]?.traceId)).toBe(traceId);
|
|
3591
|
+
});
|
|
3592
|
+
|
|
3593
|
+
it("without feedbackReview, down creates no review item", async () => {
|
|
3594
|
+
const agent = await chatAgent();
|
|
3595
|
+
const {historyId, traceId} = await chatTurn(agent);
|
|
3596
|
+
|
|
3597
|
+
await rate(agent, historyId, {rating: "down"});
|
|
3598
|
+
|
|
3599
|
+
expect(await liveScores(traceId)).toHaveLength(1);
|
|
3600
|
+
expect(await registerObsReviewItem().countDocuments({})).toBe(0);
|
|
3601
|
+
});
|
|
3602
|
+
|
|
3603
|
+
it("a row whose trace was deleted returns 200, updates the history, and writes no score", async () => {
|
|
3604
|
+
const agent = await chatAgent();
|
|
3605
|
+
const {historyId, traceId} = await chatTurn(agent);
|
|
3606
|
+
expect(traceId).toBeDefined();
|
|
3607
|
+
await registerObsTrace().deleteMany({});
|
|
3608
|
+
|
|
3609
|
+
const res = await rate(agent, historyId, {rating: "down"});
|
|
3610
|
+
|
|
3611
|
+
expect(res.status).toBe(200);
|
|
3612
|
+
expect(res.body.data).toEqual({promptIndex: 1, rating: "down"});
|
|
3613
|
+
expect(rowsOf(await loadHistory(historyId))[1]).toMatchObject({rating: "down"});
|
|
3614
|
+
expect(await registerObsScore().countDocuments({})).toBe(0);
|
|
3615
|
+
});
|
|
3616
|
+
|
|
3617
|
+
it("another user gets 403 Not authorized to access this history and no score", async () => {
|
|
3618
|
+
const owner = await chatAgent();
|
|
3619
|
+
const {historyId, traceId} = await chatTurn(owner);
|
|
3620
|
+
const other = await authAsUser(buildApp({model: createScriptedModel({steps: []})}), "admin");
|
|
3621
|
+
|
|
3622
|
+
const res = await rate(other, historyId, {rating: "down"});
|
|
3623
|
+
|
|
3624
|
+
expect(res.status).toBe(403);
|
|
3625
|
+
expect(JSON.stringify(res.body)).toContain("Not authorized to access this history");
|
|
3626
|
+
expect(rowsOf(await loadHistory(historyId))[1]?.rating).toBeUndefined();
|
|
3627
|
+
expect(await liveScores(traceId)).toHaveLength(0);
|
|
3628
|
+
});
|
|
3629
|
+
|
|
3630
|
+
it("a comment over 2000 characters returns 400 and changes nothing", async () => {
|
|
3631
|
+
const agent = await chatAgent();
|
|
3632
|
+
const {historyId, traceId} = await chatTurn(agent);
|
|
3633
|
+
|
|
3634
|
+
const res = await rate(agent, historyId, {comment: "x".repeat(2001), rating: "down"});
|
|
3635
|
+
|
|
3636
|
+
expect(res.status).toBe(400);
|
|
3637
|
+
expect(rowsOf(await loadHistory(historyId))[1]?.rating).toBeUndefined();
|
|
3638
|
+
expect(await liveScores(traceId)).toHaveLength(0);
|
|
3639
|
+
});
|
|
3640
|
+
|
|
3641
|
+
it("a comment of exactly 2000 characters is accepted", async () => {
|
|
3642
|
+
const agent = await chatAgent();
|
|
3643
|
+
const {historyId, traceId} = await chatTurn(agent);
|
|
3644
|
+
|
|
3645
|
+
const res = await rate(agent, historyId, {comment: "x".repeat(2000), rating: "down"});
|
|
3646
|
+
|
|
3647
|
+
expect(res.status).toBe(200);
|
|
3648
|
+
expect((await liveScores(traceId))[0]?.comment).toBe("x".repeat(2000));
|
|
3649
|
+
});
|
|
3650
|
+
});
|
|
3651
|
+
|
|
3652
|
+
it("an owner PATCH to the history cannot set traceId on a row", async () => {
|
|
3653
|
+
const agent = await chatAgent();
|
|
3654
|
+
const history = await GptHistory.create({
|
|
3655
|
+
prompts: [
|
|
3656
|
+
{text: "Hi", type: "user"},
|
|
3657
|
+
{text: "Hello!", type: "assistant"},
|
|
3658
|
+
],
|
|
3659
|
+
userId: ownerId,
|
|
3660
|
+
});
|
|
3661
|
+
const historyId = String(history._id);
|
|
3662
|
+
const forged = "aaaaaaaaaaaaaaaaaaaaaaaa";
|
|
3663
|
+
|
|
3664
|
+
const res = await agent.patch(`/gpt/histories/${historyId}`).send({
|
|
3665
|
+
prompts: [
|
|
3666
|
+
{text: "Hi", type: "user"},
|
|
3667
|
+
{text: "Hello!", traceId: forged, type: "assistant"},
|
|
3668
|
+
],
|
|
3669
|
+
"prompts.1.traceId": forged,
|
|
3670
|
+
title: "Renamed",
|
|
3671
|
+
});
|
|
3672
|
+
|
|
3673
|
+
expect(res.status).toBe(200);
|
|
3674
|
+
const after = await loadHistory(historyId);
|
|
3675
|
+
expect(after.title).toBe("Renamed");
|
|
3676
|
+
const rows = after.prompts as unknown as {traceId?: string}[];
|
|
3677
|
+
expect(rows.some((row) => row.traceId === forged)).toBe(false);
|
|
3678
|
+
});
|
|
3679
|
+
});
|
package/src/routes/gpt.ts
CHANGED
|
@@ -13,6 +13,8 @@ import {DateTime} from "luxon";
|
|
|
13
13
|
import type mongoose from "mongoose";
|
|
14
14
|
|
|
15
15
|
import {GptHistory} from "../models/gptHistory";
|
|
16
|
+
import {FEEDBACK_COMMENT_MAX_LENGTH, recordTraceFeedback} from "../observability/feedback";
|
|
17
|
+
import {registerObsTrace} from "../observability/local/models/obsTrace";
|
|
16
18
|
import {configureAiDatasets} from "../service/aiDatasets";
|
|
17
19
|
import {assertNoReservedToolNames, resolveAskKinds} from "../service/asks";
|
|
18
20
|
import {type ChatTurnSink, DEMO_RESPONSE, resolveAiService, runChatTurn} from "../service/chatTurn";
|
|
@@ -283,15 +285,25 @@ export const addGptRoutes = (router: express.Router, options: GptRouteOptions):
|
|
|
283
285
|
.withSummary("Rate a prompt in a GPT history")
|
|
284
286
|
.withPathParameter("id", {type: "string"})
|
|
285
287
|
.withRequestBody({
|
|
288
|
+
comment: {
|
|
289
|
+
description:
|
|
290
|
+
"Optional comment on the rating (≤ 2000 chars), recorded on the reply's trace",
|
|
291
|
+
type: "string",
|
|
292
|
+
},
|
|
286
293
|
promptIndex: {type: "number"},
|
|
287
|
-
rating: {
|
|
294
|
+
rating: {
|
|
295
|
+
// `null` clears the rating; listed so the request validator accepts it.
|
|
296
|
+
enum: ["up", "down", null] as unknown as string[],
|
|
297
|
+
nullable: true,
|
|
298
|
+
type: "string",
|
|
299
|
+
},
|
|
288
300
|
})
|
|
289
301
|
.withResponse(200, {data: {type: "object"}})
|
|
290
302
|
.build(),
|
|
291
303
|
],
|
|
292
304
|
asyncHandler(async (req: express.Request, res: express.Response) => {
|
|
293
305
|
const {id} = req.params;
|
|
294
|
-
const {promptIndex, rating} = req.body;
|
|
306
|
+
const {comment, promptIndex, rating} = req.body;
|
|
295
307
|
const userId = (req.user as {_id?: mongoose.Types.ObjectId} | undefined)?._id;
|
|
296
308
|
|
|
297
309
|
if (typeof promptIndex !== "number" || !Number.isInteger(promptIndex) || promptIndex < 0) {
|
|
@@ -300,6 +312,15 @@ export const addGptRoutes = (router: express.Router, options: GptRouteOptions):
|
|
|
300
312
|
if (rating !== null && rating !== "up" && rating !== "down") {
|
|
301
313
|
throw new APIError({status: 400, title: "rating must be 'up', 'down', or null"});
|
|
302
314
|
}
|
|
315
|
+
if (comment !== undefined && typeof comment !== "string") {
|
|
316
|
+
throw new APIError({status: 400, title: "comment must be a string"});
|
|
317
|
+
}
|
|
318
|
+
if (typeof comment === "string" && comment.trim().length > FEEDBACK_COMMENT_MAX_LENGTH) {
|
|
319
|
+
throw new APIError({
|
|
320
|
+
status: 400,
|
|
321
|
+
title: `comment must be at most ${FEEDBACK_COMMENT_MAX_LENGTH} characters`,
|
|
322
|
+
});
|
|
323
|
+
}
|
|
303
324
|
|
|
304
325
|
const history = await GptHistory.findById(id);
|
|
305
326
|
if (!history) {
|
|
@@ -319,6 +340,26 @@ export const addGptRoutes = (router: express.Router, options: GptRouteOptions):
|
|
|
319
340
|
rating === null ? {$unset: {[ratingPath]: ""}} : {$set: {[ratingPath]: rating}}
|
|
320
341
|
);
|
|
321
342
|
|
|
343
|
+
// The rating is also the reply's thumbs feedback when its trace exists; a missing trace
|
|
344
|
+
// (old history, no local trace sink) never fails the rating.
|
|
345
|
+
const traceId = history.prompts[promptIndex]?.traceId;
|
|
346
|
+
if (traceId && userId) {
|
|
347
|
+
const trimmed = typeof comment === "string" ? comment.trim() : "";
|
|
348
|
+
try {
|
|
349
|
+
if (!(await registerObsTrace().exists({_id: traceId}))) {
|
|
350
|
+
return res.json({data: {promptIndex, rating}});
|
|
351
|
+
}
|
|
352
|
+
await recordTraceFeedback({
|
|
353
|
+
input: {thumbs: rating, ...(trimmed && rating !== null ? {comment: trimmed} : {})},
|
|
354
|
+
traceId,
|
|
355
|
+
user: {id: userId.toString()},
|
|
356
|
+
});
|
|
357
|
+
} catch (error) {
|
|
358
|
+
// The rating is already saved; the trace score is best-effort.
|
|
359
|
+
logger.error("Failed to record chat rating as trace feedback", {error, traceId});
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
|
|
322
363
|
return res.json({data: {promptIndex, rating}});
|
|
323
364
|
})
|
|
324
365
|
);
|
|
@@ -149,6 +149,56 @@ const withoutClientGcsKeys = async (
|
|
|
149
149
|
) as Partial<GptHistoryDocument>;
|
|
150
150
|
};
|
|
151
151
|
|
|
152
|
+
/** The trace ids the chat turn stored on a history's assistant rows. */
|
|
153
|
+
const storedTraceIds = async (historyId: unknown): Promise<Set<string>> => {
|
|
154
|
+
if (typeof historyId !== "string") {
|
|
155
|
+
return new Set();
|
|
156
|
+
}
|
|
157
|
+
const history = await GptHistory.findById(historyId).select({"prompts.traceId": 1}).lean();
|
|
158
|
+
return new Set(
|
|
159
|
+
(history?.prompts ?? []).flatMap((prompt) => (prompt.traceId ? [prompt.traceId] : []))
|
|
160
|
+
);
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
const rowWithKnownTraceId = (row: unknown, known: Set<string>): unknown => {
|
|
164
|
+
if (!isFragment(row) || row.traceId === undefined || known.has(String(row.traceId))) {
|
|
165
|
+
return row;
|
|
166
|
+
}
|
|
167
|
+
const {traceId: _dropped, ...rest} = row;
|
|
168
|
+
return rest;
|
|
169
|
+
};
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Only the chat turn writes a row's `traceId` (feedback is recorded against it), so a client may
|
|
173
|
+
* round-trip ids the server stored but never plant a new one, including through dotted paths.
|
|
174
|
+
*/
|
|
175
|
+
const withoutClientTraceIds = async (
|
|
176
|
+
body: Partial<GptHistoryDocument>,
|
|
177
|
+
historyId?: unknown
|
|
178
|
+
): Promise<Partial<GptHistoryDocument>> => {
|
|
179
|
+
const entries = Object.entries(body);
|
|
180
|
+
if (!entries.some(([key]) => isPromptsKey(key))) {
|
|
181
|
+
return body;
|
|
182
|
+
}
|
|
183
|
+
const known = await storedTraceIds(historyId);
|
|
184
|
+
return Object.fromEntries(
|
|
185
|
+
entries.flatMap(([key, value]) => {
|
|
186
|
+
if (key === "prompts") {
|
|
187
|
+
return [
|
|
188
|
+
[key, Array.isArray(value) ? value.map((row) => rowWithKnownTraceId(row, known)) : value],
|
|
189
|
+
];
|
|
190
|
+
}
|
|
191
|
+
if (/^prompts\.\d+\.traceId$/.test(key)) {
|
|
192
|
+
return [];
|
|
193
|
+
}
|
|
194
|
+
if (/^prompts\.\d+$/.test(key)) {
|
|
195
|
+
return [[key, rowWithKnownTraceId(value, known)]];
|
|
196
|
+
}
|
|
197
|
+
return [[key, value]];
|
|
198
|
+
})
|
|
199
|
+
) as Partial<GptHistoryDocument>;
|
|
200
|
+
};
|
|
201
|
+
|
|
152
202
|
/** Server-only fields of `pendingAsk`: the paused turn's messages, replayed only to the model. */
|
|
153
203
|
const SERVER_ONLY_PENDING_ASK_FIELDS = ["promptIndex", "responseMessages"];
|
|
154
204
|
|
|
@@ -359,12 +409,15 @@ export const addGptHistoryRoutes = (
|
|
|
359
409
|
},
|
|
360
410
|
preCreate: async (body, req): Promise<GptHistoryDocument> => {
|
|
361
411
|
return {
|
|
362
|
-
...(await withoutClientGcsKeys(withoutPendingAsk(body))),
|
|
412
|
+
...(await withoutClientTraceIds(await withoutClientGcsKeys(withoutPendingAsk(body)))),
|
|
363
413
|
userId: (req.user as {_id: unknown})?._id,
|
|
364
414
|
} as GptHistoryDocument;
|
|
365
415
|
},
|
|
366
416
|
preUpdate: async (body, req): Promise<GptHistoryDocument> => {
|
|
367
|
-
const update = await
|
|
417
|
+
const update = await withoutClientTraceIds(
|
|
418
|
+
await withoutClientGcsKeys(withoutPendingAsk(body), req.params.id),
|
|
419
|
+
req.params.id
|
|
420
|
+
);
|
|
368
421
|
// A host hook that returns null still refuses the update; modelRouter checks the value.
|
|
369
422
|
return (
|
|
370
423
|
routerOptions.preUpdate ? await routerOptions.preUpdate(update, req) : update
|