@terreno/ai 58.0.5 → 58.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +58 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +6 -2
- package/dist/models/gptHistory.js +4 -0
- package/dist/observability/feedback.d.ts +31 -0
- package/dist/observability/feedback.js +218 -0
- package/dist/observability/local/datasetStore.d.ts +7 -0
- package/dist/observability/local/datasetStore.js +48 -2
- package/dist/observability/local/models/obsReviewItem.js +6 -0
- package/dist/observability/local/models/obsScore.js +12 -1
- package/dist/observability/local/promptStore.d.ts +1 -0
- package/dist/observability/local/promptStore.js +1 -0
- package/dist/observability/local/reviewStore.d.ts +5 -0
- package/dist/observability/local/reviewStore.js +17 -0
- package/dist/observability/local/traceStore.d.ts +23 -0
- package/dist/observability/local/traceStore.js +121 -10
- package/dist/observability/observabilityApp.d.ts +1 -0
- package/dist/observability/observabilityApp.js +2 -0
- package/dist/observability/observabilityAppRegistry.d.ts +1 -0
- package/dist/observability/routes/traces.js +72 -0
- package/dist/observability/types.d.ts +8 -1
- package/dist/routes/gpt.js +42 -2
- package/dist/routes/gptHistories.js +42 -2
- package/dist/service/aiService.d.ts +4 -1
- package/dist/service/aiService.js +17 -2
- package/dist/service/chatTurn.js +10 -1
- package/dist/types/index.d.ts +4 -0
- package/dist/types/observability.d.ts +1 -1
- package/package.json +9 -9
- package/src/index.ts +8 -1
- package/src/models/gptHistory.ts +5 -0
- package/src/observability/feedback.test.ts +688 -0
- package/src/observability/feedback.ts +263 -0
- package/src/observability/local/datasetStore.ts +57 -2
- package/src/observability/local/models/obsReviewItem.ts +9 -0
- package/src/observability/local/models/obsScore.ts +18 -1
- package/src/observability/local/obsModels.test.ts +10 -2
- package/src/observability/local/promptStore.test.ts +2 -0
- package/src/observability/local/promptStore.ts +2 -0
- package/src/observability/local/reviewStore.test.ts +55 -0
- package/src/observability/local/reviewStore.ts +21 -0
- package/src/observability/local/traceStore.test.ts +239 -1
- package/src/observability/local/traceStore.ts +140 -9
- package/src/observability/observabilityApp.ts +2 -0
- package/src/observability/observabilityAppRegistry.ts +1 -0
- package/src/observability/routes/datasets.test.ts +253 -0
- package/src/observability/routes/prompts.test.ts +1 -0
- package/src/observability/routes/traces.test.ts +581 -3
- package/src/observability/routes/traces.ts +87 -1
- package/src/observability/types.ts +6 -1
- package/src/routes/gpt.test.ts +263 -1
- package/src/routes/gpt.ts +43 -2
- package/src/routes/gptHistories.ts +55 -2
- package/src/service/aiService.observability.test.ts +244 -0
- package/src/service/aiService.ts +15 -2
- package/src/service/chatTurn.ts +11 -2
- package/src/types/index.ts +4 -0
- package/src/types/observability.ts +1 -1
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
import {APIError, logger} from "@terreno/api";
|
|
2
|
+
import {LocalEvaluatorStore} from "./local/evaluatorStore";
|
|
3
|
+
import {registerObsReviewItem} from "./local/models/obsReviewItem";
|
|
4
|
+
import {registerObsTrace} from "./local/models/obsTrace";
|
|
5
|
+
import {LocalReviewStore} from "./local/reviewStore";
|
|
6
|
+
import {LocalScoreSink, LocalTraceStore} from "./local/traceStore";
|
|
7
|
+
import {getObservabilityApp} from "./observabilityAppRegistry";
|
|
8
|
+
import type {ScoreRecord, ScoreSink} from "./types";
|
|
9
|
+
|
|
10
|
+
/** Passive signals a product can record about what a user did with an AI output. */
|
|
11
|
+
export const FEEDBACK_SIGNALS = [
|
|
12
|
+
"accepted",
|
|
13
|
+
"dismissed",
|
|
14
|
+
"copied",
|
|
15
|
+
"edited",
|
|
16
|
+
"regenerated",
|
|
17
|
+
] as const;
|
|
18
|
+
export type FeedbackSignal = (typeof FEEDBACK_SIGNALS)[number];
|
|
19
|
+
|
|
20
|
+
export const FEEDBACK_COMMENT_MAX_LENGTH = 2000;
|
|
21
|
+
|
|
22
|
+
/** Score name for explicit thumbs; `true` is up. */
|
|
23
|
+
const THUMBS_SCORE_NAME = "thumbs_up";
|
|
24
|
+
|
|
25
|
+
export interface TraceFeedbackInput {
|
|
26
|
+
comment?: string;
|
|
27
|
+
flagDataset?: boolean;
|
|
28
|
+
outcome?: "fn" | "fp" | "tn" | "tp";
|
|
29
|
+
signal?: FeedbackSignal;
|
|
30
|
+
thumbs?: "down" | "up" | null;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export interface TraceFeedbackResult {
|
|
34
|
+
flaggedForDataset: boolean;
|
|
35
|
+
reviewItemId?: string;
|
|
36
|
+
scores: ScoreRecord[];
|
|
37
|
+
traceId: string;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const TRACE_ID_PATTERN = /^[0-9a-f]{24}$/i;
|
|
41
|
+
const KNOWN_FIELDS = new Set(["comment", "flagDataset", "outcome", "signal", "thumbs"]);
|
|
42
|
+
const OUTCOMES = ["tp", "fp", "tn", "fn"] as const;
|
|
43
|
+
|
|
44
|
+
/** Score name for outcome class feedback (`tp` / `fp` / `tn` / `fn`). */
|
|
45
|
+
const OUTCOME_SCORE_NAME = "outcome";
|
|
46
|
+
|
|
47
|
+
interface ParsedFeedback {
|
|
48
|
+
comment?: string;
|
|
49
|
+
flagDataset?: boolean;
|
|
50
|
+
outcome?: TraceFeedbackInput["outcome"];
|
|
51
|
+
signal?: FeedbackSignal;
|
|
52
|
+
thumbs?: "down" | "up" | null;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const badRequest = (title: string): APIError => {
|
|
56
|
+
return new APIError({status: 400, title});
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
/** Validates the request body; the comment comes back trimmed. */
|
|
60
|
+
const parseInput = (input: unknown): ParsedFeedback => {
|
|
61
|
+
if (!input || typeof input !== "object" || Array.isArray(input)) {
|
|
62
|
+
throw badRequest("feedback body must be an object");
|
|
63
|
+
}
|
|
64
|
+
const body = input as Record<string, unknown>;
|
|
65
|
+
const keys = Object.keys(body).filter((key) => body[key] !== undefined);
|
|
66
|
+
if (keys.length === 0) {
|
|
67
|
+
throw badRequest("feedback needs thumbs, comment, outcome, signal, or flagDataset");
|
|
68
|
+
}
|
|
69
|
+
const unknownKey = keys.find((key) => !KNOWN_FIELDS.has(key));
|
|
70
|
+
if (unknownKey) {
|
|
71
|
+
throw badRequest(`Unknown feedback field "${unknownKey}"`);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const {flagDataset, outcome, signal, thumbs} = body;
|
|
75
|
+
if (thumbs !== undefined && thumbs !== null && thumbs !== "up" && thumbs !== "down") {
|
|
76
|
+
throw badRequest("thumbs must be 'up', 'down', or null");
|
|
77
|
+
}
|
|
78
|
+
if (outcome !== undefined && !(OUTCOMES as readonly unknown[]).includes(outcome)) {
|
|
79
|
+
throw badRequest(`outcome must be one of ${OUTCOMES.join(", ")}`);
|
|
80
|
+
}
|
|
81
|
+
if (signal !== undefined && !(FEEDBACK_SIGNALS as readonly unknown[]).includes(signal)) {
|
|
82
|
+
throw badRequest(`signal must be one of ${FEEDBACK_SIGNALS.join(", ")}`);
|
|
83
|
+
}
|
|
84
|
+
if (flagDataset !== undefined && typeof flagDataset !== "boolean") {
|
|
85
|
+
throw badRequest("flagDataset must be a boolean");
|
|
86
|
+
}
|
|
87
|
+
const parsed: ParsedFeedback = {
|
|
88
|
+
flagDataset: flagDataset as boolean | undefined,
|
|
89
|
+
outcome: outcome as ParsedFeedback["outcome"],
|
|
90
|
+
signal: signal as FeedbackSignal | undefined,
|
|
91
|
+
thumbs: thumbs as ParsedFeedback["thumbs"],
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
if (body.comment === undefined) {
|
|
95
|
+
return parsed;
|
|
96
|
+
}
|
|
97
|
+
if (typeof body.comment !== "string") {
|
|
98
|
+
throw badRequest("comment must be a string");
|
|
99
|
+
}
|
|
100
|
+
const comment = body.comment.trim();
|
|
101
|
+
if (!comment) {
|
|
102
|
+
throw badRequest("comment must not be empty");
|
|
103
|
+
}
|
|
104
|
+
if (comment.length > FEEDBACK_COMMENT_MAX_LENGTH) {
|
|
105
|
+
throw badRequest(`comment must be at most ${FEEDBACK_COMMENT_MAX_LENGTH} characters`);
|
|
106
|
+
}
|
|
107
|
+
if (thumbs !== "up" && thumbs !== "down" && parsed.outcome === undefined) {
|
|
108
|
+
throw badRequest("comment needs thumbs or outcome");
|
|
109
|
+
}
|
|
110
|
+
return {...parsed, comment};
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
/** Sends a score to every registered sink; the local sink upserts feedback scores. */
|
|
114
|
+
const exportToSinks = async (score: ScoreRecord, store: LocalTraceStore): Promise<void> => {
|
|
115
|
+
const sinks: ScoreSink[] = getObservabilityApp()?.scoreSinks ?? [];
|
|
116
|
+
if (sinks.length === 0) {
|
|
117
|
+
await store.exportScore(score);
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
const results = await Promise.allSettled(
|
|
121
|
+
sinks.map((sink) => {
|
|
122
|
+
return sink.export(score);
|
|
123
|
+
})
|
|
124
|
+
);
|
|
125
|
+
// The local store is the record the app reads, so its failure fails the request. Other sinks
|
|
126
|
+
// are best-effort, like trace export: a vendor outage must not stop review queueing.
|
|
127
|
+
results.forEach((result, index) => {
|
|
128
|
+
if (result.status === "fulfilled") {
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
131
|
+
if (sinks[index] instanceof LocalScoreSink) {
|
|
132
|
+
throw result.reason;
|
|
133
|
+
}
|
|
134
|
+
logger.error("Feedback ScoreSink.export failed", {error: result.reason, name: score.name});
|
|
135
|
+
});
|
|
136
|
+
};
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Queues a thumbs-down trace for human review against `feedbackReview.evaluatorName`, reusing an
|
|
140
|
+
* open item. Never fails the feedback request: a bad evaluator is logged and skipped.
|
|
141
|
+
*/
|
|
142
|
+
const enqueueFeedbackReview = async (traceId: string): Promise<string | undefined> => {
|
|
143
|
+
const evaluatorName = getObservabilityApp()?.feedbackReview?.evaluatorName;
|
|
144
|
+
if (!evaluatorName) {
|
|
145
|
+
return undefined;
|
|
146
|
+
}
|
|
147
|
+
try {
|
|
148
|
+
const evaluator = await new LocalEvaluatorStore().getByName(evaluatorName);
|
|
149
|
+
if (!evaluator) {
|
|
150
|
+
logger.error("feedbackReview evaluator not found; thumbs down not queued", {evaluatorName});
|
|
151
|
+
return undefined;
|
|
152
|
+
}
|
|
153
|
+
const reviewStore = new LocalReviewStore();
|
|
154
|
+
const open = await reviewStore.findOpen({evaluatorId: evaluator.id, traceId});
|
|
155
|
+
if (open) {
|
|
156
|
+
return open.id;
|
|
157
|
+
}
|
|
158
|
+
// The pending-feedback unique index must exist before the first insert can race.
|
|
159
|
+
await registerObsReviewItem().init();
|
|
160
|
+
try {
|
|
161
|
+
const [item] = await reviewStore.enqueue({
|
|
162
|
+
evaluatorId: evaluator.id,
|
|
163
|
+
reason: "feedback",
|
|
164
|
+
traceIds: [traceId],
|
|
165
|
+
});
|
|
166
|
+
return item?.id;
|
|
167
|
+
} catch (error) {
|
|
168
|
+
// A concurrent thumbs down queued it first; return that item.
|
|
169
|
+
if ((error as {code?: number}).code !== 11000) {
|
|
170
|
+
throw error;
|
|
171
|
+
}
|
|
172
|
+
return (await reviewStore.findOpen({evaluatorId: evaluator.id, traceId}))?.id;
|
|
173
|
+
}
|
|
174
|
+
} catch (error) {
|
|
175
|
+
logger.error("Failed to queue thumbs-down trace for review", {error, evaluatorName, traceId});
|
|
176
|
+
return undefined;
|
|
177
|
+
}
|
|
178
|
+
};
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* Records an end user's feedback on a trace they own. Only the trace's own user may call it:
|
|
182
|
+
* admins score traces through `POST /traces/:id/scores`, so `user-feedback` always means the
|
|
183
|
+
* user said it.
|
|
184
|
+
*/
|
|
185
|
+
export const recordTraceFeedback = async (args: {
|
|
186
|
+
input: TraceFeedbackInput;
|
|
187
|
+
traceId: string;
|
|
188
|
+
user: {admin?: boolean; id: string};
|
|
189
|
+
}): Promise<TraceFeedbackResult> => {
|
|
190
|
+
if (!TRACE_ID_PATTERN.test(args.traceId)) {
|
|
191
|
+
throw badRequest("trace id must be an object id");
|
|
192
|
+
}
|
|
193
|
+
const input = parseInput(args.input);
|
|
194
|
+
const trace = await registerObsTrace().findOneOrNone({_id: args.traceId});
|
|
195
|
+
if (!trace) {
|
|
196
|
+
throw new APIError({status: 404, title: "Trace not found"});
|
|
197
|
+
}
|
|
198
|
+
if (!trace.userId || String(trace.userId) !== args.user.id) {
|
|
199
|
+
throw new APIError({status: 403, title: "Only the trace's user can give feedback on it"});
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
const store = new LocalTraceStore();
|
|
203
|
+
const scores: ScoreRecord[] = [];
|
|
204
|
+
const commentFields = input.comment ? {comment: input.comment} : {};
|
|
205
|
+
|
|
206
|
+
if (input.thumbs === null) {
|
|
207
|
+
await store.deleteFeedbackScore({
|
|
208
|
+
name: THUMBS_SCORE_NAME,
|
|
209
|
+
source: "user-feedback",
|
|
210
|
+
traceId: args.traceId,
|
|
211
|
+
});
|
|
212
|
+
} else if (input.thumbs) {
|
|
213
|
+
scores.push({
|
|
214
|
+
dataType: "boolean",
|
|
215
|
+
name: THUMBS_SCORE_NAME,
|
|
216
|
+
source: "user-feedback",
|
|
217
|
+
traceId: args.traceId,
|
|
218
|
+
value: input.thumbs === "up",
|
|
219
|
+
...commentFields,
|
|
220
|
+
});
|
|
221
|
+
}
|
|
222
|
+
if (input.outcome) {
|
|
223
|
+
scores.push({
|
|
224
|
+
dataType: "categorical",
|
|
225
|
+
name: OUTCOME_SCORE_NAME,
|
|
226
|
+
source: "user-feedback",
|
|
227
|
+
traceId: args.traceId,
|
|
228
|
+
value: input.outcome,
|
|
229
|
+
...commentFields,
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
if (input.signal) {
|
|
233
|
+
scores.push({
|
|
234
|
+
dataType: "boolean",
|
|
235
|
+
name: input.signal,
|
|
236
|
+
source: "user-signal",
|
|
237
|
+
traceId: args.traceId,
|
|
238
|
+
value: true,
|
|
239
|
+
});
|
|
240
|
+
}
|
|
241
|
+
for (const score of scores) {
|
|
242
|
+
await exportToSinks(score, store);
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
let flaggedForDataset = Boolean(trace.flaggedForDataset);
|
|
246
|
+
if (input.flagDataset !== undefined) {
|
|
247
|
+
await registerObsTrace().updateOne(
|
|
248
|
+
{_id: trace._id},
|
|
249
|
+
{$set: {flaggedForDataset: input.flagDataset}}
|
|
250
|
+
);
|
|
251
|
+
flaggedForDataset = input.flagDataset;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
const reviewItemId =
|
|
255
|
+
input.thumbs === "down" ? await enqueueFeedbackReview(args.traceId) : undefined;
|
|
256
|
+
|
|
257
|
+
return {
|
|
258
|
+
flaggedForDataset,
|
|
259
|
+
...(reviewItemId ? {reviewItemId} : {}),
|
|
260
|
+
scores,
|
|
261
|
+
traceId: args.traceId,
|
|
262
|
+
};
|
|
263
|
+
};
|
|
@@ -18,6 +18,7 @@ import {
|
|
|
18
18
|
} from "./humanAnswers";
|
|
19
19
|
import {registerObsDataset} from "./models/obsDataset";
|
|
20
20
|
import {registerObsDatasetItem} from "./models/obsDatasetItem";
|
|
21
|
+
import {registerObsScore} from "./models/obsScore";
|
|
21
22
|
import {registerObsTrace} from "./models/obsTrace";
|
|
22
23
|
import {LocalPromptStore} from "./promptStore";
|
|
23
24
|
|
|
@@ -180,6 +181,57 @@ const computeDatasetCounts = async (datasetId: string): Promise<DatasetCounts> =
|
|
|
180
181
|
};
|
|
181
182
|
};
|
|
182
183
|
|
|
184
|
+
/** What the trace's user said and did, copied onto dataset items added from the trace. */
|
|
185
|
+
export interface DatasetItemUserFeedback {
|
|
186
|
+
comment?: string;
|
|
187
|
+
outcome?: string;
|
|
188
|
+
signals: string[];
|
|
189
|
+
thumbs?: "down" | "up";
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* Reads a trace's live end-user scores. Negative feedback means the trace output is a known-bad
|
|
194
|
+
* answer, so the item must not be treated as a gold label until a human labels it.
|
|
195
|
+
*/
|
|
196
|
+
const readUserFeedback = async (
|
|
197
|
+
traceId: mongoose.Types.ObjectId
|
|
198
|
+
): Promise<{feedback?: DatasetItemUserFeedback; negativeTags: string[]}> => {
|
|
199
|
+
const scores = await registerObsScore()
|
|
200
|
+
.find({source: {$in: ["user-feedback", "user-signal"]}, traceId})
|
|
201
|
+
.sort({created: 1});
|
|
202
|
+
if (scores.length === 0) {
|
|
203
|
+
return {negativeTags: []};
|
|
204
|
+
}
|
|
205
|
+
const feedback: DatasetItemUserFeedback = {signals: []};
|
|
206
|
+
const negativeTags: string[] = [];
|
|
207
|
+
for (const score of scores) {
|
|
208
|
+
if (score.source === "user-signal") {
|
|
209
|
+
feedback.signals.push(score.name);
|
|
210
|
+
if (score.name === "dismissed") {
|
|
211
|
+
negativeTags.push("user:dismissed");
|
|
212
|
+
}
|
|
213
|
+
continue;
|
|
214
|
+
}
|
|
215
|
+
if (score.name === "thumbs_up") {
|
|
216
|
+
feedback.thumbs = score.value === true ? "up" : "down";
|
|
217
|
+
if (score.value === false) {
|
|
218
|
+
negativeTags.push("user:thumbs-down");
|
|
219
|
+
}
|
|
220
|
+
} else if (score.name === "outcome") {
|
|
221
|
+
feedback.outcome = String(score.value);
|
|
222
|
+
if (score.value === "fp") {
|
|
223
|
+
negativeTags.push("user:false-positive");
|
|
224
|
+
} else if (score.value === "fn") {
|
|
225
|
+
negativeTags.push("user:false-negative");
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
if (score.comment && !feedback.comment) {
|
|
229
|
+
feedback.comment = score.comment;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
return {feedback, negativeTags};
|
|
233
|
+
};
|
|
234
|
+
|
|
183
235
|
export class LocalDatasetStore {
|
|
184
236
|
constructor(private readonly promptStore = new LocalPromptStore()) {}
|
|
185
237
|
|
|
@@ -408,14 +460,17 @@ export class LocalDatasetStore {
|
|
|
408
460
|
}
|
|
409
461
|
const input = trace.input ?? {};
|
|
410
462
|
await this.validateItemInput(dataset, input, 1);
|
|
463
|
+
const {feedback, negativeTags} = await readUserFeedback(trace._id);
|
|
411
464
|
const item = await registerObsDatasetItem().create({
|
|
412
465
|
datasetId: dataset._id,
|
|
413
466
|
expectedOutput: trace.output,
|
|
414
467
|
input,
|
|
415
468
|
origin: "trace",
|
|
416
|
-
|
|
469
|
+
// A user-rejected output is not a gold label: it waits for a human to relabel it.
|
|
470
|
+
proofread: !trace.sensitive && negativeTags.length === 0,
|
|
417
471
|
sourceTraceId: trace._id,
|
|
418
|
-
tags:
|
|
472
|
+
tags: negativeTags,
|
|
473
|
+
...(feedback ? {metadata: {userFeedback: feedback}} : {}),
|
|
419
474
|
});
|
|
420
475
|
created.push(toItemView(item));
|
|
421
476
|
}
|
|
@@ -76,6 +76,15 @@ obsReviewItemSchema.index(Object.fromEntries(reviewIndexSpec));
|
|
|
76
76
|
obsReviewItemSchema.index({batchId: 1, status: 1});
|
|
77
77
|
// Dataset batches skip items that already wait in an open review for the same question set.
|
|
78
78
|
obsReviewItemSchema.index({datasetItemId: 1, evaluatorId: 1});
|
|
79
|
+
// One pending user-feedback item per trace and evaluator, so concurrent thumbs downs queue once.
|
|
80
|
+
obsReviewItemSchema.index(
|
|
81
|
+
{evaluatorId: 1, traceId: 1},
|
|
82
|
+
{
|
|
83
|
+
name: "pending_feedback_unique",
|
|
84
|
+
partialFilterExpression: {deleted: false, reason: "feedback", status: "pending"},
|
|
85
|
+
unique: true,
|
|
86
|
+
}
|
|
87
|
+
);
|
|
79
88
|
|
|
80
89
|
export const registerObsReviewItem = (): ObsReviewItemModel => {
|
|
81
90
|
if (mongoose.models.ObsReviewItem) {
|
|
@@ -21,7 +21,7 @@ const obsScoreSchema = new mongoose.Schema<ObsScoreDocument, ObsScoreModel>(
|
|
|
21
21
|
name: {description: "Score dimension or name", required: true, type: String},
|
|
22
22
|
source: {
|
|
23
23
|
description: "Who wrote the score",
|
|
24
|
-
enum: ["code", "human", "llm-judge", "user-feedback"],
|
|
24
|
+
enum: ["code", "human", "llm-judge", "user-feedback", "user-signal"],
|
|
25
25
|
required: true,
|
|
26
26
|
type: String,
|
|
27
27
|
},
|
|
@@ -50,6 +50,23 @@ obsScoreSchema.plugin(isDeletedPlugin);
|
|
|
50
50
|
obsScoreSchema.plugin(findOneOrNone);
|
|
51
51
|
obsScoreSchema.plugin(findExactlyOne);
|
|
52
52
|
obsScoreSchema.index({traceId: 1});
|
|
53
|
+
// One live end-user score per (trace, source, name); a changed vote replaces the old one.
|
|
54
|
+
obsScoreSchema.index(
|
|
55
|
+
{name: 1, source: 1, traceId: 1},
|
|
56
|
+
{
|
|
57
|
+
name: "user_feedback_unique",
|
|
58
|
+
partialFilterExpression: {deleted: false, source: "user-feedback"},
|
|
59
|
+
unique: true,
|
|
60
|
+
}
|
|
61
|
+
);
|
|
62
|
+
obsScoreSchema.index(
|
|
63
|
+
{name: 1, source: 1, traceId: 1},
|
|
64
|
+
{
|
|
65
|
+
name: "user_signal_unique",
|
|
66
|
+
partialFilterExpression: {deleted: false, source: "user-signal"},
|
|
67
|
+
unique: true,
|
|
68
|
+
}
|
|
69
|
+
);
|
|
53
70
|
|
|
54
71
|
export const registerObsScore = (): ObsScoreModel => {
|
|
55
72
|
if (mongoose.models.ObsScore) {
|
|
@@ -94,12 +94,20 @@ describe("local observability models", () => {
|
|
|
94
94
|
expect(missing).toBeNull();
|
|
95
95
|
});
|
|
96
96
|
|
|
97
|
-
|
|
97
|
+
// Updated for T2 (ai-user-feedback): judge, human, and code scores may still repeat on a trace;
|
|
98
|
+
// only live end-user scores are unique per (trace, source, name).
|
|
99
|
+
it("declares unique score indexes only for live end-user sources", () => {
|
|
98
100
|
const ObsScore = registerObsScore();
|
|
99
101
|
const uniqueIndexes = ObsScore.schema
|
|
100
102
|
.indexes()
|
|
101
103
|
.filter((entry) => Boolean((entry[1] as {unique?: boolean} | undefined)?.unique));
|
|
102
|
-
expect(uniqueIndexes.
|
|
104
|
+
expect(uniqueIndexes.map((entry) => entry[1].partialFilterExpression)).toEqual([
|
|
105
|
+
{deleted: false, source: "user-feedback"},
|
|
106
|
+
{deleted: false, source: "user-signal"},
|
|
107
|
+
]);
|
|
108
|
+
for (const entry of uniqueIndexes) {
|
|
109
|
+
expect(entry[0]).toEqual({name: 1, source: 1, traceId: 1});
|
|
110
|
+
}
|
|
103
111
|
});
|
|
104
112
|
|
|
105
113
|
it("declares ObsTrace compound indexes in IP key order", () => {
|
|
@@ -67,6 +67,7 @@ describe("LocalPromptStore", () => {
|
|
|
67
67
|
expect(listed.find((row) => row.name === "summarize")?.description).toBe(
|
|
68
68
|
"Summarizes user notes for clinicians"
|
|
69
69
|
);
|
|
70
|
+
expect(listed.find((row) => row.name === "summarize")?.tags).toEqual([]);
|
|
70
71
|
expect(listed.find((row) => row.name === "legacy-no-description")?.description).toBeUndefined();
|
|
71
72
|
|
|
72
73
|
const detail = await store.getDetail("summarize");
|
|
@@ -309,6 +310,7 @@ describe("LocalPromptStore", () => {
|
|
|
309
310
|
latestVersion: 1,
|
|
310
311
|
name: "summarize",
|
|
311
312
|
production: "—",
|
|
313
|
+
tags: [],
|
|
312
314
|
type: "text",
|
|
313
315
|
usage7d: {calls: 1, costUsd: 0.4},
|
|
314
316
|
},
|
|
@@ -90,6 +90,7 @@ export interface PromptListItem {
|
|
|
90
90
|
latestVersion: number;
|
|
91
91
|
name: string;
|
|
92
92
|
production: number | "—";
|
|
93
|
+
tags: string[];
|
|
93
94
|
type: "chat" | "text";
|
|
94
95
|
usage7d?: {calls: number; costUsd?: number};
|
|
95
96
|
}
|
|
@@ -400,6 +401,7 @@ export class LocalPromptStore implements PromptRegistry {
|
|
|
400
401
|
latestVersion: latest?.version ?? 0,
|
|
401
402
|
name: prompt.name,
|
|
402
403
|
production,
|
|
404
|
+
tags: prompt.tags ?? [],
|
|
403
405
|
type: latest?.type ?? "text",
|
|
404
406
|
...(usageByName ? {usage7d: usageByName.get(prompt.name) ?? {calls: 0}} : {}),
|
|
405
407
|
});
|
|
@@ -167,4 +167,59 @@ describe("LocalReviewStore", () => {
|
|
|
167
167
|
await expect(reviewStore.list(operator)).rejects.toMatchObject({status: 400});
|
|
168
168
|
await expect(reviewStore.list("pending")).resolves.toHaveProperty("counts");
|
|
169
169
|
});
|
|
170
|
+
|
|
171
|
+
it("findOpen returns the pending or in_progress item for a trace and evaluator", async () => {
|
|
172
|
+
expect(await reviewStore.findOpen({evaluatorId, traceId})).toBeUndefined();
|
|
173
|
+
|
|
174
|
+
const [item] = await reviewStore.enqueue({
|
|
175
|
+
evaluatorId,
|
|
176
|
+
reason: "feedback",
|
|
177
|
+
traceIds: [traceId],
|
|
178
|
+
});
|
|
179
|
+
const pending = await reviewStore.findOpen({evaluatorId, traceId});
|
|
180
|
+
expect(pending).toMatchObject({
|
|
181
|
+
evaluatorId,
|
|
182
|
+
id: item.id,
|
|
183
|
+
reason: "feedback",
|
|
184
|
+
status: "pending",
|
|
185
|
+
traceId,
|
|
186
|
+
});
|
|
187
|
+
|
|
188
|
+
await reviewStore.assign(item.id, new mongoose.Types.ObjectId().toString());
|
|
189
|
+
const inProgress = await reviewStore.findOpen({evaluatorId, traceId});
|
|
190
|
+
expect(inProgress?.id).toBe(item.id);
|
|
191
|
+
expect(inProgress?.status).toBe("in_progress");
|
|
192
|
+
});
|
|
193
|
+
|
|
194
|
+
it("findOpen ignores done and skipped items and other evaluators or traces", async () => {
|
|
195
|
+
const [done] = await reviewStore.enqueue({evaluatorId, reason: "manual", traceIds: [traceId]});
|
|
196
|
+
await reviewStore.submit({id: done.id, scores: {correct: true}, sinks: [scoreSink]});
|
|
197
|
+
const [skipped] = await reviewStore.enqueue({
|
|
198
|
+
evaluatorId,
|
|
199
|
+
reason: "manual",
|
|
200
|
+
traceIds: [traceId],
|
|
201
|
+
});
|
|
202
|
+
await reviewStore.skip(skipped.id);
|
|
203
|
+
expect(await reviewStore.findOpen({evaluatorId, traceId})).toBeUndefined();
|
|
204
|
+
|
|
205
|
+
const other = await new LocalEvaluatorStore().create({
|
|
206
|
+
dimensions: [{dataType: "boolean", key: "correct", required: true}],
|
|
207
|
+
name: "other-human",
|
|
208
|
+
target: "full trace",
|
|
209
|
+
type: "human",
|
|
210
|
+
});
|
|
211
|
+
const [otherItem] = await reviewStore.enqueue({
|
|
212
|
+
evaluatorId: other.id,
|
|
213
|
+
reason: "manual",
|
|
214
|
+
traceIds: [traceId],
|
|
215
|
+
});
|
|
216
|
+
expect(await reviewStore.findOpen({evaluatorId, traceId})).toBeUndefined();
|
|
217
|
+
expect((await reviewStore.findOpen({evaluatorId: other.id, traceId}))?.id).toBe(otherItem.id);
|
|
218
|
+
expect(
|
|
219
|
+
await reviewStore.findOpen({
|
|
220
|
+
evaluatorId: other.id,
|
|
221
|
+
traceId: new mongoose.Types.ObjectId().toString(),
|
|
222
|
+
})
|
|
223
|
+
).toBeUndefined();
|
|
224
|
+
});
|
|
170
225
|
});
|
|
@@ -93,6 +93,27 @@ export class LocalReviewStore {
|
|
|
93
93
|
return created;
|
|
94
94
|
}
|
|
95
95
|
|
|
96
|
+
/** The pending or in-progress item for a trace and evaluator, if one exists. */
|
|
97
|
+
async findOpen(params: {
|
|
98
|
+
evaluatorId: string;
|
|
99
|
+
traceId: string;
|
|
100
|
+
}): Promise<ReviewListItem | undefined> {
|
|
101
|
+
// Oldest open item: an operator may already have queued the same trace more than once.
|
|
102
|
+
const [item] = await registerObsReviewItem()
|
|
103
|
+
.find({
|
|
104
|
+
evaluatorId: objectId(params.evaluatorId, "evaluatorId"),
|
|
105
|
+
status: {$in: ["pending", "in_progress"]},
|
|
106
|
+
traceId: objectId(params.traceId, "traceId"),
|
|
107
|
+
})
|
|
108
|
+
.sort({enqueuedAt: 1})
|
|
109
|
+
.limit(1);
|
|
110
|
+
if (!item) {
|
|
111
|
+
return undefined;
|
|
112
|
+
}
|
|
113
|
+
const trace = await registerObsTrace().findOneOrNone({_id: item.traceId});
|
|
114
|
+
return this.toListItem(item, trace?.name ?? "", trace?.prompts[0]?.name);
|
|
115
|
+
}
|
|
116
|
+
|
|
96
117
|
/** Queue dataset items for a human question set; items with a source trace show that trace. */
|
|
97
118
|
async enqueueDatasetItems(params: {
|
|
98
119
|
batchId: string;
|