@terreno/ai 58.0.6 → 58.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/CHANGELOG.md +52 -0
  2. package/dist/index.d.ts +2 -1
  3. package/dist/index.js +6 -2
  4. package/dist/models/gptHistory.js +4 -0
  5. package/dist/observability/feedback.d.ts +31 -0
  6. package/dist/observability/feedback.js +218 -0
  7. package/dist/observability/local/datasetStore.d.ts +7 -0
  8. package/dist/observability/local/datasetStore.js +48 -2
  9. package/dist/observability/local/models/obsReviewItem.js +6 -0
  10. package/dist/observability/local/models/obsScore.js +12 -1
  11. package/dist/observability/local/promptStore.d.ts +1 -0
  12. package/dist/observability/local/promptStore.js +1 -0
  13. package/dist/observability/local/reviewStore.d.ts +5 -0
  14. package/dist/observability/local/reviewStore.js +17 -0
  15. package/dist/observability/local/traceStore.d.ts +23 -0
  16. package/dist/observability/local/traceStore.js +121 -10
  17. package/dist/observability/observabilityApp.d.ts +1 -0
  18. package/dist/observability/observabilityApp.js +2 -0
  19. package/dist/observability/observabilityAppRegistry.d.ts +1 -0
  20. package/dist/observability/routes/traces.js +72 -0
  21. package/dist/observability/types.d.ts +8 -1
  22. package/dist/routes/gpt.js +42 -2
  23. package/dist/routes/gptHistories.js +42 -2
  24. package/dist/service/aiService.d.ts +4 -1
  25. package/dist/service/aiService.js +17 -2
  26. package/dist/service/chatTurn.js +10 -1
  27. package/dist/types/index.d.ts +4 -0
  28. package/dist/types/observability.d.ts +1 -1
  29. package/package.json +8 -8
  30. package/src/index.ts +8 -1
  31. package/src/models/gptHistory.ts +5 -0
  32. package/src/observability/feedback.test.ts +688 -0
  33. package/src/observability/feedback.ts +263 -0
  34. package/src/observability/local/datasetStore.ts +57 -2
  35. package/src/observability/local/models/obsReviewItem.ts +9 -0
  36. package/src/observability/local/models/obsScore.ts +18 -1
  37. package/src/observability/local/obsModels.test.ts +10 -2
  38. package/src/observability/local/promptStore.test.ts +2 -0
  39. package/src/observability/local/promptStore.ts +2 -0
  40. package/src/observability/local/reviewStore.test.ts +55 -0
  41. package/src/observability/local/reviewStore.ts +21 -0
  42. package/src/observability/local/traceStore.test.ts +239 -1
  43. package/src/observability/local/traceStore.ts +140 -9
  44. package/src/observability/observabilityApp.ts +2 -0
  45. package/src/observability/observabilityAppRegistry.ts +1 -0
  46. package/src/observability/routes/datasets.test.ts +253 -0
  47. package/src/observability/routes/prompts.test.ts +1 -0
  48. package/src/observability/routes/traces.test.ts +581 -3
  49. package/src/observability/routes/traces.ts +87 -1
  50. package/src/observability/types.ts +6 -1
  51. package/src/routes/gpt.test.ts +263 -1
  52. package/src/routes/gpt.ts +43 -2
  53. package/src/routes/gptHistories.ts +55 -2
  54. package/src/service/aiService.observability.test.ts +244 -0
  55. package/src/service/aiService.ts +15 -2
  56. package/src/service/chatTurn.ts +11 -2
  57. package/src/types/index.ts +4 -0
  58. package/src/types/observability.ts +1 -1
@@ -0,0 +1,263 @@
1
+ import {APIError, logger} from "@terreno/api";
2
+ import {LocalEvaluatorStore} from "./local/evaluatorStore";
3
+ import {registerObsReviewItem} from "./local/models/obsReviewItem";
4
+ import {registerObsTrace} from "./local/models/obsTrace";
5
+ import {LocalReviewStore} from "./local/reviewStore";
6
+ import {LocalScoreSink, LocalTraceStore} from "./local/traceStore";
7
+ import {getObservabilityApp} from "./observabilityAppRegistry";
8
+ import type {ScoreRecord, ScoreSink} from "./types";
9
+
10
+ /** Passive signals a product can record about what a user did with an AI output. */
11
+ export const FEEDBACK_SIGNALS = [
12
+ "accepted",
13
+ "dismissed",
14
+ "copied",
15
+ "edited",
16
+ "regenerated",
17
+ ] as const;
18
+ export type FeedbackSignal = (typeof FEEDBACK_SIGNALS)[number];
19
+
20
+ export const FEEDBACK_COMMENT_MAX_LENGTH = 2000;
21
+
22
+ /** Score name for explicit thumbs; `true` is up. */
23
+ const THUMBS_SCORE_NAME = "thumbs_up";
24
+
25
+ export interface TraceFeedbackInput {
26
+ comment?: string;
27
+ flagDataset?: boolean;
28
+ outcome?: "fn" | "fp" | "tn" | "tp";
29
+ signal?: FeedbackSignal;
30
+ thumbs?: "down" | "up" | null;
31
+ }
32
+
33
+ export interface TraceFeedbackResult {
34
+ flaggedForDataset: boolean;
35
+ reviewItemId?: string;
36
+ scores: ScoreRecord[];
37
+ traceId: string;
38
+ }
39
+
40
+ const TRACE_ID_PATTERN = /^[0-9a-f]{24}$/i;
41
+ const KNOWN_FIELDS = new Set(["comment", "flagDataset", "outcome", "signal", "thumbs"]);
42
+ const OUTCOMES = ["tp", "fp", "tn", "fn"] as const;
43
+
44
+ /** Score name for outcome class feedback (`tp` / `fp` / `tn` / `fn`). */
45
+ const OUTCOME_SCORE_NAME = "outcome";
46
+
47
+ interface ParsedFeedback {
48
+ comment?: string;
49
+ flagDataset?: boolean;
50
+ outcome?: TraceFeedbackInput["outcome"];
51
+ signal?: FeedbackSignal;
52
+ thumbs?: "down" | "up" | null;
53
+ }
54
+
55
+ const badRequest = (title: string): APIError => {
56
+ return new APIError({status: 400, title});
57
+ };
58
+
59
+ /** Validates the request body; the comment comes back trimmed. */
60
+ const parseInput = (input: unknown): ParsedFeedback => {
61
+ if (!input || typeof input !== "object" || Array.isArray(input)) {
62
+ throw badRequest("feedback body must be an object");
63
+ }
64
+ const body = input as Record<string, unknown>;
65
+ const keys = Object.keys(body).filter((key) => body[key] !== undefined);
66
+ if (keys.length === 0) {
67
+ throw badRequest("feedback needs thumbs, comment, outcome, signal, or flagDataset");
68
+ }
69
+ const unknownKey = keys.find((key) => !KNOWN_FIELDS.has(key));
70
+ if (unknownKey) {
71
+ throw badRequest(`Unknown feedback field "${unknownKey}"`);
72
+ }
73
+
74
+ const {flagDataset, outcome, signal, thumbs} = body;
75
+ if (thumbs !== undefined && thumbs !== null && thumbs !== "up" && thumbs !== "down") {
76
+ throw badRequest("thumbs must be 'up', 'down', or null");
77
+ }
78
+ if (outcome !== undefined && !(OUTCOMES as readonly unknown[]).includes(outcome)) {
79
+ throw badRequest(`outcome must be one of ${OUTCOMES.join(", ")}`);
80
+ }
81
+ if (signal !== undefined && !(FEEDBACK_SIGNALS as readonly unknown[]).includes(signal)) {
82
+ throw badRequest(`signal must be one of ${FEEDBACK_SIGNALS.join(", ")}`);
83
+ }
84
+ if (flagDataset !== undefined && typeof flagDataset !== "boolean") {
85
+ throw badRequest("flagDataset must be a boolean");
86
+ }
87
+ const parsed: ParsedFeedback = {
88
+ flagDataset: flagDataset as boolean | undefined,
89
+ outcome: outcome as ParsedFeedback["outcome"],
90
+ signal: signal as FeedbackSignal | undefined,
91
+ thumbs: thumbs as ParsedFeedback["thumbs"],
92
+ };
93
+
94
+ if (body.comment === undefined) {
95
+ return parsed;
96
+ }
97
+ if (typeof body.comment !== "string") {
98
+ throw badRequest("comment must be a string");
99
+ }
100
+ const comment = body.comment.trim();
101
+ if (!comment) {
102
+ throw badRequest("comment must not be empty");
103
+ }
104
+ if (comment.length > FEEDBACK_COMMENT_MAX_LENGTH) {
105
+ throw badRequest(`comment must be at most ${FEEDBACK_COMMENT_MAX_LENGTH} characters`);
106
+ }
107
+ if (thumbs !== "up" && thumbs !== "down" && parsed.outcome === undefined) {
108
+ throw badRequest("comment needs thumbs or outcome");
109
+ }
110
+ return {...parsed, comment};
111
+ };
112
+
113
+ /** Sends a score to every registered sink; the local sink upserts feedback scores. */
114
+ const exportToSinks = async (score: ScoreRecord, store: LocalTraceStore): Promise<void> => {
115
+ const sinks: ScoreSink[] = getObservabilityApp()?.scoreSinks ?? [];
116
+ if (sinks.length === 0) {
117
+ await store.exportScore(score);
118
+ return;
119
+ }
120
+ const results = await Promise.allSettled(
121
+ sinks.map((sink) => {
122
+ return sink.export(score);
123
+ })
124
+ );
125
+ // The local store is the record the app reads, so its failure fails the request. Other sinks
126
+ // are best-effort, like trace export: a vendor outage must not stop review queueing.
127
+ results.forEach((result, index) => {
128
+ if (result.status === "fulfilled") {
129
+ return;
130
+ }
131
+ if (sinks[index] instanceof LocalScoreSink) {
132
+ throw result.reason;
133
+ }
134
+ logger.error("Feedback ScoreSink.export failed", {error: result.reason, name: score.name});
135
+ });
136
+ };
137
+
138
+ /**
139
+ * Queues a thumbs-down trace for human review against `feedbackReview.evaluatorName`, reusing an
140
+ * open item. Never fails the feedback request: a bad evaluator is logged and skipped.
141
+ */
142
+ const enqueueFeedbackReview = async (traceId: string): Promise<string | undefined> => {
143
+ const evaluatorName = getObservabilityApp()?.feedbackReview?.evaluatorName;
144
+ if (!evaluatorName) {
145
+ return undefined;
146
+ }
147
+ try {
148
+ const evaluator = await new LocalEvaluatorStore().getByName(evaluatorName);
149
+ if (!evaluator) {
150
+ logger.error("feedbackReview evaluator not found; thumbs down not queued", {evaluatorName});
151
+ return undefined;
152
+ }
153
+ const reviewStore = new LocalReviewStore();
154
+ const open = await reviewStore.findOpen({evaluatorId: evaluator.id, traceId});
155
+ if (open) {
156
+ return open.id;
157
+ }
158
+ // The pending-feedback unique index must exist before the first insert can race.
159
+ await registerObsReviewItem().init();
160
+ try {
161
+ const [item] = await reviewStore.enqueue({
162
+ evaluatorId: evaluator.id,
163
+ reason: "feedback",
164
+ traceIds: [traceId],
165
+ });
166
+ return item?.id;
167
+ } catch (error) {
168
+ // A concurrent thumbs down queued it first; return that item.
169
+ if ((error as {code?: number}).code !== 11000) {
170
+ throw error;
171
+ }
172
+ return (await reviewStore.findOpen({evaluatorId: evaluator.id, traceId}))?.id;
173
+ }
174
+ } catch (error) {
175
+ logger.error("Failed to queue thumbs-down trace for review", {error, evaluatorName, traceId});
176
+ return undefined;
177
+ }
178
+ };
179
+
180
+ /**
181
+ * Records an end user's feedback on a trace they own. Only the trace's own user may call it:
182
+ * admins score traces through `POST /traces/:id/scores`, so `user-feedback` always means the
183
+ * user said it.
184
+ */
185
+ export const recordTraceFeedback = async (args: {
186
+ input: TraceFeedbackInput;
187
+ traceId: string;
188
+ user: {admin?: boolean; id: string};
189
+ }): Promise<TraceFeedbackResult> => {
190
+ if (!TRACE_ID_PATTERN.test(args.traceId)) {
191
+ throw badRequest("trace id must be an object id");
192
+ }
193
+ const input = parseInput(args.input);
194
+ const trace = await registerObsTrace().findOneOrNone({_id: args.traceId});
195
+ if (!trace) {
196
+ throw new APIError({status: 404, title: "Trace not found"});
197
+ }
198
+ if (!trace.userId || String(trace.userId) !== args.user.id) {
199
+ throw new APIError({status: 403, title: "Only the trace's user can give feedback on it"});
200
+ }
201
+
202
+ const store = new LocalTraceStore();
203
+ const scores: ScoreRecord[] = [];
204
+ const commentFields = input.comment ? {comment: input.comment} : {};
205
+
206
+ if (input.thumbs === null) {
207
+ await store.deleteFeedbackScore({
208
+ name: THUMBS_SCORE_NAME,
209
+ source: "user-feedback",
210
+ traceId: args.traceId,
211
+ });
212
+ } else if (input.thumbs) {
213
+ scores.push({
214
+ dataType: "boolean",
215
+ name: THUMBS_SCORE_NAME,
216
+ source: "user-feedback",
217
+ traceId: args.traceId,
218
+ value: input.thumbs === "up",
219
+ ...commentFields,
220
+ });
221
+ }
222
+ if (input.outcome) {
223
+ scores.push({
224
+ dataType: "categorical",
225
+ name: OUTCOME_SCORE_NAME,
226
+ source: "user-feedback",
227
+ traceId: args.traceId,
228
+ value: input.outcome,
229
+ ...commentFields,
230
+ });
231
+ }
232
+ if (input.signal) {
233
+ scores.push({
234
+ dataType: "boolean",
235
+ name: input.signal,
236
+ source: "user-signal",
237
+ traceId: args.traceId,
238
+ value: true,
239
+ });
240
+ }
241
+ for (const score of scores) {
242
+ await exportToSinks(score, store);
243
+ }
244
+
245
+ let flaggedForDataset = Boolean(trace.flaggedForDataset);
246
+ if (input.flagDataset !== undefined) {
247
+ await registerObsTrace().updateOne(
248
+ {_id: trace._id},
249
+ {$set: {flaggedForDataset: input.flagDataset}}
250
+ );
251
+ flaggedForDataset = input.flagDataset;
252
+ }
253
+
254
+ const reviewItemId =
255
+ input.thumbs === "down" ? await enqueueFeedbackReview(args.traceId) : undefined;
256
+
257
+ return {
258
+ flaggedForDataset,
259
+ ...(reviewItemId ? {reviewItemId} : {}),
260
+ scores,
261
+ traceId: args.traceId,
262
+ };
263
+ };
@@ -18,6 +18,7 @@ import {
18
18
  } from "./humanAnswers";
19
19
  import {registerObsDataset} from "./models/obsDataset";
20
20
  import {registerObsDatasetItem} from "./models/obsDatasetItem";
21
+ import {registerObsScore} from "./models/obsScore";
21
22
  import {registerObsTrace} from "./models/obsTrace";
22
23
  import {LocalPromptStore} from "./promptStore";
23
24
 
@@ -180,6 +181,57 @@ const computeDatasetCounts = async (datasetId: string): Promise<DatasetCounts> =
180
181
  };
181
182
  };
182
183
 
184
+ /** What the trace's user said and did, copied onto dataset items added from the trace. */
185
+ export interface DatasetItemUserFeedback {
186
+ comment?: string;
187
+ outcome?: string;
188
+ signals: string[];
189
+ thumbs?: "down" | "up";
190
+ }
191
+
192
+ /**
193
+ * Reads a trace's live end-user scores. Negative feedback means the trace output is a known-bad
194
+ * answer, so the item must not be treated as a gold label until a human labels it.
195
+ */
196
+ const readUserFeedback = async (
197
+ traceId: mongoose.Types.ObjectId
198
+ ): Promise<{feedback?: DatasetItemUserFeedback; negativeTags: string[]}> => {
199
+ const scores = await registerObsScore()
200
+ .find({source: {$in: ["user-feedback", "user-signal"]}, traceId})
201
+ .sort({created: 1});
202
+ if (scores.length === 0) {
203
+ return {negativeTags: []};
204
+ }
205
+ const feedback: DatasetItemUserFeedback = {signals: []};
206
+ const negativeTags: string[] = [];
207
+ for (const score of scores) {
208
+ if (score.source === "user-signal") {
209
+ feedback.signals.push(score.name);
210
+ if (score.name === "dismissed") {
211
+ negativeTags.push("user:dismissed");
212
+ }
213
+ continue;
214
+ }
215
+ if (score.name === "thumbs_up") {
216
+ feedback.thumbs = score.value === true ? "up" : "down";
217
+ if (score.value === false) {
218
+ negativeTags.push("user:thumbs-down");
219
+ }
220
+ } else if (score.name === "outcome") {
221
+ feedback.outcome = String(score.value);
222
+ if (score.value === "fp") {
223
+ negativeTags.push("user:false-positive");
224
+ } else if (score.value === "fn") {
225
+ negativeTags.push("user:false-negative");
226
+ }
227
+ }
228
+ if (score.comment && !feedback.comment) {
229
+ feedback.comment = score.comment;
230
+ }
231
+ }
232
+ return {feedback, negativeTags};
233
+ };
234
+
183
235
  export class LocalDatasetStore {
184
236
  constructor(private readonly promptStore = new LocalPromptStore()) {}
185
237
 
@@ -408,14 +460,17 @@ export class LocalDatasetStore {
408
460
  }
409
461
  const input = trace.input ?? {};
410
462
  await this.validateItemInput(dataset, input, 1);
463
+ const {feedback, negativeTags} = await readUserFeedback(trace._id);
411
464
  const item = await registerObsDatasetItem().create({
412
465
  datasetId: dataset._id,
413
466
  expectedOutput: trace.output,
414
467
  input,
415
468
  origin: "trace",
416
- proofread: !trace.sensitive,
469
+ // A user-rejected output is not a gold label: it waits for a human to relabel it.
470
+ proofread: !trace.sensitive && negativeTags.length === 0,
417
471
  sourceTraceId: trace._id,
418
- tags: [],
472
+ tags: negativeTags,
473
+ ...(feedback ? {metadata: {userFeedback: feedback}} : {}),
419
474
  });
420
475
  created.push(toItemView(item));
421
476
  }
@@ -76,6 +76,15 @@ obsReviewItemSchema.index(Object.fromEntries(reviewIndexSpec));
76
76
  obsReviewItemSchema.index({batchId: 1, status: 1});
77
77
  // Dataset batches skip items that already wait in an open review for the same question set.
78
78
  obsReviewItemSchema.index({datasetItemId: 1, evaluatorId: 1});
79
+ // One pending user-feedback item per trace and evaluator, so concurrent thumbs downs queue once.
80
+ obsReviewItemSchema.index(
81
+ {evaluatorId: 1, traceId: 1},
82
+ {
83
+ name: "pending_feedback_unique",
84
+ partialFilterExpression: {deleted: false, reason: "feedback", status: "pending"},
85
+ unique: true,
86
+ }
87
+ );
79
88
 
80
89
  export const registerObsReviewItem = (): ObsReviewItemModel => {
81
90
  if (mongoose.models.ObsReviewItem) {
@@ -21,7 +21,7 @@ const obsScoreSchema = new mongoose.Schema<ObsScoreDocument, ObsScoreModel>(
21
21
  name: {description: "Score dimension or name", required: true, type: String},
22
22
  source: {
23
23
  description: "Who wrote the score",
24
- enum: ["code", "human", "llm-judge", "user-feedback"],
24
+ enum: ["code", "human", "llm-judge", "user-feedback", "user-signal"],
25
25
  required: true,
26
26
  type: String,
27
27
  },
@@ -50,6 +50,23 @@ obsScoreSchema.plugin(isDeletedPlugin);
50
50
  obsScoreSchema.plugin(findOneOrNone);
51
51
  obsScoreSchema.plugin(findExactlyOne);
52
52
  obsScoreSchema.index({traceId: 1});
53
+ // One live end-user score per (trace, source, name); a changed vote replaces the old one.
54
+ obsScoreSchema.index(
55
+ {name: 1, source: 1, traceId: 1},
56
+ {
57
+ name: "user_feedback_unique",
58
+ partialFilterExpression: {deleted: false, source: "user-feedback"},
59
+ unique: true,
60
+ }
61
+ );
62
+ obsScoreSchema.index(
63
+ {name: 1, source: 1, traceId: 1},
64
+ {
65
+ name: "user_signal_unique",
66
+ partialFilterExpression: {deleted: false, source: "user-signal"},
67
+ unique: true,
68
+ }
69
+ );
53
70
 
54
71
  export const registerObsScore = (): ObsScoreModel => {
55
72
  if (mongoose.models.ObsScore) {
@@ -94,12 +94,20 @@ describe("local observability models", () => {
94
94
  expect(missing).toBeNull();
95
95
  });
96
96
 
97
- it("does not declare a unique index on scores", () => {
97
+ // Updated for T2 (ai-user-feedback): judge, human, and code scores may still repeat on a trace;
98
+ // only live end-user scores are unique per (trace, source, name).
99
+ it("declares unique score indexes only for live end-user sources", () => {
98
100
  const ObsScore = registerObsScore();
99
101
  const uniqueIndexes = ObsScore.schema
100
102
  .indexes()
101
103
  .filter((entry) => Boolean((entry[1] as {unique?: boolean} | undefined)?.unique));
102
- expect(uniqueIndexes.length).toBe(0);
104
+ expect(uniqueIndexes.map((entry) => entry[1].partialFilterExpression)).toEqual([
105
+ {deleted: false, source: "user-feedback"},
106
+ {deleted: false, source: "user-signal"},
107
+ ]);
108
+ for (const entry of uniqueIndexes) {
109
+ expect(entry[0]).toEqual({name: 1, source: 1, traceId: 1});
110
+ }
103
111
  });
104
112
 
105
113
  it("declares ObsTrace compound indexes in IP key order", () => {
@@ -67,6 +67,7 @@ describe("LocalPromptStore", () => {
67
67
  expect(listed.find((row) => row.name === "summarize")?.description).toBe(
68
68
  "Summarizes user notes for clinicians"
69
69
  );
70
+ expect(listed.find((row) => row.name === "summarize")?.tags).toEqual([]);
70
71
  expect(listed.find((row) => row.name === "legacy-no-description")?.description).toBeUndefined();
71
72
 
72
73
  const detail = await store.getDetail("summarize");
@@ -309,6 +310,7 @@ describe("LocalPromptStore", () => {
309
310
  latestVersion: 1,
310
311
  name: "summarize",
311
312
  production: "—",
313
+ tags: [],
312
314
  type: "text",
313
315
  usage7d: {calls: 1, costUsd: 0.4},
314
316
  },
@@ -90,6 +90,7 @@ export interface PromptListItem {
90
90
  latestVersion: number;
91
91
  name: string;
92
92
  production: number | "—";
93
+ tags: string[];
93
94
  type: "chat" | "text";
94
95
  usage7d?: {calls: number; costUsd?: number};
95
96
  }
@@ -400,6 +401,7 @@ export class LocalPromptStore implements PromptRegistry {
400
401
  latestVersion: latest?.version ?? 0,
401
402
  name: prompt.name,
402
403
  production,
404
+ tags: prompt.tags ?? [],
403
405
  type: latest?.type ?? "text",
404
406
  ...(usageByName ? {usage7d: usageByName.get(prompt.name) ?? {calls: 0}} : {}),
405
407
  });
@@ -167,4 +167,59 @@ describe("LocalReviewStore", () => {
167
167
  await expect(reviewStore.list(operator)).rejects.toMatchObject({status: 400});
168
168
  await expect(reviewStore.list("pending")).resolves.toHaveProperty("counts");
169
169
  });
170
+
171
+ it("findOpen returns the pending or in_progress item for a trace and evaluator", async () => {
172
+ expect(await reviewStore.findOpen({evaluatorId, traceId})).toBeUndefined();
173
+
174
+ const [item] = await reviewStore.enqueue({
175
+ evaluatorId,
176
+ reason: "feedback",
177
+ traceIds: [traceId],
178
+ });
179
+ const pending = await reviewStore.findOpen({evaluatorId, traceId});
180
+ expect(pending).toMatchObject({
181
+ evaluatorId,
182
+ id: item.id,
183
+ reason: "feedback",
184
+ status: "pending",
185
+ traceId,
186
+ });
187
+
188
+ await reviewStore.assign(item.id, new mongoose.Types.ObjectId().toString());
189
+ const inProgress = await reviewStore.findOpen({evaluatorId, traceId});
190
+ expect(inProgress?.id).toBe(item.id);
191
+ expect(inProgress?.status).toBe("in_progress");
192
+ });
193
+
194
+ it("findOpen ignores done and skipped items and other evaluators or traces", async () => {
195
+ const [done] = await reviewStore.enqueue({evaluatorId, reason: "manual", traceIds: [traceId]});
196
+ await reviewStore.submit({id: done.id, scores: {correct: true}, sinks: [scoreSink]});
197
+ const [skipped] = await reviewStore.enqueue({
198
+ evaluatorId,
199
+ reason: "manual",
200
+ traceIds: [traceId],
201
+ });
202
+ await reviewStore.skip(skipped.id);
203
+ expect(await reviewStore.findOpen({evaluatorId, traceId})).toBeUndefined();
204
+
205
+ const other = await new LocalEvaluatorStore().create({
206
+ dimensions: [{dataType: "boolean", key: "correct", required: true}],
207
+ name: "other-human",
208
+ target: "full trace",
209
+ type: "human",
210
+ });
211
+ const [otherItem] = await reviewStore.enqueue({
212
+ evaluatorId: other.id,
213
+ reason: "manual",
214
+ traceIds: [traceId],
215
+ });
216
+ expect(await reviewStore.findOpen({evaluatorId, traceId})).toBeUndefined();
217
+ expect((await reviewStore.findOpen({evaluatorId: other.id, traceId}))?.id).toBe(otherItem.id);
218
+ expect(
219
+ await reviewStore.findOpen({
220
+ evaluatorId: other.id,
221
+ traceId: new mongoose.Types.ObjectId().toString(),
222
+ })
223
+ ).toBeUndefined();
224
+ });
170
225
  });
@@ -93,6 +93,27 @@ export class LocalReviewStore {
93
93
  return created;
94
94
  }
95
95
 
96
+ /** The pending or in-progress item for a trace and evaluator, if one exists. */
97
+ async findOpen(params: {
98
+ evaluatorId: string;
99
+ traceId: string;
100
+ }): Promise<ReviewListItem | undefined> {
101
+ // Oldest open item: an operator may already have queued the same trace more than once.
102
+ const [item] = await registerObsReviewItem()
103
+ .find({
104
+ evaluatorId: objectId(params.evaluatorId, "evaluatorId"),
105
+ status: {$in: ["pending", "in_progress"]},
106
+ traceId: objectId(params.traceId, "traceId"),
107
+ })
108
+ .sort({enqueuedAt: 1})
109
+ .limit(1);
110
+ if (!item) {
111
+ return undefined;
112
+ }
113
+ const trace = await registerObsTrace().findOneOrNone({_id: item.traceId});
114
+ return this.toListItem(item, trace?.name ?? "", trace?.prompts[0]?.name);
115
+ }
116
+
96
117
  /** Queue dataset items for a human question set; items with a source trace show that trace. */
97
118
  async enqueueDatasetItems(params: {
98
119
  batchId: string;