@terreno/ai 58.0.6 → 58.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/CHANGELOG.md +52 -0
  2. package/dist/index.d.ts +2 -1
  3. package/dist/index.js +6 -2
  4. package/dist/models/gptHistory.js +4 -0
  5. package/dist/observability/feedback.d.ts +31 -0
  6. package/dist/observability/feedback.js +218 -0
  7. package/dist/observability/local/datasetStore.d.ts +7 -0
  8. package/dist/observability/local/datasetStore.js +48 -2
  9. package/dist/observability/local/models/obsReviewItem.js +6 -0
  10. package/dist/observability/local/models/obsScore.js +12 -1
  11. package/dist/observability/local/promptStore.d.ts +1 -0
  12. package/dist/observability/local/promptStore.js +1 -0
  13. package/dist/observability/local/reviewStore.d.ts +5 -0
  14. package/dist/observability/local/reviewStore.js +17 -0
  15. package/dist/observability/local/traceStore.d.ts +23 -0
  16. package/dist/observability/local/traceStore.js +121 -10
  17. package/dist/observability/observabilityApp.d.ts +1 -0
  18. package/dist/observability/observabilityApp.js +2 -0
  19. package/dist/observability/observabilityAppRegistry.d.ts +1 -0
  20. package/dist/observability/routes/traces.js +72 -0
  21. package/dist/observability/types.d.ts +8 -1
  22. package/dist/routes/gpt.js +42 -2
  23. package/dist/routes/gptHistories.js +42 -2
  24. package/dist/service/aiService.d.ts +4 -1
  25. package/dist/service/aiService.js +17 -2
  26. package/dist/service/chatTurn.js +10 -1
  27. package/dist/types/index.d.ts +4 -0
  28. package/dist/types/observability.d.ts +1 -1
  29. package/package.json +8 -8
  30. package/src/index.ts +8 -1
  31. package/src/models/gptHistory.ts +5 -0
  32. package/src/observability/feedback.test.ts +688 -0
  33. package/src/observability/feedback.ts +263 -0
  34. package/src/observability/local/datasetStore.ts +57 -2
  35. package/src/observability/local/models/obsReviewItem.ts +9 -0
  36. package/src/observability/local/models/obsScore.ts +18 -1
  37. package/src/observability/local/obsModels.test.ts +10 -2
  38. package/src/observability/local/promptStore.test.ts +2 -0
  39. package/src/observability/local/promptStore.ts +2 -0
  40. package/src/observability/local/reviewStore.test.ts +55 -0
  41. package/src/observability/local/reviewStore.ts +21 -0
  42. package/src/observability/local/traceStore.test.ts +239 -1
  43. package/src/observability/local/traceStore.ts +140 -9
  44. package/src/observability/observabilityApp.ts +2 -0
  45. package/src/observability/observabilityAppRegistry.ts +1 -0
  46. package/src/observability/routes/datasets.test.ts +253 -0
  47. package/src/observability/routes/prompts.test.ts +1 -0
  48. package/src/observability/routes/traces.test.ts +581 -3
  49. package/src/observability/routes/traces.ts +87 -1
  50. package/src/observability/types.ts +6 -1
  51. package/src/routes/gpt.test.ts +263 -1
  52. package/src/routes/gpt.ts +43 -2
  53. package/src/routes/gptHistories.ts +55 -2
  54. package/src/service/aiService.observability.test.ts +244 -0
  55. package/src/service/aiService.ts +15 -2
  56. package/src/service/chatTurn.ts +11 -2
  57. package/src/types/index.ts +4 -0
  58. package/src/types/observability.ts +1 -1
@@ -0,0 +1,688 @@
1
+ import {afterEach, beforeAll, beforeEach, describe, expect, it, spyOn} from "bun:test";
2
+ import {logger, TerrenoApp} from "@terreno/api";
3
+ import {DateTime} from "luxon";
4
+ import mongoose from "mongoose";
5
+
6
+ import {createTraceId} from "../service/aiService";
7
+ import {authAsUser, ensureTestUsers, terrenoUserModel} from "../tests/helpers";
8
+ import {FEEDBACK_SIGNALS, recordTraceFeedback} from "./feedback";
9
+ import {LocalEvaluatorStore} from "./local/evaluatorStore";
10
+ import {createLocalObservabilityPlugin} from "./local/localPlugin";
11
+ import {registerObsEvaluator} from "./local/models/obsEvaluator";
12
+ import {registerObsPrompt} from "./local/models/obsPrompt";
13
+ import {registerObsPromptLabel} from "./local/models/obsPromptLabel";
14
+ import {registerObsPromptVersion} from "./local/models/obsPromptVersion";
15
+ import {registerObsReviewItem} from "./local/models/obsReviewItem";
16
+ import {registerObsScore} from "./local/models/obsScore";
17
+ import {registerObsTrace} from "./local/models/obsTrace";
18
+ import {LocalPromptStore} from "./local/promptStore";
19
+ import {LocalReviewStore} from "./local/reviewStore";
20
+ import type {LocalTraceStore} from "./local/traceStore";
21
+ import {getObservabilityApp, ObservabilityApp, resetObservabilityApp} from "./observabilityApp";
22
+
23
+ describe("recordTraceFeedback", () => {
24
+ let store: LocalTraceStore;
25
+ let ownerId: string;
26
+ let otherId: string;
27
+ let traceId: string;
28
+
29
+ beforeAll(async () => {
30
+ const users = await ensureTestUsers();
31
+ otherId = String(users[0]?._id);
32
+ ownerId = String(users[1]?._id);
33
+ });
34
+
35
+ afterEach(() => {
36
+ resetObservabilityApp();
37
+ });
38
+
39
+ beforeEach(async () => {
40
+ await registerObsTrace().deleteMany({});
41
+ await registerObsScore().collection.deleteMany({});
42
+ const plugin = createLocalObservabilityPlugin();
43
+ store = (plugin.traceSink as unknown as {store: LocalTraceStore}).store;
44
+ new ObservabilityApp({plugins: [plugin]});
45
+ traceId = createTraceId();
46
+ await store.exportTrace({
47
+ id: traceId,
48
+ name: "t",
49
+ prompts: [],
50
+ sensitive: false,
51
+ spans: [],
52
+ startedAt: DateTime.utc().toISO() ?? "",
53
+ status: "ok",
54
+ userId: ownerId,
55
+ });
56
+ });
57
+
58
+ it("exports the allowed signal names", () => {
59
+ expect([...FEEDBACK_SIGNALS]).toEqual([
60
+ "accepted",
61
+ "dismissed",
62
+ "copied",
63
+ "edited",
64
+ "regenerated",
65
+ ]);
66
+ });
67
+
68
+ it("returns the written thumbs_up score for the owner", async () => {
69
+ const result = await recordTraceFeedback({
70
+ input: {comment: " Too long ", thumbs: "down"},
71
+ traceId,
72
+ user: {id: ownerId},
73
+ });
74
+ expect(result.traceId).toBe(traceId);
75
+ expect(result.flaggedForDataset).toBe(false);
76
+ expect(result.scores).toHaveLength(1);
77
+ expect(result.scores[0]).toMatchObject({
78
+ comment: "Too long",
79
+ dataType: "boolean",
80
+ name: "thumbs_up",
81
+ source: "user-feedback",
82
+ traceId,
83
+ value: false,
84
+ });
85
+ });
86
+
87
+ it("thumbs up then down keeps one live score with the latest value", async () => {
88
+ await recordTraceFeedback({input: {thumbs: "up"}, traceId, user: {id: ownerId}});
89
+ await recordTraceFeedback({input: {thumbs: "down"}, traceId, user: {id: ownerId}});
90
+ const live = await registerObsScore().find({traceId});
91
+ expect(live).toHaveLength(1);
92
+ expect(live[0]?.value).toBe(false);
93
+ });
94
+
95
+ it("thumbs null returns no scores and soft-deletes the live one", async () => {
96
+ await recordTraceFeedback({input: {thumbs: "up"}, traceId, user: {id: ownerId}});
97
+ const result = await recordTraceFeedback({input: {thumbs: null}, traceId, user: {id: ownerId}});
98
+ expect(result.scores).toEqual([]);
99
+ expect(await registerObsScore().find({traceId})).toHaveLength(0);
100
+ });
101
+
102
+ it("rejects a non-owner with 403, even an admin", async () => {
103
+ await expect(
104
+ recordTraceFeedback({input: {thumbs: "up"}, traceId, user: {id: otherId}})
105
+ ).rejects.toMatchObject({status: 403});
106
+ await expect(
107
+ recordTraceFeedback({input: {thumbs: "up"}, traceId, user: {admin: true, id: otherId}})
108
+ ).rejects.toMatchObject({status: 403});
109
+ });
110
+
111
+ it("rejects an unknown trace with 404", async () => {
112
+ await expect(
113
+ recordTraceFeedback({
114
+ input: {thumbs: "up"},
115
+ traceId: new mongoose.Types.ObjectId().toString(),
116
+ user: {id: ownerId},
117
+ })
118
+ ).rejects.toMatchObject({status: 404});
119
+ });
120
+
121
+ it("rejects an empty input and a comment without thumbs with 400", async () => {
122
+ await expect(
123
+ recordTraceFeedback({input: {}, traceId, user: {id: ownerId}})
124
+ ).rejects.toMatchObject({status: 400});
125
+ await expect(
126
+ recordTraceFeedback({input: {comment: "x"}, traceId, user: {id: ownerId}})
127
+ ).rejects.toMatchObject({status: 400, title: "comment needs thumbs or outcome"});
128
+ });
129
+
130
+ it("rejects a comment over 2000 characters after trim with 400", async () => {
131
+ await expect(
132
+ recordTraceFeedback({
133
+ input: {comment: "x".repeat(2001), thumbs: "down"},
134
+ traceId,
135
+ user: {id: ownerId},
136
+ })
137
+ ).rejects.toMatchObject({status: 400});
138
+ });
139
+
140
+ const liveScores = async (): Promise<{[key: string]: unknown}[]> => {
141
+ return (await registerObsScore().find({traceId})).map(
142
+ (doc) => doc.toObject() as unknown as {[key: string]: unknown}
143
+ ); // cast: Mongoose doc type has no index signature
144
+ };
145
+
146
+ it("signal dismissed writes a boolean true user-signal score", async () => {
147
+ const result = await recordTraceFeedback({
148
+ input: {signal: "dismissed"},
149
+ traceId,
150
+ user: {id: ownerId},
151
+ });
152
+ expect(result.scores).toHaveLength(1);
153
+ expect(result.scores[0]).toMatchObject({
154
+ dataType: "boolean",
155
+ name: "dismissed",
156
+ source: "user-signal",
157
+ traceId,
158
+ value: true,
159
+ });
160
+ });
161
+
162
+ it("every allowed signal is accepted and named after itself", async () => {
163
+ for (const signal of FEEDBACK_SIGNALS) {
164
+ const result = await recordTraceFeedback({input: {signal}, traceId, user: {id: ownerId}});
165
+ expect(result.scores[0]).toMatchObject({name: signal, source: "user-signal", value: true});
166
+ }
167
+ expect(await liveScores()).toHaveLength(FEEDBACK_SIGNALS.length);
168
+ });
169
+
170
+ it("the same signal twice keeps one live score", async () => {
171
+ await recordTraceFeedback({input: {signal: "copied"}, traceId, user: {id: ownerId}});
172
+ await recordTraceFeedback({input: {signal: "copied"}, traceId, user: {id: ownerId}});
173
+ const live = await liveScores();
174
+ expect(live).toHaveLength(1);
175
+ expect(live[0]).toMatchObject({name: "copied", source: "user-signal"});
176
+ });
177
+
178
+ it("concurrent identical signals leave one live score", async () => {
179
+ await Promise.all([
180
+ recordTraceFeedback({input: {signal: "copied"}, traceId, user: {id: ownerId}}),
181
+ recordTraceFeedback({input: {signal: "copied"}, traceId, user: {id: ownerId}}),
182
+ ]);
183
+ expect(await liveScores()).toHaveLength(1);
184
+ });
185
+
186
+ it("different signals coexist as separate scores", async () => {
187
+ await recordTraceFeedback({input: {signal: "copied"}, traceId, user: {id: ownerId}});
188
+ await recordTraceFeedback({input: {signal: "accepted"}, traceId, user: {id: ownerId}});
189
+ expect(await liveScores()).toHaveLength(2);
190
+ });
191
+
192
+ it("an unknown signal is rejected with 400 and the title lists the allowed signals", async () => {
193
+ const error = await recordTraceFeedback({
194
+ input: {signal: "liked"} as never,
195
+ traceId,
196
+ user: {id: ownerId},
197
+ }).catch((caught: unknown) => caught as {status?: number; title?: string});
198
+ expect(error).toMatchObject({status: 400});
199
+ for (const signal of FEEDBACK_SIGNALS) {
200
+ expect((error as {title?: string}).title).toContain(signal);
201
+ }
202
+ expect(await liveScores()).toHaveLength(0);
203
+ });
204
+
205
+ it("outcome fp with a comment writes a categorical user-feedback score", async () => {
206
+ const result = await recordTraceFeedback({
207
+ input: {comment: " Not a crisis ", outcome: "fp"},
208
+ traceId,
209
+ user: {id: ownerId},
210
+ });
211
+ expect(result.scores).toHaveLength(1);
212
+ expect(result.scores[0]).toMatchObject({
213
+ comment: "Not a crisis",
214
+ dataType: "categorical",
215
+ name: "outcome",
216
+ source: "user-feedback",
217
+ traceId,
218
+ value: "fp",
219
+ });
220
+ });
221
+
222
+ it("outcome alone, without a comment, is valid for every outcome value", async () => {
223
+ for (const outcome of ["tp", "fp", "tn", "fn"] as const) {
224
+ const result = await recordTraceFeedback({input: {outcome}, traceId, user: {id: ownerId}});
225
+ expect(result.scores[0]).toMatchObject({name: "outcome", value: outcome});
226
+ }
227
+ });
228
+
229
+ it("a second outcome replaces the first", async () => {
230
+ await recordTraceFeedback({input: {outcome: "fp"}, traceId, user: {id: ownerId}});
231
+ await recordTraceFeedback({input: {outcome: "tp"}, traceId, user: {id: ownerId}});
232
+ const live = await liveScores();
233
+ expect(live).toHaveLength(1);
234
+ expect(live[0]?.value).toBe("tp");
235
+ });
236
+
237
+ it("an invalid outcome is rejected with 400 and keeps the earlier valid outcome", async () => {
238
+ await recordTraceFeedback({input: {outcome: "tp"}, traceId, user: {id: ownerId}});
239
+ await expect(
240
+ recordTraceFeedback({input: {outcome: "xx"} as never, traceId, user: {id: ownerId}})
241
+ ).rejects.toMatchObject({status: 400});
242
+ const live = await liveScores();
243
+ expect(live).toHaveLength(1);
244
+ expect(live[0]?.value).toBe("tp");
245
+ });
246
+
247
+ it("the ObsScore model accepts source user-signal", async () => {
248
+ const doc = new (registerObsScore())({
249
+ dataType: "boolean",
250
+ name: "dismissed",
251
+ source: "user-signal",
252
+ traceId,
253
+ value: true,
254
+ });
255
+ await expect(doc.validate()).resolves.toBeUndefined();
256
+ });
257
+
258
+ it("a comment with only a signal or only flagDataset is still 400", async () => {
259
+ await expect(
260
+ recordTraceFeedback({
261
+ input: {comment: "x", signal: "copied"},
262
+ traceId,
263
+ user: {id: ownerId},
264
+ })
265
+ ).rejects.toMatchObject({status: 400, title: "comment needs thumbs or outcome"});
266
+ await expect(
267
+ recordTraceFeedback({
268
+ input: {comment: "x", flagDataset: true},
269
+ traceId,
270
+ user: {id: ownerId},
271
+ })
272
+ ).rejects.toMatchObject({status: 400, title: "comment needs thumbs or outcome"});
273
+ });
274
+
275
+ it("flagDataset true sets the flag and false clears it", async () => {
276
+ const on = await recordTraceFeedback({
277
+ input: {flagDataset: true},
278
+ traceId,
279
+ user: {id: ownerId},
280
+ });
281
+ expect(on.flaggedForDataset).toBe(true);
282
+ expect(on.scores).toEqual([]);
283
+ expect((await registerObsTrace().findById(traceId))?.flaggedForDataset).toBe(true);
284
+
285
+ const off = await recordTraceFeedback({
286
+ input: {flagDataset: false},
287
+ traceId,
288
+ user: {id: ownerId},
289
+ });
290
+ expect(off.flaggedForDataset).toBe(false);
291
+ expect((await registerObsTrace().findById(traceId))?.flaggedForDataset).toBe(false);
292
+ });
293
+
294
+ it("a request without flagDataset leaves an existing flag in place", async () => {
295
+ await recordTraceFeedback({input: {flagDataset: true}, traceId, user: {id: ownerId}});
296
+ const result = await recordTraceFeedback({
297
+ input: {thumbs: "up"},
298
+ traceId,
299
+ user: {id: ownerId},
300
+ });
301
+ expect(result.flaggedForDataset).toBe(true);
302
+ expect((await registerObsTrace().findById(traceId))?.flaggedForDataset).toBe(true);
303
+ });
304
+
305
+ it("thumbs, signal, and flagDataset combine in one call", async () => {
306
+ const result = await recordTraceFeedback({
307
+ input: {flagDataset: true, signal: "regenerated", thumbs: "down"},
308
+ traceId,
309
+ user: {id: ownerId},
310
+ });
311
+ expect(result.flaggedForDataset).toBe(true);
312
+ expect(result.scores).toHaveLength(2);
313
+ expect(result.scores.map((score) => `${score.source}:${score.name}`).sort()).toEqual([
314
+ "user-feedback:thumbs_up",
315
+ "user-signal:regenerated",
316
+ ]);
317
+ expect(await liveScores()).toHaveLength(2);
318
+ });
319
+
320
+ it("a non-owner cannot send a signal or flag a trace", async () => {
321
+ await expect(
322
+ recordTraceFeedback({input: {signal: "copied"}, traceId, user: {id: otherId}})
323
+ ).rejects.toMatchObject({status: 403});
324
+ await expect(
325
+ recordTraceFeedback({input: {flagDataset: true}, traceId, user: {id: otherId}})
326
+ ).rejects.toMatchObject({status: 403});
327
+ expect((await registerObsTrace().findById(traceId))?.flaggedForDataset).toBeFalsy();
328
+ });
329
+ });
330
+
331
+ describe("thumbs down enqueues human review (feedbackReview)", () => {
332
+ let store: LocalTraceStore;
333
+ let ownerId: string;
334
+ let traceId: string;
335
+ let humanId: string;
336
+ let judgeId: string;
337
+ const reviewStore = new LocalReviewStore();
338
+
339
+ const reviewItems = async (): Promise<{[key: string]: unknown}[]> => {
340
+ // traceId is stringified so tests compare it with the string trace id.
341
+ return (await registerObsReviewItem().find({})).map((doc) => ({
342
+ ...(doc.toObject() as unknown as {[key: string]: unknown}),
343
+ traceId: String(doc.traceId),
344
+ }));
345
+ };
346
+
347
+ const startApp = (feedbackReview?: {evaluatorName: string}): void => {
348
+ resetObservabilityApp();
349
+ const plugin = createLocalObservabilityPlugin();
350
+ store = (plugin.traceSink as unknown as {store: LocalTraceStore}).store;
351
+ new ObservabilityApp({feedbackReview, plugins: [plugin]});
352
+ };
353
+
354
+ const exportTraceFor = async (id: string): Promise<void> => {
355
+ await store.exportTrace({
356
+ id,
357
+ name: "t",
358
+ prompts: [],
359
+ sensitive: false,
360
+ spans: [],
361
+ startedAt: DateTime.utc().toISO() ?? "",
362
+ status: "ok",
363
+ userId: ownerId,
364
+ });
365
+ };
366
+
367
+ beforeAll(async () => {
368
+ const users = await ensureTestUsers();
369
+ ownerId = String(users[1]?._id);
370
+ });
371
+
372
+ afterEach(() => {
373
+ resetObservabilityApp();
374
+ });
375
+
376
+ beforeEach(async () => {
377
+ await registerObsTrace().deleteMany({});
378
+ await registerObsScore().collection.deleteMany({});
379
+ await registerObsReviewItem().deleteMany({});
380
+ await registerObsEvaluator().deleteMany({});
381
+ await Promise.all([
382
+ registerObsPrompt().deleteMany({}),
383
+ registerObsPromptVersion().deleteMany({}),
384
+ registerObsPromptLabel().deleteMany({}),
385
+ ]);
386
+ const evaluators = new LocalEvaluatorStore();
387
+ humanId = (
388
+ await evaluators.create({
389
+ dimensions: [{dataType: "boolean", key: "correct", required: true}],
390
+ name: "feedback-human",
391
+ target: "full trace",
392
+ type: "human",
393
+ })
394
+ ).id;
395
+ const prompts = new LocalPromptStore();
396
+ await prompts.create({
397
+ folder: "evaluators",
398
+ name: "feedback-judge",
399
+ outputSchema: {
400
+ properties: {correct: {type: "boolean"}},
401
+ required: ["correct"],
402
+ type: "object",
403
+ },
404
+ system: "Judge it.",
405
+ type: "chat",
406
+ });
407
+ await prompts.moveLabel("feedback-judge", {label: "production", version: 1});
408
+ judgeId = (
409
+ await evaluators.create({
410
+ dimensions: [{dataType: "boolean", key: "correct", required: true}],
411
+ judgePromptName: "feedback-judge",
412
+ name: "feedback-judge-eval",
413
+ target: "full trace",
414
+ type: "llm-judge",
415
+ })
416
+ ).id;
417
+ startApp({evaluatorName: "feedback-human"});
418
+ traceId = createTraceId();
419
+ await exportTraceFor(traceId);
420
+ });
421
+
422
+ const down = (id = traceId): ReturnType<typeof recordTraceFeedback> =>
423
+ recordTraceFeedback({input: {thumbs: "down"}, traceId: id, user: {id: ownerId}});
424
+
425
+ it("exposes feedbackReview on the registry", () => {
426
+ expect(getObservabilityApp()?.feedbackReview).toEqual({evaluatorName: "feedback-human"});
427
+ startApp();
428
+ expect(getObservabilityApp()?.feedbackReview).toBeUndefined();
429
+ });
430
+
431
+ it("thumbs down creates one pending feedback item and returns its id", async () => {
432
+ const result = await down();
433
+ const items = await reviewItems();
434
+ expect(items).toHaveLength(1);
435
+ expect(items[0]).toMatchObject({reason: "feedback", status: "pending", traceId});
436
+ expect(String(items[0]?.evaluatorId)).toBe(humanId);
437
+ expect(result.reviewItemId).toBe(String(items[0]?._id));
438
+ });
439
+
440
+ it("thumbs down again returns the same id and adds no item", async () => {
441
+ const first = await down();
442
+ const second = await down();
443
+ expect(second.reviewItemId).toBe(first.reviewItemId);
444
+ expect(await reviewItems()).toHaveLength(1);
445
+ });
446
+
447
+ it("concurrent thumbs downs queue one item and both return its id", async () => {
448
+ const results = await Promise.all([down(), down(), down()]);
449
+ expect(await reviewItems()).toHaveLength(1);
450
+ const ids = new Set(results.map((result) => result.reviewItemId));
451
+ expect(ids.size).toBe(1);
452
+ expect([...ids][0]).toBeDefined();
453
+ });
454
+
455
+ it("down, up, down keeps one open item", async () => {
456
+ const first = await down();
457
+ await recordTraceFeedback({input: {thumbs: "up"}, traceId, user: {id: ownerId}});
458
+ const third = await down();
459
+ expect(third.reviewItemId).toBe(first.reviewItemId);
460
+ expect(await reviewItems()).toHaveLength(1);
461
+ });
462
+
463
+ it("an in_progress item counts as open", async () => {
464
+ const first = await down();
465
+ await reviewStore.assign(first.reviewItemId ?? "", new mongoose.Types.ObjectId().toString());
466
+ const again = await down();
467
+ expect(again.reviewItemId).toBe(first.reviewItemId);
468
+ expect(await reviewItems()).toHaveLength(1);
469
+ });
470
+
471
+ it("a done item does not block a second pending item", async () => {
472
+ const first = await down();
473
+ await reviewStore.submit({id: first.reviewItemId ?? "", scores: {correct: true}, sinks: []});
474
+ const second = await down();
475
+ expect(second.reviewItemId).toBeDefined();
476
+ expect(second.reviewItemId).not.toBe(first.reviewItemId);
477
+ const items = await reviewItems();
478
+ expect(items).toHaveLength(2);
479
+ expect(items.filter((item) => item.status === "pending")).toHaveLength(1);
480
+ });
481
+
482
+ it("a skipped item does not block a second pending item", async () => {
483
+ const first = await down();
484
+ await reviewStore.skip(first.reviewItemId ?? "");
485
+ const second = await down();
486
+ expect(second.reviewItemId).not.toBe(first.reviewItemId);
487
+ expect(await reviewItems()).toHaveLength(2);
488
+ });
489
+
490
+ it("an open item on another trace does not block this one", async () => {
491
+ const otherTraceId = createTraceId();
492
+ await exportTraceFor(otherTraceId);
493
+ const other = await down(otherTraceId);
494
+ const mine = await down();
495
+ expect(mine.reviewItemId).not.toBe(other.reviewItemId);
496
+ expect(await reviewItems()).toHaveLength(2);
497
+ });
498
+
499
+ it("thumbs down with a comment still enqueues", async () => {
500
+ const result = await recordTraceFeedback({
501
+ input: {comment: "wrong", thumbs: "down"},
502
+ traceId,
503
+ user: {id: ownerId},
504
+ });
505
+ expect(result.reviewItemId).toBeDefined();
506
+ expect(await reviewItems()).toHaveLength(1);
507
+ });
508
+
509
+ it("thumbs up, signals, and outcomes never enqueue", async () => {
510
+ const up = await recordTraceFeedback({input: {thumbs: "up"}, traceId, user: {id: ownerId}});
511
+ expect(up.reviewItemId).toBeUndefined();
512
+ for (const signal of FEEDBACK_SIGNALS) {
513
+ const result = await recordTraceFeedback({input: {signal}, traceId, user: {id: ownerId}});
514
+ expect(result.reviewItemId).toBeUndefined();
515
+ }
516
+ for (const outcome of ["tp", "fp", "tn", "fn"] as const) {
517
+ const result = await recordTraceFeedback({input: {outcome}, traceId, user: {id: ownerId}});
518
+ expect(result.reviewItemId).toBeUndefined();
519
+ }
520
+ const cleared = await recordTraceFeedback({
521
+ input: {thumbs: null},
522
+ traceId,
523
+ user: {id: ownerId},
524
+ });
525
+ expect(cleared.reviewItemId).toBeUndefined();
526
+ expect(await reviewItems()).toHaveLength(0);
527
+ });
528
+
529
+ it("a failing external ScoreSink is logged and the thumbs down is still scored and queued", async () => {
530
+ resetObservabilityApp();
531
+ const plugin = createLocalObservabilityPlugin();
532
+ store = (plugin.traceSink as unknown as {store: LocalTraceStore}).store;
533
+ new ObservabilityApp({
534
+ feedbackReview: {evaluatorName: "feedback-human"},
535
+ plugins: [
536
+ plugin,
537
+ {
538
+ capabilities: new Set(["scores"]),
539
+ id: "vendor",
540
+ scoreSink: {
541
+ export: async (): Promise<void> => {
542
+ throw new Error("vendor down");
543
+ },
544
+ },
545
+ },
546
+ ],
547
+ });
548
+ const errorSpy = spyOn(logger, "error").mockImplementation(() => logger);
549
+ try {
550
+ const result = await down();
551
+ expect(result.scores).toHaveLength(1);
552
+ expect(typeof result.reviewItemId).toBe("string");
553
+ expect(
554
+ errorSpy.mock.calls.some((call) => call[0] === "Feedback ScoreSink.export failed")
555
+ ).toBe(true);
556
+ } finally {
557
+ errorSpy.mockRestore();
558
+ }
559
+ expect(await registerObsScore().countDocuments({deleted: false, traceId})).toBe(1);
560
+ });
561
+
562
+ it("option unset enqueues nothing", async () => {
563
+ startApp();
564
+ const result = await down();
565
+ expect(result.reviewItemId).toBeUndefined();
566
+ expect(await reviewItems()).toHaveLength(0);
567
+ });
568
+
569
+ it("a missing evaluator logs an error and still returns the score without reviewItemId", async () => {
570
+ startApp({evaluatorName: "no-such-evaluator"});
571
+ const errorSpy = spyOn(logger, "error").mockImplementation(() => logger);
572
+ try {
573
+ const result = await down();
574
+ expect(result.reviewItemId).toBeUndefined();
575
+ expect(result.scores).toHaveLength(1);
576
+ expect(errorSpy).toHaveBeenCalled();
577
+ expect(await reviewItems()).toHaveLength(0);
578
+ } finally {
579
+ errorSpy.mockRestore();
580
+ }
581
+ });
582
+
583
+ it("an llm-judge evaluator logs an error and enqueues nothing", async () => {
584
+ startApp({evaluatorName: "feedback-judge-eval"});
585
+ const errorSpy = spyOn(logger, "error").mockImplementation(() => logger);
586
+ try {
587
+ const result = await down();
588
+ expect(judgeId).toBeDefined();
589
+ expect(result.reviewItemId).toBeUndefined();
590
+ expect(result.scores).toHaveLength(1);
591
+ expect(errorSpy).toHaveBeenCalled();
592
+ expect(await reviewItems()).toHaveLength(0);
593
+ } finally {
594
+ errorSpy.mockRestore();
595
+ }
596
+ });
597
+
598
+ it("a successful enqueue logs no error", async () => {
599
+ const errorSpy = spyOn(logger, "error").mockImplementation(() => logger);
600
+ try {
601
+ await down();
602
+ expect(errorSpy).not.toHaveBeenCalled();
603
+ } finally {
604
+ errorSpy.mockRestore();
605
+ }
606
+ });
607
+
608
+ it("a non-owner is refused with 403 and enqueues nothing", async () => {
609
+ const users = await ensureTestUsers();
610
+ await expect(
611
+ recordTraceFeedback({
612
+ input: {thumbs: "down"},
613
+ traceId,
614
+ user: {admin: true, id: String(users[0]?._id)},
615
+ })
616
+ ).rejects.toMatchObject({status: 403});
617
+ expect(await reviewItems()).toHaveLength(0);
618
+ });
619
+
620
+ describe("route and review queue", () => {
621
+ // The outer beforeEach already exports `traceId`; exporting it again here hit a duplicate _id.
622
+ // Users can be recreated by earlier suites in this file, so re-own the trace by the current
623
+ // "notAdmin" user that authAsUser signs in as.
624
+ beforeEach(async () => {
625
+ const users = await ensureTestUsers();
626
+ ownerId = String(users[1]?._id);
627
+ await registerObsTrace().updateOne({_id: traceId}, {$set: {userId: users[1]?._id}});
628
+ });
629
+
630
+ const buildApp = (feedbackReview?: {
631
+ evaluatorName: string;
632
+ }): ReturnType<TerrenoApp["build"]> => {
633
+ resetObservabilityApp();
634
+ const plugin = createLocalObservabilityPlugin();
635
+ store = (plugin.traceSink as unknown as {store: LocalTraceStore}).store;
636
+ return new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
637
+ .register(new ObservabilityApp({feedbackReview, plugins: [plugin]}))
638
+ .build();
639
+ };
640
+
641
+ it("POST thumbs down returns 200 with reviewItemId, repeat returns the same id", async () => {
642
+ const app = buildApp({evaluatorName: "feedback-human"});
643
+ const agent = await authAsUser(app, "notAdmin");
644
+ const path = `/ai/observability/traces/${traceId}/feedback`;
645
+ const first = await agent.post(path).send({thumbs: "down"});
646
+ expect(first.status).toBe(200);
647
+ expect(typeof first.body.data.reviewItemId).toBe("string");
648
+ const second = await agent.post(path).send({thumbs: "down"});
649
+ expect(second.status).toBe(200);
650
+ expect(second.body.data.reviewItemId).toBe(first.body.data.reviewItemId);
651
+ });
652
+
653
+ it("GET /review lists the item under Pending with reason feedback", async () => {
654
+ const app = buildApp({evaluatorName: "feedback-human"});
655
+ const owner = await authAsUser(app, "notAdmin");
656
+ const sent = await owner
657
+ .post(`/ai/observability/traces/${traceId}/feedback`)
658
+ .send({thumbs: "down"});
659
+ const admin = await authAsUser(app, "admin");
660
+ const listed = await admin.get("/ai/observability/review?status=pending");
661
+ expect(listed.status).toBe(200);
662
+ expect(listed.body.counts.pending).toBe(1);
663
+ expect(listed.body.data).toHaveLength(1);
664
+ expect(listed.body.data[0]).toMatchObject({
665
+ id: sent.body.data.reviewItemId,
666
+ reason: "feedback",
667
+ status: "pending",
668
+ traceId,
669
+ });
670
+ });
671
+
672
+ it("POST with a missing evaluator is 200 without reviewItemId", async () => {
673
+ const app = buildApp({evaluatorName: "no-such-evaluator"});
674
+ const errorSpy = spyOn(logger, "error").mockImplementation(() => logger);
675
+ try {
676
+ const agent = await authAsUser(app, "notAdmin");
677
+ const res = await agent
678
+ .post(`/ai/observability/traces/${traceId}/feedback`)
679
+ .send({thumbs: "down"});
680
+ expect(res.status).toBe(200);
681
+ expect(res.body.data.reviewItemId).toBeUndefined();
682
+ expect(errorSpy).toHaveBeenCalled();
683
+ } finally {
684
+ errorSpy.mockRestore();
685
+ }
686
+ });
687
+ });
688
+ });