@terreno/ai 58.0.6 → 58.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/CHANGELOG.md +52 -0
  2. package/dist/index.d.ts +2 -1
  3. package/dist/index.js +6 -2
  4. package/dist/models/gptHistory.js +4 -0
  5. package/dist/observability/feedback.d.ts +31 -0
  6. package/dist/observability/feedback.js +218 -0
  7. package/dist/observability/local/datasetStore.d.ts +7 -0
  8. package/dist/observability/local/datasetStore.js +48 -2
  9. package/dist/observability/local/models/obsReviewItem.js +6 -0
  10. package/dist/observability/local/models/obsScore.js +12 -1
  11. package/dist/observability/local/promptStore.d.ts +1 -0
  12. package/dist/observability/local/promptStore.js +1 -0
  13. package/dist/observability/local/reviewStore.d.ts +5 -0
  14. package/dist/observability/local/reviewStore.js +17 -0
  15. package/dist/observability/local/traceStore.d.ts +23 -0
  16. package/dist/observability/local/traceStore.js +121 -10
  17. package/dist/observability/observabilityApp.d.ts +1 -0
  18. package/dist/observability/observabilityApp.js +2 -0
  19. package/dist/observability/observabilityAppRegistry.d.ts +1 -0
  20. package/dist/observability/routes/traces.js +72 -0
  21. package/dist/observability/types.d.ts +8 -1
  22. package/dist/routes/gpt.js +42 -2
  23. package/dist/routes/gptHistories.js +42 -2
  24. package/dist/service/aiService.d.ts +4 -1
  25. package/dist/service/aiService.js +17 -2
  26. package/dist/service/chatTurn.js +10 -1
  27. package/dist/types/index.d.ts +4 -0
  28. package/dist/types/observability.d.ts +1 -1
  29. package/package.json +8 -8
  30. package/src/index.ts +8 -1
  31. package/src/models/gptHistory.ts +5 -0
  32. package/src/observability/feedback.test.ts +688 -0
  33. package/src/observability/feedback.ts +263 -0
  34. package/src/observability/local/datasetStore.ts +57 -2
  35. package/src/observability/local/models/obsReviewItem.ts +9 -0
  36. package/src/observability/local/models/obsScore.ts +18 -1
  37. package/src/observability/local/obsModels.test.ts +10 -2
  38. package/src/observability/local/promptStore.test.ts +2 -0
  39. package/src/observability/local/promptStore.ts +2 -0
  40. package/src/observability/local/reviewStore.test.ts +55 -0
  41. package/src/observability/local/reviewStore.ts +21 -0
  42. package/src/observability/local/traceStore.test.ts +239 -1
  43. package/src/observability/local/traceStore.ts +140 -9
  44. package/src/observability/observabilityApp.ts +2 -0
  45. package/src/observability/observabilityAppRegistry.ts +1 -0
  46. package/src/observability/routes/datasets.test.ts +253 -0
  47. package/src/observability/routes/prompts.test.ts +1 -0
  48. package/src/observability/routes/traces.test.ts +581 -3
  49. package/src/observability/routes/traces.ts +87 -1
  50. package/src/observability/types.ts +6 -1
  51. package/src/routes/gpt.test.ts +263 -1
  52. package/src/routes/gpt.ts +43 -2
  53. package/src/routes/gptHistories.ts +55 -2
  54. package/src/service/aiService.observability.test.ts +244 -0
  55. package/src/service/aiService.ts +15 -2
  56. package/src/service/chatTurn.ts +11 -2
  57. package/src/types/index.ts +4 -0
  58. package/src/types/observability.ts +1 -1
@@ -1,15 +1,29 @@
1
- import {afterEach, beforeAll, beforeEach, describe, expect, it} from "bun:test";
2
- import {TerrenoApp} from "@terreno/api";
1
+ import {afterEach, beforeAll, beforeEach, describe, expect, it, mock} from "bun:test";
2
+ import {
3
+ addAuthRoutes,
4
+ apiErrorMiddleware,
5
+ apiUnauthorizedMiddleware,
6
+ createAccess,
7
+ setupAuth,
8
+ TerrenoApp,
9
+ terrenoStatements,
10
+ type UserModel as UserModelType,
11
+ } from "@terreno/api";
12
+ import {expectAPIError, getBaseServer} from "@terreno/test";
3
13
  import type express from "express";
4
14
  import {DateTime} from "luxon";
15
+ import mongoose from "mongoose";
16
+ import request from "supertest";
17
+ import {createTraceId} from "../../service/aiService";
5
18
 
6
- import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
19
+ import {authAsUser, ensureTestUsers, terrenoUserModel, UserModel} from "../../tests/helpers";
7
20
  import {createLocalObservabilityPlugin} from "../local/localPlugin";
8
21
  import {registerObsScore} from "../local/models/obsScore";
9
22
  import {registerObsSpan} from "../local/models/obsSpan";
10
23
  import {registerObsTrace} from "../local/models/obsTrace";
11
24
  import {LocalTraceStore} from "../local/traceStore";
12
25
  import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
26
+ import type {ObservabilityPlugin, ScoreRecord} from "../types";
13
27
 
14
28
  describe("observability trace routes", () => {
15
29
  let app: express.Application;
@@ -241,3 +255,567 @@ describe("observability trace routes", () => {
241
255
  expect(missingFields.status).toBe(400);
242
256
  });
243
257
  });
258
+
259
+ describe("POST /ai/observability/traces/:id/feedback", () => {
260
+ let app: express.Application;
261
+ let store: LocalTraceStore;
262
+ let ownerId: string;
263
+ let adminId: string;
264
+ let traceId: string;
265
+ const feedbackPath = (id: string): string => `/ai/observability/traces/${id}/feedback`;
266
+
267
+ const exportTraceFor = async (userId: string | undefined): Promise<string> => {
268
+ const id = createTraceId();
269
+ await store.exportTrace({
270
+ id,
271
+ name: "example-summarize",
272
+ prompts: [{name: "example-summarize", version: 1}],
273
+ sensitive: false,
274
+ spans: [],
275
+ startedAt: DateTime.utc().toISO() ?? "",
276
+ status: "ok",
277
+ userId,
278
+ });
279
+ return id;
280
+ };
281
+
282
+ const liveScores = async (id: string): Promise<{[key: string]: unknown}[]> => {
283
+ return (await registerObsScore().find({traceId: id})).map(
284
+ (doc) => doc.toObject() as unknown as {[key: string]: unknown}
285
+ ); // cast: Mongoose doc type has no index signature
286
+ };
287
+
288
+ const buildApp = (extraPlugins: ObservabilityPlugin[] = []): express.Application => {
289
+ const plugin = createLocalObservabilityPlugin();
290
+ store =
291
+ plugin.traceSink && "store" in plugin.traceSink
292
+ ? (plugin.traceSink as {store: LocalTraceStore}).store
293
+ : new LocalTraceStore();
294
+ return new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
295
+ .register(new ObservabilityApp({plugins: [plugin, ...extraPlugins]}))
296
+ .build();
297
+ };
298
+
299
+ beforeAll(async () => {
300
+ const users = await ensureTestUsers();
301
+ adminId = String(users[0]?._id);
302
+ ownerId = String(users[1]?._id);
303
+ });
304
+
305
+ afterEach(() => {
306
+ resetObservabilityApp();
307
+ });
308
+
309
+ beforeEach(async () => {
310
+ await registerObsTrace().deleteMany({});
311
+ await registerObsSpan().deleteMany({});
312
+ await registerObsScore().collection.deleteMany({});
313
+ app = buildApp();
314
+ traceId = await exportTraceFor(ownerId);
315
+ });
316
+
317
+ it("owner thumbs down with a padded comment writes one trimmed user-feedback score", async () => {
318
+ const agent = await authAsUser(app, "notAdmin");
319
+ const res = await agent
320
+ .post(feedbackPath(traceId))
321
+ .send({comment: " Dropped one of my todos ", thumbs: "down"});
322
+
323
+ expect(res.status).toBe(200);
324
+ expect(res.body.data.traceId).toBe(traceId);
325
+ expect(res.body.data.flaggedForDataset).toBe(false);
326
+ expect(res.body.data.scores).toHaveLength(1);
327
+ const scores = await liveScores(traceId);
328
+ expect(scores).toHaveLength(1);
329
+ expect(scores[0]).toMatchObject({
330
+ comment: "Dropped one of my todos",
331
+ dataType: "boolean",
332
+ name: "thumbs_up",
333
+ source: "user-feedback",
334
+ value: false,
335
+ });
336
+ });
337
+
338
+ it("owner changing a vote replaces the score and drops the comment", async () => {
339
+ const agent = await authAsUser(app, "notAdmin");
340
+ await agent.post(feedbackPath(traceId)).send({comment: "bad", thumbs: "down"});
341
+ const res = await agent.post(feedbackPath(traceId)).send({thumbs: "up"});
342
+
343
+ expect(res.status).toBe(200);
344
+ const scores = await liveScores(traceId);
345
+ expect(scores).toHaveLength(1);
346
+ expect(scores[0]?.value).toBe(true);
347
+ expect(scores[0]?.comment).toBeUndefined();
348
+ });
349
+
350
+ it("owner thumbs null returns no scores and leaves no live thumbs_up", async () => {
351
+ const agent = await authAsUser(app, "notAdmin");
352
+ await agent.post(feedbackPath(traceId)).send({thumbs: "down"});
353
+ const res = await agent.post(feedbackPath(traceId)).send({thumbs: null});
354
+
355
+ expect(res.status).toBe(200);
356
+ expect(res.body.data.scores).toEqual([]);
357
+ expect(await liveScores(traceId)).toHaveLength(0);
358
+ // Soft-deleted, not hard-deleted.
359
+ expect(
360
+ await registerObsScore().collection.countDocuments({
361
+ deleted: true,
362
+ name: "thumbs_up",
363
+ traceId: new mongoose.Types.ObjectId(traceId),
364
+ })
365
+ ).toBe(1);
366
+ });
367
+
368
+ it("thumbs null with no prior vote is a 200 no-op", async () => {
369
+ const agent = await authAsUser(app, "notAdmin");
370
+ const res = await agent.post(feedbackPath(traceId)).send({thumbs: null});
371
+ expect(res.status).toBe(200);
372
+ expect(res.body.data.scores).toEqual([]);
373
+ });
374
+
375
+ it("a vote after deselecting creates a live score again", async () => {
376
+ const agent = await authAsUser(app, "notAdmin");
377
+ await agent.post(feedbackPath(traceId)).send({thumbs: "down"});
378
+ await agent.post(feedbackPath(traceId)).send({thumbs: null});
379
+ const res = await agent.post(feedbackPath(traceId)).send({thumbs: "up"});
380
+ expect(res.status).toBe(200);
381
+ const scores = await liveScores(traceId);
382
+ expect(scores).toHaveLength(1);
383
+ expect(scores[0]?.value).toBe(true);
384
+ });
385
+
386
+ it("sends each non-null write, comment included, to an extra ScoreSink", async () => {
387
+ resetObservabilityApp();
388
+ const exported = mock(async (_score: ScoreRecord): Promise<void> => {});
389
+ app = buildApp([
390
+ {
391
+ capabilities: new Set(["scores"]),
392
+ id: "extra",
393
+ scoreSink: {export: exported},
394
+ },
395
+ ]);
396
+ traceId = await exportTraceFor(ownerId);
397
+ const agent = await authAsUser(app, "notAdmin");
398
+
399
+ await agent.post(feedbackPath(traceId)).send({comment: "Too long", thumbs: "down"});
400
+ expect(exported).toHaveBeenCalledTimes(1);
401
+ expect(exported.mock.calls[0]?.[0]).toMatchObject({
402
+ comment: "Too long",
403
+ dataType: "boolean",
404
+ name: "thumbs_up",
405
+ source: "user-feedback",
406
+ traceId,
407
+ value: false,
408
+ });
409
+
410
+ await agent.post(feedbackPath(traceId)).send({thumbs: null});
411
+ expect(exported).toHaveBeenCalledTimes(1);
412
+ });
413
+
414
+ it("owner who is not an admin gets 200 with RBAC accessControl set", async () => {
415
+ resetObservabilityApp();
416
+ const typedUserModel = UserModel as unknown as UserModelType;
417
+ const accessControl = createAccess({
418
+ connection: mongoose.connection,
419
+ resolvePermissions: async () => ({}),
420
+ statements: terrenoStatements,
421
+ });
422
+ const plugin = createLocalObservabilityPlugin();
423
+ store = (plugin.traceSink as unknown as {store: LocalTraceStore}).store;
424
+ const rbacApp = getBaseServer();
425
+ setupAuth(rbacApp, typedUserModel);
426
+ addAuthRoutes(rbacApp, typedUserModel);
427
+ new ObservabilityApp({accessControl, plugins: [plugin]}).register(rbacApp);
428
+ rbacApp.use(apiUnauthorizedMiddleware);
429
+ rbacApp.use(apiErrorMiddleware);
430
+ traceId = await exportTraceFor(ownerId);
431
+
432
+ const agent = await authAsUser(rbacApp, "notAdmin");
433
+ const res = await agent.post(feedbackPath(traceId)).send({thumbs: "up"});
434
+ expect(res.status).toBe(200);
435
+ expect(await liveScores(traceId)).toHaveLength(1);
436
+ });
437
+
438
+ it("forbids another non-admin user", async () => {
439
+ const other = await exportTraceFor(adminId);
440
+ const agent = await authAsUser(app, "notAdmin");
441
+ const res = await agent.post(feedbackPath(other)).send({thumbs: "up"});
442
+ expectAPIError(res, {status: 403});
443
+ expect(await liveScores(other)).toHaveLength(0);
444
+ });
445
+
446
+ it("forbids an admin who is not the owner", async () => {
447
+ const agent = await authAsUser(app, "admin");
448
+ const res = await agent.post(feedbackPath(traceId)).send({thumbs: "down"});
449
+ expectAPIError(res, {status: 403});
450
+ expect(await liveScores(traceId)).toHaveLength(0);
451
+ });
452
+
453
+ it("returns 401 without a token", async () => {
454
+ const res = await request(app).post(feedbackPath(traceId)).send({thumbs: "up"});
455
+ expect(res.status).toBe(401);
456
+ });
457
+
458
+ it("returns 403 for a trace with no userId", async () => {
459
+ const anonymous = await exportTraceFor(undefined);
460
+ const agent = await authAsUser(app, "notAdmin");
461
+ const res = await agent.post(feedbackPath(anonymous)).send({thumbs: "up"});
462
+ expectAPIError(res, {status: 403});
463
+ });
464
+
465
+ it("writes exactly one ObsScore per thumbs write with only the local plugin", async () => {
466
+ const agent = await authAsUser(app, "notAdmin");
467
+ await agent.post(feedbackPath(traceId)).send({thumbs: "down"});
468
+ expect(await registerObsScore().collection.countDocuments({})).toBe(1);
469
+ });
470
+
471
+ it("two concurrent thumbs-down requests leave exactly one live thumbs_up score", async () => {
472
+ const agent = await authAsUser(app, "notAdmin");
473
+ const results = await Promise.all([
474
+ agent.post(feedbackPath(traceId)).send({thumbs: "down"}),
475
+ agent.post(feedbackPath(traceId)).send({thumbs: "down"}),
476
+ ]);
477
+ expect(results.map((r) => r.status)).toEqual([200, 200]);
478
+ const scores = await liveScores(traceId);
479
+ expect(scores.filter((s) => s.name === "thumbs_up")).toHaveLength(1);
480
+ expect(await registerObsScore().collection.countDocuments({})).toBe(1);
481
+ });
482
+
483
+ it("returns 404 for an unknown trace id and 400 for a malformed one", async () => {
484
+ const agent = await authAsUser(app, "notAdmin");
485
+ const missing = await agent
486
+ .post(feedbackPath(new mongoose.Types.ObjectId().toString()))
487
+ .send({thumbs: "up"});
488
+ expectAPIError(missing, {status: 404});
489
+ const malformed = await agent.post(feedbackPath("not-an-id")).send({thumbs: "up"});
490
+ expectAPIError(malformed, {status: 400});
491
+ });
492
+
493
+ it("rejects bad bodies with 400", async () => {
494
+ const agent = await authAsUser(app, "notAdmin");
495
+ expectAPIError(await agent.post(feedbackPath(traceId)).send({}), {status: 400});
496
+ expectAPIError(await agent.post(feedbackPath(traceId)).send({comment: "x"}), {
497
+ status: 400,
498
+ title: "comment needs thumbs or outcome",
499
+ });
500
+ expectAPIError(await agent.post(feedbackPath(traceId)).send({thumbs: "meh"}), {status: 400});
501
+ expectAPIError(
502
+ await agent.post(feedbackPath(traceId)).send({comment: "x".repeat(2001), thumbs: "down"}),
503
+ {status: 400}
504
+ );
505
+ expect(await liveScores(traceId)).toHaveLength(0);
506
+ });
507
+
508
+ it("accepts a 2000-character comment and rejects a whitespace-only comment", async () => {
509
+ const agent = await authAsUser(app, "notAdmin");
510
+ const ok = await agent
511
+ .post(feedbackPath(traceId))
512
+ .send({comment: "x".repeat(2000), thumbs: "down"});
513
+ expect(ok.status).toBe(200);
514
+ const blank = await agent.post(feedbackPath(traceId)).send({comment: " ", thumbs: "down"});
515
+ expect(blank.status).toBe(400);
516
+ });
517
+
518
+ it("signal dismissed returns and stores a boolean true user-signal score", async () => {
519
+ const agent = await authAsUser(app, "notAdmin");
520
+ const res = await agent.post(feedbackPath(traceId)).send({signal: "dismissed"});
521
+ expect(res.status).toBe(200);
522
+ expect(res.body.data.scores).toHaveLength(1);
523
+ expect(res.body.data.scores[0]).toMatchObject({
524
+ dataType: "boolean",
525
+ name: "dismissed",
526
+ source: "user-signal",
527
+ value: true,
528
+ });
529
+ const scores = await liveScores(traceId);
530
+ expect(scores).toHaveLength(1);
531
+ expect(scores[0]).toMatchObject({name: "dismissed", source: "user-signal", value: true});
532
+ });
533
+
534
+ it("signal copied twice leaves one live copied score", async () => {
535
+ const agent = await authAsUser(app, "notAdmin");
536
+ expect((await agent.post(feedbackPath(traceId)).send({signal: "copied"})).status).toBe(200);
537
+ expect((await agent.post(feedbackPath(traceId)).send({signal: "copied"})).status).toBe(200);
538
+ const scores = await liveScores(traceId);
539
+ expect(scores).toHaveLength(1);
540
+ expect(scores[0]).toMatchObject({name: "copied", source: "user-signal"});
541
+ });
542
+
543
+ it("an unknown signal returns 400 listing the allowed signals", async () => {
544
+ const agent = await authAsUser(app, "notAdmin");
545
+ const res = await agent.post(feedbackPath(traceId)).send({signal: "liked"});
546
+ expectAPIError(res, {status: 400});
547
+ for (const signal of ["accepted", "dismissed", "copied", "edited", "regenerated"]) {
548
+ expect(res.body.title).toContain(signal);
549
+ }
550
+ expect(await liveScores(traceId)).toHaveLength(0);
551
+ });
552
+
553
+ it("outcome fp with a comment writes a categorical outcome score with the comment", async () => {
554
+ const agent = await authAsUser(app, "notAdmin");
555
+ const res = await agent
556
+ .post(feedbackPath(traceId))
557
+ .send({comment: "Not a crisis", outcome: "fp"});
558
+ expect(res.status).toBe(200);
559
+ expect(res.body.data.scores[0]).toMatchObject({
560
+ comment: "Not a crisis",
561
+ dataType: "categorical",
562
+ name: "outcome",
563
+ source: "user-feedback",
564
+ value: "fp",
565
+ });
566
+ const scores = await liveScores(traceId);
567
+ expect(scores).toHaveLength(1);
568
+ expect(scores[0]).toMatchObject({comment: "Not a crisis", name: "outcome", value: "fp"});
569
+ });
570
+
571
+ it("outcome alone is valid and a later outcome replaces it", async () => {
572
+ const agent = await authAsUser(app, "notAdmin");
573
+ expect((await agent.post(feedbackPath(traceId)).send({outcome: "fp"})).status).toBe(200);
574
+ expect((await agent.post(feedbackPath(traceId)).send({outcome: "tn"})).status).toBe(200);
575
+ const scores = await liveScores(traceId);
576
+ expect(scores).toHaveLength(1);
577
+ expect(scores[0]).toMatchObject({name: "outcome", value: "tn"});
578
+ });
579
+
580
+ it("an invalid outcome returns 400 and keeps the earlier valid outcome", async () => {
581
+ const agent = await authAsUser(app, "notAdmin");
582
+ expect((await agent.post(feedbackPath(traceId)).send({outcome: "tp"})).status).toBe(200);
583
+ expectAPIError(await agent.post(feedbackPath(traceId)).send({outcome: "xx"}), {status: 400});
584
+ const scores = await liveScores(traceId);
585
+ expect(scores).toHaveLength(1);
586
+ expect(scores[0]).toMatchObject({name: "outcome", value: "tp"});
587
+ });
588
+
589
+ it("flagDataset true then false toggles the flag in the response and on the trace", async () => {
590
+ const agent = await authAsUser(app, "notAdmin");
591
+ const on = await agent.post(feedbackPath(traceId)).send({flagDataset: true});
592
+ expect(on.status).toBe(200);
593
+ expect(on.body.data.flaggedForDataset).toBe(true);
594
+ expect((await registerObsTrace().findById(traceId))?.flaggedForDataset).toBe(true);
595
+
596
+ const off = await agent.post(feedbackPath(traceId)).send({flagDataset: false});
597
+ expect(off.status).toBe(200);
598
+ expect(off.body.data.flaggedForDataset).toBe(false);
599
+ expect((await registerObsTrace().findById(traceId))?.flaggedForDataset).toBe(false);
600
+ expect(await liveScores(traceId)).toHaveLength(0);
601
+ });
602
+
603
+ it("thumbs, signal, and flagDataset combine in one 200", async () => {
604
+ const agent = await authAsUser(app, "notAdmin");
605
+ const res = await agent
606
+ .post(feedbackPath(traceId))
607
+ .send({flagDataset: true, signal: "regenerated", thumbs: "down"});
608
+ expect(res.status).toBe(200);
609
+ expect(res.body.data.flaggedForDataset).toBe(true);
610
+ expect(res.body.data.scores).toHaveLength(2);
611
+ const scores = await liveScores(traceId);
612
+ expect(scores.map((score) => `${score.source}:${score.name}`).sort()).toEqual([
613
+ "user-feedback:thumbs_up",
614
+ "user-signal:regenerated",
615
+ ]);
616
+ });
617
+
618
+ it("a stranger gets 403 for a signal and for flagDataset, and nothing is written", async () => {
619
+ const stranger = await exportTraceFor(adminId);
620
+ const agent = await authAsUser(app, "notAdmin");
621
+ expectAPIError(await agent.post(feedbackPath(stranger)).send({signal: "copied"}), {
622
+ status: 403,
623
+ });
624
+ expectAPIError(await agent.post(feedbackPath(stranger)).send({flagDataset: true}), {
625
+ status: 403,
626
+ });
627
+ expect(await liveScores(stranger)).toHaveLength(0);
628
+ expect((await registerObsTrace().findById(stranger))?.flaggedForDataset).toBeFalsy();
629
+ });
630
+
631
+ it("the admin scores route accepts source user-signal", async () => {
632
+ const admin = await authAsUser(app, "admin");
633
+ const res = await admin.post(`/ai/observability/traces/${traceId}/scores`).send({
634
+ dataType: "boolean",
635
+ name: "dismissed",
636
+ source: "user-signal",
637
+ value: true,
638
+ });
639
+ expect(res.status).toBe(201);
640
+ const scores = await liveScores(traceId);
641
+ expect(scores).toHaveLength(1);
642
+ expect(scores[0]).toMatchObject({name: "dismissed", source: "user-signal", value: true});
643
+ });
644
+
645
+ it("trace list hasScore matches a thumbs-scored trace and not a deselected one", async () => {
646
+ const second = await exportTraceFor(ownerId);
647
+ const agent = await authAsUser(app, "notAdmin");
648
+ await agent.post(feedbackPath(traceId)).send({thumbs: "down"});
649
+ await agent.post(feedbackPath(second)).send({thumbs: "down"});
650
+ await agent.post(feedbackPath(second)).send({thumbs: null});
651
+
652
+ const admin = await authAsUser(app, "admin");
653
+ const listed = await admin.get("/ai/observability/traces?hasScore=true");
654
+ expect(listed.status).toBe(200);
655
+ expect(listed.body.data.map((row: {id: string}) => row.id)).toEqual([traceId]);
656
+ });
657
+
658
+ it("trace detail shows the feedback score with its comment", async () => {
659
+ const agent = await authAsUser(app, "notAdmin");
660
+ await agent.post(feedbackPath(traceId)).send({comment: "Too long", thumbs: "down"});
661
+ const admin = await authAsUser(app, "admin");
662
+ const detail = await admin.get(`/ai/observability/traces/${traceId}`);
663
+ expect(detail.status).toBe(200);
664
+ expect(detail.body.data.scores).toHaveLength(1);
665
+ expect(detail.body.data.scores[0]).toMatchObject({
666
+ comment: "Too long",
667
+ name: "thumbs_up",
668
+ source: "user-feedback",
669
+ value: false,
670
+ });
671
+ });
672
+ });
673
+
674
+ describe("GET /ai/observability/traces feedback filters", () => {
675
+ let app: express.Application;
676
+ let store: LocalTraceStore;
677
+ let a: string;
678
+ let b: string;
679
+ let c: string;
680
+ let d: string;
681
+ const listPath = "/ai/observability/traces";
682
+
683
+ const addTrace = async (promptName: string): Promise<string> => {
684
+ const id = createTraceId();
685
+ await store.exportTrace({
686
+ id,
687
+ name: promptName,
688
+ prompts: [{name: promptName, version: 1}],
689
+ sensitive: false,
690
+ spans: [],
691
+ startedAt: DateTime.utc().toISO() ?? "",
692
+ status: "ok",
693
+ });
694
+ return id;
695
+ };
696
+
697
+ const rowIds = (res: {body: {data: {id: string}[]}}): string[] =>
698
+ res.body.data.map((row) => row.id).sort();
699
+
700
+ beforeAll(async () => {
701
+ await ensureTestUsers();
702
+ });
703
+
704
+ afterEach(() => {
705
+ resetObservabilityApp();
706
+ });
707
+
708
+ beforeEach(async () => {
709
+ await registerObsTrace().deleteMany({});
710
+ await registerObsSpan().deleteMany({});
711
+ await registerObsScore().collection.deleteMany({});
712
+ const plugin = createLocalObservabilityPlugin();
713
+ store =
714
+ plugin.traceSink && "store" in plugin.traceSink
715
+ ? (plugin.traceSink as {store: LocalTraceStore}).store
716
+ : new LocalTraceStore();
717
+ app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
718
+ .register(new ObservabilityApp({plugins: [plugin]}))
719
+ .build();
720
+ a = await addTrace("example-summarize");
721
+ b = await addTrace("other");
722
+ c = await addTrace("other");
723
+ d = await addTrace("other");
724
+ await new LocalTraceStore().exportScore({
725
+ comment: "too long",
726
+ dataType: "boolean",
727
+ name: "thumbs_up",
728
+ source: "user-feedback",
729
+ traceId: a,
730
+ value: false,
731
+ });
732
+ await new LocalTraceStore().exportScore({
733
+ dataType: "boolean",
734
+ name: "thumbs_up",
735
+ source: "user-feedback",
736
+ traceId: b,
737
+ value: true,
738
+ });
739
+ await new LocalTraceStore().exportScore({
740
+ dataType: "boolean",
741
+ name: "dismissed",
742
+ source: "user-signal",
743
+ traceId: c,
744
+ value: true,
745
+ });
746
+ });
747
+
748
+ it("thumbs=down returns only the thumbs-down trace", async () => {
749
+ const agent = await authAsUser(app, "admin");
750
+ const res = await agent.get(`${listPath}?thumbs=down`);
751
+ expect(res.status).toBe(200);
752
+ expect(rowIds(res)).toEqual([a]);
753
+ expect(res.body.total).toBe(1);
754
+ });
755
+
756
+ it("thumbs=up returns only the thumbs-up trace", async () => {
757
+ const agent = await authAsUser(app, "admin");
758
+ expect(rowIds(await agent.get(`${listPath}?thumbs=up`))).toEqual([b]);
759
+ });
760
+
761
+ it("signal=dismissed returns only the dismissed trace", async () => {
762
+ const agent = await authAsUser(app, "admin");
763
+ expect(rowIds(await agent.get(`${listPath}?signal=dismissed`))).toEqual([c]);
764
+ });
765
+
766
+ it("hasComment=true returns only the commented trace", async () => {
767
+ const agent = await authAsUser(app, "admin");
768
+ expect(rowIds(await agent.get(`${listPath}?hasComment=true`))).toEqual([a]);
769
+ });
770
+
771
+ it("hasComment=false returns every trace without a comment", async () => {
772
+ const agent = await authAsUser(app, "admin");
773
+ const res = await agent.get(`${listPath}?hasComment=false`);
774
+ expect(res.status).toBe(200);
775
+ expect(rowIds(res)).toEqual([b, c, d].sort());
776
+ });
777
+
778
+ it("thumbs=down combines with prompt", async () => {
779
+ const agent = await authAsUser(app, "admin");
780
+ expect(rowIds(await agent.get(`${listPath}?thumbs=down&prompt=example-summarize`))).toEqual([
781
+ a,
782
+ ]);
783
+ expect(rowIds(await agent.get(`${listPath}?thumbs=down&prompt=other`))).toEqual([]);
784
+ });
785
+
786
+ it("a deselected thumbs score does not match thumbs=down", async () => {
787
+ await new LocalTraceStore().deleteFeedbackScore({
788
+ name: "thumbs_up",
789
+ source: "user-feedback",
790
+ traceId: a,
791
+ });
792
+ const agent = await authAsUser(app, "admin");
793
+ const res = await agent.get(`${listPath}?thumbs=down`);
794
+ expect(res.status).toBe(200);
795
+ expect(rowIds(res)).toEqual([]);
796
+ });
797
+
798
+ it("an invalid thumbs value returns 400", async () => {
799
+ const agent = await authAsUser(app, "admin");
800
+ expectAPIError(await agent.get(`${listPath}?thumbs=sideways`), {status: 400});
801
+ });
802
+
803
+ it("an unknown signal returns 400", async () => {
804
+ const agent = await authAsUser(app, "admin");
805
+ expectAPIError(await agent.get(`${listPath}?signal=liked`), {status: 400});
806
+ });
807
+
808
+ it("an invalid hasComment value returns 400", async () => {
809
+ const agent = await authAsUser(app, "admin");
810
+ expectAPIError(await agent.get(`${listPath}?hasComment=maybe`), {status: 400});
811
+ });
812
+
813
+ it("a non-admin gets 403 on a filtered list", async () => {
814
+ const agent = await authAsUser(app, "notAdmin");
815
+ expectAPIError(await agent.get(`${listPath}?thumbs=down`), {status: 403});
816
+ });
817
+
818
+ it("a request without a token gets 401 on a filtered list", async () => {
819
+ expectAPIError(await request(app).get(`${listPath}?thumbs=down`), {status: 401});
820
+ });
821
+ });