@terreno/ai 58.0.5 → 58.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/CHANGELOG.md +58 -0
  2. package/dist/index.d.ts +2 -1
  3. package/dist/index.js +6 -2
  4. package/dist/models/gptHistory.js +4 -0
  5. package/dist/observability/feedback.d.ts +31 -0
  6. package/dist/observability/feedback.js +218 -0
  7. package/dist/observability/local/datasetStore.d.ts +7 -0
  8. package/dist/observability/local/datasetStore.js +48 -2
  9. package/dist/observability/local/models/obsReviewItem.js +6 -0
  10. package/dist/observability/local/models/obsScore.js +12 -1
  11. package/dist/observability/local/promptStore.d.ts +1 -0
  12. package/dist/observability/local/promptStore.js +1 -0
  13. package/dist/observability/local/reviewStore.d.ts +5 -0
  14. package/dist/observability/local/reviewStore.js +17 -0
  15. package/dist/observability/local/traceStore.d.ts +23 -0
  16. package/dist/observability/local/traceStore.js +121 -10
  17. package/dist/observability/observabilityApp.d.ts +1 -0
  18. package/dist/observability/observabilityApp.js +2 -0
  19. package/dist/observability/observabilityAppRegistry.d.ts +1 -0
  20. package/dist/observability/routes/traces.js +72 -0
  21. package/dist/observability/types.d.ts +8 -1
  22. package/dist/routes/gpt.js +42 -2
  23. package/dist/routes/gptHistories.js +42 -2
  24. package/dist/service/aiService.d.ts +4 -1
  25. package/dist/service/aiService.js +17 -2
  26. package/dist/service/chatTurn.js +10 -1
  27. package/dist/types/index.d.ts +4 -0
  28. package/dist/types/observability.d.ts +1 -1
  29. package/package.json +9 -9
  30. package/src/index.ts +8 -1
  31. package/src/models/gptHistory.ts +5 -0
  32. package/src/observability/feedback.test.ts +688 -0
  33. package/src/observability/feedback.ts +263 -0
  34. package/src/observability/local/datasetStore.ts +57 -2
  35. package/src/observability/local/models/obsReviewItem.ts +9 -0
  36. package/src/observability/local/models/obsScore.ts +18 -1
  37. package/src/observability/local/obsModels.test.ts +10 -2
  38. package/src/observability/local/promptStore.test.ts +2 -0
  39. package/src/observability/local/promptStore.ts +2 -0
  40. package/src/observability/local/reviewStore.test.ts +55 -0
  41. package/src/observability/local/reviewStore.ts +21 -0
  42. package/src/observability/local/traceStore.test.ts +239 -1
  43. package/src/observability/local/traceStore.ts +140 -9
  44. package/src/observability/observabilityApp.ts +2 -0
  45. package/src/observability/observabilityAppRegistry.ts +1 -0
  46. package/src/observability/routes/datasets.test.ts +253 -0
  47. package/src/observability/routes/prompts.test.ts +1 -0
  48. package/src/observability/routes/traces.test.ts +581 -3
  49. package/src/observability/routes/traces.ts +87 -1
  50. package/src/observability/types.ts +6 -1
  51. package/src/routes/gpt.test.ts +263 -1
  52. package/src/routes/gpt.ts +43 -2
  53. package/src/routes/gptHistories.ts +55 -2
  54. package/src/service/aiService.observability.test.ts +244 -0
  55. package/src/service/aiService.ts +15 -2
  56. package/src/service/chatTurn.ts +11 -2
  57. package/src/types/index.ts +4 -0
  58. package/src/types/observability.ts +1 -1
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
3
3
  return (mod && mod.__esModule) ? mod : { "default": mod };
4
4
  };
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.MemoryScoreSink = exports.MemoryTraceSink = exports.LocalScoreSink = exports.LocalTraceSink = exports.LocalTraceStore = exports.parseTraceListSort = exports.DEFAULT_TRACE_LIST_SORT = exports.TRACE_LIST_SORT_KEYS = void 0;
6
+ exports.MemoryScoreSink = exports.MemoryTraceSink = exports.LocalScoreSink = exports.LocalTraceSink = exports.LocalTraceStore = exports.resetObsScoreIndexBuild = exports.parseTraceListSort = exports.DEFAULT_TRACE_LIST_SORT = exports.TRACE_LIST_SORT_KEYS = void 0;
7
7
  const api_1 = require("@terreno/api");
8
8
  const luxon_1 = require("luxon");
9
9
  const mongoose_1 = __importDefault(require("mongoose"));
@@ -113,7 +113,11 @@ const countLookupStages = (from) => {
113
113
  as: "_sortCounts",
114
114
  from,
115
115
  let: { traceId: "$_id" },
116
- pipeline: [{ $match: { $expr: { $eq: ["$traceId", "$$traceId"] } } }, { $count: "n" }],
116
+ pipeline: [
117
+ // Soft-deleted rows (e.g. a cleared thumbs vote) do not count.
118
+ { $match: { $expr: { $eq: ["$traceId", "$$traceId"] }, deleted: { $ne: true } } },
119
+ { $count: "n" },
120
+ ],
117
121
  },
118
122
  },
119
123
  { $addFields: { _sortValue: { $ifNull: [{ $arrayElemAt: ["$_sortCounts.n", 0] }, 0] } } },
@@ -189,7 +193,7 @@ const countByTraceId = async (model, ids) => {
189
193
  return counts;
190
194
  }
191
195
  const grouped = (await model.aggregate([
192
- { $match: { traceId: { $in: ids } } },
196
+ { $match: { deleted: { $ne: true }, traceId: { $in: ids } } },
193
197
  { $group: { _id: "$traceId", n: { $sum: 1 } } },
194
198
  ]));
195
199
  for (const row of grouped) {
@@ -197,12 +201,45 @@ const countByTraceId = async (model, ids) => {
197
201
  }
198
202
  return counts;
199
203
  };
204
+ /** Score sources written by end users; the local store keeps one live score per name. */
205
+ const USER_SCORE_SOURCES = new Set(["user-feedback", "user-signal"]);
206
+ let obsScoreIndexes;
207
+ /** Builds ObsScore indexes once, so the unique user-score index exists before the first upsert. */
208
+ const ensureObsScoreIndexes = () => {
209
+ // A database that already holds duplicate live end-user scores (written through the admin
210
+ // scores route before these indexes existed) cannot build them. Feedback must keep working:
211
+ // log once and upsert without the uniqueness guarantee until the duplicates are removed.
212
+ obsScoreIndexes ??= (0, obsScore_1.registerObsScore)()
213
+ .init()
214
+ .catch((error) => {
215
+ // Only existing duplicates are tolerated; any other failure retries on the next write.
216
+ if (error.code !== 11000) {
217
+ obsScoreIndexes = undefined;
218
+ throw error;
219
+ }
220
+ api_1.logger.error("ObsScore indexes failed to build; remove duplicate live user-feedback/user-signal scores per trace and name", { error });
221
+ });
222
+ return obsScoreIndexes;
223
+ };
224
+ /**
225
+ * Forgets the index build so a test can simulate a failed build.
226
+ * @internal Test seam. Production builds the indexes once per process.
227
+ */
228
+ const resetObsScoreIndexBuild = () => {
229
+ obsScoreIndexes = undefined;
230
+ };
231
+ exports.resetObsScoreIndexBuild = resetObsScoreIndexBuild;
200
232
  class LocalTraceStore {
201
233
  async exportTrace(trace) {
202
234
  const ObsTrace = (0, obsTrace_1.registerObsTrace)();
203
235
  const ObsSpan = (0, obsSpan_1.registerObsSpan)();
204
236
  const userId = objectIdOrUndefined(trace.userId);
237
+ // Only a 24-hex id is a caller-chosen trace id; other ids (uuids from older callers) are ignored.
238
+ const traceId = /^[0-9a-f]{24}$/i.test(trace.id ?? "")
239
+ ? new mongoose_1.default.Types.ObjectId(trace.id)
240
+ : undefined;
205
241
  const created = await ObsTrace.create({
242
+ ...(traceId ? { _id: traceId } : {}),
206
243
  endedAt: parseDate(trace.endedAt),
207
244
  errorSummary: firstErrorSummary(trace),
208
245
  flaggedForDataset: trace.flaggedForDataset ?? false,
@@ -250,6 +287,10 @@ class LocalTraceStore {
250
287
  }
251
288
  const spanId = objectIdOrUndefined(score.spanId);
252
289
  const evaluatorId = objectIdOrUndefined(score.evaluatorId);
290
+ if (USER_SCORE_SOURCES.has(score.source)) {
291
+ await this.upsertUserScore({ ...score, traceId: String(traceId) });
292
+ return;
293
+ }
253
294
  await (0, obsScore_1.registerObsScore)().create({
254
295
  comment: score.comment,
255
296
  confidence: score.confidence,
@@ -262,6 +303,57 @@ class LocalTraceStore {
262
303
  ...(evaluatorId ? { evaluatorId } : {}),
263
304
  });
264
305
  }
306
+ /**
307
+ * Keeps one live score per (trace, source, name) for end-user sources, so a changed vote
308
+ * replaces the old one. One atomic upsert backed by a partial unique index, so concurrent
309
+ * taps cannot leave two live scores.
310
+ */
311
+ async upsertUserScore(score) {
312
+ const ObsScore = (0, obsScore_1.registerObsScore)();
313
+ await ensureObsScoreIndexes();
314
+ const filter = {
315
+ deleted: false,
316
+ name: score.name,
317
+ source: score.source,
318
+ traceId: new mongoose_1.default.Types.ObjectId(score.traceId),
319
+ };
320
+ const set = {
321
+ dataType: score.dataType,
322
+ updated: luxon_1.DateTime.utc().toJSDate(),
323
+ value: score.value,
324
+ };
325
+ const unset = {};
326
+ if (score.comment) {
327
+ set.comment = score.comment;
328
+ }
329
+ else {
330
+ unset.comment = "";
331
+ }
332
+ const update = {
333
+ $set: set,
334
+ $setOnInsert: { created: luxon_1.DateTime.utc().toJSDate() },
335
+ ...(Object.keys(unset).length > 0 ? { $unset: unset } : {}),
336
+ };
337
+ try {
338
+ await ObsScore.collection.updateOne(filter, update, { upsert: true });
339
+ }
340
+ catch (error) {
341
+ // A concurrent upsert may insert first; the retry then matches and updates that row.
342
+ if (error.code !== 11000) {
343
+ throw error;
344
+ }
345
+ await ObsScore.collection.updateOne(filter, update, { upsert: true });
346
+ }
347
+ }
348
+ /** Soft-deletes the live end-user score for (trace, source, name), e.g. a cleared thumbs vote. */
349
+ async deleteFeedbackScore(params) {
350
+ await (0, obsScore_1.registerObsScore)().collection.updateMany({
351
+ deleted: false,
352
+ name: params.name,
353
+ source: params.source,
354
+ traceId: new mongoose_1.default.Types.ObjectId(params.traceId),
355
+ }, { $set: { deleted: true, updated: luxon_1.DateTime.utc().toJSDate() } });
356
+ }
265
357
  async list(query) {
266
358
  const page = query.page && query.page > 0 ? query.page : 1;
267
359
  const limit = query.limit && query.limit > 0 ? Math.min(query.limit, 100) : 20;
@@ -323,14 +415,33 @@ class LocalTraceStore {
323
415
  ...(to ? { $lte: to } : {}),
324
416
  };
325
417
  }
418
+ // Score-based filters each narrow the trace ids; `$and` lets several apply at once.
419
+ const idConstraints = [];
420
+ const scoredTraceIds = (scoreFilter) => {
421
+ return (0, obsScore_1.registerObsScore)().distinct("traceId", { deleted: { $ne: true }, ...scoreFilter });
422
+ };
326
423
  if (query.hasScore !== undefined) {
327
- const scoredIds = await (0, obsScore_1.registerObsScore)().distinct("traceId");
328
- if (query.hasScore) {
329
- filter._id = { $in: scoredIds };
330
- }
331
- else {
332
- filter._id = { $nin: scoredIds };
333
- }
424
+ const scoredIds = await scoredTraceIds({});
425
+ idConstraints.push({ _id: query.hasScore ? { $in: scoredIds } : { $nin: scoredIds } });
426
+ }
427
+ if (query.thumbs) {
428
+ const ids = await scoredTraceIds({
429
+ name: "thumbs_up",
430
+ source: "user-feedback",
431
+ value: query.thumbs === "up",
432
+ });
433
+ idConstraints.push({ _id: { $in: ids } });
434
+ }
435
+ if (query.signal) {
436
+ const ids = await scoredTraceIds({ name: query.signal, source: "user-signal" });
437
+ idConstraints.push({ _id: { $in: ids } });
438
+ }
439
+ if (query.hasComment !== undefined) {
440
+ const ids = await scoredTraceIds({ comment: { $nin: [null, ""] }, source: "user-feedback" });
441
+ idConstraints.push({ _id: query.hasComment ? { $in: ids } : { $nin: ids } });
442
+ }
443
+ if (idConstraints.length > 0) {
444
+ filter.$and = [...(filter.$and ?? []), ...idConstraints];
334
445
  }
335
446
  const ObsTrace = (0, obsTrace_1.registerObsTrace)();
336
447
  const total = await ObsTrace.countDocuments(filter);
@@ -13,6 +13,7 @@ export declare class ObservabilityApp implements TerrenoPlugin {
13
13
  readonly priceMap: ObservabilityAppOptions["priceMap"];
14
14
  readonly requestAiServiceFactory?: ObservabilityRequestAiServiceFactory;
15
15
  readonly sampleRate: number;
16
+ readonly feedbackReview?: ObservabilityAppOptions["feedbackReview"];
16
17
  constructor(options: ObservabilityAppOptions);
17
18
  get promptRegistry(): PromptRegistry | undefined;
18
19
  get scoreSinks(): ScoreSink[];
@@ -33,6 +33,7 @@ class ObservabilityApp {
33
33
  priceMap;
34
34
  requestAiServiceFactory;
35
35
  sampleRate;
36
+ feedbackReview;
36
37
  constructor(options) {
37
38
  this.accessControl = options.accessControl;
38
39
  this.aiService = options.aiService;
@@ -42,6 +43,7 @@ class ObservabilityApp {
42
43
  this.priceMap = options.priceMap;
43
44
  this.requestAiServiceFactory = options.requestAiServiceFactory;
44
45
  this.sampleRate = options.sampleRate ?? 0;
46
+ this.feedbackReview = options.feedbackReview;
45
47
  (0, observabilityAppRegistry_1.registerObservabilityApp)(this);
46
48
  }
47
49
  get promptRegistry() {
@@ -2,6 +2,7 @@ import type { ObservabilityAppOptions, ObservabilityControlConfig, Observability
2
2
  export interface RegisteredObservabilityApp {
3
3
  aiService?: ObservabilityGenerateClient;
4
4
  control: ObservabilityControlConfig;
5
+ feedbackReview?: ObservabilityAppOptions["feedbackReview"];
5
6
  plugins: ReadonlyArray<ObservabilityPlugin>;
6
7
  priceMap?: ObservabilityAppOptions["priceMap"];
7
8
  promptRegistry?: PromptRegistry;
@@ -2,6 +2,7 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.addObservabilityTraceRoutes = void 0;
4
4
  const api_1 = require("@terreno/api");
5
+ const feedback_1 = require("../feedback");
5
6
  const traceStore_1 = require("../local/traceStore");
6
7
  const observabilityAppRegistry_1 = require("../observabilityAppRegistry");
7
8
  const observabilityRouteAccess_1 = require("./observabilityRouteAccess");
@@ -54,6 +55,27 @@ const parseSort = (value) => {
54
55
  (0, traceStore_1.parseTraceListSort)(value);
55
56
  return value;
56
57
  };
58
+ const parseThumbs = (value) => {
59
+ if (value === undefined) {
60
+ return undefined;
61
+ }
62
+ if (value === "up" || value === "down") {
63
+ return value;
64
+ }
65
+ throw new api_1.APIError({ status: 400, title: "thumbs must be up or down" });
66
+ };
67
+ const parseSignal = (value) => {
68
+ if (value === undefined) {
69
+ return undefined;
70
+ }
71
+ if (typeof value === "string" && feedback_1.FEEDBACK_SIGNALS.includes(value)) {
72
+ return value;
73
+ }
74
+ throw new api_1.APIError({
75
+ status: 400,
76
+ title: `signal must be one of ${feedback_1.FEEDBACK_SIGNALS.join(", ")}`,
77
+ });
78
+ };
57
79
  const parsePositiveInt = (value, fallback) => {
58
80
  if (value === undefined) {
59
81
  return fallback;
@@ -75,6 +97,9 @@ const addObservabilityTraceRoutes = (router, options) => {
75
97
  .withQueryParameter("prompt", { type: "string" }, { required: false })
76
98
  .withQueryParameter("promptVersion", { type: "number" }, { required: false })
77
99
  .withQueryParameter("scope", { type: "string" }, { required: false })
100
+ .withQueryParameter("thumbs", { enum: ["up", "down"], type: "string" }, { required: false })
101
+ .withQueryParameter("signal", { enum: [...feedback_1.FEEDBACK_SIGNALS], type: "string" }, { required: false })
102
+ .withQueryParameter("hasComment", { type: "boolean" }, { required: false })
78
103
  .withQueryParameter("sort", {
79
104
  enum: traceStore_1.TRACE_LIST_SORT_KEYS.flatMap((key) => [key, `-${key}`]),
80
105
  type: "string",
@@ -88,6 +113,7 @@ const addObservabilityTraceRoutes = (router, options) => {
88
113
  const listed = await options.store.list({
89
114
  flaggedForDataset: parseBoolean(req.query.flaggedForDataset),
90
115
  from: typeof req.query.from === "string" ? req.query.from : undefined,
116
+ hasComment: parseBoolean(req.query.hasComment),
91
117
  hasScore: parseBoolean(req.query.hasScore),
92
118
  limit: parsePositiveInt(req.query.limit, 20),
93
119
  page: parsePositiveInt(req.query.page, 1),
@@ -96,8 +122,10 @@ const addObservabilityTraceRoutes = (router, options) => {
96
122
  scope: typeof req.query.scope === "string" ? req.query.scope : undefined,
97
123
  sensitive: parseBoolean(req.query.sensitive),
98
124
  sessionId: typeof req.query.sessionId === "string" ? req.query.sessionId : undefined,
125
+ signal: parseSignal(req.query.signal),
99
126
  sort: parseSort(req.query.sort),
100
127
  status: req.query.status === "ok" || req.query.status === "error" ? req.query.status : undefined,
128
+ thumbs: parseThumbs(req.query.thumbs),
101
129
  to: typeof req.query.to === "string" ? req.query.to : undefined,
102
130
  userId: typeof req.query.userId === "string" ? req.query.userId : undefined,
103
131
  });
@@ -119,6 +147,50 @@ const addObservabilityTraceRoutes = (router, options) => {
119
147
  const data = await options.store.getDetail(req.params.id);
120
148
  return res.json({ data });
121
149
  }));
150
+ // End-user feedback: authenticated and owner-only, so it skips the admin observability
151
+ // middleware every other route here uses.
152
+ router.post(`${BASE_PATH}/traces/:id/feedback`, (0, api_1.authenticateMiddleware)(), builder()
153
+ .withTags(["observability"])
154
+ .withSummary("Record the trace owner's feedback on an AI output")
155
+ .withDescription("Thumbs (with an optional comment), outcome class, a passive signal, or flag-for-dataset. " +
156
+ "Only the trace's own user may call it.")
157
+ .withPathParameter("id", { type: "string" })
158
+ .withRequestBody({
159
+ comment: {
160
+ description: "Free-text comment attached to the thumbs or outcome score (≤ 2000 chars)",
161
+ type: "string",
162
+ },
163
+ flagDataset: { description: "Flag the trace as a dataset candidate", type: "boolean" },
164
+ outcome: { enum: ["tp", "fp", "tn", "fn"], type: "string" },
165
+ signal: { enum: [...feedback_1.FEEDBACK_SIGNALS], type: "string" },
166
+ thumbs: {
167
+ // `null` clears the vote. The request validator checks enum membership even for a
168
+ // nullable field, so null must be listed (the builder's types only allow strings).
169
+ enum: ["up", "down", null],
170
+ nullable: true,
171
+ type: "string",
172
+ },
173
+ })
174
+ .withResponse(200, {
175
+ data: {
176
+ properties: {
177
+ flaggedForDataset: { type: "boolean" },
178
+ reviewItemId: { type: "string" },
179
+ scores: { items: { type: "object" }, type: "array" },
180
+ traceId: { type: "string" },
181
+ },
182
+ type: "object",
183
+ },
184
+ })
185
+ .build(), (0, api_1.asyncHandler)(async (req, res) => {
186
+ const user = req.user;
187
+ const data = await (0, feedback_1.recordTraceFeedback)({
188
+ input: req.body,
189
+ traceId: req.params.id,
190
+ user: { admin: user?.admin, id: String(user?._id) },
191
+ });
192
+ return res.json({ data });
193
+ }));
122
194
  router.post(`${BASE_PATH}/traces/:id/scores`, (0, observabilityRouteAccess_1.observabilityRouteMiddleware)(options.accessControl, { action: "score", resource: "aiReview" }, builder()
123
195
  .withTags(["observability"])
124
196
  .withSummary("Add a score to a trace")
@@ -63,7 +63,7 @@ export interface ScoreRecord {
63
63
  dataType: "boolean" | "categorical" | "numeric";
64
64
  evaluatorId?: string;
65
65
  name: string;
66
- source: "code" | "human" | "llm-judge" | "user-feedback";
66
+ source: "code" | "human" | "llm-judge" | "user-feedback" | "user-signal";
67
67
  spanId?: string;
68
68
  traceId: string;
69
69
  value: boolean | number | string;
@@ -152,6 +152,13 @@ export interface ObservabilityAppOptions {
152
152
  accessControl?: AnyTerrenoAccess;
153
153
  aiService?: ObservabilityGenerateClient;
154
154
  aiServiceFactory?: ObservabilityAiServiceFactory;
155
+ /**
156
+ * When set, an end user's thumbs down queues the trace for review (`reason: "feedback"`) against
157
+ * this human evaluator, unless the trace already has an open item for it. Unset: no queueing.
158
+ */
159
+ feedbackReview?: {
160
+ evaluatorName: string;
161
+ };
155
162
  control?: {
156
163
  datasets?: ControlPrimary;
157
164
  experiments?: ControlPrimary;
@@ -5,6 +5,8 @@ const api_1 = require("@terreno/api");
5
5
  const blocks_1 = require("@terreno/blocks");
6
6
  const luxon_1 = require("luxon");
7
7
  const gptHistory_1 = require("../models/gptHistory");
8
+ const feedback_1 = require("../observability/feedback");
9
+ const obsTrace_1 = require("../observability/local/models/obsTrace");
8
10
  const aiDatasets_1 = require("../service/aiDatasets");
9
11
  const asks_1 = require("../service/asks");
10
12
  const chatTurn_1 = require("../service/chatTurn");
@@ -228,14 +230,23 @@ const addGptRoutes = (router, options) => {
228
230
  .withSummary("Rate a prompt in a GPT history")
229
231
  .withPathParameter("id", { type: "string" })
230
232
  .withRequestBody({
233
+ comment: {
234
+ description: "Optional comment on the rating (≤ 2000 chars), recorded on the reply's trace",
235
+ type: "string",
236
+ },
231
237
  promptIndex: { type: "number" },
232
- rating: { type: "string" },
238
+ rating: {
239
+ // `null` clears the rating; listed so the request validator accepts it.
240
+ enum: ["up", "down", null],
241
+ nullable: true,
242
+ type: "string",
243
+ },
233
244
  })
234
245
  .withResponse(200, { data: { type: "object" } })
235
246
  .build(),
236
247
  ], (0, api_1.asyncHandler)(async (req, res) => {
237
248
  const { id } = req.params;
238
- const { promptIndex, rating } = req.body;
249
+ const { comment, promptIndex, rating } = req.body;
239
250
  const userId = req.user?._id;
240
251
  if (typeof promptIndex !== "number" || !Number.isInteger(promptIndex) || promptIndex < 0) {
241
252
  throw new api_1.APIError({ status: 400, title: "promptIndex must be a non-negative number" });
@@ -243,6 +254,15 @@ const addGptRoutes = (router, options) => {
243
254
  if (rating !== null && rating !== "up" && rating !== "down") {
244
255
  throw new api_1.APIError({ status: 400, title: "rating must be 'up', 'down', or null" });
245
256
  }
257
+ if (comment !== undefined && typeof comment !== "string") {
258
+ throw new api_1.APIError({ status: 400, title: "comment must be a string" });
259
+ }
260
+ if (typeof comment === "string" && comment.trim().length > feedback_1.FEEDBACK_COMMENT_MAX_LENGTH) {
261
+ throw new api_1.APIError({
262
+ status: 400,
263
+ title: `comment must be at most ${feedback_1.FEEDBACK_COMMENT_MAX_LENGTH} characters`,
264
+ });
265
+ }
246
266
  const history = await gptHistory_1.GptHistory.findById(id);
247
267
  if (!history) {
248
268
  throw new api_1.APIError({ status: 404, title: "History not found" });
@@ -256,6 +276,26 @@ const addGptRoutes = (router, options) => {
256
276
  // One field update, so rows a running turn appends meanwhile are kept.
257
277
  const ratingPath = `prompts.${promptIndex}.rating`;
258
278
  await gptHistory_1.GptHistory.updateOne({ _id: history._id }, rating === null ? { $unset: { [ratingPath]: "" } } : { $set: { [ratingPath]: rating } });
279
+ // The rating is also the reply's thumbs feedback when its trace exists; a missing trace
280
+ // (old history, no local trace sink) never fails the rating.
281
+ const traceId = history.prompts[promptIndex]?.traceId;
282
+ if (traceId && userId) {
283
+ const trimmed = typeof comment === "string" ? comment.trim() : "";
284
+ try {
285
+ if (!(await (0, obsTrace_1.registerObsTrace)().exists({ _id: traceId }))) {
286
+ return res.json({ data: { promptIndex, rating } });
287
+ }
288
+ await (0, feedback_1.recordTraceFeedback)({
289
+ input: { thumbs: rating, ...(trimmed && rating !== null ? { comment: trimmed } : {}) },
290
+ traceId,
291
+ user: { id: userId.toString() },
292
+ });
293
+ }
294
+ catch (error) {
295
+ // The rating is already saved; the trace score is best-effort.
296
+ api_1.logger.error("Failed to record chat rating as trace feedback", { error, traceId });
297
+ }
298
+ }
259
299
  return res.json({ data: { promptIndex, rating } });
260
300
  }));
261
301
  router.post("/gpt/remix", [
@@ -93,6 +93,46 @@ const withoutClientGcsKeys = async (body, historyId) => {
93
93
  return kept === undefined ? [] : [[key, kept]];
94
94
  }));
95
95
  };
96
+ /** The trace ids the chat turn stored on a history's assistant rows. */
97
+ const storedTraceIds = async (historyId) => {
98
+ if (typeof historyId !== "string") {
99
+ return new Set();
100
+ }
101
+ const history = await gptHistory_1.GptHistory.findById(historyId).select({ "prompts.traceId": 1 }).lean();
102
+ return new Set((history?.prompts ?? []).flatMap((prompt) => (prompt.traceId ? [prompt.traceId] : [])));
103
+ };
104
+ const rowWithKnownTraceId = (row, known) => {
105
+ if (!isFragment(row) || row.traceId === undefined || known.has(String(row.traceId))) {
106
+ return row;
107
+ }
108
+ const { traceId: _dropped, ...rest } = row;
109
+ return rest;
110
+ };
111
+ /**
112
+ * Only the chat turn writes a row's `traceId` (feedback is recorded against it), so a client may
113
+ * round-trip ids the server stored but never plant a new one, including through dotted paths.
114
+ */
115
+ const withoutClientTraceIds = async (body, historyId) => {
116
+ const entries = Object.entries(body);
117
+ if (!entries.some(([key]) => isPromptsKey(key))) {
118
+ return body;
119
+ }
120
+ const known = await storedTraceIds(historyId);
121
+ return Object.fromEntries(entries.flatMap(([key, value]) => {
122
+ if (key === "prompts") {
123
+ return [
124
+ [key, Array.isArray(value) ? value.map((row) => rowWithKnownTraceId(row, known)) : value],
125
+ ];
126
+ }
127
+ if (/^prompts\.\d+\.traceId$/.test(key)) {
128
+ return [];
129
+ }
130
+ if (/^prompts\.\d+$/.test(key)) {
131
+ return [[key, rowWithKnownTraceId(value, known)]];
132
+ }
133
+ return [[key, value]];
134
+ }));
135
+ };
96
136
  /** Server-only fields of `pendingAsk`: the paused turn's messages, replayed only to the model. */
97
137
  const SERVER_ONLY_PENDING_ASK_FIELDS = ["promptIndex", "responseMessages"];
98
138
  const withoutServerOnlyFields = (pendingAsk) => Object.fromEntries(Object.entries(pendingAsk).filter(([key]) => !SERVER_ONLY_PENDING_ASK_FIELDS.includes(key)));
@@ -263,12 +303,12 @@ const addGptHistoryRoutes = (router, options) => {
263
303
  },
264
304
  preCreate: async (body, req) => {
265
305
  return {
266
- ...(await withoutClientGcsKeys(withoutPendingAsk(body))),
306
+ ...(await withoutClientTraceIds(await withoutClientGcsKeys(withoutPendingAsk(body)))),
267
307
  userId: req.user?._id,
268
308
  };
269
309
  },
270
310
  preUpdate: async (body, req) => {
271
- const update = await withoutClientGcsKeys(withoutPendingAsk(body), req.params.id);
311
+ const update = await withoutClientTraceIds(await withoutClientGcsKeys(withoutPendingAsk(body), req.params.id), req.params.id);
272
312
  // A host hook that returns null still refuses the update; modelRouter checks the value.
273
313
  return (routerOptions.preUpdate ? await routerOptions.preUpdate(update, req) : update);
274
314
  },
@@ -1,6 +1,6 @@
1
1
  import { type BlocksDocument } from "@terreno/blocks";
2
2
  import type { JSONValue, LanguageModel, ModelMessage } from "ai";
3
- import type mongoose from "mongoose";
3
+ import mongoose from "mongoose";
4
4
  import type { ModelPrice, PromptVersionRef, SpanRecord } from "../observability/types";
5
5
  import type { AIRequestType, AIServiceOptions, GenerateChatStreamOptions, GenerateJsonArrayOptions, GenerateJsonObjectOptions, GenerateJsonValueOptions, GenerateObservabilityOptions, GenerateStreamOptions, GenerateTextOptions, GptHistoryPrompt, RemixOptions, SummaryOptions, TranslateOptions } from "../types";
6
6
  export declare const TemperaturePresets: {
@@ -18,7 +18,10 @@ interface ResolvedObservability {
18
18
  sessionId?: string;
19
19
  skipTrace: boolean;
20
20
  systemPrompt?: string;
21
+ traceId?: string;
21
22
  }
23
+ /** Mints an id a caller can pass as `traceId` before a generate call and return to its client. */
24
+ export declare const createTraceId: () => string;
22
25
  export declare class AIService {
23
26
  readonly model: LanguageModel;
24
27
  readonly defaultTemperature: number;
@@ -1,11 +1,15 @@
1
1
  "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
2
5
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.AIService = exports.TemperaturePresets = void 0;
6
+ exports.AIService = exports.createTraceId = exports.TemperaturePresets = void 0;
4
7
  const node_crypto_1 = require("node:crypto");
5
8
  const api_1 = require("@terreno/api");
6
9
  const blocks_1 = require("@terreno/blocks");
7
10
  const ai_1 = require("ai");
8
11
  const luxon_1 = require("luxon");
12
+ const mongoose_1 = __importDefault(require("mongoose"));
9
13
  const aiRequest_1 = require("../models/aiRequest");
10
14
  const observabilityAppRegistry_1 = require("../observability/observabilityAppRegistry");
11
15
  const jsonFenceModel_1 = require("./jsonFenceModel");
@@ -61,6 +65,12 @@ const computeCostUsd = (params) => {
61
65
  return ((price.inputPerMTok * params.inputTokens + price.outputPerMTok * params.outputTokens) /
62
66
  1_000_000);
63
67
  };
68
+ const TRACE_ID_PATTERN = /^[0-9a-f]{24}$/i;
69
+ /** Mints an id a caller can pass as `traceId` before a generate call and return to its client. */
70
+ const createTraceId = () => {
71
+ return new mongoose_1.default.Types.ObjectId().toHexString();
72
+ };
73
+ exports.createTraceId = createTraceId;
64
74
  class AIService {
65
75
  model;
66
76
  defaultTemperature;
@@ -153,6 +163,9 @@ class AIService {
153
163
  }
154
164
  }
155
165
  async resolveObservability(options) {
166
+ if (options.traceId !== undefined && !TRACE_ID_PATTERN.test(options.traceId)) {
167
+ throw new api_1.APIError({ status: 400, title: "traceId must be an object id" });
168
+ }
156
169
  let promptRef;
157
170
  let systemPrompt = options.systemPrompt;
158
171
  let sensitive = options.sensitive ?? false;
@@ -182,6 +195,7 @@ class AIService {
182
195
  sessionId: options.sessionId,
183
196
  skipTrace: options.skipTrace === true,
184
197
  systemPrompt,
198
+ traceId: options.traceId,
185
199
  };
186
200
  }
187
201
  observabilityFields(options) {
@@ -192,6 +206,7 @@ class AIService {
192
206
  sensitive: options.sensitive,
193
207
  sessionId: options.sessionId,
194
208
  skipTrace: options.skipTrace,
209
+ traceId: options.traceId,
195
210
  };
196
211
  }
197
212
  buildTraceRecord(params) {
@@ -262,7 +277,7 @@ class AIService {
262
277
  const promptRef = params.observability.promptRef;
263
278
  return {
264
279
  endedAt,
265
- id: (0, node_crypto_1.randomUUID)(),
280
+ id: params.observability.traceId ?? (0, node_crypto_1.randomUUID)(),
266
281
  input: params.prompt,
267
282
  name: spanName,
268
283
  output: params.response,
@@ -1244,12 +1244,15 @@ const runChatTurn = async ({ body, options, req, sink, }) => {
1244
1244
  systemPrompt,
1245
1245
  userId,
1246
1246
  });
1247
+ // Minted up front so the assistant row carries it and feedback on the reply can find the trace.
1248
+ const traceId = (0, observabilityAppRegistry_1.getObservabilityApp)() ? (0, aiService_1.createTraceId)() : undefined;
1247
1249
  const resolvedObservability = await aiService.resolveGenerateObservability({
1248
1250
  promptLabel,
1249
1251
  promptName,
1250
1252
  sensitive: body.sensitive === true ? true : undefined,
1251
1253
  sessionId,
1252
1254
  systemPrompt: effectiveSystemPrompt,
1255
+ traceId,
1253
1256
  });
1254
1257
  const observability = registryPrompt
1255
1258
  ? {
@@ -1360,6 +1363,7 @@ const runChatTurn = async ({ body, options, req, sink, }) => {
1360
1363
  status: "streaming",
1361
1364
  streamId,
1362
1365
  text: "",
1366
+ ...(traceId ? { traceId } : {}),
1363
1367
  type: "assistant",
1364
1368
  },
1365
1369
  ]);
@@ -1581,7 +1585,12 @@ const runChatTurn = async ({ body, options, req, sink, }) => {
1581
1585
  fullResponse: storedResponse,
1582
1586
  generatedImages,
1583
1587
  modelId: aiService.modelId,
1584
- }).map((row) => ({ ...row, status: "complete", streamId })),
1588
+ }).map((row) => ({
1589
+ ...row,
1590
+ status: "complete",
1591
+ streamId,
1592
+ ...(traceId && row.type === "assistant" ? { traceId } : {}),
1593
+ })),
1585
1594
  ...(ask
1586
1595
  ? [
1587
1596
  askCallRow(ask, "pending"),
@@ -84,6 +84,8 @@ export type GptHistoryPromptStatus = "streaming" | "complete" | "error";
84
84
  export interface GptHistoryPrompt {
85
85
  model?: string;
86
86
  rating?: "up" | "down";
87
+ /** Observability trace id of an assistant reply (written by the chat turn). */
88
+ traceId?: string;
87
89
  /** Set on assistant replies produced by /gpt/prompt. "streaming" while partial text is persisted. */
88
90
  status?: GptHistoryPromptStatus;
89
91
  /** Identifies one /gpt/prompt reply so resume clients can follow it. */
@@ -165,6 +167,8 @@ export interface GenerateObservabilityOptions {
165
167
  sensitive?: boolean;
166
168
  sessionId?: string;
167
169
  skipTrace?: boolean;
170
+ /** Id to store the trace under (from `createTraceId()`), so the caller can return it for feedback. */
171
+ traceId?: string;
168
172
  }
169
173
  export interface GenerateTextOptions extends GenerateObservabilityOptions {
170
174
  maxOutputTokens?: number;
@@ -121,7 +121,7 @@ export interface ObsScoreDocument extends mongoose.Document<mongoose.Types.Objec
121
121
  deleted: boolean;
122
122
  evaluatorId?: mongoose.Types.ObjectId;
123
123
  name: string;
124
- source: "code" | "human" | "llm-judge" | "user-feedback";
124
+ source: "code" | "human" | "llm-judge" | "user-feedback" | "user-signal";
125
125
  spanId?: mongoose.Types.ObjectId;
126
126
  traceId: mongoose.Types.ObjectId;
127
127
  updated: Date;