@terreno/ai 58.0.0-beta.2 → 58.0.0-beta.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. package/CHANGELOG.md +146 -0
  2. package/README.md +3 -3
  3. package/dist/harness/commit.d.ts +4 -3
  4. package/dist/harness/commit.js +23 -4
  5. package/dist/index.d.ts +6 -9
  6. package/dist/index.js +14 -25
  7. package/dist/observability/adminScreens.js +5 -1
  8. package/dist/observability/evaluate.d.ts +8 -1
  9. package/dist/observability/evaluate.js +10 -2
  10. package/dist/observability/judgeAgreement.d.ts +53 -0
  11. package/dist/observability/judgeAgreement.js +123 -0
  12. package/dist/observability/local/datasetStore.d.ts +5 -0
  13. package/dist/observability/local/datasetStore.js +7 -2
  14. package/dist/observability/local/evaluatorStore.d.ts +17 -0
  15. package/dist/observability/local/evaluatorStore.js +73 -2
  16. package/dist/observability/local/experimentRunner.d.ts +40 -3
  17. package/dist/observability/local/experimentRunner.js +390 -11
  18. package/dist/observability/local/humanAnswers.d.ts +57 -0
  19. package/dist/observability/local/humanAnswers.js +174 -0
  20. package/dist/observability/local/judgeRun.d.ts +56 -0
  21. package/dist/observability/local/judgeRun.js +111 -0
  22. package/dist/observability/local/localPlugin.js +4 -0
  23. package/dist/observability/local/models/obsDatasetItem.js +4 -0
  24. package/dist/observability/local/models/obsEvaluator.js +4 -0
  25. package/dist/observability/local/models/obsExperiment.js +28 -0
  26. package/dist/observability/local/models/obsExperimentItem.js +4 -0
  27. package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
  28. package/dist/observability/local/models/obsJudgeRun.js +60 -0
  29. package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
  30. package/dist/observability/local/models/obsReviewBatch.js +48 -0
  31. package/dist/observability/local/models/obsReviewItem.js +10 -2
  32. package/dist/observability/local/models/obsTrace.js +4 -0
  33. package/dist/observability/local/promptReferences.d.ts +31 -0
  34. package/dist/observability/local/promptReferences.js +114 -0
  35. package/dist/observability/local/promptStore.d.ts +55 -19
  36. package/dist/observability/local/promptStore.js +171 -7
  37. package/dist/observability/local/reviewBatchStore.d.ts +106 -0
  38. package/dist/observability/local/reviewBatchStore.js +517 -0
  39. package/dist/observability/local/reviewStore.d.ts +24 -3
  40. package/dist/observability/local/reviewStore.js +98 -22
  41. package/dist/observability/local/traceStore.d.ts +14 -1
  42. package/dist/observability/local/traceStore.js +144 -11
  43. package/dist/observability/observabilityApp.js +10 -1
  44. package/dist/observability/questionFingerprint.d.ts +7 -0
  45. package/dist/observability/questionFingerprint.js +23 -0
  46. package/dist/observability/routes/experiments.js +12 -1
  47. package/dist/observability/routes/review.js +3 -1
  48. package/dist/observability/routes/reviewBatches.d.ts +11 -0
  49. package/dist/observability/routes/reviewBatches.js +78 -0
  50. package/dist/observability/routes/traces.d.ts +1 -1
  51. package/dist/observability/routes/traces.js +34 -2
  52. package/dist/observability/types.d.ts +10 -1
  53. package/dist/service/aiService.js +4 -1
  54. package/dist/service/chatTurn.js +38 -31
  55. package/dist/types/harness.d.ts +5 -2
  56. package/dist/types/index.d.ts +8 -3
  57. package/dist/types/observability.d.ts +74 -2
  58. package/package.json +8 -24
  59. package/src/harness/agentLoop.test.ts +0 -3
  60. package/src/harness/agentRecovery.test.ts +0 -3
  61. package/src/harness/commit.ts +24 -4
  62. package/src/harness/extensions.test.ts +0 -3
  63. package/src/harness/subagent.test.ts +27 -3
  64. package/src/index.test.ts +0 -17
  65. package/src/index.ts +23 -26
  66. package/src/isolated/gptStream.isolated.ts +0 -8
  67. package/src/observability/adminScreens.test.ts +7 -0
  68. package/src/observability/adminScreens.ts +6 -1
  69. package/src/observability/evaluate.ts +12 -2
  70. package/src/observability/judgeAgreement.test.ts +111 -0
  71. package/src/observability/judgeAgreement.ts +184 -0
  72. package/src/observability/local/datasetStore.ts +30 -11
  73. package/src/observability/local/evaluatorStore.ts +96 -2
  74. package/src/observability/local/experimentRunner.ts +495 -15
  75. package/src/observability/local/humanAnswers.ts +233 -0
  76. package/src/observability/local/judgePipeline.test.ts +444 -0
  77. package/src/observability/local/judgeRun.test.ts +173 -0
  78. package/src/observability/local/judgeRun.ts +184 -0
  79. package/src/observability/local/localPlugin.ts +4 -0
  80. package/src/observability/local/models/obsDatasetItem.ts +5 -0
  81. package/src/observability/local/models/obsEvaluator.ts +4 -0
  82. package/src/observability/local/models/obsExperiment.ts +29 -0
  83. package/src/observability/local/models/obsExperimentItem.ts +4 -0
  84. package/src/observability/local/models/obsJudgeRun.ts +61 -0
  85. package/src/observability/local/models/obsReviewBatch.ts +52 -0
  86. package/src/observability/local/models/obsReviewItem.ts +11 -2
  87. package/src/observability/local/models/obsTrace.ts +5 -0
  88. package/src/observability/local/obsModels.test.ts +13 -0
  89. package/src/observability/local/promptReferences.test.ts +128 -0
  90. package/src/observability/local/promptReferences.ts +149 -0
  91. package/src/observability/local/promptStore.test.ts +291 -0
  92. package/src/observability/local/promptStore.ts +243 -32
  93. package/src/observability/local/reviewBatchStore.test.ts +251 -0
  94. package/src/observability/local/reviewBatchStore.ts +656 -0
  95. package/src/observability/local/reviewStore.ts +122 -25
  96. package/src/observability/local/traceStore.test.ts +139 -1
  97. package/src/observability/local/traceStore.ts +162 -11
  98. package/src/observability/observabilityApp.test.ts +13 -10
  99. package/src/observability/observabilityApp.ts +10 -1
  100. package/src/observability/questionFingerprint.test.ts +33 -0
  101. package/src/observability/questionFingerprint.ts +24 -0
  102. package/src/observability/routes/experiments.test.ts +37 -0
  103. package/src/observability/routes/experiments.ts +13 -1
  104. package/src/observability/routes/review.ts +3 -1
  105. package/src/observability/routes/reviewBatches.test.ts +209 -0
  106. package/src/observability/routes/reviewBatches.ts +138 -0
  107. package/src/observability/routes/status.test.ts +4 -4
  108. package/src/observability/routes/traces.test.ts +61 -0
  109. package/src/observability/routes/traces.ts +46 -4
  110. package/src/observability/types.ts +5 -2
  111. package/src/routes/routes.test.ts +101 -76
  112. package/src/service/aiService.test.ts +21 -0
  113. package/src/service/aiService.ts +4 -1
  114. package/src/service/chatTurn.ts +42 -33
  115. package/src/tests/bunSetup.ts +0 -54
  116. package/src/types/harness.ts +5 -2
  117. package/src/types/index.ts +8 -3
  118. package/src/types/observability.ts +86 -2
  119. package/dist/langfuseApp.d.ts +0 -9
  120. package/dist/langfuseApp.js +0 -53
  121. package/dist/langfuseCache.d.ts +0 -15
  122. package/dist/langfuseCache.js +0 -54
  123. package/dist/langfuseClient.d.ts +0 -6
  124. package/dist/langfuseClient.js +0 -34
  125. package/dist/langfusePrompts.d.ts +0 -12
  126. package/dist/langfusePrompts.js +0 -89
  127. package/dist/langfuseRoutesEvaluations.d.ts +0 -3
  128. package/dist/langfuseRoutesEvaluations.js +0 -32
  129. package/dist/langfuseRoutesMiddleware.d.ts +0 -2
  130. package/dist/langfuseRoutesMiddleware.js +0 -13
  131. package/dist/langfuseRoutesPlayground.d.ts +0 -2
  132. package/dist/langfuseRoutesPlayground.js +0 -36
  133. package/dist/langfuseRoutesPrompts.d.ts +0 -2
  134. package/dist/langfuseRoutesPrompts.js +0 -56
  135. package/dist/langfuseRoutesTraces.d.ts +0 -2
  136. package/dist/langfuseRoutesTraces.js +0 -30
  137. package/dist/langfuseTracing.d.ts +0 -4
  138. package/dist/langfuseTracing.js +0 -32
  139. package/dist/langfuseTypes.d.ts +0 -96
  140. package/dist/langfuseTypes.js +0 -2
  141. package/dist/langfuseVercelAi.d.ts +0 -12
  142. package/dist/langfuseVercelAi.js +0 -49
  143. package/src/isolated/langfuseClient.isolated.ts +0 -66
  144. package/src/langfuseApp.test.ts +0 -119
  145. package/src/langfuseApp.ts +0 -62
  146. package/src/langfuseCache.test.ts +0 -75
  147. package/src/langfuseCache.ts +0 -72
  148. package/src/langfuseClient.test.ts +0 -69
  149. package/src/langfuseClient.ts +0 -34
  150. package/src/langfusePrompts.test.ts +0 -188
  151. package/src/langfusePrompts.ts +0 -121
  152. package/src/langfuseRoutes.test.ts +0 -261
  153. package/src/langfuseRoutesEvaluations.ts +0 -48
  154. package/src/langfuseRoutesMiddleware.test.ts +0 -57
  155. package/src/langfuseRoutesMiddleware.ts +0 -12
  156. package/src/langfuseRoutesPlayground.ts +0 -50
  157. package/src/langfuseRoutesPrompts.ts +0 -87
  158. package/src/langfuseRoutesTraces.ts +0 -40
  159. package/src/langfuseTracing.test.ts +0 -72
  160. package/src/langfuseTracing.ts +0 -33
  161. package/src/langfuseTypes.ts +0 -103
  162. package/src/langfuseVercelAi.test.ts +0 -82
  163. package/src/langfuseVercelAi.ts +0 -72
  164. package/src/tests/fakeLangfusePrompt.ts +0 -10
@@ -8,12 +8,18 @@ const api_1 = require("@terreno/api");
8
8
  const luxon_1 = require("luxon");
9
9
  const mongoose_1 = __importDefault(require("mongoose"));
10
10
  const reviewPanels_1 = require("../reviewPanels");
11
+ const humanAnswers_1 = require("./humanAnswers");
12
+ const obsDataset_1 = require("./models/obsDataset");
13
+ const obsDatasetItem_1 = require("./models/obsDatasetItem");
11
14
  const obsEvaluator_1 = require("./models/obsEvaluator");
12
15
  const obsPrompt_1 = require("./models/obsPrompt");
16
+ const obsPromptLabel_1 = require("./models/obsPromptLabel");
13
17
  const obsPromptVersion_1 = require("./models/obsPromptVersion");
18
+ const obsReviewBatch_1 = require("./models/obsReviewBatch");
14
19
  const obsReviewItem_1 = require("./models/obsReviewItem");
15
20
  const obsTrace_1 = require("./models/obsTrace");
16
21
  const traceStore_1 = require("./traceStore");
22
+ const DATASET_ITEM_NAME = "Dataset item";
17
23
  const objectId = (value, label) => {
18
24
  if (!mongoose_1.default.Types.ObjectId.isValid(value)) {
19
25
  throw new api_1.APIError({ status: 400, title: `${label} must be an object id` });
@@ -52,24 +58,48 @@ class LocalReviewStore {
52
58
  reason: params.reason ?? "manual",
53
59
  status: "pending",
54
60
  traceId: trace._id,
61
+ ...(params.batchId ? { batchId: objectId(params.batchId, "batchId") } : {}),
55
62
  ...(params.spanId ? { spanId: objectId(params.spanId, "spanId") } : {}),
56
63
  });
57
64
  created.push(this.toListItem(item, trace.name, trace.prompts[0]?.name));
58
65
  }
59
66
  return created;
60
67
  }
61
- async list(status) {
68
+ /** Queue dataset items for a human question set; items with a source trace show that trace. */
69
+ async enqueueDatasetItems(params) {
70
+ const created = [];
71
+ for (const entry of params.items) {
72
+ const item = await (0, obsReviewItem_1.registerObsReviewItem)().create({
73
+ batchId: objectId(params.batchId, "batchId"),
74
+ datasetItemId: objectId(entry.datasetItemId, "datasetItemId"),
75
+ enqueuedAt: luxon_1.DateTime.utc().toJSDate(),
76
+ evaluatorId: objectId(params.evaluatorId, "evaluatorId"),
77
+ reason: "manual",
78
+ status: "pending",
79
+ ...(entry.traceId ? { traceId: objectId(entry.traceId, "traceId") } : {}),
80
+ });
81
+ created.push(this.toListItem(item, DATASET_ITEM_NAME));
82
+ }
83
+ return created;
84
+ }
85
+ async list(status, options = {}) {
62
86
  const ObsReviewItem = (0, obsReviewItem_1.registerObsReviewItem)();
87
+ const scope = options.batchId ? { batchId: objectId(options.batchId, "batchId") } : {};
63
88
  const counts = {
64
- done: await ObsReviewItem.countDocuments({ status: "done" }),
65
- in_progress: await ObsReviewItem.countDocuments({ status: "in_progress" }),
66
- pending: await ObsReviewItem.countDocuments({ status: "pending" }),
67
- skipped: await ObsReviewItem.countDocuments({ status: "skipped" }),
89
+ done: await ObsReviewItem.countDocuments({ ...scope, status: "done" }),
90
+ in_progress: await ObsReviewItem.countDocuments({ ...scope, status: "in_progress" }),
91
+ pending: await ObsReviewItem.countDocuments({ ...scope, status: "pending" }),
92
+ skipped: await ObsReviewItem.countDocuments({ ...scope, status: "skipped" }),
68
93
  };
69
- const filter = status ? { status } : {};
70
- const rows = await ObsReviewItem.find(filter).sort({ enqueuedAt: 1 });
94
+ const filter = status ? { ...scope, status } : scope;
95
+ // Batch items keep their batch order (enqueued together, newest trace first).
96
+ const rows = await ObsReviewItem.find(filter).sort({ _id: 1, enqueuedAt: 1 });
71
97
  const data = [];
72
98
  for (const row of rows) {
99
+ if (!row.traceId) {
100
+ data.push(this.toListItem(row, DATASET_ITEM_NAME));
101
+ continue;
102
+ }
73
103
  const trace = await (0, obsTrace_1.registerObsTrace)().findOneOrNone({ _id: row.traceId });
74
104
  data.push(this.toListItem(row, trace?.name ?? "Unknown trace", trace?.prompts[0]?.name));
75
105
  }
@@ -84,11 +114,47 @@ class LocalReviewStore {
84
114
  if (!evaluator) {
85
115
  throw new api_1.APIError({ status: 404, title: "Unknown evaluator" });
86
116
  }
87
- const trace = await (0, obsTrace_1.registerObsTrace)().findOneOrNone({ _id: item.traceId });
88
- const promptRef = trace?.prompts[0];
117
+ const trace = item.traceId
118
+ ? await (0, obsTrace_1.registerObsTrace)().findOneOrNone({ _id: item.traceId })
119
+ : undefined;
120
+ const batch = item.batchId
121
+ ? await (0, obsReviewBatch_1.registerObsReviewBatch)().findOneOrNone({ _id: item.batchId })
122
+ : undefined;
123
+ let input = trace?.input;
124
+ let output = trace?.output;
89
125
  let variables;
90
126
  let outputFieldNotes;
91
127
  let outputSchema;
128
+ // A dataset item without a (surviving) source trace shows its own input and expected output.
129
+ if (!trace && item.datasetItemId) {
130
+ const datasetItem = await (0, obsDatasetItem_1.registerObsDatasetItem)().findOneOrNone({
131
+ _id: item.datasetItemId,
132
+ });
133
+ if (datasetItem) {
134
+ const subject = (await (0, humanAnswers_1.loadDatasetItemSubjects)([datasetItem])).get(String(datasetItem._id));
135
+ input = subject?.input;
136
+ output = subject?.output;
137
+ const dataset = await (0, obsDataset_1.registerObsDataset)().findOneOrNone({ _id: datasetItem.datasetId });
138
+ if (dataset?.inputSchemaPromptName) {
139
+ const prompt = await (0, obsPrompt_1.registerObsPrompt)().findOneOrNone({
140
+ name: dataset.inputSchemaPromptName,
141
+ });
142
+ const label = prompt
143
+ ? await (0, obsPromptLabel_1.registerObsPromptLabel)().findOneOrNone({
144
+ label: "production",
145
+ promptId: prompt._id,
146
+ })
147
+ : undefined;
148
+ const version = label
149
+ ? await (0, obsPromptVersion_1.registerObsPromptVersion)().findOneOrNone({ _id: label.versionId })
150
+ : undefined;
151
+ variables = version?.variables;
152
+ outputFieldNotes = version?.outputFieldNotes;
153
+ outputSchema = version?.outputSchema;
154
+ }
155
+ }
156
+ }
157
+ const promptRef = trace?.prompts[0];
92
158
  if (promptRef) {
93
159
  const prompt = await (0, obsPrompt_1.registerObsPrompt)().findOneOrNone({ name: promptRef.name });
94
160
  if (prompt) {
@@ -102,23 +168,20 @@ class LocalReviewStore {
102
168
  }
103
169
  }
104
170
  return {
171
+ batchId: batch ? String(batch._id) : undefined,
172
+ batchName: batch?.name,
105
173
  comment: item.comment,
174
+ datasetItemId: item.datasetItemId ? String(item.datasetItemId) : undefined,
106
175
  dimensions: evaluator.dimensions,
107
176
  evaluatorId: String(evaluator._id),
108
177
  id: String(item._id),
109
178
  instructions: evaluator.instructions,
110
- panels: (0, reviewPanels_1.buildReviewPanels)({
111
- input: trace?.input,
112
- output: trace?.output,
113
- outputFieldNotes,
114
- outputSchema,
115
- variables,
116
- }),
117
- rawInput: trace?.input,
118
- rawOutput: trace?.output,
179
+ panels: (0, reviewPanels_1.buildReviewPanels)({ input, output, outputFieldNotes, outputSchema, variables }),
180
+ rawInput: input,
181
+ rawOutput: output,
119
182
  scores: item.scores,
120
183
  status: item.status,
121
- traceId: String(item.traceId),
184
+ traceId: item.traceId ? String(item.traceId) : undefined,
122
185
  };
123
186
  }
124
187
  async submit(params) {
@@ -134,7 +197,9 @@ class LocalReviewStore {
134
197
  throw new api_1.APIError({ status: 400, title: `Score "${missing.key}" is required` });
135
198
  }
136
199
  const records = [];
137
- for (const dimension of evaluator.dimensions) {
200
+ const traceId = item.traceId;
201
+ // Scores attach to traces, so a dataset item without a trace only records human answers.
202
+ for (const dimension of traceId ? evaluator.dimensions : []) {
138
203
  const value = params.scores[dimension.key];
139
204
  if (value === undefined) {
140
205
  continue;
@@ -145,7 +210,7 @@ class LocalReviewStore {
145
210
  evaluatorId: String(evaluator._id),
146
211
  name: dimension.key,
147
212
  source: "human",
148
- traceId: String(item.traceId),
213
+ traceId: String(traceId),
149
214
  value,
150
215
  });
151
216
  }
@@ -167,6 +232,15 @@ class LocalReviewStore {
167
232
  item.scores = params.scores;
168
233
  item.status = "done";
169
234
  await item.save();
235
+ if (item.datasetItemId) {
236
+ await (0, humanAnswers_1.writeHumanAnswer)({
237
+ datasetItemId: item.datasetItemId,
238
+ evaluator,
239
+ reviewItemId: String(item._id),
240
+ scores: params.scores,
241
+ userId: params.userId,
242
+ });
243
+ }
170
244
  }
171
245
  async skip(id) {
172
246
  const item = await this.requireItem(id);
@@ -189,13 +263,15 @@ class LocalReviewStore {
189
263
  toListItem(row, traceName, promptName) {
190
264
  return {
191
265
  assigneeId: row.assigneeId ? String(row.assigneeId) : undefined,
266
+ batchId: row.batchId ? String(row.batchId) : undefined,
267
+ datasetItemId: row.datasetItemId ? String(row.datasetItemId) : undefined,
192
268
  enqueuedAt: toIso(row.enqueuedAt),
193
269
  evaluatorId: String(row.evaluatorId),
194
270
  id: String(row._id),
195
271
  promptName,
196
272
  reason: row.reason,
197
273
  status: row.status,
198
- traceId: String(row.traceId),
274
+ traceId: row.traceId ? String(row.traceId) : undefined,
199
275
  traceName,
200
276
  };
201
277
  }
@@ -1,15 +1,27 @@
1
1
  import type { ScoreRecord, ScoreSink, TraceRecord, TraceSink } from "../types";
2
+ /** Columns the trace list can sort by. Prefix with `-` for descending. */
3
+ export declare const TRACE_LIST_SORT_KEYS: readonly ["cost", "latency", "name", "scores", "spans", "startedAt", "status", "tokens"];
4
+ export type TraceListSortKey = (typeof TRACE_LIST_SORT_KEYS)[number];
5
+ export interface TraceListSort {
6
+ direction: 1 | -1;
7
+ key: TraceListSortKey;
8
+ }
9
+ /** Newest first, matching the admin Traces screen default. */
10
+ export declare const DEFAULT_TRACE_LIST_SORT = "-startedAt";
2
11
  export interface TraceListQuery {
3
12
  flaggedForDataset?: boolean;
4
13
  from?: string;
5
14
  hasScore?: boolean;
6
15
  limit?: number;
7
16
  page?: number;
8
- prompt?: string;
17
+ /** One prompt name, or several to match traces that used any of them. */
18
+ prompt?: string | string[];
9
19
  promptVersion?: number;
10
20
  scope?: string;
11
21
  sensitive?: boolean;
12
22
  sessionId?: string;
23
+ /** One of `TRACE_LIST_SORT_KEYS`, optionally prefixed with `-` for descending. */
24
+ sort?: string;
13
25
  status?: "error" | "ok";
14
26
  to?: string;
15
27
  userId?: string;
@@ -69,6 +81,7 @@ export interface TraceListItem {
69
81
  usage?: TraceRecord["usage"];
70
82
  userId?: string;
71
83
  }
84
+ export declare const parseTraceListSort: (value?: string) => TraceListSort;
72
85
  export declare class LocalTraceStore {
73
86
  exportTrace(trace: TraceRecord): Promise<{
74
87
  id: string;
@@ -3,13 +3,26 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
3
3
  return (mod && mod.__esModule) ? mod : { "default": mod };
4
4
  };
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.MemoryScoreSink = exports.MemoryTraceSink = exports.LocalScoreSink = exports.LocalTraceSink = exports.LocalTraceStore = void 0;
6
+ exports.MemoryScoreSink = exports.MemoryTraceSink = exports.LocalScoreSink = exports.LocalTraceSink = exports.LocalTraceStore = exports.parseTraceListSort = exports.DEFAULT_TRACE_LIST_SORT = exports.TRACE_LIST_SORT_KEYS = void 0;
7
7
  const api_1 = require("@terreno/api");
8
8
  const luxon_1 = require("luxon");
9
9
  const mongoose_1 = __importDefault(require("mongoose"));
10
10
  const obsScore_1 = require("./models/obsScore");
11
11
  const obsSpan_1 = require("./models/obsSpan");
12
12
  const obsTrace_1 = require("./models/obsTrace");
13
+ /** Columns the trace list can sort by. Prefix with `-` for descending. */
14
+ exports.TRACE_LIST_SORT_KEYS = [
15
+ "cost",
16
+ "latency",
17
+ "name",
18
+ "scores",
19
+ "spans",
20
+ "startedAt",
21
+ "status",
22
+ "tokens",
23
+ ];
24
+ /** Newest first, matching the admin Traces screen default. */
25
+ exports.DEFAULT_TRACE_LIST_SORT = "-startedAt";
13
26
  const toIso = (value) => {
14
27
  if (!value) {
15
28
  return undefined;
@@ -51,6 +64,105 @@ const usageWithoutUndefinedCost = (usage) => {
51
64
  }
52
65
  return usage;
53
66
  };
67
+ const isTraceListSortKey = (value) => {
68
+ return exports.TRACE_LIST_SORT_KEYS.includes(value);
69
+ };
70
+ const parseTraceListSort = (value) => {
71
+ const raw = value?.trim() || exports.DEFAULT_TRACE_LIST_SORT;
72
+ const descending = raw.startsWith("-");
73
+ const key = descending ? raw.slice(1) : raw;
74
+ if (!isTraceListSortKey(key)) {
75
+ throw new api_1.APIError({
76
+ detail: `Use one of ${exports.TRACE_LIST_SORT_KEYS.join(", ")}, optionally prefixed with "-".`,
77
+ status: 400,
78
+ title: "Unknown trace sort key",
79
+ });
80
+ }
81
+ return { direction: descending ? -1 : 1, key };
82
+ };
83
+ exports.parseTraceListSort = parseTraceListSort;
84
+ const promptNames = (prompt) => {
85
+ const values = Array.isArray(prompt) ? prompt : prompt === undefined ? [] : [prompt];
86
+ return values.map((entry) => entry.trim()).filter((entry) => entry !== "");
87
+ };
88
+ /** Count rows in `from` whose traceId matches the trace, for span and score sorts. */
89
+ const countLookupStages = (from) => {
90
+ return [
91
+ {
92
+ $lookup: {
93
+ as: "_sortCounts",
94
+ from,
95
+ let: { traceId: "$_id" },
96
+ pipeline: [{ $match: { $expr: { $eq: ["$traceId", "$$traceId"] } } }, { $count: "n" }],
97
+ },
98
+ },
99
+ { $addFields: { _sortValue: { $ifNull: [{ $arrayElemAt: ["$_sortCounts.n", 0] }, 0] } } },
100
+ ];
101
+ };
102
+ const missingAware = (expression) => {
103
+ return [
104
+ { $addFields: { _sortValue: expression } },
105
+ { $addFields: { _sortMissing: { $cond: [{ $eq: ["$_sortValue", null] }, 1, 0] } } },
106
+ ];
107
+ };
108
+ const isMissing = (path) => {
109
+ return { $eq: [{ $type: path }, "missing"] };
110
+ };
111
+ /**
112
+ * Stages that compute `_sortValue` (and `_sortMissing` for usage-derived values, so traces
113
+ * without usage or an end time always list last) before the sort runs.
114
+ */
115
+ const sortValueStages = (key) => {
116
+ if (key === "cost") {
117
+ return missingAware({ $ifNull: ["$usage.costUsd", null] });
118
+ }
119
+ if (key === "tokens") {
120
+ return missingAware({
121
+ $cond: [
122
+ { $and: [isMissing("$usage.inputTokens"), isMissing("$usage.outputTokens")] },
123
+ null,
124
+ { $add: [{ $ifNull: ["$usage.inputTokens", 0] }, { $ifNull: ["$usage.outputTokens", 0] }] },
125
+ ],
126
+ });
127
+ }
128
+ if (key === "latency") {
129
+ return missingAware({
130
+ $cond: [
131
+ { $or: [{ $eq: [{ $ifNull: ["$endedAt", null] }, null] }, { $eq: ["$startedAt", null] }] },
132
+ null,
133
+ { $subtract: ["$endedAt", "$startedAt"] },
134
+ ],
135
+ });
136
+ }
137
+ if (key === "spans") {
138
+ return countLookupStages((0, obsSpan_1.registerObsSpan)().collection.name);
139
+ }
140
+ if (key === "scores") {
141
+ return countLookupStages((0, obsScore_1.registerObsScore)().collection.name);
142
+ }
143
+ return [];
144
+ };
145
+ /** `$sort` keys are order-significant, so the spec is built from an ordered entry list. */
146
+ const sortSpec = (sort) => {
147
+ const dir = sort.direction;
148
+ if (sort.key === "startedAt" || sort.key === "name" || sort.key === "status") {
149
+ return Object.fromEntries([
150
+ [sort.key, dir],
151
+ ["_id", dir],
152
+ ]);
153
+ }
154
+ if (sort.key === "spans" || sort.key === "scores") {
155
+ return Object.fromEntries([
156
+ ["_sortValue", dir],
157
+ ["_id", dir],
158
+ ]);
159
+ }
160
+ return Object.fromEntries([
161
+ ["_sortMissing", 1],
162
+ ["_sortValue", dir],
163
+ ["_id", dir],
164
+ ]);
165
+ };
54
166
  const countByTraceId = async (model, ids) => {
55
167
  const counts = new Map();
56
168
  if (ids.length === 0) {
@@ -143,16 +255,24 @@ class LocalTraceStore {
143
255
  if (query.scope) {
144
256
  filter.scope = query.scope;
145
257
  }
146
- if (query.promptVersion !== undefined && !query.prompt) {
258
+ const sort = (0, exports.parseTraceListSort)(query.sort);
259
+ const prompts = promptNames(query.prompt);
260
+ if (query.promptVersion !== undefined && prompts.length === 0) {
147
261
  throw new api_1.APIError({ status: 400, title: "prompt is required when promptVersion is set" });
148
262
  }
149
- if (query.prompt && query.promptVersion !== undefined) {
263
+ if (query.promptVersion !== undefined && prompts.length > 1) {
264
+ throw new api_1.APIError({ status: 400, title: "promptVersion needs exactly one prompt" });
265
+ }
266
+ if (prompts.length === 1 && query.promptVersion !== undefined) {
150
267
  filter.prompts = {
151
- $elemMatch: { name: query.prompt, version: query.promptVersion },
268
+ $elemMatch: { name: prompts[0], version: query.promptVersion },
152
269
  };
153
270
  }
154
- else if (query.prompt) {
155
- filter["prompts.name"] = query.prompt;
271
+ else if (prompts.length === 1) {
272
+ filter["prompts.name"] = prompts[0];
273
+ }
274
+ else if (prompts.length > 1) {
275
+ filter["prompts.name"] = { $in: prompts };
156
276
  }
157
277
  if (query.sensitive !== undefined) {
158
278
  filter.sensitive = query.sensitive;
@@ -186,11 +306,24 @@ class LocalTraceStore {
186
306
  }
187
307
  const ObsTrace = (0, obsTrace_1.registerObsTrace)();
188
308
  const total = await ObsTrace.countDocuments(filter);
189
- const rows = await ObsTrace.find(filter)
190
- .sort({ created: -1 })
191
- .skip((page - 1) * limit)
192
- .limit(limit);
193
- const ids = rows.map((row) => row._id);
309
+ // Sort in Mongo before skip/limit so derived columns (latency, counts) order every page.
310
+ const aggregate = ObsTrace.aggregate([
311
+ { $match: { deleted: { $ne: true }, ...filter } },
312
+ ...sortValueStages(sort.key),
313
+ { $sort: sortSpec(sort) },
314
+ { $skip: (page - 1) * limit },
315
+ { $limit: limit },
316
+ { $project: { _id: 1 } },
317
+ ]);
318
+ if (sort.key === "name") {
319
+ aggregate.collation({ locale: "en" });
320
+ }
321
+ const ordered = await aggregate.exec();
322
+ const ids = ordered.map((row) => row._id);
323
+ const position = new Map(ids.map((id, index) => [String(id), index]));
324
+ const rows = (await ObsTrace.find({ _id: { $in: ids } })).sort((a, b) => {
325
+ return (position.get(String(a._id)) ?? 0) - (position.get(String(b._id)) ?? 0);
326
+ });
194
327
  const spanCountById = await countByTraceId((0, obsSpan_1.registerObsSpan)(), ids);
195
328
  const scoreCountById = await countByTraceId((0, obsScore_1.registerObsScore)(), ids);
196
329
  return {
@@ -7,6 +7,7 @@ const datasetStore_1 = require("./local/datasetStore");
7
7
  const evaluatorStore_1 = require("./local/evaluatorStore");
8
8
  const experimentRunner_1 = require("./local/experimentRunner");
9
9
  const promptStore_1 = require("./local/promptStore");
10
+ const reviewBatchStore_1 = require("./local/reviewBatchStore");
10
11
  const reviewStore_1 = require("./local/reviewStore");
11
12
  const traceStore_1 = require("./local/traceStore");
12
13
  const observabilityAppRegistry_1 = require("./observabilityAppRegistry");
@@ -17,6 +18,7 @@ const evaluators_1 = require("./routes/evaluators");
17
18
  const experiments_1 = require("./routes/experiments");
18
19
  const prompts_1 = require("./routes/prompts");
19
20
  const review_1 = require("./routes/review");
21
+ const reviewBatches_1 = require("./routes/reviewBatches");
20
22
  const status_1 = require("./routes/status");
21
23
  const testMultiStage_1 = require("./routes/testMultiStage");
22
24
  const traces_1 = require("./routes/traces");
@@ -120,9 +122,16 @@ class ObservabilityApp {
120
122
  ...routeAccess,
121
123
  store: evaluatorStore,
122
124
  });
125
+ const reviewStore = new reviewStore_1.LocalReviewStore();
126
+ (0, reviewBatches_1.addObservabilityReviewBatchRoutes)(app, {
127
+ ...routeAccess,
128
+ aiService: this.aiService,
129
+ requestAiServiceFactory: this.requestAiServiceFactory,
130
+ store: new reviewBatchStore_1.LocalReviewBatchStore(promptStore ?? new promptStore_1.LocalPromptStore(), reviewStore),
131
+ });
123
132
  (0, review_1.addObservabilityReviewRoutes)(app, {
124
133
  ...routeAccess,
125
- store: new reviewStore_1.LocalReviewStore(),
134
+ store: reviewStore,
126
135
  });
127
136
  if (localPlugin.datasetStore instanceof datasetStore_1.LocalDatasetStore) {
128
137
  (0, datasets_1.addObservabilityDatasetRoutes)(app, {
@@ -0,0 +1,7 @@
1
+ import type { EvaluatorDimension } from "../types/observability";
2
+ /**
3
+ * Stable fingerprint of a question set's questions. Human answers stay reusable while the
4
+ * fingerprint matches; editing a question's key, type, wording, or range makes earlier answers
5
+ * stale. Dimension order and the `required` flag do not change the fingerprint.
6
+ */
7
+ export declare const questionSetFingerprint: (dimensions: Pick<EvaluatorDimension, "dataType" | "key" | "question" | "range">[]) => string;
@@ -0,0 +1,23 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.questionSetFingerprint = void 0;
4
+ const node_crypto_1 = require("node:crypto");
5
+ /**
6
+ * Stable fingerprint of a question set's questions. Human answers stay reusable while the
7
+ * fingerprint matches; editing a question's key, type, wording, or range makes earlier answers
8
+ * stale. Dimension order and the `required` flag do not change the fingerprint.
9
+ */
10
+ const questionSetFingerprint = (dimensions) => {
11
+ const canonical = [...dimensions]
12
+ .map((dimension) => {
13
+ return {
14
+ dataType: dimension.dataType,
15
+ key: dimension.key,
16
+ question: dimension.question?.trim() ?? null,
17
+ range: dimension.range?.trim() ?? null,
18
+ };
19
+ })
20
+ .sort((left, right) => left.key.localeCompare(right.key));
21
+ return (0, node_crypto_1.createHash)("sha256").update(JSON.stringify(canonical)).digest("hex");
22
+ };
23
+ exports.questionSetFingerprint = questionSetFingerprint;
@@ -18,6 +18,15 @@ const parseOptionalPromptName = (value) => {
18
18
  }
19
19
  return trimmed;
20
20
  };
21
+ const parseKind = (value) => {
22
+ if (value === undefined || value === "") {
23
+ return undefined;
24
+ }
25
+ if (value !== "judge" && value !== "prompt") {
26
+ throw new api_1.APIError({ status: 400, title: "kind must be prompt or judge" });
27
+ }
28
+ return value;
29
+ };
21
30
  const parsePositiveInt = (value, fallback) => {
22
31
  if (value === undefined) {
23
32
  return fallback;
@@ -45,11 +54,13 @@ const addObservabilityExperimentRoutes = (router, options) => {
45
54
  .withTags(["observability"])
46
55
  .withSummary("List experiments")
47
56
  .withQueryParameter("promptName", { type: "string" }, { required: false })
57
+ .withQueryParameter("kind", { enum: ["judge", "prompt"], type: "string" }, { required: false })
48
58
  .withQueryParameter("page", { type: "number" }, { required: false })
49
59
  .withQueryParameter("limit", { type: "number" }, { required: false })
50
60
  .withResponse(200, {})
51
61
  .build()), (0, api_1.asyncHandler)(async (req, res) => {
52
62
  const listed = await options.runner.list({
63
+ kind: parseKind(req.query.kind),
53
64
  limit: parsePositiveInt(req.query.limit, experimentRunner_1.EXPERIMENT_LIST_DEFAULT_LIMIT),
54
65
  page: parsePositiveInt(req.query.page, 1),
55
66
  promptName: parseOptionalPromptName(req.query.promptName),
@@ -65,7 +76,7 @@ const addObservabilityExperimentRoutes = (router, options) => {
65
76
  }));
66
77
  router.post(`${BASE_PATH}/experiments`, (0, observabilityRouteAccess_1.observabilityRouteMiddleware)(options.accessControl, { action: "create", resource: "aiExperiment" }, builder()
67
78
  .withTags(["observability"])
68
- .withSummary("Create an experiment")
79
+ .withSummary("Create a prompt comparison or judge calibration experiment")
69
80
  .withResponse(201, {})
70
81
  .build()), (0, api_1.asyncHandler)(async (req, res) => {
71
82
  const data = await options.runner.create(req.body);
@@ -32,7 +32,8 @@ const addObservabilityReviewRoutes = (router, options) => {
32
32
  .withResponse(200, { data: { type: "array" } })
33
33
  .build()), (0, api_1.asyncHandler)(async (req, res) => {
34
34
  const status = req.query.status;
35
- return res.json({ ...(await options.store.list(status)), more: false });
35
+ const batchId = typeof req.query.batchId === "string" ? req.query.batchId : undefined;
36
+ return res.json({ ...(await options.store.list(status, { batchId })), more: false });
36
37
  }));
37
38
  router.get(`${BASE_PATH}/review/:id`, (0, observabilityRouteAccess_1.observabilityRouteMiddleware)(options.accessControl, { action: "read", resource: "aiReview" }, builder()
38
39
  .withTags(["observability"])
@@ -59,6 +60,7 @@ const addObservabilityReviewRoutes = (router, options) => {
59
60
  id: req.params.id,
60
61
  scores: body.scores ?? {},
61
62
  sinks: (0, observabilityAppRegistry_1.getObservabilityApp)()?.scoreSinks ?? [],
63
+ userId: req.user?._id ? String(req.user._id) : undefined,
62
64
  });
63
65
  }
64
66
  else if (body.action === "skip") {
@@ -0,0 +1,11 @@
1
+ import type express from "express";
2
+ import type { LocalReviewBatchStore } from "../local/reviewBatchStore";
3
+ import type { ObservabilityGenerateClient, ObservabilityRequestAiServiceFactory } from "../types";
4
+ import { type ObservabilityRouteAccessOptions } from "./observabilityRouteAccess";
5
+ export interface ObservabilityReviewBatchRouteOptions extends ObservabilityRouteAccessOptions {
6
+ aiService?: ObservabilityGenerateClient;
7
+ requestAiServiceFactory?: ObservabilityRequestAiServiceFactory;
8
+ store: LocalReviewBatchStore;
9
+ }
10
+ /** Register before the review item routes so `/review/batches` is not read as an item id. */
11
+ export declare const addObservabilityReviewBatchRoutes: (router: express.Application, options: ObservabilityReviewBatchRouteOptions) => void;
@@ -0,0 +1,78 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.addObservabilityReviewBatchRoutes = void 0;
4
+ const api_1 = require("@terreno/api");
5
+ const observabilityRouteAccess_1 = require("./observabilityRouteAccess");
6
+ const requestAiService_1 = require("./requestAiService");
7
+ const BASE_PATH = "/ai/observability";
8
+ /** Register before the review item routes so `/review/batches` is not read as an item id. */
9
+ const addObservabilityReviewBatchRoutes = (router, options) => {
10
+ const openApiOptions = options.openApi ? { openApi: options.openApi } : {};
11
+ const builder = () => {
12
+ return (0, api_1.createOpenApiBuilder)(openApiOptions);
13
+ };
14
+ router.get(`${BASE_PATH}/review/batches`, (0, observabilityRouteAccess_1.observabilityRouteMiddleware)(options.accessControl, { action: "list", resource: "aiReview" }, builder()
15
+ .withTags(["observability"])
16
+ .withSummary("List human review batches with judge agreement")
17
+ .withResponse(200, { data: { type: "array" } })
18
+ .build()), (0, api_1.asyncHandler)(async (_req, res) => {
19
+ return res.json({ data: await options.store.list(), more: false });
20
+ }));
21
+ router.post(`${BASE_PATH}/review/batches`, (0, observabilityRouteAccess_1.observabilityRouteMiddleware)(options.accessControl, { action: "assign", resource: "aiReview" }, builder()
22
+ .withTags(["observability"])
23
+ .withSummary("Create a review batch from traces, the most recent traces, or a dataset's unanswered items")
24
+ .withResponse(201, { data: { type: "object" } })
25
+ .build()), (0, api_1.asyncHandler)(async (req, res) => {
26
+ const body = req.body;
27
+ if (!body.evaluatorId) {
28
+ throw new api_1.APIError({ status: 400, title: "evaluatorId is required" });
29
+ }
30
+ if (body.traceIds !== undefined && !Array.isArray(body.traceIds)) {
31
+ throw new api_1.APIError({ status: 400, title: "traceIds must be an array" });
32
+ }
33
+ if (body.datasetId !== undefined && typeof body.datasetId !== "string") {
34
+ throw new api_1.APIError({ status: 400, title: "datasetId must be a string" });
35
+ }
36
+ const data = await options.store.create({
37
+ datasetId: body.datasetId || undefined,
38
+ description: body.description,
39
+ evaluatorId: body.evaluatorId,
40
+ limit: body.limit,
41
+ name: body.name,
42
+ promptName: body.promptName || undefined,
43
+ traceIds: body.traceIds,
44
+ });
45
+ return res.status(201).json({ data });
46
+ }));
47
+ router.get(`${BASE_PATH}/review/batches/:id`, (0, observabilityRouteAccess_1.observabilityRouteMiddleware)(options.accessControl, { action: "read", resource: "aiReview" }, builder()
48
+ .withTags(["observability"])
49
+ .withSummary("Get a review batch with human answers and judge agreement")
50
+ .withPathParameter("id", { type: "string" })
51
+ .withResponse(200, { data: { type: "object" } })
52
+ .build()), (0, api_1.asyncHandler)(async (req, res) => {
53
+ return res.json({ data: await options.store.get(req.params.id) });
54
+ }));
55
+ router.post(`${BASE_PATH}/review/batches/:id/judgeRuns`, (0, observabilityRouteAccess_1.observabilityRouteMiddleware)(options.accessControl, { action: "assign", resource: "aiReview" }, builder()
56
+ .withTags(["observability"])
57
+ .withSummary("Run an LLM judge over a review batch")
58
+ .withPathParameter("id", { type: "string" })
59
+ .withResponse(201, { data: { type: "object" } })
60
+ .build()), (0, api_1.asyncHandler)(async (req, res) => {
61
+ const body = req.body;
62
+ if (!body.evaluatorId) {
63
+ throw new api_1.APIError({ status: 400, title: "evaluatorId is required" });
64
+ }
65
+ const ai = (0, requestAiService_1.resolveRequestAiService)({
66
+ aiService: options.aiService,
67
+ req,
68
+ requestAiServiceFactory: options.requestAiServiceFactory,
69
+ });
70
+ const data = await options.store.runJudge({
71
+ ai,
72
+ batchId: req.params.id,
73
+ evaluatorId: body.evaluatorId,
74
+ });
75
+ return res.status(201).json({ data });
76
+ }));
77
+ };
78
+ exports.addObservabilityReviewBatchRoutes = addObservabilityReviewBatchRoutes;
@@ -1,5 +1,5 @@
1
1
  import type express from "express";
2
- import type { LocalTraceStore } from "../local/traceStore";
2
+ import { type LocalTraceStore } from "../local/traceStore";
3
3
  import { type ObservabilityRouteAccessOptions } from "./observabilityRouteAccess";
4
4
  export interface ObservabilityTraceRouteOptions extends ObservabilityRouteAccessOptions {
5
5
  store: LocalTraceStore;