@terreno/ai 57.20.0 → 57.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. package/CHANGELOG.md +81 -0
  2. package/README.md +3 -3
  3. package/biome.jsonc +3 -2
  4. package/dist/harness/agentLoop.js +1 -0
  5. package/dist/harness/commit.d.ts +4 -3
  6. package/dist/harness/commit.js +23 -4
  7. package/dist/index.d.ts +6 -9
  8. package/dist/index.js +14 -25
  9. package/dist/observability/adminScreens.js +5 -1
  10. package/dist/observability/evaluate.d.ts +8 -1
  11. package/dist/observability/evaluate.js +10 -2
  12. package/dist/observability/judgeAgreement.d.ts +53 -0
  13. package/dist/observability/judgeAgreement.js +123 -0
  14. package/dist/observability/local/datasetStore.d.ts +5 -0
  15. package/dist/observability/local/datasetStore.js +7 -2
  16. package/dist/observability/local/evaluatorStore.d.ts +17 -0
  17. package/dist/observability/local/evaluatorStore.js +73 -2
  18. package/dist/observability/local/experimentRunner.d.ts +42 -8
  19. package/dist/observability/local/experimentRunner.js +390 -11
  20. package/dist/observability/local/humanAnswers.d.ts +57 -0
  21. package/dist/observability/local/humanAnswers.js +174 -0
  22. package/dist/observability/local/judgeRun.d.ts +56 -0
  23. package/dist/observability/local/judgeRun.js +111 -0
  24. package/dist/observability/local/localPlugin.js +4 -0
  25. package/dist/observability/local/models/obsDatasetItem.js +4 -0
  26. package/dist/observability/local/models/obsEvaluator.js +4 -0
  27. package/dist/observability/local/models/obsExperiment.js +28 -0
  28. package/dist/observability/local/models/obsExperimentItem.js +4 -0
  29. package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
  30. package/dist/observability/local/models/obsJudgeRun.js +60 -0
  31. package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
  32. package/dist/observability/local/models/obsReviewBatch.js +48 -0
  33. package/dist/observability/local/models/obsReviewItem.js +10 -2
  34. package/dist/observability/local/models/obsTrace.js +4 -0
  35. package/dist/observability/local/promptReferences.d.ts +31 -0
  36. package/dist/observability/local/promptReferences.js +114 -0
  37. package/dist/observability/local/promptStore.d.ts +55 -19
  38. package/dist/observability/local/promptStore.js +171 -7
  39. package/dist/observability/local/reviewBatchStore.d.ts +106 -0
  40. package/dist/observability/local/reviewBatchStore.js +517 -0
  41. package/dist/observability/local/reviewStore.d.ts +24 -3
  42. package/dist/observability/local/reviewStore.js +98 -22
  43. package/dist/observability/local/traceStore.d.ts +14 -1
  44. package/dist/observability/local/traceStore.js +144 -11
  45. package/dist/observability/observabilityApp.js +10 -1
  46. package/dist/observability/questionFingerprint.d.ts +7 -0
  47. package/dist/observability/questionFingerprint.js +23 -0
  48. package/dist/observability/routes/experiments.js +12 -1
  49. package/dist/observability/routes/review.js +3 -1
  50. package/dist/observability/routes/reviewBatches.d.ts +11 -0
  51. package/dist/observability/routes/reviewBatches.js +78 -0
  52. package/dist/observability/routes/traces.d.ts +1 -1
  53. package/dist/observability/routes/traces.js +34 -2
  54. package/dist/observability/types.d.ts +10 -1
  55. package/dist/service/aiService.js +4 -1
  56. package/dist/service/chatTurn.js +38 -31
  57. package/dist/types/harness.d.ts +10 -4
  58. package/dist/types/index.d.ts +8 -3
  59. package/dist/types/observability.d.ts +74 -2
  60. package/e2e/smoke.ts +227 -0
  61. package/package.json +14 -28
  62. package/src/aiApp.test.ts +22 -17
  63. package/src/harness/agentLoop.test.ts +0 -3
  64. package/src/harness/agentLoop.ts +2 -1
  65. package/src/harness/agentRecovery.test.ts +30 -30
  66. package/src/harness/approvals.test.ts +11 -9
  67. package/src/harness/commit.ts +24 -4
  68. package/src/harness/events.test.ts +10 -6
  69. package/src/harness/extensions.test.ts +6 -8
  70. package/src/harness/ownership.test.ts +20 -9
  71. package/src/harness/retries.test.ts +6 -3
  72. package/src/harness/routes/conversations.test.ts +2 -2
  73. package/src/harness/routes/tasks.test.ts +2 -2
  74. package/src/harness/runners/jobsRunner.test.ts +1 -1
  75. package/src/harness/subagent.test.ts +29 -4
  76. package/src/harness/waits.test.ts +6 -4
  77. package/src/index.test.ts +3 -20
  78. package/src/index.ts +23 -26
  79. package/src/isolated/gptStream.isolated.ts +25 -28
  80. package/src/isolated/lazyProjectModel.isolated.ts +2 -2
  81. package/src/models/gptHistory.test.ts +10 -6
  82. package/src/observability/adminScreens.test.ts +7 -0
  83. package/src/observability/adminScreens.ts +6 -1
  84. package/src/observability/evaluate.test.ts +12 -10
  85. package/src/observability/evaluate.ts +12 -2
  86. package/src/observability/judgeAgreement.test.ts +111 -0
  87. package/src/observability/judgeAgreement.ts +184 -0
  88. package/src/observability/local/datasetStore.ts +30 -11
  89. package/src/observability/local/evaluatorStore.ts +96 -2
  90. package/src/observability/local/experimentRunner.test.ts +9 -12
  91. package/src/observability/local/experimentRunner.ts +497 -17
  92. package/src/observability/local/humanAnswers.ts +233 -0
  93. package/src/observability/local/judgePipeline.test.ts +444 -0
  94. package/src/observability/local/judgeRun.test.ts +173 -0
  95. package/src/observability/local/judgeRun.ts +184 -0
  96. package/src/observability/local/localPlugin.ts +4 -0
  97. package/src/observability/local/models/obsDatasetItem.ts +5 -0
  98. package/src/observability/local/models/obsEvaluator.ts +4 -0
  99. package/src/observability/local/models/obsExperiment.ts +29 -0
  100. package/src/observability/local/models/obsExperimentItem.ts +4 -0
  101. package/src/observability/local/models/obsJudgeRun.ts +61 -0
  102. package/src/observability/local/models/obsReviewBatch.ts +52 -0
  103. package/src/observability/local/models/obsReviewItem.ts +11 -2
  104. package/src/observability/local/models/obsTrace.ts +5 -0
  105. package/src/observability/local/obsModels.test.ts +13 -0
  106. package/src/observability/local/promptReferences.test.ts +128 -0
  107. package/src/observability/local/promptReferences.ts +149 -0
  108. package/src/observability/local/promptStore.test.ts +295 -3
  109. package/src/observability/local/promptStore.ts +243 -32
  110. package/src/observability/local/reviewBatchStore.test.ts +251 -0
  111. package/src/observability/local/reviewBatchStore.ts +656 -0
  112. package/src/observability/local/reviewStore.ts +122 -25
  113. package/src/observability/local/traceStore.test.ts +139 -1
  114. package/src/observability/local/traceStore.ts +162 -11
  115. package/src/observability/observabilityApp.test.ts +13 -10
  116. package/src/observability/observabilityApp.ts +10 -1
  117. package/src/observability/questionFingerprint.test.ts +33 -0
  118. package/src/observability/questionFingerprint.ts +24 -0
  119. package/src/observability/routes/datasets.test.ts +2 -2
  120. package/src/observability/routes/evaluators.test.ts +2 -2
  121. package/src/observability/routes/experiments.test.ts +39 -2
  122. package/src/observability/routes/experiments.ts +13 -1
  123. package/src/observability/routes/prompts.test.ts +3 -3
  124. package/src/observability/routes/review.test.ts +2 -2
  125. package/src/observability/routes/review.ts +3 -1
  126. package/src/observability/routes/reviewBatches.test.ts +209 -0
  127. package/src/observability/routes/reviewBatches.ts +138 -0
  128. package/src/observability/routes/status.test.ts +8 -8
  129. package/src/observability/routes/testMultiStage.test.ts +6 -6
  130. package/src/observability/routes/traces.test.ts +63 -2
  131. package/src/observability/routes/traces.ts +46 -4
  132. package/src/observability/types.ts +5 -2
  133. package/src/routes/files.test.ts +5 -3
  134. package/src/routes/gpt.test.ts +21 -14
  135. package/src/routes/gptActions.test.ts +7 -8
  136. package/src/routes/gptDatasets.test.ts +1 -1
  137. package/src/routes/gptFiles.test.ts +1 -1
  138. package/src/routes/gptHistories.test.ts +15 -9
  139. package/src/routes/mcp.test.ts +3 -3
  140. package/src/routes/projects.test.ts +5 -3
  141. package/src/routes/routes.test.ts +156 -115
  142. package/src/service/aiService.test.ts +22 -1
  143. package/src/service/aiService.ts +4 -1
  144. package/src/service/asks.test.ts +14 -6
  145. package/src/service/chatTurn.ts +42 -33
  146. package/src/service/fileStorage.test.ts +5 -2
  147. package/src/service/getMCPTools.test.ts +9 -5
  148. package/src/service/mcpService.test.ts +6 -3
  149. package/src/tests/bunSetup.ts +0 -54
  150. package/src/tests/chatHarness.ts +4 -4
  151. package/src/tests/fakeJudge.ts +8 -0
  152. package/src/tests/helpers.ts +26 -4
  153. package/src/types/harness.ts +11 -4
  154. package/src/types/index.ts +8 -3
  155. package/src/types/observability.ts +86 -2
  156. package/tsconfig.typecheck.json +18 -0
  157. package/dist/langfuseApp.d.ts +0 -9
  158. package/dist/langfuseApp.js +0 -53
  159. package/dist/langfuseCache.d.ts +0 -15
  160. package/dist/langfuseCache.js +0 -54
  161. package/dist/langfuseClient.d.ts +0 -6
  162. package/dist/langfuseClient.js +0 -34
  163. package/dist/langfusePrompts.d.ts +0 -12
  164. package/dist/langfusePrompts.js +0 -89
  165. package/dist/langfuseRoutesEvaluations.d.ts +0 -3
  166. package/dist/langfuseRoutesEvaluations.js +0 -32
  167. package/dist/langfuseRoutesMiddleware.d.ts +0 -2
  168. package/dist/langfuseRoutesMiddleware.js +0 -13
  169. package/dist/langfuseRoutesPlayground.d.ts +0 -2
  170. package/dist/langfuseRoutesPlayground.js +0 -36
  171. package/dist/langfuseRoutesPrompts.d.ts +0 -2
  172. package/dist/langfuseRoutesPrompts.js +0 -56
  173. package/dist/langfuseRoutesTraces.d.ts +0 -2
  174. package/dist/langfuseRoutesTraces.js +0 -30
  175. package/dist/langfuseTracing.d.ts +0 -4
  176. package/dist/langfuseTracing.js +0 -32
  177. package/dist/langfuseTypes.d.ts +0 -96
  178. package/dist/langfuseTypes.js +0 -2
  179. package/dist/langfuseVercelAi.d.ts +0 -12
  180. package/dist/langfuseVercelAi.js +0 -49
  181. package/src/isolated/langfuseClient.isolated.ts +0 -60
  182. package/src/langfuseApp.test.ts +0 -135
  183. package/src/langfuseApp.ts +0 -62
  184. package/src/langfuseCache.test.ts +0 -75
  185. package/src/langfuseCache.ts +0 -72
  186. package/src/langfuseClient.test.ts +0 -69
  187. package/src/langfuseClient.ts +0 -34
  188. package/src/langfusePrompts.test.ts +0 -188
  189. package/src/langfusePrompts.ts +0 -121
  190. package/src/langfuseRoutes.test.ts +0 -256
  191. package/src/langfuseRoutesEvaluations.ts +0 -48
  192. package/src/langfuseRoutesMiddleware.test.ts +0 -57
  193. package/src/langfuseRoutesMiddleware.ts +0 -12
  194. package/src/langfuseRoutesPlayground.ts +0 -50
  195. package/src/langfuseRoutesPrompts.ts +0 -87
  196. package/src/langfuseRoutesTraces.ts +0 -40
  197. package/src/langfuseTracing.test.ts +0 -68
  198. package/src/langfuseTracing.ts +0 -33
  199. package/src/langfuseTypes.ts +0 -103
  200. package/src/langfuseVercelAi.test.ts +0 -79
  201. package/src/langfuseVercelAi.ts +0 -72
@@ -5,9 +5,14 @@ import mongoose from "mongoose";
5
5
  import type {EvaluatorDimension} from "../../types/observability";
6
6
  import {buildReviewPanels, type ReviewPanels} from "../reviewPanels";
7
7
  import type {ScoreRecord, ScoreSink} from "../types";
8
+ import {loadDatasetItemSubjects, writeHumanAnswer} from "./humanAnswers";
9
+ import {registerObsDataset} from "./models/obsDataset";
10
+ import {registerObsDatasetItem} from "./models/obsDatasetItem";
8
11
  import {registerObsEvaluator} from "./models/obsEvaluator";
9
12
  import {registerObsPrompt} from "./models/obsPrompt";
13
+ import {registerObsPromptLabel} from "./models/obsPromptLabel";
10
14
  import {registerObsPromptVersion} from "./models/obsPromptVersion";
15
+ import {registerObsReviewBatch} from "./models/obsReviewBatch";
11
16
  import {registerObsReviewItem} from "./models/obsReviewItem";
12
17
  import {registerObsTrace} from "./models/obsTrace";
13
18
  import {LocalTraceStore} from "./traceStore";
@@ -16,16 +21,22 @@ export type ReviewStatus = "done" | "in_progress" | "pending" | "skipped";
16
21
 
17
22
  export interface ReviewListItem {
18
23
  assigneeId?: string;
24
+ batchId?: string;
25
+ /** Dataset item under review, when the item came from a dataset review batch. */
26
+ datasetItemId?: string;
19
27
  enqueuedAt: string;
20
28
  evaluatorId: string;
21
29
  id: string;
22
30
  promptName?: string;
23
31
  reason: string;
24
32
  status: ReviewStatus;
25
- traceId: string;
33
+ /** Unset when the reviewed dataset item has no source trace. */
34
+ traceId?: string;
26
35
  traceName: string;
27
36
  }
28
37
 
38
+ const DATASET_ITEM_NAME = "Dataset item";
39
+
29
40
  const objectId = (value: string, label: string): mongoose.Types.ObjectId => {
30
41
  if (!mongoose.Types.ObjectId.isValid(value)) {
31
42
  throw new APIError({status: 400, title: `${label} must be an object id`});
@@ -39,6 +50,7 @@ const toIso = (value: Date): string => {
39
50
 
40
51
  export class LocalReviewStore {
41
52
  async enqueue(params: {
53
+ batchId?: string;
42
54
  evaluatorId: string;
43
55
  reason?: "dataset_candidate" | "eval" | "feedback" | "manual";
44
56
  spanId?: string;
@@ -71,6 +83,7 @@ export class LocalReviewStore {
71
83
  reason: params.reason ?? "manual",
72
84
  status: "pending",
73
85
  traceId: trace._id,
86
+ ...(params.batchId ? {batchId: objectId(params.batchId, "batchId")} : {}),
74
87
  ...(params.spanId ? {spanId: objectId(params.spanId, "spanId")} : {}),
75
88
  });
76
89
  created.push(this.toListItem(item, trace.name, trace.prompts[0]?.name));
@@ -78,21 +91,52 @@ export class LocalReviewStore {
78
91
  return created;
79
92
  }
80
93
 
81
- async list(status?: ReviewStatus): Promise<{
94
+ /** Queue dataset items for a human question set; items with a source trace show that trace. */
95
+ async enqueueDatasetItems(params: {
96
+ batchId: string;
97
+ evaluatorId: string;
98
+ items: Array<{datasetItemId: string; traceId?: string}>;
99
+ }): Promise<ReviewListItem[]> {
100
+ const created: ReviewListItem[] = [];
101
+ for (const entry of params.items) {
102
+ const item = await registerObsReviewItem().create({
103
+ batchId: objectId(params.batchId, "batchId"),
104
+ datasetItemId: objectId(entry.datasetItemId, "datasetItemId"),
105
+ enqueuedAt: DateTime.utc().toJSDate(),
106
+ evaluatorId: objectId(params.evaluatorId, "evaluatorId"),
107
+ reason: "manual",
108
+ status: "pending",
109
+ ...(entry.traceId ? {traceId: objectId(entry.traceId, "traceId")} : {}),
110
+ });
111
+ created.push(this.toListItem(item, DATASET_ITEM_NAME));
112
+ }
113
+ return created;
114
+ }
115
+
116
+ async list(
117
+ status?: ReviewStatus,
118
+ options: {batchId?: string} = {}
119
+ ): Promise<{
82
120
  counts: Record<ReviewStatus, number>;
83
121
  data: ReviewListItem[];
84
122
  }> {
85
123
  const ObsReviewItem = registerObsReviewItem();
124
+ const scope = options.batchId ? {batchId: objectId(options.batchId, "batchId")} : {};
86
125
  const counts = {
87
- done: await ObsReviewItem.countDocuments({status: "done"}),
88
- in_progress: await ObsReviewItem.countDocuments({status: "in_progress"}),
89
- pending: await ObsReviewItem.countDocuments({status: "pending"}),
90
- skipped: await ObsReviewItem.countDocuments({status: "skipped"}),
126
+ done: await ObsReviewItem.countDocuments({...scope, status: "done"}),
127
+ in_progress: await ObsReviewItem.countDocuments({...scope, status: "in_progress"}),
128
+ pending: await ObsReviewItem.countDocuments({...scope, status: "pending"}),
129
+ skipped: await ObsReviewItem.countDocuments({...scope, status: "skipped"}),
91
130
  };
92
- const filter = status ? {status} : {};
93
- const rows = await ObsReviewItem.find(filter).sort({enqueuedAt: 1});
131
+ const filter = status ? {...scope, status} : scope;
132
+ // Batch items keep their batch order (enqueued together, newest trace first).
133
+ const rows = await ObsReviewItem.find(filter).sort({_id: 1, enqueuedAt: 1});
94
134
  const data: ReviewListItem[] = [];
95
135
  for (const row of rows) {
136
+ if (!row.traceId) {
137
+ data.push(this.toListItem(row, DATASET_ITEM_NAME));
138
+ continue;
139
+ }
96
140
  const trace = await registerObsTrace().findOneOrNone({_id: row.traceId});
97
141
  data.push(this.toListItem(row, trace?.name ?? "Unknown trace", trace?.prompts[0]?.name));
98
142
  }
@@ -100,7 +144,10 @@ export class LocalReviewStore {
100
144
  }
101
145
 
102
146
  async getDetail(id: string): Promise<{
147
+ batchId?: string;
148
+ batchName?: string;
103
149
  comment?: string;
150
+ datasetItemId?: string;
104
151
  dimensions: EvaluatorDimension[];
105
152
  evaluatorId: string;
106
153
  id: string;
@@ -110,7 +157,7 @@ export class LocalReviewStore {
110
157
  rawOutput?: unknown;
111
158
  scores?: Record<string, boolean | number | string>;
112
159
  status: ReviewStatus;
113
- traceId: string;
160
+ traceId?: string;
114
161
  }> {
115
162
  const item = await registerObsReviewItem().findOneOrNone({_id: objectId(id, "id")});
116
163
  if (!item) {
@@ -120,13 +167,49 @@ export class LocalReviewStore {
120
167
  if (!evaluator) {
121
168
  throw new APIError({status: 404, title: "Unknown evaluator"});
122
169
  }
123
- const trace = await registerObsTrace().findOneOrNone({_id: item.traceId});
124
- const promptRef = trace?.prompts[0];
170
+ const trace = item.traceId
171
+ ? await registerObsTrace().findOneOrNone({_id: item.traceId})
172
+ : undefined;
173
+ const batch = item.batchId
174
+ ? await registerObsReviewBatch().findOneOrNone({_id: item.batchId})
175
+ : undefined;
176
+ let input = trace?.input;
177
+ let output = trace?.output;
125
178
  let variables:
126
179
  | {key: string; label?: string; required: boolean; reviewerNote?: string}[]
127
180
  | undefined;
128
181
  let outputFieldNotes: Record<string, string> | undefined;
129
182
  let outputSchema: Record<string, unknown> | undefined;
183
+ // A dataset item without a (surviving) source trace shows its own input and expected output.
184
+ if (!trace && item.datasetItemId) {
185
+ const datasetItem = await registerObsDatasetItem().findOneOrNone({
186
+ _id: item.datasetItemId,
187
+ });
188
+ if (datasetItem) {
189
+ const subject = (await loadDatasetItemSubjects([datasetItem])).get(String(datasetItem._id));
190
+ input = subject?.input;
191
+ output = subject?.output;
192
+ const dataset = await registerObsDataset().findOneOrNone({_id: datasetItem.datasetId});
193
+ if (dataset?.inputSchemaPromptName) {
194
+ const prompt = await registerObsPrompt().findOneOrNone({
195
+ name: dataset.inputSchemaPromptName,
196
+ });
197
+ const label = prompt
198
+ ? await registerObsPromptLabel().findOneOrNone({
199
+ label: "production",
200
+ promptId: prompt._id,
201
+ })
202
+ : undefined;
203
+ const version = label
204
+ ? await registerObsPromptVersion().findOneOrNone({_id: label.versionId})
205
+ : undefined;
206
+ variables = version?.variables;
207
+ outputFieldNotes = version?.outputFieldNotes;
208
+ outputSchema = version?.outputSchema;
209
+ }
210
+ }
211
+ }
212
+ const promptRef = trace?.prompts[0];
130
213
  if (promptRef) {
131
214
  const prompt = await registerObsPrompt().findOneOrNone({name: promptRef.name});
132
215
  if (prompt) {
@@ -140,23 +223,20 @@ export class LocalReviewStore {
140
223
  }
141
224
  }
142
225
  return {
226
+ batchId: batch ? String(batch._id) : undefined,
227
+ batchName: batch?.name,
143
228
  comment: item.comment,
229
+ datasetItemId: item.datasetItemId ? String(item.datasetItemId) : undefined,
144
230
  dimensions: evaluator.dimensions,
145
231
  evaluatorId: String(evaluator._id),
146
232
  id: String(item._id),
147
233
  instructions: evaluator.instructions,
148
- panels: buildReviewPanels({
149
- input: trace?.input,
150
- output: trace?.output,
151
- outputFieldNotes,
152
- outputSchema,
153
- variables,
154
- }),
155
- rawInput: trace?.input,
156
- rawOutput: trace?.output,
234
+ panels: buildReviewPanels({input, output, outputFieldNotes, outputSchema, variables}),
235
+ rawInput: input,
236
+ rawOutput: output,
157
237
  scores: item.scores,
158
238
  status: item.status,
159
- traceId: String(item.traceId),
239
+ traceId: item.traceId ? String(item.traceId) : undefined,
160
240
  };
161
241
  }
162
242
 
@@ -165,6 +245,8 @@ export class LocalReviewStore {
165
245
  id: string;
166
246
  scores: Record<string, boolean | number | string>;
167
247
  sinks: ScoreSink[];
248
+ /** Reviewer, recorded on the dataset item's human answers. */
249
+ userId?: string;
168
250
  }): Promise<void> {
169
251
  const item = await this.requireItem(params.id);
170
252
  const evaluator = await registerObsEvaluator().findOneOrNone({_id: item.evaluatorId});
@@ -178,7 +260,9 @@ export class LocalReviewStore {
178
260
  throw new APIError({status: 400, title: `Score "${missing.key}" is required`});
179
261
  }
180
262
  const records: ScoreRecord[] = [];
181
- for (const dimension of evaluator.dimensions) {
263
+ const traceId = item.traceId;
264
+ // Scores attach to traces, so a dataset item without a trace only records human answers.
265
+ for (const dimension of traceId ? evaluator.dimensions : []) {
182
266
  const value = params.scores[dimension.key];
183
267
  if (value === undefined) {
184
268
  continue;
@@ -189,7 +273,7 @@ export class LocalReviewStore {
189
273
  evaluatorId: String(evaluator._id),
190
274
  name: dimension.key,
191
275
  source: "human" as const,
192
- traceId: String(item.traceId),
276
+ traceId: String(traceId),
193
277
  value,
194
278
  });
195
279
  }
@@ -213,6 +297,15 @@ export class LocalReviewStore {
213
297
  item.scores = params.scores;
214
298
  item.status = "done";
215
299
  await item.save();
300
+ if (item.datasetItemId) {
301
+ await writeHumanAnswer({
302
+ datasetItemId: item.datasetItemId,
303
+ evaluator,
304
+ reviewItemId: String(item._id),
305
+ scores: params.scores,
306
+ userId: params.userId,
307
+ });
308
+ }
216
309
  }
217
310
 
218
311
  async skip(id: string): Promise<void> {
@@ -240,24 +333,28 @@ export class LocalReviewStore {
240
333
  row: {
241
334
  _id: mongoose.Types.ObjectId;
242
335
  assigneeId?: mongoose.Types.ObjectId;
336
+ batchId?: mongoose.Types.ObjectId;
337
+ datasetItemId?: mongoose.Types.ObjectId;
243
338
  enqueuedAt: Date;
244
339
  evaluatorId: mongoose.Types.ObjectId;
245
340
  reason: string;
246
341
  status: ReviewStatus;
247
- traceId: mongoose.Types.ObjectId;
342
+ traceId?: mongoose.Types.ObjectId;
248
343
  },
249
344
  traceName: string,
250
345
  promptName?: string
251
346
  ): ReviewListItem {
252
347
  return {
253
348
  assigneeId: row.assigneeId ? String(row.assigneeId) : undefined,
349
+ batchId: row.batchId ? String(row.batchId) : undefined,
350
+ datasetItemId: row.datasetItemId ? String(row.datasetItemId) : undefined,
254
351
  enqueuedAt: toIso(row.enqueuedAt),
255
352
  evaluatorId: String(row.evaluatorId),
256
353
  id: String(row._id),
257
354
  promptName,
258
355
  reason: row.reason,
259
356
  status: row.status,
260
- traceId: String(row.traceId),
357
+ traceId: row.traceId ? String(row.traceId) : undefined,
261
358
  traceName,
262
359
  };
263
360
  }
@@ -8,7 +8,13 @@ import {createLocalObservabilityPlugin} from "./localPlugin";
8
8
  import {registerObsScore} from "./models/obsScore";
9
9
  import {registerObsSpan} from "./models/obsSpan";
10
10
  import {registerObsTrace} from "./models/obsTrace";
11
- import {LocalTraceSink, LocalTraceStore, MemoryScoreSink, MemoryTraceSink} from "./traceStore";
11
+ import {
12
+ LocalTraceSink,
13
+ LocalTraceStore,
14
+ MemoryScoreSink,
15
+ MemoryTraceSink,
16
+ parseTraceListSort,
17
+ } from "./traceStore";
12
18
 
13
19
  const iso = (msAgo: number): string => {
14
20
  return DateTime.utc().minus({milliseconds: msAgo}).toISO() ?? "";
@@ -244,6 +250,138 @@ describe("LocalTraceStore", () => {
244
250
  expect(sinkDetail.name).toBe("sink");
245
251
  });
246
252
 
253
+ describe("sorting", () => {
254
+ const at = (base: DateTime, ms: number): string => base.plus({milliseconds: ms}).toISO() ?? "";
255
+
256
+ const span = (id: string, startedAt: string): TraceRecord["spans"][number] => {
257
+ return {id, kind: "LLM", name: id, startedAt, status: "ok"};
258
+ };
259
+
260
+ const score = (traceId: string): ScoreRecord => {
261
+ return {dataType: "boolean", name: "correct", source: "human", traceId, value: true};
262
+ };
263
+
264
+ // bravo: 15 tokens, $0.002, 300 ms, 1 span, 2 scores
265
+ // alpha: 150 tokens, $0.0001, 100 ms, 3 spans, 0 scores
266
+ // Charlie: no usage, no end time, 0 spans, 1 score
267
+ const seed = async (): Promise<{alpha: string; bravo: string; charlie: string}> => {
268
+ const base = DateTime.utc().minus({hours: 1});
269
+ const bravo = await store.exportTrace(
270
+ baseTrace({
271
+ endedAt: at(base, 300),
272
+ name: "bravo",
273
+ spans: [span("b1", at(base, 0))],
274
+ startedAt: at(base, 0),
275
+ usage: {costUsd: 0.002, inputTokens: 10, outputTokens: 5},
276
+ })
277
+ );
278
+ const alpha = await store.exportTrace(
279
+ baseTrace({
280
+ endedAt: at(base, 1100),
281
+ name: "alpha",
282
+ spans: [
283
+ span("a1", at(base, 1000)),
284
+ span("a2", at(base, 1000)),
285
+ span("a3", at(base, 1000)),
286
+ ],
287
+ startedAt: at(base, 1000),
288
+ usage: {costUsd: 0.0001, inputTokens: 100, outputTokens: 50},
289
+ })
290
+ );
291
+ const charlie = await store.exportTrace(
292
+ baseTrace({
293
+ endedAt: undefined,
294
+ name: "Charlie",
295
+ spans: [],
296
+ startedAt: at(base, 2000),
297
+ usage: undefined,
298
+ })
299
+ );
300
+ await store.exportScore(score(bravo.id));
301
+ await store.exportScore(score(bravo.id));
302
+ await store.exportScore(score(charlie.id));
303
+ return {alpha: alpha.id, bravo: bravo.id, charlie: charlie.id};
304
+ };
305
+
306
+ const names = async (sort?: string): Promise<string[]> => {
307
+ const listed = await store.list({sort});
308
+ return listed.data.map((row) => row.name);
309
+ };
310
+
311
+ const pagedNames = async (sort: string): Promise<string[]> => {
312
+ const first = await store.list({limit: 2, page: 1, sort});
313
+ const second = await store.list({limit: 2, page: 2, sort});
314
+ expect(first.meta.total).toBe(3);
315
+ return [...first.data, ...second.data].map((row) => row.name);
316
+ };
317
+
318
+ const cases: Array<[string, string[]]> = [
319
+ ["startedAt", ["bravo", "alpha", "Charlie"]],
320
+ ["-startedAt", ["Charlie", "alpha", "bravo"]],
321
+ ["name", ["alpha", "bravo", "Charlie"]],
322
+ ["-name", ["Charlie", "bravo", "alpha"]],
323
+ ["tokens", ["bravo", "alpha", "Charlie"]],
324
+ ["-tokens", ["alpha", "bravo", "Charlie"]],
325
+ ["cost", ["alpha", "bravo", "Charlie"]],
326
+ ["-cost", ["bravo", "alpha", "Charlie"]],
327
+ ["latency", ["alpha", "bravo", "Charlie"]],
328
+ ["-latency", ["bravo", "alpha", "Charlie"]],
329
+ ["spans", ["Charlie", "bravo", "alpha"]],
330
+ ["-spans", ["alpha", "bravo", "Charlie"]],
331
+ ["scores", ["alpha", "Charlie", "bravo"]],
332
+ ["-scores", ["bravo", "Charlie", "alpha"]],
333
+ ];
334
+
335
+ for (const [sort, expected] of cases) {
336
+ it(`orders every page by ${sort}`, async () => {
337
+ await seed();
338
+ expect(await names(sort)).toEqual(expected);
339
+ expect(await pagedNames(sort)).toEqual(expected);
340
+ });
341
+ }
342
+
343
+ it("defaults to newest first and keeps counts on sorted rows", async () => {
344
+ const ids = await seed();
345
+ expect(await names()).toEqual(["Charlie", "alpha", "bravo"]);
346
+ const bySpans = await store.list({sort: "-spans"});
347
+ expect(bySpans.data[0]).toMatchObject({id: ids.alpha, scoreCount: 0, spanCount: 3});
348
+ });
349
+
350
+ it("orders by status in both directions", async () => {
351
+ await seed();
352
+ await store.exportTrace(baseTrace({name: "zulu", status: "error"}));
353
+ const asc = await store.list({sort: "status"});
354
+ expect(asc.data.map((row) => row.status)).toEqual(["error", "ok", "ok", "ok"]);
355
+ const desc = await store.list({limit: 3, page: 2, sort: "-status"});
356
+ expect(desc.data.map((row) => row.name)).toEqual(["zulu"]);
357
+ });
358
+
359
+ it("sorts inside the filtered set", async () => {
360
+ await seed();
361
+ await store.exportTrace(baseTrace({name: "zulu", status: "error"}));
362
+ const okOnly = await store.list({sort: "-tokens", status: "ok"});
363
+ expect(okOnly.data.map((row) => row.name)).toEqual(["alpha", "bravo", "Charlie"]);
364
+ });
365
+
366
+ it("rejects unknown sort keys with a 400", async () => {
367
+ await expect(store.list({sort: "created"})).rejects.toMatchObject({status: 400});
368
+ await expect(store.list({sort: "-"})).rejects.toMatchObject({status: 400});
369
+ expect(parseTraceListSort(undefined)).toEqual({direction: -1, key: "startedAt"});
370
+ expect(parseTraceListSort("cost")).toEqual({direction: 1, key: "cost"});
371
+ });
372
+ });
373
+
374
+ it("filters by any of several prompt names", async () => {
375
+ await store.exportTrace(baseTrace({name: "one", prompts: [{name: "summarize", version: 1}]}));
376
+ await store.exportTrace(baseTrace({name: "two", prompts: [{name: "title", version: 1}]}));
377
+ await store.exportTrace(baseTrace({name: "three", prompts: [{name: "other", version: 1}]}));
378
+ const listed = await store.list({prompt: ["summarize", "title"], sort: "name"});
379
+ expect(listed.data.map((row) => row.name)).toEqual(["one", "two"]);
380
+ await expect(store.list({prompt: ["summarize", "title"], promptVersion: 1})).rejects.toThrow(
381
+ /exactly one prompt/
382
+ );
383
+ });
384
+
247
385
  it("throws when loading an unknown trace", async () => {
248
386
  await expect(store.getDetail(new mongoose.Types.ObjectId().toString())).rejects.toThrow(
249
387
  /Unknown trace/
@@ -7,17 +7,42 @@ import {registerObsScore} from "./models/obsScore";
7
7
  import {registerObsSpan} from "./models/obsSpan";
8
8
  import {registerObsTrace} from "./models/obsTrace";
9
9
 
10
+ /** Columns the trace list can sort by. Prefix with `-` for descending. */
11
+ export const TRACE_LIST_SORT_KEYS = [
12
+ "cost",
13
+ "latency",
14
+ "name",
15
+ "scores",
16
+ "spans",
17
+ "startedAt",
18
+ "status",
19
+ "tokens",
20
+ ] as const;
21
+
22
+ export type TraceListSortKey = (typeof TRACE_LIST_SORT_KEYS)[number];
23
+
24
+ export interface TraceListSort {
25
+ direction: 1 | -1;
26
+ key: TraceListSortKey;
27
+ }
28
+
29
+ /** Newest first, matching the admin Traces screen default. */
30
+ export const DEFAULT_TRACE_LIST_SORT = "-startedAt";
31
+
10
32
  export interface TraceListQuery {
11
33
  flaggedForDataset?: boolean;
12
34
  from?: string;
13
35
  hasScore?: boolean;
14
36
  limit?: number;
15
37
  page?: number;
16
- prompt?: string;
38
+ /** One prompt name, or several to match traces that used any of them. */
39
+ prompt?: string | string[];
17
40
  promptVersion?: number;
18
41
  scope?: string;
19
42
  sensitive?: boolean;
20
43
  sessionId?: string;
44
+ /** One of `TRACE_LIST_SORT_KEYS`, optionally prefixed with `-` for descending. */
45
+ sort?: string;
21
46
  status?: "error" | "ok";
22
47
  to?: string;
23
48
  userId?: string;
@@ -129,6 +154,112 @@ const usageWithoutUndefinedCost = (
129
154
  return usage;
130
155
  };
131
156
 
157
+ const isTraceListSortKey = (value: string): value is TraceListSortKey => {
158
+ return (TRACE_LIST_SORT_KEYS as readonly string[]).includes(value);
159
+ };
160
+
161
+ export const parseTraceListSort = (value?: string): TraceListSort => {
162
+ const raw = value?.trim() || DEFAULT_TRACE_LIST_SORT;
163
+ const descending = raw.startsWith("-");
164
+ const key = descending ? raw.slice(1) : raw;
165
+ if (!isTraceListSortKey(key)) {
166
+ throw new APIError({
167
+ detail: `Use one of ${TRACE_LIST_SORT_KEYS.join(", ")}, optionally prefixed with "-".`,
168
+ status: 400,
169
+ title: "Unknown trace sort key",
170
+ });
171
+ }
172
+ return {direction: descending ? -1 : 1, key};
173
+ };
174
+
175
+ const promptNames = (prompt?: string | string[]): string[] => {
176
+ const values = Array.isArray(prompt) ? prompt : prompt === undefined ? [] : [prompt];
177
+ return values.map((entry) => entry.trim()).filter((entry) => entry !== "");
178
+ };
179
+
180
+ /** Count rows in `from` whose traceId matches the trace, for span and score sorts. */
181
+ const countLookupStages = (from: string): mongoose.PipelineStage[] => {
182
+ return [
183
+ {
184
+ $lookup: {
185
+ as: "_sortCounts",
186
+ from,
187
+ let: {traceId: "$_id"},
188
+ pipeline: [{$match: {$expr: {$eq: ["$traceId", "$$traceId"]}}}, {$count: "n"}],
189
+ },
190
+ },
191
+ {$addFields: {_sortValue: {$ifNull: [{$arrayElemAt: ["$_sortCounts.n", 0]}, 0]}}},
192
+ ];
193
+ };
194
+
195
+ const missingAware = (expression: unknown): mongoose.PipelineStage[] => {
196
+ return [
197
+ {$addFields: {_sortValue: expression}},
198
+ {$addFields: {_sortMissing: {$cond: [{$eq: ["$_sortValue", null]}, 1, 0]}}},
199
+ ];
200
+ };
201
+
202
+ const isMissing = (path: string): Record<string, unknown> => {
203
+ return {$eq: [{$type: path}, "missing"]};
204
+ };
205
+
206
+ /**
207
+ * Stages that compute `_sortValue` (and `_sortMissing` for usage-derived values, so traces
208
+ * without usage or an end time always list last) before the sort runs.
209
+ */
210
+ const sortValueStages = (key: TraceListSortKey): mongoose.PipelineStage[] => {
211
+ if (key === "cost") {
212
+ return missingAware({$ifNull: ["$usage.costUsd", null]});
213
+ }
214
+ if (key === "tokens") {
215
+ return missingAware({
216
+ $cond: [
217
+ {$and: [isMissing("$usage.inputTokens"), isMissing("$usage.outputTokens")]},
218
+ null,
219
+ {$add: [{$ifNull: ["$usage.inputTokens", 0]}, {$ifNull: ["$usage.outputTokens", 0]}]},
220
+ ],
221
+ });
222
+ }
223
+ if (key === "latency") {
224
+ return missingAware({
225
+ $cond: [
226
+ {$or: [{$eq: [{$ifNull: ["$endedAt", null]}, null]}, {$eq: ["$startedAt", null]}]},
227
+ null,
228
+ {$subtract: ["$endedAt", "$startedAt"]},
229
+ ],
230
+ });
231
+ }
232
+ if (key === "spans") {
233
+ return countLookupStages(registerObsSpan().collection.name);
234
+ }
235
+ if (key === "scores") {
236
+ return countLookupStages(registerObsScore().collection.name);
237
+ }
238
+ return [];
239
+ };
240
+
241
+ /** `$sort` keys are order-significant, so the spec is built from an ordered entry list. */
242
+ const sortSpec = (sort: TraceListSort): Record<string, 1 | -1> => {
243
+ const dir = sort.direction;
244
+ if (sort.key === "startedAt" || sort.key === "name" || sort.key === "status") {
245
+ return Object.fromEntries([
246
+ [sort.key, dir],
247
+ ["_id", dir],
248
+ ]);
249
+ }
250
+ if (sort.key === "spans" || sort.key === "scores") {
251
+ return Object.fromEntries([
252
+ ["_sortValue", dir],
253
+ ["_id", dir],
254
+ ]);
255
+ }
256
+ return Object.fromEntries([
257
+ ["_sortMissing", 1],
258
+ ["_sortValue", dir],
259
+ ["_id", dir],
260
+ ]);
261
+ };
262
+
132
263
  const countByTraceId = async (
133
264
  model: mongoose.Model<mongoose.Document>,
134
265
  ids: mongoose.Types.ObjectId[]
@@ -229,15 +360,22 @@ export class LocalTraceStore {
229
360
  if (query.scope) {
230
361
  filter.scope = query.scope;
231
362
  }
232
- if (query.promptVersion !== undefined && !query.prompt) {
363
+ const sort = parseTraceListSort(query.sort);
364
+ const prompts = promptNames(query.prompt);
365
+ if (query.promptVersion !== undefined && prompts.length === 0) {
233
366
  throw new APIError({status: 400, title: "prompt is required when promptVersion is set"});
234
367
  }
235
- if (query.prompt && query.promptVersion !== undefined) {
368
+ if (query.promptVersion !== undefined && prompts.length > 1) {
369
+ throw new APIError({status: 400, title: "promptVersion needs exactly one prompt"});
370
+ }
371
+ if (prompts.length === 1 && query.promptVersion !== undefined) {
236
372
  filter.prompts = {
237
- $elemMatch: {name: query.prompt, version: query.promptVersion},
373
+ $elemMatch: {name: prompts[0], version: query.promptVersion},
238
374
  };
239
- } else if (query.prompt) {
240
- filter["prompts.name"] = query.prompt;
375
+ } else if (prompts.length === 1) {
376
+ filter["prompts.name"] = prompts[0];
377
+ } else if (prompts.length > 1) {
378
+ filter["prompts.name"] = {$in: prompts};
241
379
  }
242
380
  if (query.sensitive !== undefined) {
243
381
  filter.sensitive = query.sensitive;
@@ -270,11 +408,24 @@ export class LocalTraceStore {
270
408
  }
271
409
  const ObsTrace = registerObsTrace();
272
410
  const total = await ObsTrace.countDocuments(filter);
273
- const rows = await ObsTrace.find(filter)
274
- .sort({created: -1})
275
- .skip((page - 1) * limit)
276
- .limit(limit);
277
- const ids = rows.map((row) => row._id);
411
+ // Sort in Mongo before skip/limit so derived columns (latency, counts) order every page.
412
+ const aggregate = ObsTrace.aggregate<{_id: mongoose.Types.ObjectId}>([
413
+ {$match: {deleted: {$ne: true}, ...filter}},
414
+ ...sortValueStages(sort.key),
415
+ {$sort: sortSpec(sort)},
416
+ {$skip: (page - 1) * limit},
417
+ {$limit: limit},
418
+ {$project: {_id: 1}},
419
+ ]);
420
+ if (sort.key === "name") {
421
+ aggregate.collation({locale: "en"});
422
+ }
423
+ const ordered = await aggregate.exec();
424
+ const ids = ordered.map((row) => row._id);
425
+ const position = new Map(ids.map((id, index) => [String(id), index]));
426
+ const rows = (await ObsTrace.find({_id: {$in: ids}})).sort((a, b) => {
427
+ return (position.get(String(a._id)) ?? 0) - (position.get(String(b._id)) ?? 0);
428
+ });
278
429
  const spanCountById = await countByTraceId(registerObsSpan(), ids);
279
430
  const scoreCountById = await countByTraceId(registerObsScore(), ids);
280
431
  return {