@terreno/ai 57.20.0 → 57.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. package/CHANGELOG.md +81 -0
  2. package/README.md +3 -3
  3. package/biome.jsonc +3 -2
  4. package/dist/harness/agentLoop.js +1 -0
  5. package/dist/harness/commit.d.ts +4 -3
  6. package/dist/harness/commit.js +23 -4
  7. package/dist/index.d.ts +6 -9
  8. package/dist/index.js +14 -25
  9. package/dist/observability/adminScreens.js +5 -1
  10. package/dist/observability/evaluate.d.ts +8 -1
  11. package/dist/observability/evaluate.js +10 -2
  12. package/dist/observability/judgeAgreement.d.ts +53 -0
  13. package/dist/observability/judgeAgreement.js +123 -0
  14. package/dist/observability/local/datasetStore.d.ts +5 -0
  15. package/dist/observability/local/datasetStore.js +7 -2
  16. package/dist/observability/local/evaluatorStore.d.ts +17 -0
  17. package/dist/observability/local/evaluatorStore.js +73 -2
  18. package/dist/observability/local/experimentRunner.d.ts +42 -8
  19. package/dist/observability/local/experimentRunner.js +390 -11
  20. package/dist/observability/local/humanAnswers.d.ts +57 -0
  21. package/dist/observability/local/humanAnswers.js +174 -0
  22. package/dist/observability/local/judgeRun.d.ts +56 -0
  23. package/dist/observability/local/judgeRun.js +111 -0
  24. package/dist/observability/local/localPlugin.js +4 -0
  25. package/dist/observability/local/models/obsDatasetItem.js +4 -0
  26. package/dist/observability/local/models/obsEvaluator.js +4 -0
  27. package/dist/observability/local/models/obsExperiment.js +28 -0
  28. package/dist/observability/local/models/obsExperimentItem.js +4 -0
  29. package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
  30. package/dist/observability/local/models/obsJudgeRun.js +60 -0
  31. package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
  32. package/dist/observability/local/models/obsReviewBatch.js +48 -0
  33. package/dist/observability/local/models/obsReviewItem.js +10 -2
  34. package/dist/observability/local/models/obsTrace.js +4 -0
  35. package/dist/observability/local/promptReferences.d.ts +31 -0
  36. package/dist/observability/local/promptReferences.js +114 -0
  37. package/dist/observability/local/promptStore.d.ts +55 -19
  38. package/dist/observability/local/promptStore.js +171 -7
  39. package/dist/observability/local/reviewBatchStore.d.ts +106 -0
  40. package/dist/observability/local/reviewBatchStore.js +517 -0
  41. package/dist/observability/local/reviewStore.d.ts +24 -3
  42. package/dist/observability/local/reviewStore.js +98 -22
  43. package/dist/observability/local/traceStore.d.ts +14 -1
  44. package/dist/observability/local/traceStore.js +144 -11
  45. package/dist/observability/observabilityApp.js +10 -1
  46. package/dist/observability/questionFingerprint.d.ts +7 -0
  47. package/dist/observability/questionFingerprint.js +23 -0
  48. package/dist/observability/routes/experiments.js +12 -1
  49. package/dist/observability/routes/review.js +3 -1
  50. package/dist/observability/routes/reviewBatches.d.ts +11 -0
  51. package/dist/observability/routes/reviewBatches.js +78 -0
  52. package/dist/observability/routes/traces.d.ts +1 -1
  53. package/dist/observability/routes/traces.js +34 -2
  54. package/dist/observability/types.d.ts +10 -1
  55. package/dist/service/aiService.js +4 -1
  56. package/dist/service/chatTurn.js +38 -31
  57. package/dist/types/harness.d.ts +10 -4
  58. package/dist/types/index.d.ts +8 -3
  59. package/dist/types/observability.d.ts +74 -2
  60. package/e2e/smoke.ts +227 -0
  61. package/package.json +14 -28
  62. package/src/aiApp.test.ts +22 -17
  63. package/src/harness/agentLoop.test.ts +0 -3
  64. package/src/harness/agentLoop.ts +2 -1
  65. package/src/harness/agentRecovery.test.ts +30 -30
  66. package/src/harness/approvals.test.ts +11 -9
  67. package/src/harness/commit.ts +24 -4
  68. package/src/harness/events.test.ts +10 -6
  69. package/src/harness/extensions.test.ts +6 -8
  70. package/src/harness/ownership.test.ts +20 -9
  71. package/src/harness/retries.test.ts +6 -3
  72. package/src/harness/routes/conversations.test.ts +2 -2
  73. package/src/harness/routes/tasks.test.ts +2 -2
  74. package/src/harness/runners/jobsRunner.test.ts +1 -1
  75. package/src/harness/subagent.test.ts +29 -4
  76. package/src/harness/waits.test.ts +6 -4
  77. package/src/index.test.ts +3 -20
  78. package/src/index.ts +23 -26
  79. package/src/isolated/gptStream.isolated.ts +25 -28
  80. package/src/isolated/lazyProjectModel.isolated.ts +2 -2
  81. package/src/models/gptHistory.test.ts +10 -6
  82. package/src/observability/adminScreens.test.ts +7 -0
  83. package/src/observability/adminScreens.ts +6 -1
  84. package/src/observability/evaluate.test.ts +12 -10
  85. package/src/observability/evaluate.ts +12 -2
  86. package/src/observability/judgeAgreement.test.ts +111 -0
  87. package/src/observability/judgeAgreement.ts +184 -0
  88. package/src/observability/local/datasetStore.ts +30 -11
  89. package/src/observability/local/evaluatorStore.ts +96 -2
  90. package/src/observability/local/experimentRunner.test.ts +9 -12
  91. package/src/observability/local/experimentRunner.ts +497 -17
  92. package/src/observability/local/humanAnswers.ts +233 -0
  93. package/src/observability/local/judgePipeline.test.ts +444 -0
  94. package/src/observability/local/judgeRun.test.ts +173 -0
  95. package/src/observability/local/judgeRun.ts +184 -0
  96. package/src/observability/local/localPlugin.ts +4 -0
  97. package/src/observability/local/models/obsDatasetItem.ts +5 -0
  98. package/src/observability/local/models/obsEvaluator.ts +4 -0
  99. package/src/observability/local/models/obsExperiment.ts +29 -0
  100. package/src/observability/local/models/obsExperimentItem.ts +4 -0
  101. package/src/observability/local/models/obsJudgeRun.ts +61 -0
  102. package/src/observability/local/models/obsReviewBatch.ts +52 -0
  103. package/src/observability/local/models/obsReviewItem.ts +11 -2
  104. package/src/observability/local/models/obsTrace.ts +5 -0
  105. package/src/observability/local/obsModels.test.ts +13 -0
  106. package/src/observability/local/promptReferences.test.ts +128 -0
  107. package/src/observability/local/promptReferences.ts +149 -0
  108. package/src/observability/local/promptStore.test.ts +295 -3
  109. package/src/observability/local/promptStore.ts +243 -32
  110. package/src/observability/local/reviewBatchStore.test.ts +251 -0
  111. package/src/observability/local/reviewBatchStore.ts +656 -0
  112. package/src/observability/local/reviewStore.ts +122 -25
  113. package/src/observability/local/traceStore.test.ts +139 -1
  114. package/src/observability/local/traceStore.ts +162 -11
  115. package/src/observability/observabilityApp.test.ts +13 -10
  116. package/src/observability/observabilityApp.ts +10 -1
  117. package/src/observability/questionFingerprint.test.ts +33 -0
  118. package/src/observability/questionFingerprint.ts +24 -0
  119. package/src/observability/routes/datasets.test.ts +2 -2
  120. package/src/observability/routes/evaluators.test.ts +2 -2
  121. package/src/observability/routes/experiments.test.ts +39 -2
  122. package/src/observability/routes/experiments.ts +13 -1
  123. package/src/observability/routes/prompts.test.ts +3 -3
  124. package/src/observability/routes/review.test.ts +2 -2
  125. package/src/observability/routes/review.ts +3 -1
  126. package/src/observability/routes/reviewBatches.test.ts +209 -0
  127. package/src/observability/routes/reviewBatches.ts +138 -0
  128. package/src/observability/routes/status.test.ts +8 -8
  129. package/src/observability/routes/testMultiStage.test.ts +6 -6
  130. package/src/observability/routes/traces.test.ts +63 -2
  131. package/src/observability/routes/traces.ts +46 -4
  132. package/src/observability/types.ts +5 -2
  133. package/src/routes/files.test.ts +5 -3
  134. package/src/routes/gpt.test.ts +21 -14
  135. package/src/routes/gptActions.test.ts +7 -8
  136. package/src/routes/gptDatasets.test.ts +1 -1
  137. package/src/routes/gptFiles.test.ts +1 -1
  138. package/src/routes/gptHistories.test.ts +15 -9
  139. package/src/routes/mcp.test.ts +3 -3
  140. package/src/routes/projects.test.ts +5 -3
  141. package/src/routes/routes.test.ts +156 -115
  142. package/src/service/aiService.test.ts +22 -1
  143. package/src/service/aiService.ts +4 -1
  144. package/src/service/asks.test.ts +14 -6
  145. package/src/service/chatTurn.ts +42 -33
  146. package/src/service/fileStorage.test.ts +5 -2
  147. package/src/service/getMCPTools.test.ts +9 -5
  148. package/src/service/mcpService.test.ts +6 -3
  149. package/src/tests/bunSetup.ts +0 -54
  150. package/src/tests/chatHarness.ts +4 -4
  151. package/src/tests/fakeJudge.ts +8 -0
  152. package/src/tests/helpers.ts +26 -4
  153. package/src/types/harness.ts +11 -4
  154. package/src/types/index.ts +8 -3
  155. package/src/types/observability.ts +86 -2
  156. package/tsconfig.typecheck.json +18 -0
  157. package/dist/langfuseApp.d.ts +0 -9
  158. package/dist/langfuseApp.js +0 -53
  159. package/dist/langfuseCache.d.ts +0 -15
  160. package/dist/langfuseCache.js +0 -54
  161. package/dist/langfuseClient.d.ts +0 -6
  162. package/dist/langfuseClient.js +0 -34
  163. package/dist/langfusePrompts.d.ts +0 -12
  164. package/dist/langfusePrompts.js +0 -89
  165. package/dist/langfuseRoutesEvaluations.d.ts +0 -3
  166. package/dist/langfuseRoutesEvaluations.js +0 -32
  167. package/dist/langfuseRoutesMiddleware.d.ts +0 -2
  168. package/dist/langfuseRoutesMiddleware.js +0 -13
  169. package/dist/langfuseRoutesPlayground.d.ts +0 -2
  170. package/dist/langfuseRoutesPlayground.js +0 -36
  171. package/dist/langfuseRoutesPrompts.d.ts +0 -2
  172. package/dist/langfuseRoutesPrompts.js +0 -56
  173. package/dist/langfuseRoutesTraces.d.ts +0 -2
  174. package/dist/langfuseRoutesTraces.js +0 -30
  175. package/dist/langfuseTracing.d.ts +0 -4
  176. package/dist/langfuseTracing.js +0 -32
  177. package/dist/langfuseTypes.d.ts +0 -96
  178. package/dist/langfuseTypes.js +0 -2
  179. package/dist/langfuseVercelAi.d.ts +0 -12
  180. package/dist/langfuseVercelAi.js +0 -49
  181. package/src/isolated/langfuseClient.isolated.ts +0 -60
  182. package/src/langfuseApp.test.ts +0 -135
  183. package/src/langfuseApp.ts +0 -62
  184. package/src/langfuseCache.test.ts +0 -75
  185. package/src/langfuseCache.ts +0 -72
  186. package/src/langfuseClient.test.ts +0 -69
  187. package/src/langfuseClient.ts +0 -34
  188. package/src/langfusePrompts.test.ts +0 -188
  189. package/src/langfusePrompts.ts +0 -121
  190. package/src/langfuseRoutes.test.ts +0 -256
  191. package/src/langfuseRoutesEvaluations.ts +0 -48
  192. package/src/langfuseRoutesMiddleware.test.ts +0 -57
  193. package/src/langfuseRoutesMiddleware.ts +0 -12
  194. package/src/langfuseRoutesPlayground.ts +0 -50
  195. package/src/langfuseRoutesPrompts.ts +0 -87
  196. package/src/langfuseRoutesTraces.ts +0 -40
  197. package/src/langfuseTracing.test.ts +0 -68
  198. package/src/langfuseTracing.ts +0 -33
  199. package/src/langfuseTypes.ts +0 -103
  200. package/src/langfuseVercelAi.test.ts +0 -79
  201. package/src/langfuseVercelAi.ts +0 -72
@@ -2,7 +2,7 @@ import {APIError} from "@terreno/api";
2
2
  import {DateTime} from "luxon";
3
3
  import type mongoose from "mongoose";
4
4
 
5
- import type {ObsPromptVariable} from "../../types/observability";
5
+ import type {ObsPromptRef, ObsPromptVariable} from "../../types/observability";
6
6
  import {compileTemplate} from "../compileTemplate";
7
7
  import type {ModelPrice, PromptRegistry, PromptVersionRef} from "../types";
8
8
  import {registerObsExperiment} from "./models/obsExperiment";
@@ -10,6 +10,12 @@ import {registerObsPrompt} from "./models/obsPrompt";
10
10
  import {registerObsPromptLabel} from "./models/obsPromptLabel";
11
11
  import {registerObsPromptVersion} from "./models/obsPromptVersion";
12
12
  import {registerObsTrace} from "./models/obsTrace";
13
+ import {
14
+ type PromptReferenceLookup,
15
+ promptIncludeKey,
16
+ type ResolvedPromptText,
17
+ resolvePromptReferences,
18
+ } from "./promptReferences";
13
19
 
14
20
  const PROMPT_RELATIONSHIP_LIMIT = 20;
15
21
 
@@ -45,6 +51,20 @@ export interface PromptRelationships {
45
51
  };
46
52
  }
47
53
 
54
+ /** Another prompt whose saved system or template contains `{{> name}}`. */
55
+ export interface PromptUsedBy {
56
+ folder: string;
57
+ name: string;
58
+ versions: number[];
59
+ }
60
+
61
+ /** Bounded used-by list, same cap as trace and experiment relationships. */
62
+ export interface PromptUsedByPage {
63
+ items: PromptUsedBy[];
64
+ limit: number;
65
+ total: number;
66
+ }
67
+
48
68
  export interface PromptVersionFields {
49
69
  config?: Record<string, unknown>;
50
70
  inputSchema?: Record<string, unknown>;
@@ -80,11 +100,36 @@ export interface MoveLabelResult {
80
100
  version: number;
81
101
  }
82
102
 
103
+ /** A label of the prompt being saved, as it will read after the save. */
104
+ interface PendingLabel {
105
+ body: string;
106
+ label: string;
107
+ version: number;
108
+ }
109
+
110
+ interface ResolvedVersionText {
111
+ includes: ObsPromptRef[];
112
+ system?: string;
113
+ template?: string;
114
+ }
115
+
83
116
  export interface CompiledMessage {
84
117
  content: string;
85
118
  role: "system" | "user";
86
119
  }
87
120
 
121
+ /** A stored version with prompt references in `system` and `template` resolved. */
122
+ export interface ResolvedPromptVersion {
123
+ includes: ObsPromptRef[];
124
+ inputSchema?: Record<string, unknown>;
125
+ outputSchema?: Record<string, unknown>;
126
+ system?: string;
127
+ template?: string;
128
+ type: "chat" | "text";
129
+ variables: ObsPromptVariable[];
130
+ version: number;
131
+ }
132
+
88
133
  export interface PlaygroundRunResult {
89
134
  compiledMessages: CompiledMessage[];
90
135
  costUsd?: number;
@@ -113,6 +158,11 @@ const versionBody = (system?: string, template?: string): string => {
113
158
  return template ?? "";
114
159
  };
115
160
 
161
+ /** Matches `{{> name}}`, `{{> name label=…}}`, and `{{> name version=n}}`. */
162
+ const promptReferencePattern = (name: string): RegExp => {
163
+ return new RegExp(`\\{\\{>\\s*${escapeRegex(name)}(\\s|\\}\\})`);
164
+ };
165
+
116
166
  const escapeRegex = (value: string): string => {
117
167
  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
118
168
  };
@@ -146,6 +196,7 @@ export class LocalPromptStore implements PromptRegistry {
146
196
  if (existing) {
147
197
  throw new APIError({status: 409, title: `Prompt "${input.name}" already exists`});
148
198
  }
199
+ await this.validateReferences(input.name, input);
149
200
  try {
150
201
  const prompt = await ObsPrompt.create({
151
202
  ...(input.description !== undefined ? {description: input.description} : {}),
@@ -174,6 +225,13 @@ export class LocalPromptStore implements PromptRegistry {
174
225
  .sort({version: -1})
175
226
  .limit(1);
176
227
  const nextVersion = (latestDocs[0]?.version ?? 0) + 1;
228
+ await this.validateReferences(prompt.name, fields);
229
+ // The new version takes the `latest` label, so prompts that include `latest` must still resolve.
230
+ await this.validateDependents(prompt.name, {
231
+ body: versionBody(fields.system, fields.template),
232
+ label: "latest",
233
+ version: nextVersion,
234
+ });
177
235
  const created = await this.insertVersion(prompt._id, nextVersion, fields);
178
236
  await this.upsertLabel(prompt._id, "latest", created._id);
179
237
  return {name: prompt.name, version: nextVersion};
@@ -198,6 +256,13 @@ export class LocalPromptStore implements PromptRegistry {
198
256
  title: `Unknown version ${params.version} for prompt "${name}"`,
199
257
  });
200
258
  }
259
+ await this.validateReferences(prompt.name, versionDoc);
260
+ // Other prompts may include this label, so moving it can nest them too deeply.
261
+ await this.validateDependents(prompt.name, {
262
+ body: versionBody(versionDoc.system, versionDoc.template),
263
+ label: params.label,
264
+ version: versionDoc.version,
265
+ });
201
266
  const outgoingVersion = await this.upsertLabel(prompt._id, params.label, versionDoc._id);
202
267
  return {label: params.label, outgoingVersion, version: params.version};
203
268
  }
@@ -219,8 +284,13 @@ export class LocalPromptStore implements PromptRegistry {
219
284
  if (!version) {
220
285
  return undefined;
221
286
  }
287
+ const resolved = await this.resolveText(
288
+ prompt.name,
289
+ versionBody(version.system, version.template)
290
+ );
222
291
  return {
223
- body: versionBody(version.system, version.template),
292
+ body: resolved.text,
293
+ ...(resolved.includes.length > 0 ? {includes: resolved.includes} : {}),
224
294
  label,
225
295
  name: prompt.name,
226
296
  sensitive: version.sensitive,
@@ -238,9 +308,13 @@ export class LocalPromptStore implements PromptRegistry {
238
308
  name: string;
239
309
  relationships: PromptRelationships;
240
310
  tags: string[];
311
+ usedBy: PromptUsedByPage;
241
312
  versions: Array<{
242
313
  config?: Record<string, unknown>;
243
314
  created: string;
315
+ inputSchema?: Record<string, unknown>;
316
+ outputFieldNotes?: Record<string, string>;
317
+ outputSchema?: Record<string, unknown>;
244
318
  sensitive: boolean;
245
319
  system?: string;
246
320
  template?: string;
@@ -264,10 +338,14 @@ export class LocalPromptStore implements PromptRegistry {
264
338
  name: prompt.name,
265
339
  relationships: await this.loadRelationships(prompt.name, options?.promptVersion),
266
340
  tags: prompt.tags,
341
+ usedBy: await this.loadUsedBy(prompt.name),
267
342
  versions: versions.map((row) => {
268
343
  return {
269
344
  config: row.config,
270
345
  created: DateTime.fromJSDate(row.created).toUTC().toISO() ?? "",
346
+ ...(row.inputSchema ? {inputSchema: row.inputSchema} : {}),
347
+ ...(row.outputFieldNotes ? {outputFieldNotes: row.outputFieldNotes} : {}),
348
+ ...(row.outputSchema ? {outputSchema: row.outputSchema} : {}),
271
349
  sensitive: row.sensitive,
272
350
  system: row.system,
273
351
  template: row.template,
@@ -361,9 +439,10 @@ export class LocalPromptStore implements PromptRegistry {
361
439
  }): Promise<PlaygroundRunResult> {
362
440
  const prompt = await this.requirePrompt(params.name);
363
441
  const versionDoc = await this.resolvePlaygroundVersion(prompt._id, params.version);
442
+ const resolved = await this.resolveVersionText(prompt.name, versionDoc);
364
443
  const compiledMessages = this.compile({
365
- system: versionDoc.system,
366
- template: versionDoc.template,
444
+ system: resolved.system,
445
+ template: resolved.template,
367
446
  userPrompt: params.userPrompt,
368
447
  variables: params.variables,
369
448
  });
@@ -503,18 +582,7 @@ export class LocalPromptStore implements PromptRegistry {
503
582
  async getVersionByLabel(
504
583
  name: string,
505
584
  label = "production"
506
- ): Promise<
507
- | {
508
- inputSchema?: Record<string, unknown>;
509
- outputSchema?: Record<string, unknown>;
510
- system?: string;
511
- template?: string;
512
- type: "chat" | "text";
513
- variables: ObsPromptVariable[];
514
- version: number;
515
- }
516
- | undefined
517
- > {
585
+ ): Promise<ResolvedPromptVersion | undefined> {
518
586
  const prompt = await registerObsPrompt().findOneOrNone({name});
519
587
  if (!prompt) {
520
588
  return undefined;
@@ -530,11 +598,13 @@ export class LocalPromptStore implements PromptRegistry {
530
598
  if (!version) {
531
599
  return undefined;
532
600
  }
601
+ const resolved = await this.resolveVersionText(prompt.name, version);
533
602
  return {
603
+ includes: resolved.includes,
534
604
  inputSchema: version.inputSchema,
535
605
  outputSchema: version.outputSchema,
536
- system: version.system,
537
- template: version.template,
606
+ system: resolved.system,
607
+ template: resolved.template,
538
608
  type: version.type,
539
609
  variables: version.variables,
540
610
  version: version.version,
@@ -544,18 +614,7 @@ export class LocalPromptStore implements PromptRegistry {
544
614
  async getVersionByNumber(
545
615
  name: string,
546
616
  versionNumber: number
547
- ): Promise<
548
- | {
549
- inputSchema?: Record<string, unknown>;
550
- outputSchema?: Record<string, unknown>;
551
- system?: string;
552
- template?: string;
553
- type: "chat" | "text";
554
- variables: ObsPromptVariable[];
555
- version: number;
556
- }
557
- | undefined
558
- > {
617
+ ): Promise<ResolvedPromptVersion | undefined> {
559
618
  const prompt = await registerObsPrompt().findOneOrNone({name});
560
619
  if (!prompt) {
561
620
  return undefined;
@@ -567,17 +626,60 @@ export class LocalPromptStore implements PromptRegistry {
567
626
  if (!version) {
568
627
  return undefined;
569
628
  }
629
+ const resolved = await this.resolveVersionText(prompt.name, version);
570
630
  return {
631
+ includes: resolved.includes,
571
632
  inputSchema: version.inputSchema,
572
633
  outputSchema: version.outputSchema,
573
- system: version.system,
574
- template: version.template,
634
+ system: resolved.system,
635
+ template: resolved.template,
575
636
  type: version.type,
576
637
  variables: version.variables,
577
638
  version: version.version,
578
639
  };
579
640
  }
580
641
 
642
+ /**
643
+ * Prompts that mention this name in a saved `{{> name}}` tag, with the versions that do.
644
+ * The current prompt is omitted so a self-tag does not list itself.
645
+ */
646
+ private async loadUsedBy(name: string): Promise<PromptUsedByPage> {
647
+ const versions = await registerObsPromptVersion().find({
648
+ $or: [{system: promptReferencePattern(name)}, {template: promptReferencePattern(name)}],
649
+ });
650
+ const versionsByPromptId = new Map<string, number[]>();
651
+ for (const version of versions) {
652
+ const promptId = String(version.promptId);
653
+ const current = versionsByPromptId.get(promptId) ?? [];
654
+ current.push(version.version);
655
+ versionsByPromptId.set(promptId, current);
656
+ }
657
+ const empty = {items: [], limit: PROMPT_RELATIONSHIP_LIMIT, total: 0};
658
+ if (versionsByPromptId.size === 0) {
659
+ return empty;
660
+ }
661
+ const prompts = await registerObsPrompt().find({
662
+ _id: {$in: [...versionsByPromptId.keys()]},
663
+ });
664
+ const items = prompts
665
+ .filter((prompt) => prompt.name !== name)
666
+ .map((prompt) => {
667
+ return {
668
+ folder: prompt.folder,
669
+ name: prompt.name,
670
+ versions: [...(versionsByPromptId.get(String(prompt._id)) ?? [])].sort(
671
+ (left, right) => left - right
672
+ ),
673
+ };
674
+ })
675
+ .sort((left, right) => left.name.localeCompare(right.name));
676
+ return {
677
+ items: items.slice(0, PROMPT_RELATIONSHIP_LIMIT),
678
+ limit: PROMPT_RELATIONSHIP_LIMIT,
679
+ total: items.length,
680
+ };
681
+ }
682
+
581
683
  private async loadRelationships(
582
684
  promptName: string,
583
685
  promptVersion?: number
@@ -640,6 +742,115 @@ export class LocalPromptStore implements PromptRegistry {
640
742
  };
641
743
  }
642
744
 
745
+ /**
746
+ * Loads a referenced prompt by pinned version or by label. `pending` stands in for a label
747
+ * of the prompt being saved, as it will read once the save lands, when checking the prompts
748
+ * that include it.
749
+ */
750
+ private referenceLookup(pending?: PendingLabel & {name: string}): PromptReferenceLookup {
751
+ return async (reference) => {
752
+ if (pending && reference.name === pending.name && reference.label === pending.label) {
753
+ return pending;
754
+ }
755
+ const prompt = await registerObsPrompt().findOneOrNone({name: reference.name});
756
+ if (!prompt) {
757
+ return undefined;
758
+ }
759
+ const ObsPromptVersion = registerObsPromptVersion();
760
+ if (reference.version !== undefined) {
761
+ const pinned = await ObsPromptVersion.findOneOrNone({
762
+ promptId: prompt._id,
763
+ version: reference.version,
764
+ });
765
+ return pinned
766
+ ? {body: versionBody(pinned.system, pinned.template), version: pinned.version}
767
+ : undefined;
768
+ }
769
+ const labelDoc = await registerObsPromptLabel().findOneOrNone({
770
+ label: reference.label,
771
+ promptId: prompt._id,
772
+ });
773
+ const labelled = labelDoc
774
+ ? await ObsPromptVersion.findOneOrNone({_id: labelDoc.versionId})
775
+ : undefined;
776
+ return labelled
777
+ ? {
778
+ body: versionBody(labelled.system, labelled.template),
779
+ label: reference.label,
780
+ version: labelled.version,
781
+ }
782
+ : undefined;
783
+ };
784
+ }
785
+
786
+ private async resolveText(name: string, text: string): Promise<ResolvedPromptText> {
787
+ return resolvePromptReferences({ancestors: [name], lookup: this.referenceLookup(), text});
788
+ }
789
+
790
+ private async resolveVersionText(
791
+ name: string,
792
+ version: {system?: string; template?: string}
793
+ ): Promise<ResolvedVersionText> {
794
+ const system =
795
+ version.system === undefined ? undefined : await this.resolveText(name, version.system);
796
+ const template =
797
+ version.template === undefined ? undefined : await this.resolveText(name, version.template);
798
+ const includes = new Map<string, ObsPromptRef>();
799
+ for (const ref of [...(system?.includes ?? []), ...(template?.includes ?? [])]) {
800
+ includes.set(promptIncludeKey(ref), ref);
801
+ }
802
+ return {includes: [...includes.values()], system: system?.text, template: template?.text};
803
+ }
804
+
805
+ /**
806
+ * Rejects a malformed, unknown, cyclic, or too-deep reference before a version or label is
807
+ * saved, so every saved reference resolves at call time. A label must already exist on the
808
+ * referenced prompt: promote the shared prompt before referencing it by label.
809
+ */
810
+ private async validateReferences(
811
+ name: string,
812
+ fields: {system?: string; template?: string},
813
+ lookup: PromptReferenceLookup = this.referenceLookup()
814
+ ): Promise<void> {
815
+ for (const text of [fields.system, fields.template]) {
816
+ if (text) {
817
+ await resolvePromptReferences({ancestors: [name], lookup, text});
818
+ }
819
+ }
820
+ }
821
+
822
+ /**
823
+ * Re-resolves every version of each prompt that includes `name`, directly or through another
824
+ * prompt, as it will read once `pending` lands. A new body for a shared prompt can push a prompt
825
+ * that includes it past the nesting limit, and experiments still run unlabelled versions.
826
+ */
827
+ private async validateDependents(name: string, pending: PendingLabel): Promise<void> {
828
+ const lookup = this.referenceLookup({...pending, name});
829
+ const ObsPrompt = registerObsPrompt();
830
+ const ObsPromptVersion = registerObsPromptVersion();
831
+ const seen = new Set([name]);
832
+ const queue = [name];
833
+ while (queue.length > 0) {
834
+ const target = queue.shift() as string;
835
+ const includesTarget = promptReferencePattern(target);
836
+ const promptIds = await ObsPromptVersion.distinct("promptId", {
837
+ $or: [{system: includesTarget}, {template: includesTarget}],
838
+ });
839
+ const dependents = await ObsPrompt.find({_id: {$in: promptIds}});
840
+ for (const dependent of dependents) {
841
+ if (seen.has(dependent.name)) {
842
+ continue;
843
+ }
844
+ seen.add(dependent.name);
845
+ queue.push(dependent.name);
846
+ const versions = await ObsPromptVersion.find({promptId: dependent._id});
847
+ for (const version of versions) {
848
+ await this.validateReferences(dependent.name, version, lookup);
849
+ }
850
+ }
851
+ }
852
+ }
853
+
643
854
  private async requirePrompt(name: string): Promise<{
644
855
  _id: mongoose.Types.ObjectId;
645
856
  description?: string;
@@ -0,0 +1,251 @@
1
+ import {afterEach, beforeEach, describe, expect, it} from "bun:test";
2
+ import {DateTime} from "luxon";
3
+
4
+ import type {EvaluatorAiClient} from "../evaluate";
5
+ import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
6
+ import {LocalEvaluatorStore} from "./evaluatorStore";
7
+ import {createLocalObservabilityPlugin} from "./localPlugin";
8
+ import {registerObsEvaluator} from "./models/obsEvaluator";
9
+ import {registerObsJudgeRun} from "./models/obsJudgeRun";
10
+ import {registerObsPrompt} from "./models/obsPrompt";
11
+ import {registerObsPromptLabel} from "./models/obsPromptLabel";
12
+ import {registerObsPromptVersion} from "./models/obsPromptVersion";
13
+ import {registerObsReviewBatch} from "./models/obsReviewBatch";
14
+ import {registerObsReviewItem} from "./models/obsReviewItem";
15
+ import {registerObsTrace} from "./models/obsTrace";
16
+ import {LocalPromptStore} from "./promptStore";
17
+ import {LocalReviewBatchStore} from "./reviewBatchStore";
18
+ import {LocalReviewStore} from "./reviewStore";
19
+ import {LocalTraceStore, MemoryScoreSink} from "./traceStore";
20
+
21
+ const QUESTIONS = [
22
+ {dataType: "boolean" as const, key: "faithful", question: "Is it faithful?", required: true},
23
+ {dataType: "boolean" as const, key: "complete", question: "Is it complete?", required: true},
24
+ ];
25
+
26
+ const exportTrace = async (params: {
27
+ hoursAgo: number;
28
+ output?: string;
29
+ prompt: string;
30
+ status?: "error" | "ok";
31
+ }): Promise<string> => {
32
+ const startedAt = DateTime.utc().minus({hours: params.hoursAgo}).toISO() ?? "";
33
+ const exported = await new LocalTraceStore().exportTrace({
34
+ id: `t-${params.hoursAgo}`,
35
+ input: `input ${params.hoursAgo}`,
36
+ name: params.prompt,
37
+ output: params.output,
38
+ prompts: [{name: params.prompt, version: 1}],
39
+ sensitive: false,
40
+ spans: [{id: "s", kind: "LLM", name: "llm", startedAt, status: params.status ?? "ok"}],
41
+ startedAt,
42
+ status: params.status ?? "ok",
43
+ });
44
+ return exported.id;
45
+ };
46
+
47
+ // Judge that answers "faithful" from the output text and always says "complete".
48
+ const fakeJudge: EvaluatorAiClient & {modelId: string} = {
49
+ generateJsonObject: async <OBJECT>(options: {prompt: string}) => {
50
+ return {complete: true, confidence: 0.9, faithful: !options.prompt.includes("bad")} as OBJECT;
51
+ },
52
+ modelId: "fake-judge",
53
+ };
54
+
55
+ describe("LocalReviewBatchStore", () => {
56
+ let promptStore: LocalPromptStore;
57
+ let store: LocalReviewBatchStore;
58
+ let questionSetId: string;
59
+ let judgeId: string;
60
+
61
+ afterEach(() => {
62
+ resetObservabilityApp();
63
+ });
64
+
65
+ beforeEach(async () => {
66
+ const plugin = createLocalObservabilityPlugin();
67
+ plugin.scoreSink = new MemoryScoreSink();
68
+ new ObservabilityApp({plugins: [plugin]});
69
+ for (const model of [
70
+ registerObsReviewItem(),
71
+ registerObsReviewBatch(),
72
+ registerObsJudgeRun(),
73
+ registerObsEvaluator(),
74
+ registerObsTrace(),
75
+ registerObsPrompt(),
76
+ registerObsPromptVersion(),
77
+ registerObsPromptLabel(),
78
+ ]) {
79
+ await (model as unknown as {deleteMany: (filter: object) => Promise<unknown>}).deleteMany({});
80
+ }
81
+ promptStore = new LocalPromptStore();
82
+ store = new LocalReviewBatchStore(promptStore);
83
+ await promptStore.create({
84
+ folder: "evaluators",
85
+ name: "judge-questions",
86
+ outputSchema: {
87
+ properties: {
88
+ complete: {type: "boolean"},
89
+ confidence: {type: "number"},
90
+ faithful: {type: "boolean"},
91
+ },
92
+ required: ["faithful", "complete"],
93
+ type: "object",
94
+ },
95
+ system: "Answer each question.",
96
+ type: "chat",
97
+ });
98
+ await promptStore.moveLabel("judge-questions", {label: "production", version: 1});
99
+ const evaluators = new LocalEvaluatorStore(promptStore);
100
+ questionSetId = (
101
+ await evaluators.create({
102
+ dimensions: QUESTIONS,
103
+ name: "summary-questions",
104
+ target: "full trace",
105
+ type: "human",
106
+ })
107
+ ).id;
108
+ judgeId = (
109
+ await evaluators.create({
110
+ dimensions: QUESTIONS.map(({question: _question, ...row}) => row),
111
+ judgePromptName: "judge-questions",
112
+ name: "summary-judge",
113
+ target: "full trace",
114
+ type: "llm-judge",
115
+ })
116
+ ).id;
117
+ });
118
+
119
+ it("builds a batch from the newest successful traces for a prompt", async () => {
120
+ for (let hoursAgo = 1; hoursAgo <= 12; hoursAgo += 1) {
121
+ await exportTrace({hoursAgo, output: `summary ${hoursAgo}`, prompt: "summarize"});
122
+ }
123
+ await exportTrace({hoursAgo: 0, prompt: "summarize", status: "error"});
124
+ await exportTrace({hoursAgo: 0, output: "other", prompt: "jokes"});
125
+
126
+ const batch = await store.create({evaluatorId: questionSetId, promptName: "summarize"});
127
+ expect(batch.name).toBe("summarize — batch 1");
128
+ expect(batch.counts).toEqual({done: 0, pending: 10, skipped: 0, total: 10});
129
+ expect(batch.items.map((item) => item.output)).toEqual(
130
+ Array.from({length: 10}, (_value, index) => `summary ${index + 1}`)
131
+ );
132
+ expect(batch.dimensions.map((row) => row.question)).toEqual([
133
+ "Is it faithful?",
134
+ "Is it complete?",
135
+ ]);
136
+
137
+ // A second batch skips traces already batched for this question set.
138
+ const next = await store.create({evaluatorId: questionSetId, promptName: "summarize"});
139
+ expect(next.items.map((item) => item.output)).toEqual(["summary 11", "summary 12"]);
140
+ await expect(
141
+ store.create({evaluatorId: questionSetId, promptName: "summarize"})
142
+ ).rejects.toThrow(/No recent "summarize" traces/);
143
+
144
+ const listed = await store.list();
145
+ expect(listed.map((row) => [row.name, row.counts.total, row.judgeCount])).toEqual([
146
+ ["summarize — batch 2", 2, 0],
147
+ ["summarize — batch 1", 10, 0],
148
+ ]);
149
+
150
+ const pending = await new LocalReviewStore().list("pending", {batchId: batch.id});
151
+ expect(pending.data).toHaveLength(10);
152
+ expect(pending.counts.pending).toBe(10);
153
+ const detail = await new LocalReviewStore().getDetail(pending.data[0]?.id ?? "");
154
+ expect(detail.batchName).toBe("summarize — batch 1");
155
+ });
156
+
157
+ it("leaves no batch behind when traces are unknown or enqueue fails", async () => {
158
+ const trace = await exportTrace({hoursAgo: 1, output: "x", prompt: "summarize"});
159
+ await expect(
160
+ store.create({evaluatorId: questionSetId, traceIds: [trace, "64b000000000000000000000"]})
161
+ ).rejects.toThrow(/Unknown trace ids/);
162
+
163
+ const failingReviews = new LocalReviewStore();
164
+ failingReviews.enqueue = async (): Promise<never> => {
165
+ throw new Error("enqueue failed");
166
+ };
167
+ const failingStore = new LocalReviewBatchStore(promptStore, failingReviews);
168
+ await expect(failingStore.create({evaluatorId: questionSetId})).rejects.toThrow(
169
+ "enqueue failed"
170
+ );
171
+ expect(await store.list()).toEqual([]);
172
+
173
+ // The trace was not claimed by the failed batch, so grabbing recent traces still finds it.
174
+ const batch = await store.create({evaluatorId: questionSetId});
175
+ expect(batch.items.map((item) => item.traceId)).toEqual([trace]);
176
+ });
177
+
178
+ it("rejects non-human question sets and out-of-range limits", async () => {
179
+ await expect(store.create({evaluatorId: judgeId})).rejects.toThrow(/human evaluator/);
180
+ await expect(store.create({evaluatorId: questionSetId, limit: 51})).rejects.toThrow(/limit/);
181
+ });
182
+
183
+ it("compares judge answers with human answers question by question", async () => {
184
+ const good = await exportTrace({hoursAgo: 2, output: "good summary", prompt: "summarize"});
185
+ const bad = await exportTrace({hoursAgo: 1, output: "bad summary", prompt: "summarize"});
186
+ const batch = await store.create({
187
+ evaluatorId: questionSetId,
188
+ name: "Hand picked",
189
+ traceIds: [good, bad],
190
+ });
191
+ const reviews = new LocalReviewStore();
192
+ const [goodItem, badItem] = batch.items;
193
+ await reviews.submit({
194
+ id: goodItem?.reviewItemId ?? "",
195
+ scores: {complete: true, faithful: true},
196
+ sinks: [new MemoryScoreSink()],
197
+ });
198
+ await reviews.submit({
199
+ id: badItem?.reviewItemId ?? "",
200
+ scores: {complete: false, faithful: false},
201
+ sinks: [new MemoryScoreSink()],
202
+ });
203
+
204
+ const judged = await store.runJudge({ai: fakeJudge, batchId: batch.id, evaluatorId: judgeId});
205
+ expect(judged.judgeCount).toBe(1);
206
+ const [summary] = judged.judges;
207
+ expect(summary?.evaluatorName).toBe("summary-judge");
208
+ expect(summary?.modelId).toBe("fake-judge");
209
+ expect(summary?.agreement.rate).toBe(0.75);
210
+ expect(summary?.agreement.questions.map((row) => row.rate)).toEqual([1, 0.5]);
211
+ expect(summary?.agreement.disagreements).toEqual([
212
+ {human: false, judge: true, key: "complete", traceId: bad},
213
+ ]);
214
+ expect(judged.items[1]?.judgeAnswers[judgeId]?.scores).toEqual({
215
+ complete: true,
216
+ faithful: false,
217
+ });
218
+
219
+ // Re-running keeps only the latest run per judge.
220
+ await store.runJudge({ai: fakeJudge, batchId: batch.id, evaluatorId: judgeId});
221
+ const listed = await store.list();
222
+ expect(listed[0]?.judgeCount).toBe(1);
223
+ expect(listed[0]?.bestJudge).toEqual({evaluatorName: "summary-judge", rate: 0.75});
224
+ expect(listed[0]?.counts.done).toBe(2);
225
+ });
226
+
227
+ it("refuses judges that share no questions with the batch", async () => {
228
+ await promptStore.create({
229
+ folder: "evaluators",
230
+ name: "judge-other",
231
+ outputSchema: {properties: {tone: {type: "string"}}, required: ["tone"], type: "object"},
232
+ type: "chat",
233
+ });
234
+ await promptStore.moveLabel("judge-other", {label: "production", version: 1});
235
+ const other = await new LocalEvaluatorStore(promptStore).create({
236
+ dimensions: [{dataType: "categorical", key: "tone", required: true}],
237
+ judgePromptName: "judge-other",
238
+ name: "tone-judge",
239
+ target: "full trace",
240
+ type: "llm-judge",
241
+ });
242
+ const trace = await exportTrace({hoursAgo: 1, output: "x", prompt: "summarize"});
243
+ const batch = await store.create({evaluatorId: questionSetId, traceIds: [trace]});
244
+ await expect(
245
+ store.runJudge({ai: fakeJudge, batchId: batch.id, evaluatorId: other.id})
246
+ ).rejects.toThrow(/answers none of this batch's questions/);
247
+ await expect(
248
+ store.runJudge({ai: fakeJudge, batchId: batch.id, evaluatorId: questionSetId})
249
+ ).rejects.toThrow(/Only llm-judge/);
250
+ });
251
+ });