@terreno/ai 58.0.0-beta.3 → 58.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. package/CHANGELOG.md +166 -0
  2. package/README.md +3 -3
  3. package/dist/harness/commit.d.ts +4 -3
  4. package/dist/harness/commit.js +23 -4
  5. package/dist/index.d.ts +6 -9
  6. package/dist/index.js +14 -25
  7. package/dist/observability/adminScreens.js +5 -1
  8. package/dist/observability/evaluate.d.ts +8 -1
  9. package/dist/observability/evaluate.js +10 -2
  10. package/dist/observability/judgeAgreement.d.ts +53 -0
  11. package/dist/observability/judgeAgreement.js +123 -0
  12. package/dist/observability/local/datasetStore.d.ts +5 -0
  13. package/dist/observability/local/datasetStore.js +7 -2
  14. package/dist/observability/local/evaluatorStore.d.ts +17 -0
  15. package/dist/observability/local/evaluatorStore.js +73 -2
  16. package/dist/observability/local/experimentRunner.d.ts +40 -3
  17. package/dist/observability/local/experimentRunner.js +390 -11
  18. package/dist/observability/local/humanAnswers.d.ts +57 -0
  19. package/dist/observability/local/humanAnswers.js +174 -0
  20. package/dist/observability/local/judgeRun.d.ts +56 -0
  21. package/dist/observability/local/judgeRun.js +111 -0
  22. package/dist/observability/local/localPlugin.js +4 -0
  23. package/dist/observability/local/models/obsDatasetItem.js +4 -0
  24. package/dist/observability/local/models/obsEvaluator.js +4 -0
  25. package/dist/observability/local/models/obsExperiment.js +28 -0
  26. package/dist/observability/local/models/obsExperimentItem.js +4 -0
  27. package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
  28. package/dist/observability/local/models/obsJudgeRun.js +60 -0
  29. package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
  30. package/dist/observability/local/models/obsReviewBatch.js +48 -0
  31. package/dist/observability/local/models/obsReviewItem.js +10 -2
  32. package/dist/observability/local/models/obsTrace.js +4 -0
  33. package/dist/observability/local/promptReferences.d.ts +31 -0
  34. package/dist/observability/local/promptReferences.js +114 -0
  35. package/dist/observability/local/promptStore.d.ts +55 -19
  36. package/dist/observability/local/promptStore.js +171 -7
  37. package/dist/observability/local/reviewBatchStore.d.ts +106 -0
  38. package/dist/observability/local/reviewBatchStore.js +517 -0
  39. package/dist/observability/local/reviewStore.d.ts +24 -3
  40. package/dist/observability/local/reviewStore.js +98 -22
  41. package/dist/observability/local/traceStore.d.ts +14 -1
  42. package/dist/observability/local/traceStore.js +144 -11
  43. package/dist/observability/observabilityApp.js +10 -1
  44. package/dist/observability/questionFingerprint.d.ts +7 -0
  45. package/dist/observability/questionFingerprint.js +23 -0
  46. package/dist/observability/routes/experiments.js +12 -1
  47. package/dist/observability/routes/review.js +3 -1
  48. package/dist/observability/routes/reviewBatches.d.ts +11 -0
  49. package/dist/observability/routes/reviewBatches.js +78 -0
  50. package/dist/observability/routes/traces.d.ts +1 -1
  51. package/dist/observability/routes/traces.js +34 -2
  52. package/dist/observability/types.d.ts +10 -1
  53. package/dist/service/aiService.js +4 -1
  54. package/dist/service/chatTurn.js +38 -31
  55. package/dist/types/harness.d.ts +5 -2
  56. package/dist/types/index.d.ts +8 -3
  57. package/dist/types/observability.d.ts +74 -2
  58. package/package.json +8 -24
  59. package/src/harness/agentLoop.test.ts +0 -3
  60. package/src/harness/agentRecovery.test.ts +0 -3
  61. package/src/harness/commit.ts +24 -4
  62. package/src/harness/extensions.test.ts +0 -3
  63. package/src/harness/subagent.test.ts +27 -3
  64. package/src/index.test.ts +0 -17
  65. package/src/index.ts +23 -26
  66. package/src/isolated/gptStream.isolated.ts +0 -8
  67. package/src/observability/adminScreens.test.ts +7 -0
  68. package/src/observability/adminScreens.ts +6 -1
  69. package/src/observability/evaluate.ts +12 -2
  70. package/src/observability/judgeAgreement.test.ts +111 -0
  71. package/src/observability/judgeAgreement.ts +184 -0
  72. package/src/observability/local/datasetStore.ts +30 -11
  73. package/src/observability/local/evaluatorStore.ts +96 -2
  74. package/src/observability/local/experimentRunner.ts +495 -15
  75. package/src/observability/local/humanAnswers.ts +233 -0
  76. package/src/observability/local/judgePipeline.test.ts +444 -0
  77. package/src/observability/local/judgeRun.test.ts +173 -0
  78. package/src/observability/local/judgeRun.ts +184 -0
  79. package/src/observability/local/localPlugin.ts +4 -0
  80. package/src/observability/local/models/obsDatasetItem.ts +5 -0
  81. package/src/observability/local/models/obsEvaluator.ts +4 -0
  82. package/src/observability/local/models/obsExperiment.ts +29 -0
  83. package/src/observability/local/models/obsExperimentItem.ts +4 -0
  84. package/src/observability/local/models/obsJudgeRun.ts +61 -0
  85. package/src/observability/local/models/obsReviewBatch.ts +52 -0
  86. package/src/observability/local/models/obsReviewItem.ts +11 -2
  87. package/src/observability/local/models/obsTrace.ts +5 -0
  88. package/src/observability/local/obsModels.test.ts +13 -0
  89. package/src/observability/local/promptReferences.test.ts +128 -0
  90. package/src/observability/local/promptReferences.ts +149 -0
  91. package/src/observability/local/promptStore.test.ts +291 -0
  92. package/src/observability/local/promptStore.ts +243 -32
  93. package/src/observability/local/reviewBatchStore.test.ts +251 -0
  94. package/src/observability/local/reviewBatchStore.ts +656 -0
  95. package/src/observability/local/reviewStore.ts +122 -25
  96. package/src/observability/local/traceStore.test.ts +139 -1
  97. package/src/observability/local/traceStore.ts +162 -11
  98. package/src/observability/observabilityApp.test.ts +13 -10
  99. package/src/observability/observabilityApp.ts +10 -1
  100. package/src/observability/questionFingerprint.test.ts +33 -0
  101. package/src/observability/questionFingerprint.ts +24 -0
  102. package/src/observability/routes/experiments.test.ts +37 -0
  103. package/src/observability/routes/experiments.ts +13 -1
  104. package/src/observability/routes/review.ts +3 -1
  105. package/src/observability/routes/reviewBatches.test.ts +209 -0
  106. package/src/observability/routes/reviewBatches.ts +138 -0
  107. package/src/observability/routes/status.test.ts +4 -4
  108. package/src/observability/routes/traces.test.ts +61 -0
  109. package/src/observability/routes/traces.ts +46 -4
  110. package/src/observability/types.ts +5 -2
  111. package/src/routes/routes.test.ts +101 -76
  112. package/src/service/aiService.test.ts +21 -0
  113. package/src/service/aiService.ts +4 -1
  114. package/src/service/chatTurn.ts +42 -33
  115. package/src/tests/bunSetup.ts +0 -54
  116. package/src/types/harness.ts +5 -2
  117. package/src/types/index.ts +8 -3
  118. package/src/types/observability.ts +86 -2
  119. package/dist/langfuseApp.d.ts +0 -9
  120. package/dist/langfuseApp.js +0 -53
  121. package/dist/langfuseCache.d.ts +0 -15
  122. package/dist/langfuseCache.js +0 -54
  123. package/dist/langfuseClient.d.ts +0 -6
  124. package/dist/langfuseClient.js +0 -34
  125. package/dist/langfusePrompts.d.ts +0 -12
  126. package/dist/langfusePrompts.js +0 -89
  127. package/dist/langfuseRoutesEvaluations.d.ts +0 -3
  128. package/dist/langfuseRoutesEvaluations.js +0 -32
  129. package/dist/langfuseRoutesMiddleware.d.ts +0 -2
  130. package/dist/langfuseRoutesMiddleware.js +0 -13
  131. package/dist/langfuseRoutesPlayground.d.ts +0 -2
  132. package/dist/langfuseRoutesPlayground.js +0 -36
  133. package/dist/langfuseRoutesPrompts.d.ts +0 -2
  134. package/dist/langfuseRoutesPrompts.js +0 -56
  135. package/dist/langfuseRoutesTraces.d.ts +0 -2
  136. package/dist/langfuseRoutesTraces.js +0 -30
  137. package/dist/langfuseTracing.d.ts +0 -4
  138. package/dist/langfuseTracing.js +0 -32
  139. package/dist/langfuseTypes.d.ts +0 -96
  140. package/dist/langfuseTypes.js +0 -2
  141. package/dist/langfuseVercelAi.d.ts +0 -12
  142. package/dist/langfuseVercelAi.js +0 -49
  143. package/src/isolated/langfuseClient.isolated.ts +0 -66
  144. package/src/langfuseApp.test.ts +0 -119
  145. package/src/langfuseApp.ts +0 -62
  146. package/src/langfuseCache.test.ts +0 -75
  147. package/src/langfuseCache.ts +0 -72
  148. package/src/langfuseClient.test.ts +0 -69
  149. package/src/langfuseClient.ts +0 -34
  150. package/src/langfusePrompts.test.ts +0 -188
  151. package/src/langfusePrompts.ts +0 -121
  152. package/src/langfuseRoutes.test.ts +0 -261
  153. package/src/langfuseRoutesEvaluations.ts +0 -48
  154. package/src/langfuseRoutesMiddleware.test.ts +0 -57
  155. package/src/langfuseRoutesMiddleware.ts +0 -12
  156. package/src/langfuseRoutesPlayground.ts +0 -50
  157. package/src/langfuseRoutesPrompts.ts +0 -87
  158. package/src/langfuseRoutesTraces.ts +0 -40
  159. package/src/langfuseTracing.test.ts +0 -72
  160. package/src/langfuseTracing.ts +0 -33
  161. package/src/langfuseTypes.ts +0 -103
  162. package/src/langfuseVercelAi.test.ts +0 -82
  163. package/src/langfuseVercelAi.ts +0 -72
  164. package/src/tests/fakeLangfusePrompt.ts +0 -10
@@ -0,0 +1,114 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.resolvePromptReferences = exports.promptIncludeKey = void 0;
4
+ const api_1 = require("@terreno/api");
5
+ /**
6
+ * A reference to another registry prompt, written like a template partial:
7
+ * `{{> base-rules}}`, `{{> base-rules label=staging}}`, or `{{> base-rules version=3}}`.
8
+ */
9
+ const PROMPT_REFERENCE = /\{\{>([^}]*)\}\}/g;
10
+ const MAX_PROMPT_REFERENCE_DEPTH = 5;
11
+ const DEFAULT_PROMPT_REFERENCE_LABEL = "production";
12
+ /** Identity of one recorded include, for deduping. */
13
+ const promptIncludeKey = (ref) => {
14
+ return [ref.includedBy ?? "", ref.name, ref.version].join("\u0000");
15
+ };
16
+ exports.promptIncludeKey = promptIncludeKey;
17
+ const invalidReference = (tag, detail) => {
18
+ return new api_1.APIError({
19
+ detail: `${tag}: ${detail}`,
20
+ status: 400,
21
+ title: "Invalid prompt reference",
22
+ });
23
+ };
24
+ const parseReference = (tag, body) => {
25
+ const [name, ...options] = body.trim().split(/\s+/);
26
+ if (!name || name.includes("=")) {
27
+ throw invalidReference(tag, "a prompt name must follow {{>");
28
+ }
29
+ const fields = new Map();
30
+ for (const option of options) {
31
+ const separator = option.indexOf("=");
32
+ const key = option.slice(0, separator);
33
+ if (separator < 1 || (key !== "label" && key !== "version") || fields.has(key)) {
34
+ throw invalidReference(tag, `expected one label=<label> or version=<n>, got "${option}"`);
35
+ }
36
+ fields.set(key, option.slice(separator + 1));
37
+ }
38
+ const label = fields.get("label");
39
+ const versionText = fields.get("version");
40
+ if (label !== undefined && versionText !== undefined) {
41
+ throw invalidReference(tag, "use label or version, not both");
42
+ }
43
+ if (label === "") {
44
+ throw invalidReference(tag, "label= needs a label name");
45
+ }
46
+ if (versionText === undefined) {
47
+ return { label: label ?? DEFAULT_PROMPT_REFERENCE_LABEL, name, tag };
48
+ }
49
+ const version = Number(versionText);
50
+ if (!Number.isInteger(version) || version < 1) {
51
+ throw invalidReference(tag, "version must be a positive integer");
52
+ }
53
+ return { name, tag, version };
54
+ };
55
+ /** The references in `text`, in order of appearance. Throws 400 on a malformed tag. */
56
+ const parsePromptReferences = (text) => {
57
+ return [...text.matchAll(PROMPT_REFERENCE)].map((match) => {
58
+ return parseReference(match[0], match[1]);
59
+ });
60
+ };
61
+ /**
62
+ * Replaces every reference in `text` with the referenced prompt's body, recursively. `ancestors`
63
+ * is the chain of prompt names already being resolved (start with the including prompt's own
64
+ * name), so a cycle or a chain deeper than MAX_PROMPT_REFERENCE_DEPTH fails with 400.
65
+ */
66
+ const resolvePromptReferences = async (params) => {
67
+ const includes = new Map();
68
+ const resolve = async (text, ancestors) => {
69
+ const references = parsePromptReferences(text);
70
+ if (references.length === 0) {
71
+ return text;
72
+ }
73
+ const includedBy = ancestors[ancestors.length - 1];
74
+ const replacements = new Map();
75
+ for (const reference of references) {
76
+ if (ancestors.includes(reference.name)) {
77
+ throw new api_1.APIError({
78
+ detail: [...ancestors, reference.name].join(" → "),
79
+ status: 400,
80
+ title: "Prompt reference cycle",
81
+ });
82
+ }
83
+ if (ancestors.length > MAX_PROMPT_REFERENCE_DEPTH) {
84
+ throw new api_1.APIError({
85
+ detail: `${ancestors.join(" → ")} exceeds ${MAX_PROMPT_REFERENCE_DEPTH} levels`,
86
+ status: 400,
87
+ title: "Prompt references nested too deeply",
88
+ });
89
+ }
90
+ const found = await params.lookup({
91
+ label: reference.label,
92
+ name: reference.name,
93
+ version: reference.version,
94
+ });
95
+ if (!found) {
96
+ throw invalidReference(reference.tag, "no prompt matches this name and label or version");
97
+ }
98
+ const ref = {
99
+ includedBy,
100
+ ...(found.label !== undefined ? { label: found.label } : {}),
101
+ name: reference.name,
102
+ version: found.version,
103
+ };
104
+ includes.set((0, exports.promptIncludeKey)(ref), ref);
105
+ replacements.set(reference.tag, await resolve(found.body, [...ancestors, reference.name]));
106
+ }
107
+ return text.replace(PROMPT_REFERENCE, (tag) => {
108
+ return replacements.get(tag) ?? tag;
109
+ });
110
+ };
111
+ const text = await resolve(params.text, params.ancestors);
112
+ return { includes: [...includes.values()], text };
113
+ };
114
+ exports.resolvePromptReferences = resolvePromptReferences;
@@ -1,5 +1,5 @@
1
1
  import type mongoose from "mongoose";
2
- import type { ObsPromptVariable } from "../../types/observability";
2
+ import type { ObsPromptRef, ObsPromptVariable } from "../../types/observability";
3
3
  import type { ModelPrice, PromptRegistry, PromptVersionRef } from "../types";
4
4
  export interface PromptRelatedExperimentSummary {
5
5
  created: string;
@@ -30,6 +30,18 @@ export interface PromptRelationships {
30
30
  total: number;
31
31
  };
32
32
  }
33
+ /** Another prompt whose saved system or template contains `{{> name}}`. */
34
+ export interface PromptUsedBy {
35
+ folder: string;
36
+ name: string;
37
+ versions: number[];
38
+ }
39
+ /** Bounded used-by list, same cap as trace and experiment relationships. */
40
+ export interface PromptUsedByPage {
41
+ items: PromptUsedBy[];
42
+ limit: number;
43
+ total: number;
44
+ }
33
45
  export interface PromptVersionFields {
34
46
  config?: Record<string, unknown>;
35
47
  inputSchema?: Record<string, unknown>;
@@ -68,6 +80,17 @@ export interface CompiledMessage {
68
80
  content: string;
69
81
  role: "system" | "user";
70
82
  }
83
+ /** A stored version with prompt references in `system` and `template` resolved. */
84
+ export interface ResolvedPromptVersion {
85
+ includes: ObsPromptRef[];
86
+ inputSchema?: Record<string, unknown>;
87
+ outputSchema?: Record<string, unknown>;
88
+ system?: string;
89
+ template?: string;
90
+ type: "chat" | "text";
91
+ variables: ObsPromptVariable[];
92
+ version: number;
93
+ }
71
94
  export interface PlaygroundRunResult {
72
95
  compiledMessages: CompiledMessage[];
73
96
  costUsd?: number;
@@ -120,9 +143,13 @@ export declare class LocalPromptStore implements PromptRegistry {
120
143
  name: string;
121
144
  relationships: PromptRelationships;
122
145
  tags: string[];
146
+ usedBy: PromptUsedByPage;
123
147
  versions: Array<{
124
148
  config?: Record<string, unknown>;
125
149
  created: string;
150
+ inputSchema?: Record<string, unknown>;
151
+ outputFieldNotes?: Record<string, string>;
152
+ outputSchema?: Record<string, unknown>;
126
153
  sensitive: boolean;
127
154
  system?: string;
128
155
  template?: string;
@@ -156,25 +183,34 @@ export declare class LocalPromptStore implements PromptRegistry {
156
183
  private usageLast7d;
157
184
  private insertVersion;
158
185
  private upsertLabel;
159
- getVersionByLabel(name: string, label?: string): Promise<{
160
- inputSchema?: Record<string, unknown>;
161
- outputSchema?: Record<string, unknown>;
162
- system?: string;
163
- template?: string;
164
- type: "chat" | "text";
165
- variables: ObsPromptVariable[];
166
- version: number;
167
- } | undefined>;
168
- getVersionByNumber(name: string, versionNumber: number): Promise<{
169
- inputSchema?: Record<string, unknown>;
170
- outputSchema?: Record<string, unknown>;
171
- system?: string;
172
- template?: string;
173
- type: "chat" | "text";
174
- variables: ObsPromptVariable[];
175
- version: number;
176
- } | undefined>;
186
+ getVersionByLabel(name: string, label?: string): Promise<ResolvedPromptVersion | undefined>;
187
+ getVersionByNumber(name: string, versionNumber: number): Promise<ResolvedPromptVersion | undefined>;
188
+ /**
189
+ * Prompts that mention this name in a saved `{{> name}}` tag, with the versions that do.
190
+ * The current prompt is omitted so a self-tag does not list itself.
191
+ */
192
+ private loadUsedBy;
177
193
  private loadRelationships;
194
+ /**
195
+ * Loads a referenced prompt by pinned version or by label. `pending` stands in for a label
196
+ * of the prompt being saved, as it will read once the save lands, when checking the prompts
197
+ * that include it.
198
+ */
199
+ private referenceLookup;
200
+ private resolveText;
201
+ private resolveVersionText;
202
+ /**
203
+ * Rejects a malformed, unknown, cyclic, or too-deep reference before a version or label is
204
+ * saved, so every saved reference resolves at call time. A label must already exist on the
205
+ * referenced prompt: promote the shared prompt before referencing it by label.
206
+ */
207
+ private validateReferences;
208
+ /**
209
+ * Re-resolves every version of each prompt that includes `name`, directly or through another
210
+ * prompt, as it will read once `pending` lands. A new body for a shared prompt can push a prompt
211
+ * that includes it past the nesting limit, and experiments still run unlabelled versions.
212
+ */
213
+ private validateDependents;
178
214
  private requirePrompt;
179
215
  private isDuplicateKey;
180
216
  }
@@ -9,6 +9,7 @@ const obsPrompt_1 = require("./models/obsPrompt");
9
9
  const obsPromptLabel_1 = require("./models/obsPromptLabel");
10
10
  const obsPromptVersion_1 = require("./models/obsPromptVersion");
11
11
  const obsTrace_1 = require("./models/obsTrace");
12
+ const promptReferences_1 = require("./promptReferences");
12
13
  const PROMPT_RELATIONSHIP_LIMIT = 20;
13
14
  const versionBody = (system, template) => {
14
15
  if (system && system.length > 0) {
@@ -16,6 +17,10 @@ const versionBody = (system, template) => {
16
17
  }
17
18
  return template ?? "";
18
19
  };
20
+ /** Matches `{{> name}}`, `{{> name label=…}}`, and `{{> name version=n}}`. */
21
+ const promptReferencePattern = (name) => {
22
+ return new RegExp(`\\{\\{>\\s*${escapeRegex(name)}(\\s|\\}\\})`);
23
+ };
19
24
  const escapeRegex = (value) => {
20
25
  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
21
26
  };
@@ -40,6 +45,7 @@ class LocalPromptStore {
40
45
  if (existing) {
41
46
  throw new api_1.APIError({ status: 409, title: `Prompt "${input.name}" already exists` });
42
47
  }
48
+ await this.validateReferences(input.name, input);
43
49
  try {
44
50
  const prompt = await ObsPrompt.create({
45
51
  ...(input.description !== undefined ? { description: input.description } : {}),
@@ -65,6 +71,13 @@ class LocalPromptStore {
65
71
  .sort({ version: -1 })
66
72
  .limit(1);
67
73
  const nextVersion = (latestDocs[0]?.version ?? 0) + 1;
74
+ await this.validateReferences(prompt.name, fields);
75
+ // The new version takes the `latest` label, so prompts that include `latest` must still resolve.
76
+ await this.validateDependents(prompt.name, {
77
+ body: versionBody(fields.system, fields.template),
78
+ label: "latest",
79
+ version: nextVersion,
80
+ });
68
81
  const created = await this.insertVersion(prompt._id, nextVersion, fields);
69
82
  await this.upsertLabel(prompt._id, "latest", created._id);
70
83
  return { name: prompt.name, version: nextVersion };
@@ -85,6 +98,13 @@ class LocalPromptStore {
85
98
  title: `Unknown version ${params.version} for prompt "${name}"`,
86
99
  });
87
100
  }
101
+ await this.validateReferences(prompt.name, versionDoc);
102
+ // Other prompts may include this label, so moving it can nest them too deeply.
103
+ await this.validateDependents(prompt.name, {
104
+ body: versionBody(versionDoc.system, versionDoc.template),
105
+ label: params.label,
106
+ version: versionDoc.version,
107
+ });
88
108
  const outgoingVersion = await this.upsertLabel(prompt._id, params.label, versionDoc._id);
89
109
  return { label: params.label, outgoingVersion, version: params.version };
90
110
  }
@@ -105,8 +125,10 @@ class LocalPromptStore {
105
125
  if (!version) {
106
126
  return undefined;
107
127
  }
128
+ const resolved = await this.resolveText(prompt.name, versionBody(version.system, version.template));
108
129
  return {
109
- body: versionBody(version.system, version.template),
130
+ body: resolved.text,
131
+ ...(resolved.includes.length > 0 ? { includes: resolved.includes } : {}),
110
132
  label,
111
133
  name: prompt.name,
112
134
  sensitive: version.sensitive,
@@ -129,10 +151,14 @@ class LocalPromptStore {
129
151
  name: prompt.name,
130
152
  relationships: await this.loadRelationships(prompt.name, options?.promptVersion),
131
153
  tags: prompt.tags,
154
+ usedBy: await this.loadUsedBy(prompt.name),
132
155
  versions: versions.map((row) => {
133
156
  return {
134
157
  config: row.config,
135
158
  created: luxon_1.DateTime.fromJSDate(row.created).toUTC().toISO() ?? "",
159
+ ...(row.inputSchema ? { inputSchema: row.inputSchema } : {}),
160
+ ...(row.outputFieldNotes ? { outputFieldNotes: row.outputFieldNotes } : {}),
161
+ ...(row.outputSchema ? { outputSchema: row.outputSchema } : {}),
136
162
  sensitive: row.sensitive,
137
163
  system: row.system,
138
164
  template: row.template,
@@ -204,9 +230,10 @@ class LocalPromptStore {
204
230
  async runPlayground(params) {
205
231
  const prompt = await this.requirePrompt(params.name);
206
232
  const versionDoc = await this.resolvePlaygroundVersion(prompt._id, params.version);
233
+ const resolved = await this.resolveVersionText(prompt.name, versionDoc);
207
234
  const compiledMessages = this.compile({
208
- system: versionDoc.system,
209
- template: versionDoc.template,
235
+ system: resolved.system,
236
+ template: resolved.template,
210
237
  userPrompt: params.userPrompt,
211
238
  variables: params.variables,
212
239
  });
@@ -340,11 +367,13 @@ class LocalPromptStore {
340
367
  if (!version) {
341
368
  return undefined;
342
369
  }
370
+ const resolved = await this.resolveVersionText(prompt.name, version);
343
371
  return {
372
+ includes: resolved.includes,
344
373
  inputSchema: version.inputSchema,
345
374
  outputSchema: version.outputSchema,
346
- system: version.system,
347
- template: version.template,
375
+ system: resolved.system,
376
+ template: resolved.template,
348
377
  type: version.type,
349
378
  variables: version.variables,
350
379
  version: version.version,
@@ -362,16 +391,56 @@ class LocalPromptStore {
362
391
  if (!version) {
363
392
  return undefined;
364
393
  }
394
+ const resolved = await this.resolveVersionText(prompt.name, version);
365
395
  return {
396
+ includes: resolved.includes,
366
397
  inputSchema: version.inputSchema,
367
398
  outputSchema: version.outputSchema,
368
- system: version.system,
369
- template: version.template,
399
+ system: resolved.system,
400
+ template: resolved.template,
370
401
  type: version.type,
371
402
  variables: version.variables,
372
403
  version: version.version,
373
404
  };
374
405
  }
406
+ /**
407
+ * Prompts that mention this name in a saved `{{> name}}` tag, with the versions that do.
408
+ * The current prompt is omitted so a self-tag does not list itself.
409
+ */
410
+ async loadUsedBy(name) {
411
+ const versions = await (0, obsPromptVersion_1.registerObsPromptVersion)().find({
412
+ $or: [{ system: promptReferencePattern(name) }, { template: promptReferencePattern(name) }],
413
+ });
414
+ const versionsByPromptId = new Map();
415
+ for (const version of versions) {
416
+ const promptId = String(version.promptId);
417
+ const current = versionsByPromptId.get(promptId) ?? [];
418
+ current.push(version.version);
419
+ versionsByPromptId.set(promptId, current);
420
+ }
421
+ const empty = { items: [], limit: PROMPT_RELATIONSHIP_LIMIT, total: 0 };
422
+ if (versionsByPromptId.size === 0) {
423
+ return empty;
424
+ }
425
+ const prompts = await (0, obsPrompt_1.registerObsPrompt)().find({
426
+ _id: { $in: [...versionsByPromptId.keys()] },
427
+ });
428
+ const items = prompts
429
+ .filter((prompt) => prompt.name !== name)
430
+ .map((prompt) => {
431
+ return {
432
+ folder: prompt.folder,
433
+ name: prompt.name,
434
+ versions: [...(versionsByPromptId.get(String(prompt._id)) ?? [])].sort((left, right) => left - right),
435
+ };
436
+ })
437
+ .sort((left, right) => left.name.localeCompare(right.name));
438
+ return {
439
+ items: items.slice(0, PROMPT_RELATIONSHIP_LIMIT),
440
+ limit: PROMPT_RELATIONSHIP_LIMIT,
441
+ total: items.length,
442
+ };
443
+ }
375
444
  async loadRelationships(promptName, promptVersion) {
376
445
  const traceFilter = promptVersion === undefined
377
446
  ? { "prompts.name": promptName }
@@ -427,6 +496,101 @@ class LocalPromptStore {
427
496
  },
428
497
  };
429
498
  }
499
+ /**
500
+ * Loads a referenced prompt by pinned version or by label. `pending` stands in for a label
501
+ * of the prompt being saved, as it will read once the save lands, when checking the prompts
502
+ * that include it.
503
+ */
504
+ referenceLookup(pending) {
505
+ return async (reference) => {
506
+ if (pending && reference.name === pending.name && reference.label === pending.label) {
507
+ return pending;
508
+ }
509
+ const prompt = await (0, obsPrompt_1.registerObsPrompt)().findOneOrNone({ name: reference.name });
510
+ if (!prompt) {
511
+ return undefined;
512
+ }
513
+ const ObsPromptVersion = (0, obsPromptVersion_1.registerObsPromptVersion)();
514
+ if (reference.version !== undefined) {
515
+ const pinned = await ObsPromptVersion.findOneOrNone({
516
+ promptId: prompt._id,
517
+ version: reference.version,
518
+ });
519
+ return pinned
520
+ ? { body: versionBody(pinned.system, pinned.template), version: pinned.version }
521
+ : undefined;
522
+ }
523
+ const labelDoc = await (0, obsPromptLabel_1.registerObsPromptLabel)().findOneOrNone({
524
+ label: reference.label,
525
+ promptId: prompt._id,
526
+ });
527
+ const labelled = labelDoc
528
+ ? await ObsPromptVersion.findOneOrNone({ _id: labelDoc.versionId })
529
+ : undefined;
530
+ return labelled
531
+ ? {
532
+ body: versionBody(labelled.system, labelled.template),
533
+ label: reference.label,
534
+ version: labelled.version,
535
+ }
536
+ : undefined;
537
+ };
538
+ }
539
+ async resolveText(name, text) {
540
+ return (0, promptReferences_1.resolvePromptReferences)({ ancestors: [name], lookup: this.referenceLookup(), text });
541
+ }
542
+ async resolveVersionText(name, version) {
543
+ const system = version.system === undefined ? undefined : await this.resolveText(name, version.system);
544
+ const template = version.template === undefined ? undefined : await this.resolveText(name, version.template);
545
+ const includes = new Map();
546
+ for (const ref of [...(system?.includes ?? []), ...(template?.includes ?? [])]) {
547
+ includes.set((0, promptReferences_1.promptIncludeKey)(ref), ref);
548
+ }
549
+ return { includes: [...includes.values()], system: system?.text, template: template?.text };
550
+ }
551
+ /**
552
+ * Rejects a malformed, unknown, cyclic, or too-deep reference before a version or label is
553
+ * saved, so every saved reference resolves at call time. A label must already exist on the
554
+ * referenced prompt: promote the shared prompt before referencing it by label.
555
+ */
556
+ async validateReferences(name, fields, lookup = this.referenceLookup()) {
557
+ for (const text of [fields.system, fields.template]) {
558
+ if (text) {
559
+ await (0, promptReferences_1.resolvePromptReferences)({ ancestors: [name], lookup, text });
560
+ }
561
+ }
562
+ }
563
+ /**
564
+ * Re-resolves every version of each prompt that includes `name`, directly or through another
565
+ * prompt, as it will read once `pending` lands. A new body for a shared prompt can push a prompt
566
+ * that includes it past the nesting limit, and experiments still run unlabelled versions.
567
+ */
568
+ async validateDependents(name, pending) {
569
+ const lookup = this.referenceLookup({ ...pending, name });
570
+ const ObsPrompt = (0, obsPrompt_1.registerObsPrompt)();
571
+ const ObsPromptVersion = (0, obsPromptVersion_1.registerObsPromptVersion)();
572
+ const seen = new Set([name]);
573
+ const queue = [name];
574
+ while (queue.length > 0) {
575
+ const target = queue.shift();
576
+ const includesTarget = promptReferencePattern(target);
577
+ const promptIds = await ObsPromptVersion.distinct("promptId", {
578
+ $or: [{ system: includesTarget }, { template: includesTarget }],
579
+ });
580
+ const dependents = await ObsPrompt.find({ _id: { $in: promptIds } });
581
+ for (const dependent of dependents) {
582
+ if (seen.has(dependent.name)) {
583
+ continue;
584
+ }
585
+ seen.add(dependent.name);
586
+ queue.push(dependent.name);
587
+ const versions = await ObsPromptVersion.find({ promptId: dependent._id });
588
+ for (const version of versions) {
589
+ await this.validateReferences(dependent.name, version, lookup);
590
+ }
591
+ }
592
+ }
593
+ }
430
594
  async requirePrompt(name) {
431
595
  const prompt = await (0, obsPrompt_1.registerObsPrompt)().findOneOrNone({ name });
432
596
  if (!prompt) {
@@ -0,0 +1,106 @@
1
+ import type { EvaluatorDimension } from "../../types/observability";
2
+ import type { EvaluatorAiClient } from "../evaluate";
3
+ import { type JudgeAgreement, type ScoreValue } from "../judgeAgreement";
4
+ import { LocalPromptStore } from "./promptStore";
5
+ import { LocalReviewStore, type ReviewStatus } from "./reviewStore";
6
+ export declare const DEFAULT_REVIEW_BATCH_SIZE = 10;
7
+ export declare const MAX_REVIEW_BATCH_SIZE = 50;
8
+ export interface ReviewBatchCounts {
9
+ done: number;
10
+ pending: number;
11
+ skipped: number;
12
+ total: number;
13
+ }
14
+ export interface ReviewBatchJudgeSummary {
15
+ agreement: JudgeAgreement;
16
+ completedAt: string;
17
+ evaluatorId: string;
18
+ evaluatorName: string;
19
+ judgePromptName?: string;
20
+ modelId?: string;
21
+ runId: string;
22
+ }
23
+ export interface ReviewBatchListItem {
24
+ bestJudge?: {
25
+ evaluatorName: string;
26
+ rate?: number;
27
+ };
28
+ counts: ReviewBatchCounts;
29
+ created: string;
30
+ /** Dataset the batch judges, when it was built from a dataset. */
31
+ datasetId?: string;
32
+ datasetName?: string;
33
+ description?: string;
34
+ evaluatorId: string;
35
+ evaluatorName: string;
36
+ id: string;
37
+ judgeCount: number;
38
+ name: string;
39
+ promptName?: string;
40
+ /** Question-set fingerprint when the batch was created (dataset batches). */
41
+ questionsFingerprint?: string;
42
+ /** True when the question set's questions changed since this dataset batch was created. */
43
+ questionsStale?: boolean;
44
+ }
45
+ export interface ReviewBatchItem {
46
+ datasetItemId?: string;
47
+ input?: unknown;
48
+ judgeAnswers: Record<string, {
49
+ error?: string;
50
+ scores?: Record<string, ScoreValue>;
51
+ }>;
52
+ output?: unknown;
53
+ reviewItemId: string;
54
+ scores?: Record<string, ScoreValue>;
55
+ status: ReviewStatus;
56
+ /** Unset for a dataset item without a source trace. */
57
+ traceId?: string;
58
+ traceName: string;
59
+ }
60
+ export interface ReviewBatchDetail extends ReviewBatchListItem {
61
+ dimensions: EvaluatorDimension[];
62
+ instructions?: string;
63
+ items: ReviewBatchItem[];
64
+ judges: ReviewBatchJudgeSummary[];
65
+ }
66
+ /**
67
+ * Review batches pair a fixed set of traces (one input and one output each) with a human question
68
+ * set. Reviewers answer every question per item; operators then run LLM judges over the same
69
+ * traces and compare judge answers with the human answers question by question.
70
+ */
71
+ export declare class LocalReviewBatchStore {
72
+ private readonly promptStore;
73
+ private readonly reviewStore;
74
+ constructor(promptStore?: LocalPromptStore, reviewStore?: LocalReviewStore);
75
+ create(params: {
76
+ datasetId?: string;
77
+ description?: string;
78
+ evaluatorId: string;
79
+ limit?: number;
80
+ name?: string;
81
+ promptName?: string;
82
+ traceIds?: string[];
83
+ }): Promise<ReviewBatchDetail>;
84
+ /**
85
+ * One review item per dataset item that lacks a current answer from this question set and is
86
+ * not already waiting in an open review for it.
87
+ */
88
+ private createFromDataset;
89
+ list(): Promise<ReviewBatchListItem[]>;
90
+ get(id: string): Promise<ReviewBatchDetail>;
91
+ /** Run an LLM judge over every batch trace and store its answers for comparison. */
92
+ runJudge(params: {
93
+ ai: EvaluatorAiClient & {
94
+ modelId?: string;
95
+ };
96
+ batchId: string;
97
+ evaluatorId: string;
98
+ }): Promise<ReviewBatchDetail>;
99
+ private answerIds;
100
+ private loadSubjects;
101
+ private datasetNames;
102
+ private latestJudgeRuns;
103
+ private summarizeJudges;
104
+ private toListItem;
105
+ private recentTraceIds;
106
+ }