@terreno/ai 57.21.0 → 57.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/README.md +3 -3
- package/biome.jsonc +2 -2
- package/dist/harness/commit.d.ts +4 -3
- package/dist/harness/commit.js +23 -4
- package/dist/index.d.ts +6 -9
- package/dist/index.js +14 -25
- package/dist/observability/adminScreens.js +5 -1
- package/dist/observability/evaluate.d.ts +8 -1
- package/dist/observability/evaluate.js +10 -2
- package/dist/observability/judgeAgreement.d.ts +53 -0
- package/dist/observability/judgeAgreement.js +123 -0
- package/dist/observability/local/datasetStore.d.ts +5 -0
- package/dist/observability/local/datasetStore.js +7 -2
- package/dist/observability/local/evaluatorStore.d.ts +17 -0
- package/dist/observability/local/evaluatorStore.js +73 -2
- package/dist/observability/local/experimentRunner.d.ts +40 -3
- package/dist/observability/local/experimentRunner.js +390 -11
- package/dist/observability/local/humanAnswers.d.ts +57 -0
- package/dist/observability/local/humanAnswers.js +174 -0
- package/dist/observability/local/judgeRun.d.ts +56 -0
- package/dist/observability/local/judgeRun.js +111 -0
- package/dist/observability/local/localPlugin.js +4 -0
- package/dist/observability/local/models/obsDatasetItem.js +4 -0
- package/dist/observability/local/models/obsEvaluator.js +4 -0
- package/dist/observability/local/models/obsExperiment.js +28 -0
- package/dist/observability/local/models/obsExperimentItem.js +4 -0
- package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
- package/dist/observability/local/models/obsJudgeRun.js +60 -0
- package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
- package/dist/observability/local/models/obsReviewBatch.js +48 -0
- package/dist/observability/local/models/obsReviewItem.js +10 -2
- package/dist/observability/local/models/obsTrace.js +4 -0
- package/dist/observability/local/promptReferences.d.ts +31 -0
- package/dist/observability/local/promptReferences.js +114 -0
- package/dist/observability/local/promptStore.d.ts +55 -19
- package/dist/observability/local/promptStore.js +171 -7
- package/dist/observability/local/reviewBatchStore.d.ts +106 -0
- package/dist/observability/local/reviewBatchStore.js +517 -0
- package/dist/observability/local/reviewStore.d.ts +24 -3
- package/dist/observability/local/reviewStore.js +98 -22
- package/dist/observability/local/traceStore.d.ts +14 -1
- package/dist/observability/local/traceStore.js +144 -11
- package/dist/observability/observabilityApp.js +10 -1
- package/dist/observability/questionFingerprint.d.ts +7 -0
- package/dist/observability/questionFingerprint.js +23 -0
- package/dist/observability/routes/experiments.js +12 -1
- package/dist/observability/routes/review.js +3 -1
- package/dist/observability/routes/reviewBatches.d.ts +11 -0
- package/dist/observability/routes/reviewBatches.js +78 -0
- package/dist/observability/routes/traces.d.ts +1 -1
- package/dist/observability/routes/traces.js +34 -2
- package/dist/observability/types.d.ts +10 -1
- package/dist/service/aiService.js +4 -1
- package/dist/service/chatTurn.js +38 -31
- package/dist/types/harness.d.ts +5 -2
- package/dist/types/index.d.ts +8 -3
- package/dist/types/observability.d.ts +74 -2
- package/package.json +8 -24
- package/src/harness/agentLoop.test.ts +0 -3
- package/src/harness/agentRecovery.test.ts +0 -3
- package/src/harness/commit.ts +24 -4
- package/src/harness/extensions.test.ts +0 -3
- package/src/harness/subagent.test.ts +27 -3
- package/src/index.test.ts +0 -17
- package/src/index.ts +23 -26
- package/src/isolated/gptStream.isolated.ts +0 -8
- package/src/observability/adminScreens.test.ts +7 -0
- package/src/observability/adminScreens.ts +6 -1
- package/src/observability/evaluate.ts +12 -2
- package/src/observability/judgeAgreement.test.ts +111 -0
- package/src/observability/judgeAgreement.ts +184 -0
- package/src/observability/local/datasetStore.ts +30 -11
- package/src/observability/local/evaluatorStore.ts +96 -2
- package/src/observability/local/experimentRunner.ts +495 -15
- package/src/observability/local/humanAnswers.ts +233 -0
- package/src/observability/local/judgePipeline.test.ts +444 -0
- package/src/observability/local/judgeRun.test.ts +173 -0
- package/src/observability/local/judgeRun.ts +184 -0
- package/src/observability/local/localPlugin.ts +4 -0
- package/src/observability/local/models/obsDatasetItem.ts +5 -0
- package/src/observability/local/models/obsEvaluator.ts +4 -0
- package/src/observability/local/models/obsExperiment.ts +29 -0
- package/src/observability/local/models/obsExperimentItem.ts +4 -0
- package/src/observability/local/models/obsJudgeRun.ts +61 -0
- package/src/observability/local/models/obsReviewBatch.ts +52 -0
- package/src/observability/local/models/obsReviewItem.ts +11 -2
- package/src/observability/local/models/obsTrace.ts +5 -0
- package/src/observability/local/obsModels.test.ts +13 -0
- package/src/observability/local/promptReferences.test.ts +128 -0
- package/src/observability/local/promptReferences.ts +149 -0
- package/src/observability/local/promptStore.test.ts +291 -0
- package/src/observability/local/promptStore.ts +243 -32
- package/src/observability/local/reviewBatchStore.test.ts +251 -0
- package/src/observability/local/reviewBatchStore.ts +656 -0
- package/src/observability/local/reviewStore.ts +122 -25
- package/src/observability/local/traceStore.test.ts +139 -1
- package/src/observability/local/traceStore.ts +162 -11
- package/src/observability/observabilityApp.test.ts +13 -10
- package/src/observability/observabilityApp.ts +10 -1
- package/src/observability/questionFingerprint.test.ts +33 -0
- package/src/observability/questionFingerprint.ts +24 -0
- package/src/observability/routes/experiments.test.ts +37 -0
- package/src/observability/routes/experiments.ts +13 -1
- package/src/observability/routes/review.ts +3 -1
- package/src/observability/routes/reviewBatches.test.ts +209 -0
- package/src/observability/routes/reviewBatches.ts +138 -0
- package/src/observability/routes/status.test.ts +4 -4
- package/src/observability/routes/traces.test.ts +61 -0
- package/src/observability/routes/traces.ts +46 -4
- package/src/observability/types.ts +5 -2
- package/src/routes/routes.test.ts +101 -76
- package/src/service/aiService.test.ts +21 -0
- package/src/service/aiService.ts +4 -1
- package/src/service/chatTurn.ts +42 -33
- package/src/tests/bunSetup.ts +0 -54
- package/src/types/harness.ts +5 -2
- package/src/types/index.ts +8 -3
- package/src/types/observability.ts +86 -2
- package/dist/langfuseApp.d.ts +0 -9
- package/dist/langfuseApp.js +0 -53
- package/dist/langfuseCache.d.ts +0 -15
- package/dist/langfuseCache.js +0 -54
- package/dist/langfuseClient.d.ts +0 -6
- package/dist/langfuseClient.js +0 -34
- package/dist/langfusePrompts.d.ts +0 -12
- package/dist/langfusePrompts.js +0 -89
- package/dist/langfuseRoutesEvaluations.d.ts +0 -3
- package/dist/langfuseRoutesEvaluations.js +0 -32
- package/dist/langfuseRoutesMiddleware.d.ts +0 -2
- package/dist/langfuseRoutesMiddleware.js +0 -13
- package/dist/langfuseRoutesPlayground.d.ts +0 -2
- package/dist/langfuseRoutesPlayground.js +0 -36
- package/dist/langfuseRoutesPrompts.d.ts +0 -2
- package/dist/langfuseRoutesPrompts.js +0 -56
- package/dist/langfuseRoutesTraces.d.ts +0 -2
- package/dist/langfuseRoutesTraces.js +0 -30
- package/dist/langfuseTracing.d.ts +0 -4
- package/dist/langfuseTracing.js +0 -32
- package/dist/langfuseTypes.d.ts +0 -96
- package/dist/langfuseTypes.js +0 -2
- package/dist/langfuseVercelAi.d.ts +0 -12
- package/dist/langfuseVercelAi.js +0 -49
- package/src/isolated/langfuseClient.isolated.ts +0 -66
- package/src/langfuseApp.test.ts +0 -119
- package/src/langfuseApp.ts +0 -62
- package/src/langfuseCache.test.ts +0 -75
- package/src/langfuseCache.ts +0 -72
- package/src/langfuseClient.test.ts +0 -69
- package/src/langfuseClient.ts +0 -34
- package/src/langfusePrompts.test.ts +0 -188
- package/src/langfusePrompts.ts +0 -121
- package/src/langfuseRoutes.test.ts +0 -261
- package/src/langfuseRoutesEvaluations.ts +0 -48
- package/src/langfuseRoutesMiddleware.test.ts +0 -57
- package/src/langfuseRoutesMiddleware.ts +0 -12
- package/src/langfuseRoutesPlayground.ts +0 -50
- package/src/langfuseRoutesPrompts.ts +0 -87
- package/src/langfuseRoutesTraces.ts +0 -40
- package/src/langfuseTracing.test.ts +0 -72
- package/src/langfuseTracing.ts +0 -33
- package/src/langfuseTypes.ts +0 -103
- package/src/langfuseVercelAi.test.ts +0 -82
- package/src/langfuseVercelAi.ts +0 -72
- package/src/tests/fakeLangfusePrompt.ts +0 -10
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.resolvePromptReferences = exports.promptIncludeKey = void 0;
|
|
4
|
+
const api_1 = require("@terreno/api");
|
|
5
|
+
/**
|
|
6
|
+
* A reference to another registry prompt, written like a template partial:
|
|
7
|
+
* `{{> base-rules}}`, `{{> base-rules label=staging}}`, or `{{> base-rules version=3}}`.
|
|
8
|
+
*/
|
|
9
|
+
const PROMPT_REFERENCE = /\{\{>([^}]*)\}\}/g;
|
|
10
|
+
const MAX_PROMPT_REFERENCE_DEPTH = 5;
|
|
11
|
+
const DEFAULT_PROMPT_REFERENCE_LABEL = "production";
|
|
12
|
+
/** Identity of one recorded include, for deduping. */
|
|
13
|
+
const promptIncludeKey = (ref) => {
|
|
14
|
+
return [ref.includedBy ?? "", ref.name, ref.version].join("\u0000");
|
|
15
|
+
};
|
|
16
|
+
exports.promptIncludeKey = promptIncludeKey;
|
|
17
|
+
const invalidReference = (tag, detail) => {
|
|
18
|
+
return new api_1.APIError({
|
|
19
|
+
detail: `${tag}: ${detail}`,
|
|
20
|
+
status: 400,
|
|
21
|
+
title: "Invalid prompt reference",
|
|
22
|
+
});
|
|
23
|
+
};
|
|
24
|
+
const parseReference = (tag, body) => {
|
|
25
|
+
const [name, ...options] = body.trim().split(/\s+/);
|
|
26
|
+
if (!name || name.includes("=")) {
|
|
27
|
+
throw invalidReference(tag, "a prompt name must follow {{>");
|
|
28
|
+
}
|
|
29
|
+
const fields = new Map();
|
|
30
|
+
for (const option of options) {
|
|
31
|
+
const separator = option.indexOf("=");
|
|
32
|
+
const key = option.slice(0, separator);
|
|
33
|
+
if (separator < 1 || (key !== "label" && key !== "version") || fields.has(key)) {
|
|
34
|
+
throw invalidReference(tag, `expected one label=<label> or version=<n>, got "${option}"`);
|
|
35
|
+
}
|
|
36
|
+
fields.set(key, option.slice(separator + 1));
|
|
37
|
+
}
|
|
38
|
+
const label = fields.get("label");
|
|
39
|
+
const versionText = fields.get("version");
|
|
40
|
+
if (label !== undefined && versionText !== undefined) {
|
|
41
|
+
throw invalidReference(tag, "use label or version, not both");
|
|
42
|
+
}
|
|
43
|
+
if (label === "") {
|
|
44
|
+
throw invalidReference(tag, "label= needs a label name");
|
|
45
|
+
}
|
|
46
|
+
if (versionText === undefined) {
|
|
47
|
+
return { label: label ?? DEFAULT_PROMPT_REFERENCE_LABEL, name, tag };
|
|
48
|
+
}
|
|
49
|
+
const version = Number(versionText);
|
|
50
|
+
if (!Number.isInteger(version) || version < 1) {
|
|
51
|
+
throw invalidReference(tag, "version must be a positive integer");
|
|
52
|
+
}
|
|
53
|
+
return { name, tag, version };
|
|
54
|
+
};
|
|
55
|
+
/** The references in `text`, in order of appearance. Throws 400 on a malformed tag. */
|
|
56
|
+
const parsePromptReferences = (text) => {
|
|
57
|
+
return [...text.matchAll(PROMPT_REFERENCE)].map((match) => {
|
|
58
|
+
return parseReference(match[0], match[1]);
|
|
59
|
+
});
|
|
60
|
+
};
|
|
61
|
+
/**
|
|
62
|
+
* Replaces every reference in `text` with the referenced prompt's body, recursively. `ancestors`
|
|
63
|
+
* is the chain of prompt names already being resolved (start with the including prompt's own
|
|
64
|
+
* name), so a cycle or a chain deeper than MAX_PROMPT_REFERENCE_DEPTH fails with 400.
|
|
65
|
+
*/
|
|
66
|
+
const resolvePromptReferences = async (params) => {
|
|
67
|
+
const includes = new Map();
|
|
68
|
+
const resolve = async (text, ancestors) => {
|
|
69
|
+
const references = parsePromptReferences(text);
|
|
70
|
+
if (references.length === 0) {
|
|
71
|
+
return text;
|
|
72
|
+
}
|
|
73
|
+
const includedBy = ancestors[ancestors.length - 1];
|
|
74
|
+
const replacements = new Map();
|
|
75
|
+
for (const reference of references) {
|
|
76
|
+
if (ancestors.includes(reference.name)) {
|
|
77
|
+
throw new api_1.APIError({
|
|
78
|
+
detail: [...ancestors, reference.name].join(" → "),
|
|
79
|
+
status: 400,
|
|
80
|
+
title: "Prompt reference cycle",
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
if (ancestors.length > MAX_PROMPT_REFERENCE_DEPTH) {
|
|
84
|
+
throw new api_1.APIError({
|
|
85
|
+
detail: `${ancestors.join(" → ")} exceeds ${MAX_PROMPT_REFERENCE_DEPTH} levels`,
|
|
86
|
+
status: 400,
|
|
87
|
+
title: "Prompt references nested too deeply",
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
const found = await params.lookup({
|
|
91
|
+
label: reference.label,
|
|
92
|
+
name: reference.name,
|
|
93
|
+
version: reference.version,
|
|
94
|
+
});
|
|
95
|
+
if (!found) {
|
|
96
|
+
throw invalidReference(reference.tag, "no prompt matches this name and label or version");
|
|
97
|
+
}
|
|
98
|
+
const ref = {
|
|
99
|
+
includedBy,
|
|
100
|
+
...(found.label !== undefined ? { label: found.label } : {}),
|
|
101
|
+
name: reference.name,
|
|
102
|
+
version: found.version,
|
|
103
|
+
};
|
|
104
|
+
includes.set((0, exports.promptIncludeKey)(ref), ref);
|
|
105
|
+
replacements.set(reference.tag, await resolve(found.body, [...ancestors, reference.name]));
|
|
106
|
+
}
|
|
107
|
+
return text.replace(PROMPT_REFERENCE, (tag) => {
|
|
108
|
+
return replacements.get(tag) ?? tag;
|
|
109
|
+
});
|
|
110
|
+
};
|
|
111
|
+
const text = await resolve(params.text, params.ancestors);
|
|
112
|
+
return { includes: [...includes.values()], text };
|
|
113
|
+
};
|
|
114
|
+
exports.resolvePromptReferences = resolvePromptReferences;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type mongoose from "mongoose";
|
|
2
|
-
import type { ObsPromptVariable } from "../../types/observability";
|
|
2
|
+
import type { ObsPromptRef, ObsPromptVariable } from "../../types/observability";
|
|
3
3
|
import type { ModelPrice, PromptRegistry, PromptVersionRef } from "../types";
|
|
4
4
|
export interface PromptRelatedExperimentSummary {
|
|
5
5
|
created: string;
|
|
@@ -30,6 +30,18 @@ export interface PromptRelationships {
|
|
|
30
30
|
total: number;
|
|
31
31
|
};
|
|
32
32
|
}
|
|
33
|
+
/** Another prompt whose saved system or template contains `{{> name}}`. */
|
|
34
|
+
export interface PromptUsedBy {
|
|
35
|
+
folder: string;
|
|
36
|
+
name: string;
|
|
37
|
+
versions: number[];
|
|
38
|
+
}
|
|
39
|
+
/** Bounded used-by list, same cap as trace and experiment relationships. */
|
|
40
|
+
export interface PromptUsedByPage {
|
|
41
|
+
items: PromptUsedBy[];
|
|
42
|
+
limit: number;
|
|
43
|
+
total: number;
|
|
44
|
+
}
|
|
33
45
|
export interface PromptVersionFields {
|
|
34
46
|
config?: Record<string, unknown>;
|
|
35
47
|
inputSchema?: Record<string, unknown>;
|
|
@@ -68,6 +80,17 @@ export interface CompiledMessage {
|
|
|
68
80
|
content: string;
|
|
69
81
|
role: "system" | "user";
|
|
70
82
|
}
|
|
83
|
+
/** A stored version with prompt references in `system` and `template` resolved. */
|
|
84
|
+
export interface ResolvedPromptVersion {
|
|
85
|
+
includes: ObsPromptRef[];
|
|
86
|
+
inputSchema?: Record<string, unknown>;
|
|
87
|
+
outputSchema?: Record<string, unknown>;
|
|
88
|
+
system?: string;
|
|
89
|
+
template?: string;
|
|
90
|
+
type: "chat" | "text";
|
|
91
|
+
variables: ObsPromptVariable[];
|
|
92
|
+
version: number;
|
|
93
|
+
}
|
|
71
94
|
export interface PlaygroundRunResult {
|
|
72
95
|
compiledMessages: CompiledMessage[];
|
|
73
96
|
costUsd?: number;
|
|
@@ -120,9 +143,13 @@ export declare class LocalPromptStore implements PromptRegistry {
|
|
|
120
143
|
name: string;
|
|
121
144
|
relationships: PromptRelationships;
|
|
122
145
|
tags: string[];
|
|
146
|
+
usedBy: PromptUsedByPage;
|
|
123
147
|
versions: Array<{
|
|
124
148
|
config?: Record<string, unknown>;
|
|
125
149
|
created: string;
|
|
150
|
+
inputSchema?: Record<string, unknown>;
|
|
151
|
+
outputFieldNotes?: Record<string, string>;
|
|
152
|
+
outputSchema?: Record<string, unknown>;
|
|
126
153
|
sensitive: boolean;
|
|
127
154
|
system?: string;
|
|
128
155
|
template?: string;
|
|
@@ -156,25 +183,34 @@ export declare class LocalPromptStore implements PromptRegistry {
|
|
|
156
183
|
private usageLast7d;
|
|
157
184
|
private insertVersion;
|
|
158
185
|
private upsertLabel;
|
|
159
|
-
getVersionByLabel(name: string, label?: string): Promise<
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
version: number;
|
|
167
|
-
} | undefined>;
|
|
168
|
-
getVersionByNumber(name: string, versionNumber: number): Promise<{
|
|
169
|
-
inputSchema?: Record<string, unknown>;
|
|
170
|
-
outputSchema?: Record<string, unknown>;
|
|
171
|
-
system?: string;
|
|
172
|
-
template?: string;
|
|
173
|
-
type: "chat" | "text";
|
|
174
|
-
variables: ObsPromptVariable[];
|
|
175
|
-
version: number;
|
|
176
|
-
} | undefined>;
|
|
186
|
+
getVersionByLabel(name: string, label?: string): Promise<ResolvedPromptVersion | undefined>;
|
|
187
|
+
getVersionByNumber(name: string, versionNumber: number): Promise<ResolvedPromptVersion | undefined>;
|
|
188
|
+
/**
|
|
189
|
+
* Prompts that mention this name in a saved `{{> name}}` tag, with the versions that do.
|
|
190
|
+
* The current prompt is omitted so a self-tag does not list itself.
|
|
191
|
+
*/
|
|
192
|
+
private loadUsedBy;
|
|
177
193
|
private loadRelationships;
|
|
194
|
+
/**
|
|
195
|
+
* Loads a referenced prompt by pinned version or by label. `pending` stands in for a label
|
|
196
|
+
* of the prompt being saved, as it will read once the save lands, when checking the prompts
|
|
197
|
+
* that include it.
|
|
198
|
+
*/
|
|
199
|
+
private referenceLookup;
|
|
200
|
+
private resolveText;
|
|
201
|
+
private resolveVersionText;
|
|
202
|
+
/**
|
|
203
|
+
* Rejects a malformed, unknown, cyclic, or too-deep reference before a version or label is
|
|
204
|
+
* saved, so every saved reference resolves at call time. A label must already exist on the
|
|
205
|
+
* referenced prompt: promote the shared prompt before referencing it by label.
|
|
206
|
+
*/
|
|
207
|
+
private validateReferences;
|
|
208
|
+
/**
|
|
209
|
+
* Re-resolves every version of each prompt that includes `name`, directly or through another
|
|
210
|
+
* prompt, as it will read once `pending` lands. A new body for a shared prompt can push a prompt
|
|
211
|
+
* that includes it past the nesting limit, and experiments still run unlabelled versions.
|
|
212
|
+
*/
|
|
213
|
+
private validateDependents;
|
|
178
214
|
private requirePrompt;
|
|
179
215
|
private isDuplicateKey;
|
|
180
216
|
}
|
|
@@ -9,6 +9,7 @@ const obsPrompt_1 = require("./models/obsPrompt");
|
|
|
9
9
|
const obsPromptLabel_1 = require("./models/obsPromptLabel");
|
|
10
10
|
const obsPromptVersion_1 = require("./models/obsPromptVersion");
|
|
11
11
|
const obsTrace_1 = require("./models/obsTrace");
|
|
12
|
+
const promptReferences_1 = require("./promptReferences");
|
|
12
13
|
const PROMPT_RELATIONSHIP_LIMIT = 20;
|
|
13
14
|
const versionBody = (system, template) => {
|
|
14
15
|
if (system && system.length > 0) {
|
|
@@ -16,6 +17,10 @@ const versionBody = (system, template) => {
|
|
|
16
17
|
}
|
|
17
18
|
return template ?? "";
|
|
18
19
|
};
|
|
20
|
+
/** Matches `{{> name}}`, `{{> name label=…}}`, and `{{> name version=n}}`. */
|
|
21
|
+
const promptReferencePattern = (name) => {
|
|
22
|
+
return new RegExp(`\\{\\{>\\s*${escapeRegex(name)}(\\s|\\}\\})`);
|
|
23
|
+
};
|
|
19
24
|
const escapeRegex = (value) => {
|
|
20
25
|
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
21
26
|
};
|
|
@@ -40,6 +45,7 @@ class LocalPromptStore {
|
|
|
40
45
|
if (existing) {
|
|
41
46
|
throw new api_1.APIError({ status: 409, title: `Prompt "${input.name}" already exists` });
|
|
42
47
|
}
|
|
48
|
+
await this.validateReferences(input.name, input);
|
|
43
49
|
try {
|
|
44
50
|
const prompt = await ObsPrompt.create({
|
|
45
51
|
...(input.description !== undefined ? { description: input.description } : {}),
|
|
@@ -65,6 +71,13 @@ class LocalPromptStore {
|
|
|
65
71
|
.sort({ version: -1 })
|
|
66
72
|
.limit(1);
|
|
67
73
|
const nextVersion = (latestDocs[0]?.version ?? 0) + 1;
|
|
74
|
+
await this.validateReferences(prompt.name, fields);
|
|
75
|
+
// The new version takes the `latest` label, so prompts that include `latest` must still resolve.
|
|
76
|
+
await this.validateDependents(prompt.name, {
|
|
77
|
+
body: versionBody(fields.system, fields.template),
|
|
78
|
+
label: "latest",
|
|
79
|
+
version: nextVersion,
|
|
80
|
+
});
|
|
68
81
|
const created = await this.insertVersion(prompt._id, nextVersion, fields);
|
|
69
82
|
await this.upsertLabel(prompt._id, "latest", created._id);
|
|
70
83
|
return { name: prompt.name, version: nextVersion };
|
|
@@ -85,6 +98,13 @@ class LocalPromptStore {
|
|
|
85
98
|
title: `Unknown version ${params.version} for prompt "${name}"`,
|
|
86
99
|
});
|
|
87
100
|
}
|
|
101
|
+
await this.validateReferences(prompt.name, versionDoc);
|
|
102
|
+
// Other prompts may include this label, so moving it can nest them too deeply.
|
|
103
|
+
await this.validateDependents(prompt.name, {
|
|
104
|
+
body: versionBody(versionDoc.system, versionDoc.template),
|
|
105
|
+
label: params.label,
|
|
106
|
+
version: versionDoc.version,
|
|
107
|
+
});
|
|
88
108
|
const outgoingVersion = await this.upsertLabel(prompt._id, params.label, versionDoc._id);
|
|
89
109
|
return { label: params.label, outgoingVersion, version: params.version };
|
|
90
110
|
}
|
|
@@ -105,8 +125,10 @@ class LocalPromptStore {
|
|
|
105
125
|
if (!version) {
|
|
106
126
|
return undefined;
|
|
107
127
|
}
|
|
128
|
+
const resolved = await this.resolveText(prompt.name, versionBody(version.system, version.template));
|
|
108
129
|
return {
|
|
109
|
-
body:
|
|
130
|
+
body: resolved.text,
|
|
131
|
+
...(resolved.includes.length > 0 ? { includes: resolved.includes } : {}),
|
|
110
132
|
label,
|
|
111
133
|
name: prompt.name,
|
|
112
134
|
sensitive: version.sensitive,
|
|
@@ -129,10 +151,14 @@ class LocalPromptStore {
|
|
|
129
151
|
name: prompt.name,
|
|
130
152
|
relationships: await this.loadRelationships(prompt.name, options?.promptVersion),
|
|
131
153
|
tags: prompt.tags,
|
|
154
|
+
usedBy: await this.loadUsedBy(prompt.name),
|
|
132
155
|
versions: versions.map((row) => {
|
|
133
156
|
return {
|
|
134
157
|
config: row.config,
|
|
135
158
|
created: luxon_1.DateTime.fromJSDate(row.created).toUTC().toISO() ?? "",
|
|
159
|
+
...(row.inputSchema ? { inputSchema: row.inputSchema } : {}),
|
|
160
|
+
...(row.outputFieldNotes ? { outputFieldNotes: row.outputFieldNotes } : {}),
|
|
161
|
+
...(row.outputSchema ? { outputSchema: row.outputSchema } : {}),
|
|
136
162
|
sensitive: row.sensitive,
|
|
137
163
|
system: row.system,
|
|
138
164
|
template: row.template,
|
|
@@ -204,9 +230,10 @@ class LocalPromptStore {
|
|
|
204
230
|
async runPlayground(params) {
|
|
205
231
|
const prompt = await this.requirePrompt(params.name);
|
|
206
232
|
const versionDoc = await this.resolvePlaygroundVersion(prompt._id, params.version);
|
|
233
|
+
const resolved = await this.resolveVersionText(prompt.name, versionDoc);
|
|
207
234
|
const compiledMessages = this.compile({
|
|
208
|
-
system:
|
|
209
|
-
template:
|
|
235
|
+
system: resolved.system,
|
|
236
|
+
template: resolved.template,
|
|
210
237
|
userPrompt: params.userPrompt,
|
|
211
238
|
variables: params.variables,
|
|
212
239
|
});
|
|
@@ -340,11 +367,13 @@ class LocalPromptStore {
|
|
|
340
367
|
if (!version) {
|
|
341
368
|
return undefined;
|
|
342
369
|
}
|
|
370
|
+
const resolved = await this.resolveVersionText(prompt.name, version);
|
|
343
371
|
return {
|
|
372
|
+
includes: resolved.includes,
|
|
344
373
|
inputSchema: version.inputSchema,
|
|
345
374
|
outputSchema: version.outputSchema,
|
|
346
|
-
system:
|
|
347
|
-
template:
|
|
375
|
+
system: resolved.system,
|
|
376
|
+
template: resolved.template,
|
|
348
377
|
type: version.type,
|
|
349
378
|
variables: version.variables,
|
|
350
379
|
version: version.version,
|
|
@@ -362,16 +391,56 @@ class LocalPromptStore {
|
|
|
362
391
|
if (!version) {
|
|
363
392
|
return undefined;
|
|
364
393
|
}
|
|
394
|
+
const resolved = await this.resolveVersionText(prompt.name, version);
|
|
365
395
|
return {
|
|
396
|
+
includes: resolved.includes,
|
|
366
397
|
inputSchema: version.inputSchema,
|
|
367
398
|
outputSchema: version.outputSchema,
|
|
368
|
-
system:
|
|
369
|
-
template:
|
|
399
|
+
system: resolved.system,
|
|
400
|
+
template: resolved.template,
|
|
370
401
|
type: version.type,
|
|
371
402
|
variables: version.variables,
|
|
372
403
|
version: version.version,
|
|
373
404
|
};
|
|
374
405
|
}
|
|
406
|
+
/**
|
|
407
|
+
* Prompts that mention this name in a saved `{{> name}}` tag, with the versions that do.
|
|
408
|
+
* The current prompt is omitted so a self-tag does not list itself.
|
|
409
|
+
*/
|
|
410
|
+
async loadUsedBy(name) {
|
|
411
|
+
const versions = await (0, obsPromptVersion_1.registerObsPromptVersion)().find({
|
|
412
|
+
$or: [{ system: promptReferencePattern(name) }, { template: promptReferencePattern(name) }],
|
|
413
|
+
});
|
|
414
|
+
const versionsByPromptId = new Map();
|
|
415
|
+
for (const version of versions) {
|
|
416
|
+
const promptId = String(version.promptId);
|
|
417
|
+
const current = versionsByPromptId.get(promptId) ?? [];
|
|
418
|
+
current.push(version.version);
|
|
419
|
+
versionsByPromptId.set(promptId, current);
|
|
420
|
+
}
|
|
421
|
+
const empty = { items: [], limit: PROMPT_RELATIONSHIP_LIMIT, total: 0 };
|
|
422
|
+
if (versionsByPromptId.size === 0) {
|
|
423
|
+
return empty;
|
|
424
|
+
}
|
|
425
|
+
const prompts = await (0, obsPrompt_1.registerObsPrompt)().find({
|
|
426
|
+
_id: { $in: [...versionsByPromptId.keys()] },
|
|
427
|
+
});
|
|
428
|
+
const items = prompts
|
|
429
|
+
.filter((prompt) => prompt.name !== name)
|
|
430
|
+
.map((prompt) => {
|
|
431
|
+
return {
|
|
432
|
+
folder: prompt.folder,
|
|
433
|
+
name: prompt.name,
|
|
434
|
+
versions: [...(versionsByPromptId.get(String(prompt._id)) ?? [])].sort((left, right) => left - right),
|
|
435
|
+
};
|
|
436
|
+
})
|
|
437
|
+
.sort((left, right) => left.name.localeCompare(right.name));
|
|
438
|
+
return {
|
|
439
|
+
items: items.slice(0, PROMPT_RELATIONSHIP_LIMIT),
|
|
440
|
+
limit: PROMPT_RELATIONSHIP_LIMIT,
|
|
441
|
+
total: items.length,
|
|
442
|
+
};
|
|
443
|
+
}
|
|
375
444
|
async loadRelationships(promptName, promptVersion) {
|
|
376
445
|
const traceFilter = promptVersion === undefined
|
|
377
446
|
? { "prompts.name": promptName }
|
|
@@ -427,6 +496,101 @@ class LocalPromptStore {
|
|
|
427
496
|
},
|
|
428
497
|
};
|
|
429
498
|
}
|
|
499
|
+
/**
|
|
500
|
+
* Loads a referenced prompt by pinned version or by label. `pending` stands in for a label
|
|
501
|
+
* of the prompt being saved, as it will read once the save lands, when checking the prompts
|
|
502
|
+
* that include it.
|
|
503
|
+
*/
|
|
504
|
+
referenceLookup(pending) {
|
|
505
|
+
return async (reference) => {
|
|
506
|
+
if (pending && reference.name === pending.name && reference.label === pending.label) {
|
|
507
|
+
return pending;
|
|
508
|
+
}
|
|
509
|
+
const prompt = await (0, obsPrompt_1.registerObsPrompt)().findOneOrNone({ name: reference.name });
|
|
510
|
+
if (!prompt) {
|
|
511
|
+
return undefined;
|
|
512
|
+
}
|
|
513
|
+
const ObsPromptVersion = (0, obsPromptVersion_1.registerObsPromptVersion)();
|
|
514
|
+
if (reference.version !== undefined) {
|
|
515
|
+
const pinned = await ObsPromptVersion.findOneOrNone({
|
|
516
|
+
promptId: prompt._id,
|
|
517
|
+
version: reference.version,
|
|
518
|
+
});
|
|
519
|
+
return pinned
|
|
520
|
+
? { body: versionBody(pinned.system, pinned.template), version: pinned.version }
|
|
521
|
+
: undefined;
|
|
522
|
+
}
|
|
523
|
+
const labelDoc = await (0, obsPromptLabel_1.registerObsPromptLabel)().findOneOrNone({
|
|
524
|
+
label: reference.label,
|
|
525
|
+
promptId: prompt._id,
|
|
526
|
+
});
|
|
527
|
+
const labelled = labelDoc
|
|
528
|
+
? await ObsPromptVersion.findOneOrNone({ _id: labelDoc.versionId })
|
|
529
|
+
: undefined;
|
|
530
|
+
return labelled
|
|
531
|
+
? {
|
|
532
|
+
body: versionBody(labelled.system, labelled.template),
|
|
533
|
+
label: reference.label,
|
|
534
|
+
version: labelled.version,
|
|
535
|
+
}
|
|
536
|
+
: undefined;
|
|
537
|
+
};
|
|
538
|
+
}
|
|
539
|
+
async resolveText(name, text) {
|
|
540
|
+
return (0, promptReferences_1.resolvePromptReferences)({ ancestors: [name], lookup: this.referenceLookup(), text });
|
|
541
|
+
}
|
|
542
|
+
async resolveVersionText(name, version) {
|
|
543
|
+
const system = version.system === undefined ? undefined : await this.resolveText(name, version.system);
|
|
544
|
+
const template = version.template === undefined ? undefined : await this.resolveText(name, version.template);
|
|
545
|
+
const includes = new Map();
|
|
546
|
+
for (const ref of [...(system?.includes ?? []), ...(template?.includes ?? [])]) {
|
|
547
|
+
includes.set((0, promptReferences_1.promptIncludeKey)(ref), ref);
|
|
548
|
+
}
|
|
549
|
+
return { includes: [...includes.values()], system: system?.text, template: template?.text };
|
|
550
|
+
}
|
|
551
|
+
/**
|
|
552
|
+
* Rejects a malformed, unknown, cyclic, or too-deep reference before a version or label is
|
|
553
|
+
* saved, so every saved reference resolves at call time. A label must already exist on the
|
|
554
|
+
* referenced prompt: promote the shared prompt before referencing it by label.
|
|
555
|
+
*/
|
|
556
|
+
async validateReferences(name, fields, lookup = this.referenceLookup()) {
|
|
557
|
+
for (const text of [fields.system, fields.template]) {
|
|
558
|
+
if (text) {
|
|
559
|
+
await (0, promptReferences_1.resolvePromptReferences)({ ancestors: [name], lookup, text });
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
/**
|
|
564
|
+
* Re-resolves every version of each prompt that includes `name`, directly or through another
|
|
565
|
+
* prompt, as it will read once `pending` lands. A new body for a shared prompt can push a prompt
|
|
566
|
+
* that includes it past the nesting limit, and experiments still run unlabelled versions.
|
|
567
|
+
*/
|
|
568
|
+
async validateDependents(name, pending) {
|
|
569
|
+
const lookup = this.referenceLookup({ ...pending, name });
|
|
570
|
+
const ObsPrompt = (0, obsPrompt_1.registerObsPrompt)();
|
|
571
|
+
const ObsPromptVersion = (0, obsPromptVersion_1.registerObsPromptVersion)();
|
|
572
|
+
const seen = new Set([name]);
|
|
573
|
+
const queue = [name];
|
|
574
|
+
while (queue.length > 0) {
|
|
575
|
+
const target = queue.shift();
|
|
576
|
+
const includesTarget = promptReferencePattern(target);
|
|
577
|
+
const promptIds = await ObsPromptVersion.distinct("promptId", {
|
|
578
|
+
$or: [{ system: includesTarget }, { template: includesTarget }],
|
|
579
|
+
});
|
|
580
|
+
const dependents = await ObsPrompt.find({ _id: { $in: promptIds } });
|
|
581
|
+
for (const dependent of dependents) {
|
|
582
|
+
if (seen.has(dependent.name)) {
|
|
583
|
+
continue;
|
|
584
|
+
}
|
|
585
|
+
seen.add(dependent.name);
|
|
586
|
+
queue.push(dependent.name);
|
|
587
|
+
const versions = await ObsPromptVersion.find({ promptId: dependent._id });
|
|
588
|
+
for (const version of versions) {
|
|
589
|
+
await this.validateReferences(dependent.name, version, lookup);
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
}
|
|
430
594
|
async requirePrompt(name) {
|
|
431
595
|
const prompt = await (0, obsPrompt_1.registerObsPrompt)().findOneOrNone({ name });
|
|
432
596
|
if (!prompt) {
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import type { EvaluatorDimension } from "../../types/observability";
|
|
2
|
+
import type { EvaluatorAiClient } from "../evaluate";
|
|
3
|
+
import { type JudgeAgreement, type ScoreValue } from "../judgeAgreement";
|
|
4
|
+
import { LocalPromptStore } from "./promptStore";
|
|
5
|
+
import { LocalReviewStore, type ReviewStatus } from "./reviewStore";
|
|
6
|
+
export declare const DEFAULT_REVIEW_BATCH_SIZE = 10;
|
|
7
|
+
export declare const MAX_REVIEW_BATCH_SIZE = 50;
|
|
8
|
+
export interface ReviewBatchCounts {
|
|
9
|
+
done: number;
|
|
10
|
+
pending: number;
|
|
11
|
+
skipped: number;
|
|
12
|
+
total: number;
|
|
13
|
+
}
|
|
14
|
+
export interface ReviewBatchJudgeSummary {
|
|
15
|
+
agreement: JudgeAgreement;
|
|
16
|
+
completedAt: string;
|
|
17
|
+
evaluatorId: string;
|
|
18
|
+
evaluatorName: string;
|
|
19
|
+
judgePromptName?: string;
|
|
20
|
+
modelId?: string;
|
|
21
|
+
runId: string;
|
|
22
|
+
}
|
|
23
|
+
export interface ReviewBatchListItem {
|
|
24
|
+
bestJudge?: {
|
|
25
|
+
evaluatorName: string;
|
|
26
|
+
rate?: number;
|
|
27
|
+
};
|
|
28
|
+
counts: ReviewBatchCounts;
|
|
29
|
+
created: string;
|
|
30
|
+
/** Dataset the batch judges, when it was built from a dataset. */
|
|
31
|
+
datasetId?: string;
|
|
32
|
+
datasetName?: string;
|
|
33
|
+
description?: string;
|
|
34
|
+
evaluatorId: string;
|
|
35
|
+
evaluatorName: string;
|
|
36
|
+
id: string;
|
|
37
|
+
judgeCount: number;
|
|
38
|
+
name: string;
|
|
39
|
+
promptName?: string;
|
|
40
|
+
/** Question-set fingerprint when the batch was created (dataset batches). */
|
|
41
|
+
questionsFingerprint?: string;
|
|
42
|
+
/** True when the question set's questions changed since this dataset batch was created. */
|
|
43
|
+
questionsStale?: boolean;
|
|
44
|
+
}
|
|
45
|
+
export interface ReviewBatchItem {
|
|
46
|
+
datasetItemId?: string;
|
|
47
|
+
input?: unknown;
|
|
48
|
+
judgeAnswers: Record<string, {
|
|
49
|
+
error?: string;
|
|
50
|
+
scores?: Record<string, ScoreValue>;
|
|
51
|
+
}>;
|
|
52
|
+
output?: unknown;
|
|
53
|
+
reviewItemId: string;
|
|
54
|
+
scores?: Record<string, ScoreValue>;
|
|
55
|
+
status: ReviewStatus;
|
|
56
|
+
/** Unset for a dataset item without a source trace. */
|
|
57
|
+
traceId?: string;
|
|
58
|
+
traceName: string;
|
|
59
|
+
}
|
|
60
|
+
export interface ReviewBatchDetail extends ReviewBatchListItem {
|
|
61
|
+
dimensions: EvaluatorDimension[];
|
|
62
|
+
instructions?: string;
|
|
63
|
+
items: ReviewBatchItem[];
|
|
64
|
+
judges: ReviewBatchJudgeSummary[];
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Review batches pair a fixed set of traces (one input and one output each) with a human question
|
|
68
|
+
* set. Reviewers answer every question per item; operators then run LLM judges over the same
|
|
69
|
+
* traces and compare judge answers with the human answers question by question.
|
|
70
|
+
*/
|
|
71
|
+
export declare class LocalReviewBatchStore {
|
|
72
|
+
private readonly promptStore;
|
|
73
|
+
private readonly reviewStore;
|
|
74
|
+
constructor(promptStore?: LocalPromptStore, reviewStore?: LocalReviewStore);
|
|
75
|
+
create(params: {
|
|
76
|
+
datasetId?: string;
|
|
77
|
+
description?: string;
|
|
78
|
+
evaluatorId: string;
|
|
79
|
+
limit?: number;
|
|
80
|
+
name?: string;
|
|
81
|
+
promptName?: string;
|
|
82
|
+
traceIds?: string[];
|
|
83
|
+
}): Promise<ReviewBatchDetail>;
|
|
84
|
+
/**
|
|
85
|
+
* One review item per dataset item that lacks a current answer from this question set and is
|
|
86
|
+
* not already waiting in an open review for it.
|
|
87
|
+
*/
|
|
88
|
+
private createFromDataset;
|
|
89
|
+
list(): Promise<ReviewBatchListItem[]>;
|
|
90
|
+
get(id: string): Promise<ReviewBatchDetail>;
|
|
91
|
+
/** Run an LLM judge over every batch trace and store its answers for comparison. */
|
|
92
|
+
runJudge(params: {
|
|
93
|
+
ai: EvaluatorAiClient & {
|
|
94
|
+
modelId?: string;
|
|
95
|
+
};
|
|
96
|
+
batchId: string;
|
|
97
|
+
evaluatorId: string;
|
|
98
|
+
}): Promise<ReviewBatchDetail>;
|
|
99
|
+
private answerIds;
|
|
100
|
+
private loadSubjects;
|
|
101
|
+
private datasetNames;
|
|
102
|
+
private latestJudgeRuns;
|
|
103
|
+
private summarizeJudges;
|
|
104
|
+
private toListItem;
|
|
105
|
+
private recentTraceIds;
|
|
106
|
+
}
|