@terreno/ai 58.0.0-beta.2 → 58.0.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +146 -0
- package/README.md +3 -3
- package/dist/harness/commit.d.ts +4 -3
- package/dist/harness/commit.js +23 -4
- package/dist/index.d.ts +6 -9
- package/dist/index.js +14 -25
- package/dist/observability/adminScreens.js +5 -1
- package/dist/observability/evaluate.d.ts +8 -1
- package/dist/observability/evaluate.js +10 -2
- package/dist/observability/judgeAgreement.d.ts +53 -0
- package/dist/observability/judgeAgreement.js +123 -0
- package/dist/observability/local/datasetStore.d.ts +5 -0
- package/dist/observability/local/datasetStore.js +7 -2
- package/dist/observability/local/evaluatorStore.d.ts +17 -0
- package/dist/observability/local/evaluatorStore.js +73 -2
- package/dist/observability/local/experimentRunner.d.ts +40 -3
- package/dist/observability/local/experimentRunner.js +390 -11
- package/dist/observability/local/humanAnswers.d.ts +57 -0
- package/dist/observability/local/humanAnswers.js +174 -0
- package/dist/observability/local/judgeRun.d.ts +56 -0
- package/dist/observability/local/judgeRun.js +111 -0
- package/dist/observability/local/localPlugin.js +4 -0
- package/dist/observability/local/models/obsDatasetItem.js +4 -0
- package/dist/observability/local/models/obsEvaluator.js +4 -0
- package/dist/observability/local/models/obsExperiment.js +28 -0
- package/dist/observability/local/models/obsExperimentItem.js +4 -0
- package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
- package/dist/observability/local/models/obsJudgeRun.js +60 -0
- package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
- package/dist/observability/local/models/obsReviewBatch.js +48 -0
- package/dist/observability/local/models/obsReviewItem.js +10 -2
- package/dist/observability/local/models/obsTrace.js +4 -0
- package/dist/observability/local/promptReferences.d.ts +31 -0
- package/dist/observability/local/promptReferences.js +114 -0
- package/dist/observability/local/promptStore.d.ts +55 -19
- package/dist/observability/local/promptStore.js +171 -7
- package/dist/observability/local/reviewBatchStore.d.ts +106 -0
- package/dist/observability/local/reviewBatchStore.js +517 -0
- package/dist/observability/local/reviewStore.d.ts +24 -3
- package/dist/observability/local/reviewStore.js +98 -22
- package/dist/observability/local/traceStore.d.ts +14 -1
- package/dist/observability/local/traceStore.js +144 -11
- package/dist/observability/observabilityApp.js +10 -1
- package/dist/observability/questionFingerprint.d.ts +7 -0
- package/dist/observability/questionFingerprint.js +23 -0
- package/dist/observability/routes/experiments.js +12 -1
- package/dist/observability/routes/review.js +3 -1
- package/dist/observability/routes/reviewBatches.d.ts +11 -0
- package/dist/observability/routes/reviewBatches.js +78 -0
- package/dist/observability/routes/traces.d.ts +1 -1
- package/dist/observability/routes/traces.js +34 -2
- package/dist/observability/types.d.ts +10 -1
- package/dist/service/aiService.js +4 -1
- package/dist/service/chatTurn.js +38 -31
- package/dist/types/harness.d.ts +5 -2
- package/dist/types/index.d.ts +8 -3
- package/dist/types/observability.d.ts +74 -2
- package/package.json +8 -24
- package/src/harness/agentLoop.test.ts +0 -3
- package/src/harness/agentRecovery.test.ts +0 -3
- package/src/harness/commit.ts +24 -4
- package/src/harness/extensions.test.ts +0 -3
- package/src/harness/subagent.test.ts +27 -3
- package/src/index.test.ts +0 -17
- package/src/index.ts +23 -26
- package/src/isolated/gptStream.isolated.ts +0 -8
- package/src/observability/adminScreens.test.ts +7 -0
- package/src/observability/adminScreens.ts +6 -1
- package/src/observability/evaluate.ts +12 -2
- package/src/observability/judgeAgreement.test.ts +111 -0
- package/src/observability/judgeAgreement.ts +184 -0
- package/src/observability/local/datasetStore.ts +30 -11
- package/src/observability/local/evaluatorStore.ts +96 -2
- package/src/observability/local/experimentRunner.ts +495 -15
- package/src/observability/local/humanAnswers.ts +233 -0
- package/src/observability/local/judgePipeline.test.ts +444 -0
- package/src/observability/local/judgeRun.test.ts +173 -0
- package/src/observability/local/judgeRun.ts +184 -0
- package/src/observability/local/localPlugin.ts +4 -0
- package/src/observability/local/models/obsDatasetItem.ts +5 -0
- package/src/observability/local/models/obsEvaluator.ts +4 -0
- package/src/observability/local/models/obsExperiment.ts +29 -0
- package/src/observability/local/models/obsExperimentItem.ts +4 -0
- package/src/observability/local/models/obsJudgeRun.ts +61 -0
- package/src/observability/local/models/obsReviewBatch.ts +52 -0
- package/src/observability/local/models/obsReviewItem.ts +11 -2
- package/src/observability/local/models/obsTrace.ts +5 -0
- package/src/observability/local/obsModels.test.ts +13 -0
- package/src/observability/local/promptReferences.test.ts +128 -0
- package/src/observability/local/promptReferences.ts +149 -0
- package/src/observability/local/promptStore.test.ts +291 -0
- package/src/observability/local/promptStore.ts +243 -32
- package/src/observability/local/reviewBatchStore.test.ts +251 -0
- package/src/observability/local/reviewBatchStore.ts +656 -0
- package/src/observability/local/reviewStore.ts +122 -25
- package/src/observability/local/traceStore.test.ts +139 -1
- package/src/observability/local/traceStore.ts +162 -11
- package/src/observability/observabilityApp.test.ts +13 -10
- package/src/observability/observabilityApp.ts +10 -1
- package/src/observability/questionFingerprint.test.ts +33 -0
- package/src/observability/questionFingerprint.ts +24 -0
- package/src/observability/routes/experiments.test.ts +37 -0
- package/src/observability/routes/experiments.ts +13 -1
- package/src/observability/routes/review.ts +3 -1
- package/src/observability/routes/reviewBatches.test.ts +209 -0
- package/src/observability/routes/reviewBatches.ts +138 -0
- package/src/observability/routes/status.test.ts +4 -4
- package/src/observability/routes/traces.test.ts +61 -0
- package/src/observability/routes/traces.ts +46 -4
- package/src/observability/types.ts +5 -2
- package/src/routes/routes.test.ts +101 -76
- package/src/service/aiService.test.ts +21 -0
- package/src/service/aiService.ts +4 -1
- package/src/service/chatTurn.ts +42 -33
- package/src/tests/bunSetup.ts +0 -54
- package/src/types/harness.ts +5 -2
- package/src/types/index.ts +8 -3
- package/src/types/observability.ts +86 -2
- package/dist/langfuseApp.d.ts +0 -9
- package/dist/langfuseApp.js +0 -53
- package/dist/langfuseCache.d.ts +0 -15
- package/dist/langfuseCache.js +0 -54
- package/dist/langfuseClient.d.ts +0 -6
- package/dist/langfuseClient.js +0 -34
- package/dist/langfusePrompts.d.ts +0 -12
- package/dist/langfusePrompts.js +0 -89
- package/dist/langfuseRoutesEvaluations.d.ts +0 -3
- package/dist/langfuseRoutesEvaluations.js +0 -32
- package/dist/langfuseRoutesMiddleware.d.ts +0 -2
- package/dist/langfuseRoutesMiddleware.js +0 -13
- package/dist/langfuseRoutesPlayground.d.ts +0 -2
- package/dist/langfuseRoutesPlayground.js +0 -36
- package/dist/langfuseRoutesPrompts.d.ts +0 -2
- package/dist/langfuseRoutesPrompts.js +0 -56
- package/dist/langfuseRoutesTraces.d.ts +0 -2
- package/dist/langfuseRoutesTraces.js +0 -30
- package/dist/langfuseTracing.d.ts +0 -4
- package/dist/langfuseTracing.js +0 -32
- package/dist/langfuseTypes.d.ts +0 -96
- package/dist/langfuseTypes.js +0 -2
- package/dist/langfuseVercelAi.d.ts +0 -12
- package/dist/langfuseVercelAi.js +0 -49
- package/src/isolated/langfuseClient.isolated.ts +0 -66
- package/src/langfuseApp.test.ts +0 -119
- package/src/langfuseApp.ts +0 -62
- package/src/langfuseCache.test.ts +0 -75
- package/src/langfuseCache.ts +0 -72
- package/src/langfuseClient.test.ts +0 -69
- package/src/langfuseClient.ts +0 -34
- package/src/langfusePrompts.test.ts +0 -188
- package/src/langfusePrompts.ts +0 -121
- package/src/langfuseRoutes.test.ts +0 -261
- package/src/langfuseRoutesEvaluations.ts +0 -48
- package/src/langfuseRoutesMiddleware.test.ts +0 -57
- package/src/langfuseRoutesMiddleware.ts +0 -12
- package/src/langfuseRoutesPlayground.ts +0 -50
- package/src/langfuseRoutesPrompts.ts +0 -87
- package/src/langfuseRoutesTraces.ts +0 -40
- package/src/langfuseTracing.test.ts +0 -72
- package/src/langfuseTracing.ts +0 -33
- package/src/langfuseTypes.ts +0 -103
- package/src/langfuseVercelAi.test.ts +0 -82
- package/src/langfuseVercelAi.ts +0 -72
- package/src/tests/fakeLangfusePrompt.ts +0 -10
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import {describe, expect, it} from "bun:test";
|
|
2
|
+
|
|
3
|
+
import {type PromptReferenceLookup, resolvePromptReferences} from "./promptReferences";
|
|
4
|
+
|
|
5
|
+
// Mirrors the module's nesting limit, which the docs promise.
|
|
6
|
+
const MAX_PROMPT_REFERENCE_DEPTH = 5;
|
|
7
|
+
|
|
8
|
+
const tag = (body: string): string => `{{> ${body}}}`;
|
|
9
|
+
|
|
10
|
+
const lookupFrom = (
|
|
11
|
+
bodies: Record<string, {body: string; version: number}>
|
|
12
|
+
): PromptReferenceLookup => {
|
|
13
|
+
return async ({label, name, version}) => {
|
|
14
|
+
const found = bodies[name];
|
|
15
|
+
if (!found || (version !== undefined && version !== found.version)) {
|
|
16
|
+
return undefined;
|
|
17
|
+
}
|
|
18
|
+
return {...found, ...(label !== undefined ? {label} : {})};
|
|
19
|
+
};
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
describe("prompt reference tags", () => {
|
|
23
|
+
const seen = async (text: string): Promise<Array<{label?: string; version?: number}>> => {
|
|
24
|
+
const lookups: Array<{label?: string; version?: number}> = [];
|
|
25
|
+
await resolvePromptReferences({
|
|
26
|
+
ancestors: ["caller"],
|
|
27
|
+
lookup: async ({label, version}) => {
|
|
28
|
+
lookups.push({label, version});
|
|
29
|
+
return {body: "", version: version ?? 1};
|
|
30
|
+
},
|
|
31
|
+
text,
|
|
32
|
+
});
|
|
33
|
+
return lookups;
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
it("defaults a reference without label or version to production", async () => {
|
|
37
|
+
expect(await seen(tag("base-rules"))).toEqual([{label: "production", version: undefined}]);
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
it("reads a pinned version and an explicit label", async () => {
|
|
41
|
+
expect(await seen(`${tag("a version=3")} ${tag("b label=staging")}`)).toEqual([
|
|
42
|
+
{label: undefined, version: 3},
|
|
43
|
+
{label: "staging", version: undefined},
|
|
44
|
+
]);
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
it("rejects a missing name, a bad version, label plus version, and an unknown option", async () => {
|
|
48
|
+
for (const body of [
|
|
49
|
+
"label=production",
|
|
50
|
+
"a version=0",
|
|
51
|
+
"a label=production version=1",
|
|
52
|
+
"a tone=warm",
|
|
53
|
+
"a label=",
|
|
54
|
+
"",
|
|
55
|
+
]) {
|
|
56
|
+
await expect(seen(tag(body))).rejects.toThrow("Invalid prompt reference");
|
|
57
|
+
}
|
|
58
|
+
});
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
describe("resolvePromptReferences", () => {
|
|
62
|
+
it("inlines nested references and records each include with the prompt that pulled it in", async () => {
|
|
63
|
+
const resolved = await resolvePromptReferences({
|
|
64
|
+
ancestors: ["triage"],
|
|
65
|
+
lookup: lookupFrom({
|
|
66
|
+
"base-rules": {body: `Be kind.\n${tag("safety")}`, version: 4},
|
|
67
|
+
safety: {body: "Escalate risk.", version: 2},
|
|
68
|
+
}),
|
|
69
|
+
text: `${tag("base-rules")}\nTriage the {{message}}.`,
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
expect(resolved.text).toBe("Be kind.\nEscalate risk.\nTriage the {{message}}.");
|
|
73
|
+
expect(resolved.includes).toEqual([
|
|
74
|
+
{includedBy: "triage", label: "production", name: "base-rules", version: 4},
|
|
75
|
+
{includedBy: "base-rules", label: "production", name: "safety", version: 2},
|
|
76
|
+
]);
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
it("returns text without references unchanged", async () => {
|
|
80
|
+
const resolved = await resolvePromptReferences({
|
|
81
|
+
ancestors: ["plain"],
|
|
82
|
+
lookup: lookupFrom({}),
|
|
83
|
+
text: "No references here.",
|
|
84
|
+
});
|
|
85
|
+
expect(resolved).toEqual({includes: [], text: "No references here."});
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
it("records a reference used twice once", async () => {
|
|
89
|
+
const resolved = await resolvePromptReferences({
|
|
90
|
+
ancestors: ["twice"],
|
|
91
|
+
lookup: lookupFrom({base: {body: "B", version: 1}}),
|
|
92
|
+
text: `${tag("base")} and ${tag("base")}`,
|
|
93
|
+
});
|
|
94
|
+
expect(resolved.text).toBe("B and B");
|
|
95
|
+
expect(resolved.includes).toHaveLength(1);
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
it("rejects an unknown reference", async () => {
|
|
99
|
+
await expect(
|
|
100
|
+
resolvePromptReferences({
|
|
101
|
+
ancestors: ["triage"],
|
|
102
|
+
lookup: lookupFrom({}),
|
|
103
|
+
text: tag("missing"),
|
|
104
|
+
})
|
|
105
|
+
).rejects.toThrow("Invalid prompt reference");
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
it("rejects a cycle", async () => {
|
|
109
|
+
await expect(
|
|
110
|
+
resolvePromptReferences({
|
|
111
|
+
ancestors: ["a"],
|
|
112
|
+
lookup: lookupFrom({b: {body: tag("a"), version: 1}}),
|
|
113
|
+
text: tag("b"),
|
|
114
|
+
})
|
|
115
|
+
).rejects.toThrow("Prompt reference cycle");
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
it("rejects a chain deeper than the limit", async () => {
|
|
119
|
+
const bodies: Record<string, {body: string; version: number}> = {};
|
|
120
|
+
for (let level = 1; level <= MAX_PROMPT_REFERENCE_DEPTH + 1; level++) {
|
|
121
|
+
bodies[`p${level}`] = {body: tag(`p${level + 1}`), version: 1};
|
|
122
|
+
}
|
|
123
|
+
bodies[`p${MAX_PROMPT_REFERENCE_DEPTH + 2}`] = {body: "leaf", version: 1};
|
|
124
|
+
await expect(
|
|
125
|
+
resolvePromptReferences({ancestors: ["p0"], lookup: lookupFrom(bodies), text: tag("p1")})
|
|
126
|
+
).rejects.toThrow("Prompt references nested too deeply");
|
|
127
|
+
});
|
|
128
|
+
});
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import {APIError} from "@terreno/api";
|
|
2
|
+
|
|
3
|
+
import type {ObsPromptRef} from "../../types/observability";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* A reference to another registry prompt, written like a template partial:
|
|
7
|
+
* `{{> base-rules}}`, `{{> base-rules label=staging}}`, or `{{> base-rules version=3}}`.
|
|
8
|
+
*/
|
|
9
|
+
const PROMPT_REFERENCE = /\{\{>([^}]*)\}\}/g;
|
|
10
|
+
|
|
11
|
+
const MAX_PROMPT_REFERENCE_DEPTH = 5;
|
|
12
|
+
|
|
13
|
+
const DEFAULT_PROMPT_REFERENCE_LABEL = "production";
|
|
14
|
+
|
|
15
|
+
export interface PromptReference {
|
|
16
|
+
label?: string;
|
|
17
|
+
name: string;
|
|
18
|
+
tag: string;
|
|
19
|
+
version?: number;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export interface ResolvedPromptReference {
|
|
23
|
+
body: string;
|
|
24
|
+
label?: string;
|
|
25
|
+
version: number;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Loads the referenced prompt's raw body; undefined when the name, label, or version is unknown. */
|
|
29
|
+
export type PromptReferenceLookup = (
|
|
30
|
+
reference: Omit<PromptReference, "tag">
|
|
31
|
+
) => Promise<ResolvedPromptReference | undefined>;
|
|
32
|
+
|
|
33
|
+
export interface ResolvedPromptText {
|
|
34
|
+
/** Every prompt version pulled in, depth-first, once each, tagged with the prompt that included it. */
|
|
35
|
+
includes: ObsPromptRef[];
|
|
36
|
+
text: string;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Identity of one recorded include, for deduping. */
|
|
40
|
+
export const promptIncludeKey = (ref: ObsPromptRef): string => {
|
|
41
|
+
return [ref.includedBy ?? "", ref.name, ref.version].join("\u0000");
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
const invalidReference = (tag: string, detail: string): APIError => {
|
|
45
|
+
return new APIError({
|
|
46
|
+
detail: `${tag}: ${detail}`,
|
|
47
|
+
status: 400,
|
|
48
|
+
title: "Invalid prompt reference",
|
|
49
|
+
});
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
const parseReference = (tag: string, body: string): PromptReference => {
|
|
53
|
+
const [name, ...options] = body.trim().split(/\s+/);
|
|
54
|
+
if (!name || name.includes("=")) {
|
|
55
|
+
throw invalidReference(tag, "a prompt name must follow {{>");
|
|
56
|
+
}
|
|
57
|
+
const fields = new Map<string, string>();
|
|
58
|
+
for (const option of options) {
|
|
59
|
+
const separator = option.indexOf("=");
|
|
60
|
+
const key = option.slice(0, separator);
|
|
61
|
+
if (separator < 1 || (key !== "label" && key !== "version") || fields.has(key)) {
|
|
62
|
+
throw invalidReference(tag, `expected one label=<label> or version=<n>, got "${option}"`);
|
|
63
|
+
}
|
|
64
|
+
fields.set(key, option.slice(separator + 1));
|
|
65
|
+
}
|
|
66
|
+
const label = fields.get("label");
|
|
67
|
+
const versionText = fields.get("version");
|
|
68
|
+
if (label !== undefined && versionText !== undefined) {
|
|
69
|
+
throw invalidReference(tag, "use label or version, not both");
|
|
70
|
+
}
|
|
71
|
+
if (label === "") {
|
|
72
|
+
throw invalidReference(tag, "label= needs a label name");
|
|
73
|
+
}
|
|
74
|
+
if (versionText === undefined) {
|
|
75
|
+
return {label: label ?? DEFAULT_PROMPT_REFERENCE_LABEL, name, tag};
|
|
76
|
+
}
|
|
77
|
+
const version = Number(versionText);
|
|
78
|
+
if (!Number.isInteger(version) || version < 1) {
|
|
79
|
+
throw invalidReference(tag, "version must be a positive integer");
|
|
80
|
+
}
|
|
81
|
+
return {name, tag, version};
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
/** The references in `text`, in order of appearance. Throws 400 on a malformed tag. */
|
|
85
|
+
const parsePromptReferences = (text: string): PromptReference[] => {
|
|
86
|
+
return [...text.matchAll(PROMPT_REFERENCE)].map((match) => {
|
|
87
|
+
return parseReference(match[0], match[1]);
|
|
88
|
+
});
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Replaces every reference in `text` with the referenced prompt's body, recursively. `ancestors`
|
|
93
|
+
* is the chain of prompt names already being resolved (start with the including prompt's own
|
|
94
|
+
* name), so a cycle or a chain deeper than MAX_PROMPT_REFERENCE_DEPTH fails with 400.
|
|
95
|
+
*/
|
|
96
|
+
export const resolvePromptReferences = async (params: {
|
|
97
|
+
ancestors: string[];
|
|
98
|
+
lookup: PromptReferenceLookup;
|
|
99
|
+
text: string;
|
|
100
|
+
}): Promise<ResolvedPromptText> => {
|
|
101
|
+
const includes = new Map<string, ObsPromptRef>();
|
|
102
|
+
|
|
103
|
+
const resolve = async (text: string, ancestors: string[]): Promise<string> => {
|
|
104
|
+
const references = parsePromptReferences(text);
|
|
105
|
+
if (references.length === 0) {
|
|
106
|
+
return text;
|
|
107
|
+
}
|
|
108
|
+
const includedBy = ancestors[ancestors.length - 1];
|
|
109
|
+
const replacements = new Map<string, string>();
|
|
110
|
+
for (const reference of references) {
|
|
111
|
+
if (ancestors.includes(reference.name)) {
|
|
112
|
+
throw new APIError({
|
|
113
|
+
detail: [...ancestors, reference.name].join(" → "),
|
|
114
|
+
status: 400,
|
|
115
|
+
title: "Prompt reference cycle",
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
if (ancestors.length > MAX_PROMPT_REFERENCE_DEPTH) {
|
|
119
|
+
throw new APIError({
|
|
120
|
+
detail: `${ancestors.join(" → ")} exceeds ${MAX_PROMPT_REFERENCE_DEPTH} levels`,
|
|
121
|
+
status: 400,
|
|
122
|
+
title: "Prompt references nested too deeply",
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
const found = await params.lookup({
|
|
126
|
+
label: reference.label,
|
|
127
|
+
name: reference.name,
|
|
128
|
+
version: reference.version,
|
|
129
|
+
});
|
|
130
|
+
if (!found) {
|
|
131
|
+
throw invalidReference(reference.tag, "no prompt matches this name and label or version");
|
|
132
|
+
}
|
|
133
|
+
const ref: ObsPromptRef = {
|
|
134
|
+
includedBy,
|
|
135
|
+
...(found.label !== undefined ? {label: found.label} : {}),
|
|
136
|
+
name: reference.name,
|
|
137
|
+
version: found.version,
|
|
138
|
+
};
|
|
139
|
+
includes.set(promptIncludeKey(ref), ref);
|
|
140
|
+
replacements.set(reference.tag, await resolve(found.body, [...ancestors, reference.name]));
|
|
141
|
+
}
|
|
142
|
+
return text.replace(PROMPT_REFERENCE, (tag) => {
|
|
143
|
+
return replacements.get(tag) ?? tag;
|
|
144
|
+
});
|
|
145
|
+
};
|
|
146
|
+
|
|
147
|
+
const text = await resolve(params.text, params.ancestors);
|
|
148
|
+
return {includes: [...includes.values()], text};
|
|
149
|
+
};
|
|
@@ -29,6 +29,25 @@ describe("LocalPromptStore", () => {
|
|
|
29
29
|
await registerObsExperiment().deleteMany({});
|
|
30
30
|
});
|
|
31
31
|
|
|
32
|
+
it("returns version schemas from prompt detail", async () => {
|
|
33
|
+
const outputSchema = {
|
|
34
|
+
properties: {helpfulness: {type: "number"}},
|
|
35
|
+
required: ["helpfulness"],
|
|
36
|
+
type: "object",
|
|
37
|
+
};
|
|
38
|
+
await store.create({
|
|
39
|
+
folder: "evaluators",
|
|
40
|
+
name: "eval-judge-helpfulness",
|
|
41
|
+
outputFieldNotes: {helpfulness: "0 to 1"},
|
|
42
|
+
outputSchema,
|
|
43
|
+
system: "Score helpfulness",
|
|
44
|
+
type: "chat",
|
|
45
|
+
});
|
|
46
|
+
const detail = await store.getDetail("eval-judge-helpfulness");
|
|
47
|
+
assert.deepEqual(detail.versions[0]?.outputSchema, outputSchema);
|
|
48
|
+
assert.deepEqual(detail.versions[0]?.outputFieldNotes, {helpfulness: "0 to 1"});
|
|
49
|
+
});
|
|
50
|
+
|
|
32
51
|
it("round-trips an optional description on create, list, and detail", async () => {
|
|
33
52
|
await store.create({
|
|
34
53
|
description: "Summarizes user notes for clinicians",
|
|
@@ -123,6 +142,51 @@ describe("LocalPromptStore", () => {
|
|
|
123
142
|
).toBe(2);
|
|
124
143
|
});
|
|
125
144
|
|
|
145
|
+
it("lists prompts whose saved text references this prompt", async () => {
|
|
146
|
+
await store.create({
|
|
147
|
+
folder: "shared",
|
|
148
|
+
name: "base-rules",
|
|
149
|
+
template: "Be brief.",
|
|
150
|
+
type: "text",
|
|
151
|
+
});
|
|
152
|
+
await store.moveLabel("base-rules", {label: "production", version: 1});
|
|
153
|
+
await store.create({
|
|
154
|
+
folder: "examples",
|
|
155
|
+
name: "triage-intake",
|
|
156
|
+
template: "{{> base-rules}}\nTriage the form.",
|
|
157
|
+
type: "text",
|
|
158
|
+
});
|
|
159
|
+
await store.createVersion("triage-intake", {
|
|
160
|
+
template: "No shared rules.",
|
|
161
|
+
type: "text",
|
|
162
|
+
});
|
|
163
|
+
await store.create({
|
|
164
|
+
folder: "examples",
|
|
165
|
+
name: "pinned-caller",
|
|
166
|
+
template: "{{> base-rules version=1}}",
|
|
167
|
+
type: "text",
|
|
168
|
+
});
|
|
169
|
+
await store.create({
|
|
170
|
+
folder: "examples",
|
|
171
|
+
name: "unrelated",
|
|
172
|
+
template: "base-rules is only mentioned in prose.",
|
|
173
|
+
type: "text",
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
const detail = await store.getDetail("base-rules");
|
|
177
|
+
expect(detail.usedBy).toEqual({
|
|
178
|
+
items: [
|
|
179
|
+
{folder: "examples", name: "pinned-caller", versions: [1]},
|
|
180
|
+
{folder: "examples", name: "triage-intake", versions: [1]},
|
|
181
|
+
],
|
|
182
|
+
limit: 20,
|
|
183
|
+
total: 2,
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
const caller = await store.getDetail("triage-intake");
|
|
187
|
+
expect(caller.usedBy).toEqual({items: [], limit: 20, total: 0});
|
|
188
|
+
});
|
|
189
|
+
|
|
126
190
|
it("filters hub traces by prompt version and keeps unfiltered detail", async () => {
|
|
127
191
|
await store.create({
|
|
128
192
|
folder: "examples",
|
|
@@ -349,4 +413,231 @@ describe("LocalPromptStore", () => {
|
|
|
349
413
|
assert.isUndefined(await store.get({name: "missing"}));
|
|
350
414
|
assert.isUndefined(await store.getVersionByNumber("missing", 1));
|
|
351
415
|
});
|
|
416
|
+
|
|
417
|
+
describe("shared prompt references", () => {
|
|
418
|
+
const baseTag = "{{> base-rules}}";
|
|
419
|
+
|
|
420
|
+
beforeEach(async () => {
|
|
421
|
+
await store.create({
|
|
422
|
+
folder: "shared",
|
|
423
|
+
name: "base-rules",
|
|
424
|
+
template: "Answer in plain language for {{audience}}.",
|
|
425
|
+
type: "text",
|
|
426
|
+
});
|
|
427
|
+
await store.moveLabel("base-rules", {label: "production", version: 1});
|
|
428
|
+
});
|
|
429
|
+
|
|
430
|
+
it("resolves a shared prompt into every prompt that references it and lists it as an include", async () => {
|
|
431
|
+
await store.create({
|
|
432
|
+
folder: "triage",
|
|
433
|
+
name: "triage-intake",
|
|
434
|
+
system: `${baseTag}\nTriage the intake form.`,
|
|
435
|
+
type: "text",
|
|
436
|
+
});
|
|
437
|
+
await store.moveLabel("triage-intake", {label: "production", version: 1});
|
|
438
|
+
|
|
439
|
+
const resolved = await store.get({name: "triage-intake"});
|
|
440
|
+
expect(resolved?.body).toBe(
|
|
441
|
+
"Answer in plain language for {{audience}}.\nTriage the intake form."
|
|
442
|
+
);
|
|
443
|
+
expect(resolved?.includes).toEqual([
|
|
444
|
+
{includedBy: "triage-intake", label: "production", name: "base-rules", version: 1},
|
|
445
|
+
]);
|
|
446
|
+
|
|
447
|
+
const byNumber = await store.getVersionByNumber("triage-intake", 1);
|
|
448
|
+
expect(byNumber?.system).toBe(resolved?.body);
|
|
449
|
+
expect(byNumber?.includes).toEqual(resolved?.includes ?? []);
|
|
450
|
+
|
|
451
|
+
// The editor keeps the reference so the shared text stays in one place.
|
|
452
|
+
const detail = await store.getDetail("triage-intake");
|
|
453
|
+
expect(detail.versions[0].system).toBe(`${baseTag}\nTriage the intake form.`);
|
|
454
|
+
});
|
|
455
|
+
|
|
456
|
+
it("picks up a promoted shared version without a new version of the including prompt", async () => {
|
|
457
|
+
await store.create({folder: "triage", name: "triage-intake", system: baseTag, type: "text"});
|
|
458
|
+
await store.moveLabel("triage-intake", {label: "production", version: 1});
|
|
459
|
+
await store.createVersion("base-rules", {template: "Be brief.", type: "text"});
|
|
460
|
+
await store.moveLabel("base-rules", {label: "production", version: 2});
|
|
461
|
+
|
|
462
|
+
const resolved = await store.get({name: "triage-intake"});
|
|
463
|
+
expect(resolved?.body).toBe("Be brief.");
|
|
464
|
+
expect(resolved?.version).toBe(1);
|
|
465
|
+
expect(resolved?.includes?.[0]?.version).toBe(2);
|
|
466
|
+
});
|
|
467
|
+
|
|
468
|
+
it("pins a shared version with version=", async () => {
|
|
469
|
+
await store.createVersion("base-rules", {template: "Be brief.", type: "text"});
|
|
470
|
+
await store.moveLabel("base-rules", {label: "production", version: 2});
|
|
471
|
+
await store.create({
|
|
472
|
+
folder: "triage",
|
|
473
|
+
name: "pinned",
|
|
474
|
+
system: "{{> base-rules version=1}}",
|
|
475
|
+
type: "text",
|
|
476
|
+
});
|
|
477
|
+
await store.moveLabel("pinned", {label: "production", version: 1});
|
|
478
|
+
|
|
479
|
+
const resolved = await store.get({name: "pinned"});
|
|
480
|
+
expect(resolved?.body).toBe("Answer in plain language for {{audience}}.");
|
|
481
|
+
expect(resolved?.includes).toEqual([{includedBy: "pinned", name: "base-rules", version: 1}]);
|
|
482
|
+
});
|
|
483
|
+
|
|
484
|
+
it("rejects unknown and cyclic references when a version is saved", async () => {
|
|
485
|
+
await expect(
|
|
486
|
+
store.create({
|
|
487
|
+
folder: "triage",
|
|
488
|
+
name: "broken",
|
|
489
|
+
system: "{{> missing}}",
|
|
490
|
+
type: "text",
|
|
491
|
+
})
|
|
492
|
+
).rejects.toThrow("Invalid prompt reference");
|
|
493
|
+
assert.isNull(await registerObsPrompt().findOneOrNone({name: "broken"}));
|
|
494
|
+
|
|
495
|
+
await store.create({folder: "triage", name: "triage-intake", system: baseTag, type: "text"});
|
|
496
|
+
await expect(
|
|
497
|
+
store.createVersion("base-rules", {
|
|
498
|
+
template: "{{> triage-intake label=latest}}",
|
|
499
|
+
type: "text",
|
|
500
|
+
})
|
|
501
|
+
).rejects.toThrow("Prompt reference cycle");
|
|
502
|
+
});
|
|
503
|
+
|
|
504
|
+
it("rejects a save that references a label the shared prompt does not have", async () => {
|
|
505
|
+
await store.create({folder: "shared", name: "draft-rules", template: "Draft", type: "text"});
|
|
506
|
+
await expect(
|
|
507
|
+
store.create({
|
|
508
|
+
folder: "triage",
|
|
509
|
+
name: "uses-draft",
|
|
510
|
+
system: "{{> draft-rules}}",
|
|
511
|
+
type: "text",
|
|
512
|
+
})
|
|
513
|
+
).rejects.toThrow("Invalid prompt reference");
|
|
514
|
+
await expect(
|
|
515
|
+
store.createVersion("base-rules", {
|
|
516
|
+
template: "{{> draft-rules label=staging}}",
|
|
517
|
+
type: "text",
|
|
518
|
+
})
|
|
519
|
+
).rejects.toThrow("Invalid prompt reference");
|
|
520
|
+
await expect(
|
|
521
|
+
store.createVersion("base-rules", {template: "{{> draft-rules version=9}}", type: "text"})
|
|
522
|
+
).rejects.toThrow("Invalid prompt reference");
|
|
523
|
+
|
|
524
|
+
await store.moveLabel("draft-rules", {label: "production", version: 1});
|
|
525
|
+
await store.create({
|
|
526
|
+
folder: "triage",
|
|
527
|
+
name: "uses-draft",
|
|
528
|
+
system: "{{> draft-rules}}",
|
|
529
|
+
type: "text",
|
|
530
|
+
});
|
|
531
|
+
await store.moveLabel("uses-draft", {label: "production", version: 1});
|
|
532
|
+
expect((await store.get({name: "uses-draft"}))?.body).toBe("Draft");
|
|
533
|
+
});
|
|
534
|
+
|
|
535
|
+
it("rejects a label move that would close a reference cycle", async () => {
|
|
536
|
+
await store.create({folder: "triage", name: "triage-intake", system: "Plain", type: "text"});
|
|
537
|
+
await store.moveLabel("triage-intake", {label: "staging", version: 1});
|
|
538
|
+
await store.createVersion("base-rules", {
|
|
539
|
+
template: "{{> triage-intake label=staging}}",
|
|
540
|
+
type: "text",
|
|
541
|
+
});
|
|
542
|
+
// Valid on its own: base-rules production is still v1, which references nothing.
|
|
543
|
+
await store.createVersion("triage-intake", {system: baseTag, type: "text"});
|
|
544
|
+
|
|
545
|
+
// base-rules latest (v2) includes triage-intake staging, which would now include base-rules.
|
|
546
|
+
await expect(
|
|
547
|
+
store.moveLabel("triage-intake", {label: "staging", version: 2})
|
|
548
|
+
).rejects.toThrow("Prompt reference cycle");
|
|
549
|
+
expect((await store.get({label: "staging", name: "triage-intake"}))?.version).toBe(1);
|
|
550
|
+
expect((await store.get({label: "latest", name: "base-rules"}))?.body).toBe("Plain");
|
|
551
|
+
});
|
|
552
|
+
|
|
553
|
+
it("rejects a new version whose latest label would close a reference cycle", async () => {
|
|
554
|
+
await store.create({
|
|
555
|
+
folder: "triage",
|
|
556
|
+
name: "triage-intake",
|
|
557
|
+
system: "{{> base-rules label=latest}}",
|
|
558
|
+
type: "text",
|
|
559
|
+
});
|
|
560
|
+
await expect(
|
|
561
|
+
store.createVersion("base-rules", {
|
|
562
|
+
template: "{{> triage-intake label=latest}}",
|
|
563
|
+
type: "text",
|
|
564
|
+
})
|
|
565
|
+
).rejects.toThrow("Prompt reference cycle");
|
|
566
|
+
});
|
|
567
|
+
|
|
568
|
+
describe("nesting below a prompt that includes the saved one", () => {
|
|
569
|
+
// top → p1 → … → p5 is exactly MAX_PROMPT_REFERENCE_DEPTH levels deep.
|
|
570
|
+
const chain = ["p1", "p2", "p3", "p4", "p5"];
|
|
571
|
+
|
|
572
|
+
beforeEach(async () => {
|
|
573
|
+
await store.create({folder: "shared", name: "p5", template: "Leaf", type: "text"});
|
|
574
|
+
await store.moveLabel("p5", {label: "production", version: 1});
|
|
575
|
+
for (let index = chain.length - 2; index >= 0; index--) {
|
|
576
|
+
const name = chain[index];
|
|
577
|
+
const next = chain[index + 1];
|
|
578
|
+
const tag = next === "p5" ? "{{> p5 label=latest}}" : `{{> ${next}}}`;
|
|
579
|
+
await store.create({folder: "shared", name, template: tag, type: "text"});
|
|
580
|
+
await store.moveLabel(name, {label: "production", version: 1});
|
|
581
|
+
}
|
|
582
|
+
await store.create({folder: "triage", name: "top", system: "{{> p1}}", type: "text"});
|
|
583
|
+
await store.moveLabel("top", {label: "production", version: 1});
|
|
584
|
+
});
|
|
585
|
+
|
|
586
|
+
it("rejects a new version that would push an including prompt past the limit", async () => {
|
|
587
|
+
await expect(store.createVersion("p5", {template: baseTag, type: "text"})).rejects.toThrow(
|
|
588
|
+
"Prompt references nested too deeply"
|
|
589
|
+
);
|
|
590
|
+
expect((await store.get({label: "latest", name: "p5"}))?.version).toBe(1);
|
|
591
|
+
expect((await store.get({name: "top"}))?.body).toBe("Leaf");
|
|
592
|
+
});
|
|
593
|
+
|
|
594
|
+
it("rejects a new version that would push an unlabelled including version past the limit", async () => {
|
|
595
|
+
// top v1 keeps the five-level chain but no longer holds a label.
|
|
596
|
+
await store.createVersion("top", {system: "Plain", type: "text"});
|
|
597
|
+
await store.moveLabel("top", {label: "production", version: 2});
|
|
598
|
+
|
|
599
|
+
await expect(store.createVersion("p5", {template: baseTag, type: "text"})).rejects.toThrow(
|
|
600
|
+
"Prompt references nested too deeply"
|
|
601
|
+
);
|
|
602
|
+
expect((await store.getVersionByNumber("top", 1))?.system).toBe("Leaf");
|
|
603
|
+
});
|
|
604
|
+
|
|
605
|
+
it("rejects a label move that would push an including prompt past the limit", async () => {
|
|
606
|
+
// p4 now includes p5 production, so a referencing p5 version can land as `latest`.
|
|
607
|
+
await store.createVersion("p4", {template: "{{> p5}}", type: "text"});
|
|
608
|
+
await store.moveLabel("p4", {label: "production", version: 2});
|
|
609
|
+
await store.createVersion("p5", {template: baseTag, type: "text"});
|
|
610
|
+
|
|
611
|
+
await expect(store.moveLabel("p5", {label: "production", version: 2})).rejects.toThrow(
|
|
612
|
+
"Prompt references nested too deeply"
|
|
613
|
+
);
|
|
614
|
+
expect((await store.get({name: "top"}))?.body).toBe("Leaf");
|
|
615
|
+
});
|
|
616
|
+
});
|
|
617
|
+
|
|
618
|
+
it("compiles shared text and variables in the playground", async () => {
|
|
619
|
+
await store.create({
|
|
620
|
+
folder: "triage",
|
|
621
|
+
name: "triage-intake",
|
|
622
|
+
system: baseTag,
|
|
623
|
+
template: "Intake: {{form}}",
|
|
624
|
+
type: "text",
|
|
625
|
+
});
|
|
626
|
+
|
|
627
|
+
const result = await store.runPlayground({
|
|
628
|
+
generator: {
|
|
629
|
+
generate: async () => {
|
|
630
|
+
return {latencyMs: 1, output: "ok"};
|
|
631
|
+
},
|
|
632
|
+
},
|
|
633
|
+
modelId: "mock-model",
|
|
634
|
+
name: "triage-intake",
|
|
635
|
+
variables: {audience: "teens", form: "sleep trouble"},
|
|
636
|
+
});
|
|
637
|
+
expect(result.compiledMessages).toEqual([
|
|
638
|
+
{content: "Answer in plain language for teens.", role: "system"},
|
|
639
|
+
{content: "Intake: sleep trouble", role: "user"},
|
|
640
|
+
]);
|
|
641
|
+
});
|
|
642
|
+
});
|
|
352
643
|
});
|