@vertesia/studio-utils 1.6.0-dev.20260923.085325Z → 1.6.0-dev.20261004.085740Z
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/index.d.ts +2 -1
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +4 -1
- package/lib/index.js.map +1 -1
- package/lib/prompts/extract-vars.d.ts +44 -12
- package/lib/prompts/extract-vars.d.ts.map +1 -1
- package/lib/prompts/extract-vars.js +165 -78
- package/lib/prompts/extract-vars.js.map +1 -1
- package/lib/prompts/render.d.ts +7 -9
- package/lib/prompts/render.d.ts.map +1 -1
- package/lib/prompts/render.js +12 -15
- package/lib/prompts/render.js.map +1 -1
- package/lib/prompts/validate.d.ts +8 -2
- package/lib/prompts/validate.d.ts.map +1 -1
- package/lib/prompts/validate.js +87 -51
- package/lib/prompts/validate.js.map +1 -1
- package/lib/roles/agent-runs.d.ts +8 -0
- package/lib/roles/agent-runs.d.ts.map +1 -0
- package/lib/roles/agent-runs.js +36 -0
- package/lib/roles/agent-runs.js.map +1 -0
- package/lib/roles/classes.d.ts +15 -1
- package/lib/roles/classes.d.ts.map +1 -1
- package/lib/roles/classes.js +10 -1
- package/lib/roles/classes.js.map +1 -1
- package/lib/roles/content.d.ts.map +1 -1
- package/lib/roles/content.js +14 -4
- package/lib/roles/content.js.map +1 -1
- package/lib/roles/index.d.ts +4 -3
- package/lib/roles/index.d.ts.map +1 -1
- package/lib/roles/index.js +7 -5
- package/lib/roles/index.js.map +1 -1
- package/lib/vertesia-studio-utils.js +2 -1
- package/lib/vertesia-studio-utils.js.map +1 -1
- package/package.json +11 -10
- package/src/index.ts +20 -1
- package/src/prompts/extract-vars.ts +212 -71
- package/src/prompts/render.ts +24 -15
- package/src/prompts/validate.ts +113 -57
- package/src/roles/agent-runs.ts +41 -0
- package/src/roles/classes.ts +7 -0
- package/src/roles/content.ts +21 -5
- package/src/roles/index.ts +7 -5
- package/src/apps/index.test.ts +0 -166
- package/src/conditions/match.test.ts +0 -134
- package/src/prompts/render.test.ts +0 -110
- package/src/prompts/validate.test.ts +0 -309
- package/src/roles/index.test.ts +0 -221
|
@@ -1,113 +1,254 @@
|
|
|
1
|
+
import { HANDLEBARS_CONTEXT_HELPERS, isHandlebarsHelper, isTemplateSystemVariable } from '@vertesia/jst';
|
|
1
2
|
import Handlebars from 'handlebars';
|
|
2
3
|
|
|
4
|
+
/** A read of a root-level template variable. */
|
|
5
|
+
export interface HandlebarsVariableReference {
|
|
6
|
+
/** Root identifier: `customer` for `{{customer.name}}`, `{{../customer}}` or `{{@root.customer}}`. */
|
|
7
|
+
name: string;
|
|
8
|
+
/** The path as written, e.g. `customer.name`. */
|
|
9
|
+
expression: string;
|
|
10
|
+
/** True when the path reads a property of the variable (`customer.name`). */
|
|
11
|
+
hasPath: boolean;
|
|
12
|
+
/** Source of the tag that contains the read, e.g. `{{#if customer}}`. */
|
|
13
|
+
tag: string;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** A helper name used where Handlebars will not call it. */
|
|
17
|
+
export interface HandlebarsHelperMisuse {
|
|
18
|
+
helper: string;
|
|
19
|
+
/**
|
|
20
|
+
* - `used_as_value`: passed as an argument (`{{#if stringify}}`), where Handlebars reads the data.
|
|
21
|
+
* - `missing_arguments`: called bare (`{{stringify}}`) with none of the arguments it needs.
|
|
22
|
+
*/
|
|
23
|
+
problem: 'used_as_value' | 'missing_arguments';
|
|
24
|
+
tag: string;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export interface HandlebarsTemplateAnalysis {
|
|
28
|
+
references: HandlebarsVariableReference[];
|
|
29
|
+
helperMisuses: HandlebarsHelperMisuse[];
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
interface PathNode {
|
|
33
|
+
type: 'PathExpression';
|
|
34
|
+
original: string;
|
|
35
|
+
parts: string[];
|
|
36
|
+
depth: number;
|
|
37
|
+
data: boolean;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
interface SourceLocation {
|
|
41
|
+
start: { line: number; column: number };
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
type Node = { type: string; loc?: SourceLocation } & Record<string, unknown>;
|
|
45
|
+
|
|
46
|
+
/** Context a template section renders with: the input root, or an item whose shape is unknown. */
|
|
47
|
+
type ContextKind = 'root' | 'item';
|
|
48
|
+
|
|
49
|
+
function isPath(node: unknown): node is PathNode {
|
|
50
|
+
return !!node && typeof node === 'object' && (node as Node).type === 'PathExpression';
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** `this.x`, `./x` and `this` read the context explicitly — Handlebars never treats them as helpers. */
|
|
54
|
+
function isScoped(path: PathNode): boolean {
|
|
55
|
+
return /^\.|this\b/.test(path.original);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** A single bare identifier — the only form Handlebars may resolve to a helper. */
|
|
59
|
+
function isSimpleId(path: PathNode): boolean {
|
|
60
|
+
return path.parts.length === 1 && !isScoped(path) && !path.data && path.depth === 0;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function lineOffsets(template: string): number[] {
|
|
64
|
+
const offsets = [0];
|
|
65
|
+
for (let i = 0; i < template.length; i++) {
|
|
66
|
+
if (template[i] === '\n') offsets.push(i + 1);
|
|
67
|
+
}
|
|
68
|
+
return offsets;
|
|
69
|
+
}
|
|
70
|
+
|
|
3
71
|
/**
|
|
4
|
-
*
|
|
72
|
+
* Analyze the variables and helpers a Handlebars template uses, resolving names the way the
|
|
73
|
+
* prompt renderer does (helpers and system variables come from `@vertesia/jst`).
|
|
5
74
|
*
|
|
6
|
-
*
|
|
7
|
-
* - `{{
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* - `
|
|
11
|
-
* - `{{@index}}
|
|
12
|
-
*
|
|
13
|
-
* Helper names in mustaches and block heads (`{{customHelper x}}`, `{{#if x}}`) are NOT added to
|
|
14
|
-
* the set — only the arguments and inner template references are. This matches the typical use
|
|
15
|
-
* case for prompt validation: detect which schema properties the template actually reads.
|
|
75
|
+
* Scoping rules:
|
|
76
|
+
* - Inside `{{#each}}` / `{{#with}}` (and `{{#section}}` over data), bare names read the current item,
|
|
77
|
+
* whose shape the input schema does not describe — they are not reported. `../x` and `@root.x`
|
|
78
|
+
* read the enclosing or root context and are reported.
|
|
79
|
+
* - `as |x|` block params are local bindings, not variables.
|
|
80
|
+
* - `{{@index}}` and other `@`-data (except `@root.x`) are runtime values, not variables.
|
|
16
81
|
*
|
|
17
|
-
* Returns
|
|
18
|
-
* (e.g. `executeHandlebars`) to surface parse errors separately.
|
|
82
|
+
* Returns null when the template does not parse; the render check reports the syntax error.
|
|
19
83
|
*/
|
|
20
|
-
export function
|
|
84
|
+
export function analyzeHandlebarsTemplate(template: string): HandlebarsTemplateAnalysis | null {
|
|
21
85
|
let ast: hbs.AST.Program;
|
|
22
86
|
try {
|
|
23
87
|
ast = Handlebars.parse(template);
|
|
24
88
|
} catch {
|
|
25
|
-
return
|
|
89
|
+
return null;
|
|
26
90
|
}
|
|
27
91
|
|
|
28
|
-
const
|
|
29
|
-
const
|
|
92
|
+
const offsets = lineOffsets(template);
|
|
93
|
+
const tagAt = (loc: SourceLocation | undefined): string => {
|
|
94
|
+
if (!loc) return '';
|
|
95
|
+
const start = (offsets[loc.start.line - 1] ?? 0) + loc.start.column;
|
|
96
|
+
const end = template.indexOf('}}', start);
|
|
97
|
+
if (end < 0) return template.slice(start);
|
|
98
|
+
return template.slice(start, template[end + 2] === '}' ? end + 3 : end + 2);
|
|
99
|
+
};
|
|
30
100
|
|
|
31
|
-
const
|
|
101
|
+
const references: HandlebarsVariableReference[] = [];
|
|
102
|
+
const helperMisuses: HandlebarsHelperMisuse[] = [];
|
|
103
|
+
const contexts: ContextKind[] = ['root'];
|
|
104
|
+
const blockParams: Array<Set<string>> = [];
|
|
105
|
+
let tag = '';
|
|
32
106
|
|
|
33
|
-
const
|
|
34
|
-
if (!node || typeof node !== 'object') return;
|
|
35
|
-
const n = node as { type: string } & Record<string, unknown>;
|
|
107
|
+
const isBlockParam = (name: string): boolean => blockParams.some((scope) => scope.has(name));
|
|
36
108
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
109
|
+
/** Record a data read of `path`, if it reads the root input. */
|
|
110
|
+
const readData = (path: PathNode): void => {
|
|
111
|
+
if (path.data) {
|
|
112
|
+
// `@root.x` reads the input root; `@index`, `@key`, ... are runtime values.
|
|
113
|
+
if (path.parts[0] === 'root' && path.parts.length > 1) {
|
|
114
|
+
references.push({
|
|
115
|
+
name: path.parts[1],
|
|
116
|
+
expression: path.original,
|
|
117
|
+
hasPath: path.parts.length > 2,
|
|
118
|
+
tag,
|
|
119
|
+
});
|
|
42
120
|
}
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
if (path.parts.length === 0) return; // `this` / `.`
|
|
124
|
+
const root = path.parts[0];
|
|
125
|
+
if (path.depth === 0 && !isScoped(path) && isBlockParam(root)) return;
|
|
126
|
+
// Handlebars stops at the root context when `../` goes past it.
|
|
127
|
+
const context = contexts[Math.max(0, contexts.length - 1 - path.depth)];
|
|
128
|
+
if (context !== 'root') return;
|
|
129
|
+
references.push({ name: root, expression: path.original, hasPath: path.parts.length > 1, tag });
|
|
130
|
+
};
|
|
43
131
|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
132
|
+
/** A path in argument position (params, hash values): Handlebars reads data, never calls a helper. */
|
|
133
|
+
const visitArgument = (node: unknown): void => {
|
|
134
|
+
if (isPath(node)) {
|
|
135
|
+
const name = node.parts[0];
|
|
136
|
+
// System variables are injected as data too, so they resolve as arguments.
|
|
137
|
+
if (
|
|
138
|
+
isSimpleId(node) &&
|
|
139
|
+
!isBlockParam(name) &&
|
|
140
|
+
isHandlebarsHelper(name) &&
|
|
141
|
+
!isTemplateSystemVariable(name)
|
|
142
|
+
) {
|
|
143
|
+
helperMisuses.push({ helper: name, problem: 'used_as_value', tag });
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
readData(node);
|
|
147
|
+
return;
|
|
148
|
+
}
|
|
149
|
+
visit(node);
|
|
150
|
+
};
|
|
50
151
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
152
|
+
const visitCall = (n: Node): void => {
|
|
153
|
+
const params = (n.params as unknown[] | undefined) ?? [];
|
|
154
|
+
for (const p of params) visitArgument(p);
|
|
155
|
+
const pairs = (n.hash as { pairs?: Array<{ value: unknown }> } | undefined)?.pairs ?? [];
|
|
156
|
+
for (const pair of pairs) visitArgument(pair.value);
|
|
157
|
+
};
|
|
56
158
|
|
|
57
|
-
|
|
58
|
-
|
|
159
|
+
const hasArguments = (n: Node): boolean =>
|
|
160
|
+
((n.params as unknown[] | undefined)?.length ?? 0) > 0 ||
|
|
161
|
+
((n.hash as { pairs?: unknown[] } | undefined)?.pairs?.length ?? 0) > 0;
|
|
59
162
|
|
|
60
|
-
|
|
163
|
+
const visit = (node: unknown): void => {
|
|
164
|
+
if (!node || typeof node !== 'object') return;
|
|
165
|
+
const n = node as Node;
|
|
166
|
+
|
|
167
|
+
switch (n.type) {
|
|
168
|
+
case 'Program': {
|
|
169
|
+
for (const stmt of (n.body as unknown[] | undefined) ?? []) visit(stmt);
|
|
61
170
|
break;
|
|
62
171
|
}
|
|
63
172
|
|
|
64
173
|
case 'MustacheStatement': {
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
174
|
+
tag = tagAt(n.loc);
|
|
175
|
+
const path = n.path;
|
|
176
|
+
if (hasArguments(n)) {
|
|
177
|
+
// `{{helper arg}}`: the head is a helper name (a missing one fails the render check).
|
|
178
|
+
visitCall(n);
|
|
179
|
+
} else if (isPath(path)) {
|
|
180
|
+
// A bare `{{name}}` calls a helper of that name when there is one, else reads data.
|
|
181
|
+
// System variables are helpers returning their injected value, meant to be used bare.
|
|
182
|
+
const name = path.parts[0];
|
|
183
|
+
if (isSimpleId(path) && !isBlockParam(name) && isHandlebarsHelper(name)) {
|
|
184
|
+
if (!isTemplateSystemVariable(name)) {
|
|
185
|
+
helperMisuses.push({ helper: name, problem: 'missing_arguments', tag });
|
|
186
|
+
}
|
|
187
|
+
} else {
|
|
188
|
+
readData(path);
|
|
189
|
+
}
|
|
190
|
+
} else {
|
|
191
|
+
visit(path);
|
|
75
192
|
}
|
|
76
|
-
if (params) for (const p of params) visit(p);
|
|
77
|
-
visit(hash);
|
|
78
193
|
break;
|
|
79
194
|
}
|
|
80
195
|
|
|
81
|
-
case '
|
|
82
|
-
|
|
83
|
-
const
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
196
|
+
case 'BlockStatement': {
|
|
197
|
+
tag = tagAt(n.loc);
|
|
198
|
+
const path = n.path;
|
|
199
|
+
const helper = isPath(path) && isSimpleId(path) ? path.parts[0] : undefined;
|
|
200
|
+
let itemContext: boolean;
|
|
201
|
+
if (hasArguments(n) || (helper && isHandlebarsHelper(helper))) {
|
|
202
|
+
visitCall(n);
|
|
203
|
+
itemContext = !!helper && HANDLEBARS_CONTEXT_HELPERS.includes(helper);
|
|
204
|
+
} else {
|
|
205
|
+
// `{{#section}}…{{/section}}` over data: iterates or enters `section`.
|
|
206
|
+
if (isPath(path)) readData(path);
|
|
207
|
+
itemContext = true;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
const program = n.program as { blockParams?: string[] } | undefined;
|
|
211
|
+
const inverse = n.inverse as { blockParams?: string[] } | undefined;
|
|
212
|
+
blockParams.push(new Set(program?.blockParams ?? []));
|
|
213
|
+
contexts.push(itemContext ? 'item' : contexts[contexts.length - 1]);
|
|
214
|
+
visit(n.program);
|
|
215
|
+
contexts.pop();
|
|
216
|
+
blockParams.pop();
|
|
88
217
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
const root = parts[0];
|
|
94
|
-
if (isLocal(root)) break;
|
|
95
|
-
variables.add(root);
|
|
218
|
+
// `{{else}}` renders with the enclosing context.
|
|
219
|
+
blockParams.push(new Set(inverse?.blockParams ?? []));
|
|
220
|
+
visit(n.inverse);
|
|
221
|
+
blockParams.pop();
|
|
96
222
|
break;
|
|
97
223
|
}
|
|
98
224
|
|
|
99
|
-
case '
|
|
100
|
-
|
|
101
|
-
|
|
225
|
+
case 'SubExpression': {
|
|
226
|
+
// `(helper arg)`: the head is a call, like a bare `{{helper}}`.
|
|
227
|
+
visitCall(n);
|
|
102
228
|
break;
|
|
103
229
|
}
|
|
104
230
|
|
|
105
|
-
// ContentStatement, CommentStatement,
|
|
231
|
+
// ContentStatement, CommentStatement, literals, partials — no variables.
|
|
106
232
|
default:
|
|
107
233
|
break;
|
|
108
234
|
}
|
|
109
235
|
};
|
|
110
236
|
|
|
111
237
|
visit(ast);
|
|
112
|
-
return
|
|
238
|
+
return { references, helperMisuses };
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* Extract the set of root-level input variable names a Handlebars template reads.
|
|
243
|
+
*
|
|
244
|
+
* - `{{foo}}` → `foo`; `{{obj.bar.baz}}` → `obj`
|
|
245
|
+
* - `{{#each items as |item|}}{{item.name}}{{/each}}` → `items` (`item` is a block param)
|
|
246
|
+
* - `{{lookup obj key}}` → `obj`, `key` (helper name skipped)
|
|
247
|
+
* - `{{@index}}`, `{{this}}`, and bare names inside `#each`/`#with` → ignored
|
|
248
|
+
*
|
|
249
|
+
* Returns an empty set on parse failure.
|
|
250
|
+
*/
|
|
251
|
+
export function extractHandlebarsVariables(template: string): Set<string> {
|
|
252
|
+
const analysis = analyzeHandlebarsTemplate(template);
|
|
253
|
+
return new Set(analysis?.references.map((r) => r.name) ?? []);
|
|
113
254
|
}
|
package/src/prompts/render.ts
CHANGED
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
import type { JSONObject, JSONSchema, PromptSegment } from '@llumiverse/common';
|
|
2
2
|
import { type PromptSegmentDef, PromptSegmentDefType, type PromptTemplate, TemplateType } from '@vertesia/common';
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
CompositeError,
|
|
5
|
+
renderHandlebarsTemplate,
|
|
6
|
+
renderJsTemplate,
|
|
7
|
+
type TemplateSystemContext,
|
|
8
|
+
withTemplateSystemVariables,
|
|
9
|
+
} from '@vertesia/jst';
|
|
4
10
|
|
|
5
11
|
export interface SegmentPreview {
|
|
6
12
|
error?: Error;
|
|
@@ -12,29 +18,32 @@ export interface SegmentPreview {
|
|
|
12
18
|
/**
|
|
13
19
|
* Render a prompt template with the given input data.
|
|
14
20
|
*
|
|
15
|
-
* Handlebars templates use {{variable}} interpolation against `data`.
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
* studio-server executor
|
|
19
|
-
*
|
|
20
|
-
* and `apps/studio-server/src/executor/rendering/template.ts`). Listing it here keeps
|
|
21
|
-
* the Playground preview and `validatePrompt` in sync with runtime resolution — a JST
|
|
22
|
-
* template referencing `_model` validates fine here and renders fine in production.
|
|
21
|
+
* Handlebars templates use {{variable}} interpolation against `data`. JST (JavaScript template)
|
|
22
|
+
* bodies evaluate against `data` with the schema's top-level property names exposed as globals.
|
|
23
|
+
* Both see the system variables (`TEMPLATE_SYSTEM_VARIABLES` in `@vertesia/jst`), added here the
|
|
24
|
+
* same way the studio-server executor adds them, so a preview renders what an execution would.
|
|
25
|
+
* `_model` is only set when the caller passes `system.model` or the data already carries it.
|
|
23
26
|
*
|
|
24
27
|
* For `TemplateType.text`, the content is returned verbatim — it is static text, not a
|
|
25
28
|
* template — matching the studio-server executor (see `apps/studio-server/src/executor/
|
|
26
29
|
* rendering/template.ts`). Routing it through the JST evaluator would compile the prose as
|
|
27
30
|
* JavaScript and throw on any plain sentence (e.g. "You are a helpful assistant.").
|
|
28
31
|
*/
|
|
29
|
-
export function renderTemplate(
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
32
|
+
export function renderTemplate(
|
|
33
|
+
code: string,
|
|
34
|
+
contentType: TemplateType,
|
|
35
|
+
schema: JSONSchema,
|
|
36
|
+
data: JSONObject,
|
|
37
|
+
system?: TemplateSystemContext,
|
|
38
|
+
): string {
|
|
33
39
|
if (contentType === TemplateType.text) {
|
|
34
40
|
return code;
|
|
35
41
|
}
|
|
36
|
-
const
|
|
37
|
-
|
|
42
|
+
const input = withTemplateSystemVariables(data, system);
|
|
43
|
+
if (contentType === TemplateType.handlebars) {
|
|
44
|
+
return renderHandlebarsTemplate(code, input);
|
|
45
|
+
}
|
|
46
|
+
return renderJsTemplate(code, schema.properties ? Object.keys(schema.properties) : [], input);
|
|
38
47
|
}
|
|
39
48
|
|
|
40
49
|
/**
|
package/src/prompts/validate.ts
CHANGED
|
@@ -1,13 +1,25 @@
|
|
|
1
1
|
import type { JSONObject } from '@llumiverse/common';
|
|
2
2
|
import { type JSONSchema, TemplateType } from '@vertesia/common';
|
|
3
|
-
import {
|
|
4
|
-
|
|
3
|
+
import {
|
|
4
|
+
describeTemplateSystemVariables,
|
|
5
|
+
getFreeVariables,
|
|
6
|
+
isTemplateSystemVariable,
|
|
7
|
+
JST_TEMPLATE_GLOBALS,
|
|
8
|
+
renderJsTemplate,
|
|
9
|
+
TEMPLATE_SYSTEM_VARIABLES,
|
|
10
|
+
withTemplateSystemVariables,
|
|
11
|
+
} from '@vertesia/jst';
|
|
12
|
+
import { analyzeHandlebarsTemplate } from './extract-vars.js';
|
|
5
13
|
import { generateMockData } from './mock-data.js';
|
|
6
14
|
import { executeHandlebars } from './render.js';
|
|
7
15
|
|
|
8
16
|
export type PromptValidationIssueType =
|
|
9
17
|
| 'undeclared_template_variable'
|
|
10
18
|
| 'unused_schema_variable'
|
|
19
|
+
| 'reserved_variable_declared'
|
|
20
|
+
| 'system_variable_property_access'
|
|
21
|
+
| 'helper_used_as_value'
|
|
22
|
+
| 'helper_missing_arguments'
|
|
11
23
|
| 'handlebars_render_error'
|
|
12
24
|
| 'jst_unsafe_construct'
|
|
13
25
|
| 'jst_render_error';
|
|
@@ -43,16 +55,6 @@ export interface PromptValidationInput {
|
|
|
43
55
|
inputSchema?: JSONSchema;
|
|
44
56
|
}
|
|
45
57
|
|
|
46
|
-
// JST's renderJsTemplate auto-adds `_` (helpers object) and runtime injects `Set` and `Array`
|
|
47
|
-
// — treat them as globals so they don't appear as free vars in user templates.
|
|
48
|
-
// Globals always available to JST templates regardless of schema:
|
|
49
|
-
// - `_`, `Array`, `Set`: runtime-injected by `renderJsTemplate` (jst library)
|
|
50
|
-
// - `_model`: runtime-injected by the studio-server executor as `{ ..._model: run.modelId }`
|
|
51
|
-
// (see ExecutionRequest.ts:313 and executor/rendering/template.ts:13)
|
|
52
|
-
// Keeping these in sync with `renderTemplate` in ./render.ts so a JST template that runs in
|
|
53
|
-
// production also passes the validator.
|
|
54
|
-
const JST_AUTO_GLOBALS = ['_', 'Array', 'Set', '_model'];
|
|
55
|
-
|
|
56
58
|
function countSeverities(issues: PromptValidationIssue[]): { error_count: number; warning_count: number } {
|
|
57
59
|
let error_count = 0;
|
|
58
60
|
let warning_count = 0;
|
|
@@ -66,43 +68,110 @@ function countSeverities(issues: PromptValidationIssue[]): { error_count: number
|
|
|
66
68
|
return { error_count, warning_count };
|
|
67
69
|
}
|
|
68
70
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
const
|
|
72
|
-
const
|
|
71
|
+
/** Mock data for the render smoke test: schema-shaped input plus the runtime system values. */
|
|
72
|
+
function buildMockInput(inputSchema: JSONSchema): Record<string, unknown> {
|
|
73
|
+
const mockData = generateMockData(inputSchema);
|
|
74
|
+
const mockObject: JSONObject =
|
|
75
|
+
typeof mockData === 'object' && mockData !== null && !Array.isArray(mockData) ? (mockData as JSONObject) : {};
|
|
76
|
+
return withTemplateSystemVariables(mockObject, { model: 'validation-model' });
|
|
77
|
+
}
|
|
73
78
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
79
|
+
/**
|
|
80
|
+
* Issues shared by both template languages: reserved names declared in the schema, and
|
|
81
|
+
* unused schema properties. `usedVars` holds the input variables the template reads.
|
|
82
|
+
*/
|
|
83
|
+
function checkDeclarations(
|
|
84
|
+
declaredVars: Set<string>,
|
|
85
|
+
usedVars: Set<string>,
|
|
86
|
+
usageHint: (name: string) => string,
|
|
87
|
+
): PromptValidationIssue[] {
|
|
88
|
+
const issues: PromptValidationIssue[] = [];
|
|
89
|
+
for (const declared of declaredVars) {
|
|
90
|
+
if (isTemplateSystemVariable(declared)) {
|
|
77
91
|
issues.push({
|
|
78
|
-
type: '
|
|
79
|
-
severity: '
|
|
80
|
-
variable:
|
|
81
|
-
message: `
|
|
92
|
+
type: 'reserved_variable_declared',
|
|
93
|
+
severity: 'warning',
|
|
94
|
+
variable: declared,
|
|
95
|
+
message: `Schema declares '${declared}', a system variable the runtime supplies and overrides. Remove it from input_schema.`,
|
|
82
96
|
});
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
for (const declared of declaredVars) {
|
|
87
|
-
if (!usedVars.has(declared)) {
|
|
97
|
+
} else if (!usedVars.has(declared)) {
|
|
88
98
|
issues.push({
|
|
89
99
|
type: 'unused_schema_variable',
|
|
90
100
|
severity: 'warning',
|
|
91
101
|
variable: declared,
|
|
92
|
-
message: `Schema declares property '${declared}' but the template never references it. Remove it from input_schema or
|
|
102
|
+
message: `Schema declares property '${declared}' but the template never references it. Remove it from input_schema or ${usageHint(declared)}.`,
|
|
93
103
|
});
|
|
94
104
|
}
|
|
95
105
|
}
|
|
106
|
+
return issues;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function undeclaredVariableIssue(name: string, where: string): PromptValidationIssue {
|
|
110
|
+
return {
|
|
111
|
+
type: 'undeclared_template_variable',
|
|
112
|
+
severity: 'error',
|
|
113
|
+
variable: name,
|
|
114
|
+
message:
|
|
115
|
+
`Template reads variable '${name}' ${where} but it is not declared in input_schema.properties. ` +
|
|
116
|
+
`Add '${name}' to the schema with an appropriate type. ` +
|
|
117
|
+
`System variables need no declaration: ${describeTemplateSystemVariables()}.`,
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function validateHandlebarsPrompt(content: string, inputSchema?: JSONSchema): PromptValidationIssue[] {
|
|
122
|
+
const issues: PromptValidationIssue[] = [];
|
|
123
|
+
const analysis = analyzeHandlebarsTemplate(content);
|
|
124
|
+
const declaredVars = new Set<string>(inputSchema?.properties ? Object.keys(inputSchema.properties) : []);
|
|
125
|
+
const usedVars = new Set<string>();
|
|
126
|
+
const reported = new Set<string>();
|
|
127
|
+
|
|
128
|
+
for (const ref of analysis?.references ?? []) {
|
|
129
|
+
if (isTemplateSystemVariable(ref.name)) {
|
|
130
|
+
const variable = TEMPLATE_SYSTEM_VARIABLES.find((v) => v.name === ref.name);
|
|
131
|
+
if (ref.hasPath && variable?.type === 'string' && !reported.has(`path:${ref.expression}`)) {
|
|
132
|
+
reported.add(`path:${ref.expression}`);
|
|
133
|
+
issues.push({
|
|
134
|
+
type: 'system_variable_property_access',
|
|
135
|
+
severity: 'error',
|
|
136
|
+
variable: ref.name,
|
|
137
|
+
message: `'${ref.expression}' in ${ref.tag} reads a property of system variable '${ref.name}', which is a string (${variable.description}). Use '${ref.name}' directly.`,
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
usedVars.add(ref.name);
|
|
143
|
+
if (!declaredVars.has(ref.name) && !reported.has(ref.name)) {
|
|
144
|
+
reported.add(ref.name);
|
|
145
|
+
issues.push(undeclaredVariableIssue(ref.name, `in ${ref.tag}`));
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
for (const misuse of analysis?.helperMisuses ?? []) {
|
|
150
|
+
issues.push(
|
|
151
|
+
misuse.problem === 'used_as_value'
|
|
152
|
+
? {
|
|
153
|
+
type: 'helper_used_as_value',
|
|
154
|
+
severity: 'error',
|
|
155
|
+
variable: misuse.helper,
|
|
156
|
+
message: `'${misuse.helper}' in ${misuse.tag} is a helper, but as an argument Handlebars reads it as data, which is empty. Call it as a subexpression: (${misuse.helper} ...).`,
|
|
157
|
+
}
|
|
158
|
+
: {
|
|
159
|
+
type: 'helper_missing_arguments',
|
|
160
|
+
severity: 'error',
|
|
161
|
+
variable: misuse.helper,
|
|
162
|
+
message: `${misuse.tag} calls helper '${misuse.helper}' without arguments. Pass the value it operates on, e.g. {{${misuse.helper} value}}.`,
|
|
163
|
+
},
|
|
164
|
+
);
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
issues.push(...checkDeclarations(declaredVars, usedVars, (name) => `use it via {{${name}}}`));
|
|
96
168
|
|
|
97
169
|
// Render-time smoke test — always runs so syntax errors and failing helper calls are
|
|
98
170
|
// surfaced even when undeclared-variable errors are already in the list. Handlebars renders
|
|
99
171
|
// missing vars as empty strings (non-strict by default), so the render check does NOT echo
|
|
100
172
|
// the var errors — anything it reports is a distinct template problem worth showing.
|
|
101
173
|
const renderSchema = inputSchema ?? ({} as JSONSchema);
|
|
102
|
-
const
|
|
103
|
-
const mockObject: JSONObject =
|
|
104
|
-
typeof mockData === 'object' && mockData !== null && !Array.isArray(mockData) ? (mockData as JSONObject) : {};
|
|
105
|
-
const renderResult = executeHandlebars(content, renderSchema, mockObject);
|
|
174
|
+
const renderResult = executeHandlebars(content, renderSchema, buildMockInput(renderSchema) as JSONObject);
|
|
106
175
|
if (!renderResult.success) {
|
|
107
176
|
issues.push({
|
|
108
177
|
type: 'handlebars_render_error',
|
|
@@ -121,7 +190,7 @@ function validateJstPrompt(content: string, inputSchema?: JSONSchema): PromptVal
|
|
|
121
190
|
let referenced: Set<string>;
|
|
122
191
|
try {
|
|
123
192
|
const result = getFreeVariables(content, {
|
|
124
|
-
globals:
|
|
193
|
+
globals: [...JST_TEMPLATE_GLOBALS],
|
|
125
194
|
acorn: { allowReturnOutsideFunction: true, locations: true },
|
|
126
195
|
});
|
|
127
196
|
referenced = result.vars;
|
|
@@ -144,38 +213,19 @@ function validateJstPrompt(content: string, inputSchema?: JSONSchema): PromptVal
|
|
|
144
213
|
|
|
145
214
|
for (const used of referenced) {
|
|
146
215
|
if (!declaredVars.has(used)) {
|
|
147
|
-
issues.push(
|
|
148
|
-
type: 'undeclared_template_variable',
|
|
149
|
-
severity: 'error',
|
|
150
|
-
variable: used,
|
|
151
|
-
message: `Template references variable '${used}' but it is not declared in input_schema.properties. Add '${used}' to the schema with an appropriate type.`,
|
|
152
|
-
});
|
|
216
|
+
issues.push(undeclaredVariableIssue(used, 'in the template'));
|
|
153
217
|
}
|
|
154
218
|
}
|
|
155
219
|
|
|
156
|
-
|
|
157
|
-
if (!referenced.has(declared)) {
|
|
158
|
-
issues.push({
|
|
159
|
-
type: 'unused_schema_variable',
|
|
160
|
-
severity: 'warning',
|
|
161
|
-
variable: declared,
|
|
162
|
-
message: `Schema declares property '${declared}' but the template never references it. Remove it from input_schema or use it in the template.`,
|
|
163
|
-
});
|
|
164
|
-
}
|
|
165
|
-
}
|
|
220
|
+
issues.push(...checkDeclarations(declaredVars, referenced, () => 'use it in the template'));
|
|
166
221
|
|
|
167
222
|
// Render-time smoke test — only if there are no blocking errors so far, otherwise
|
|
168
223
|
// the failure mode would just echo what we already reported.
|
|
169
224
|
const blockingSoFar = issues.some((i) => i.severity === 'error');
|
|
170
225
|
if (!blockingSoFar) {
|
|
171
226
|
const renderSchema = inputSchema ?? ({} as JSONSchema);
|
|
172
|
-
const mockData = generateMockData(renderSchema);
|
|
173
|
-
const mockObject: JSONObject =
|
|
174
|
-
typeof mockData === 'object' && mockData !== null && !Array.isArray(mockData)
|
|
175
|
-
? (mockData as JSONObject)
|
|
176
|
-
: {};
|
|
177
227
|
try {
|
|
178
|
-
renderJsTemplate(content, [...declaredVars],
|
|
228
|
+
renderJsTemplate(content, [...declaredVars], buildMockInput(renderSchema));
|
|
179
229
|
} catch (renderError) {
|
|
180
230
|
issues.push({
|
|
181
231
|
type: 'jst_render_error',
|
|
@@ -193,9 +243,15 @@ function validateJstPrompt(content: string, inputSchema?: JSONSchema): PromptVal
|
|
|
193
243
|
*
|
|
194
244
|
* For `handlebars` and `jst` templates, the following checks are performed:
|
|
195
245
|
* 1. Every variable referenced in the template must be declared as a top-level property
|
|
196
|
-
* in `inputSchema.properties` (else → `undeclared_template_variable` error).
|
|
246
|
+
* in `inputSchema.properties` (else → `undeclared_template_variable` error). System variables
|
|
247
|
+
* (`TEMPLATE_SYSTEM_VARIABLES` in `@vertesia/jst`) are supplied at runtime and need no declaration;
|
|
248
|
+
* declaring one → `reserved_variable_declared` warning, reading a property of one →
|
|
249
|
+
* `system_variable_property_access` error.
|
|
197
250
|
* 2. Every property declared in `inputSchema.properties` should be referenced by the template
|
|
198
251
|
* (else → `unused_schema_variable` warning — non-blocking).
|
|
252
|
+
* For Handlebars only: a helper passed as an argument (`{{#if stringify}}`) →
|
|
253
|
+
* `helper_used_as_value` error; a helper called bare without the arguments it needs
|
|
254
|
+
* (`{{stringify}}`) → `helper_missing_arguments` error.
|
|
199
255
|
* 3. The template must render successfully against schema-derived mock data
|
|
200
256
|
* (else → `handlebars_render_error` / `jst_render_error` error).
|
|
201
257
|
* 4. For JST only: unsafe constructs (`with`, `for`, `while`, `import`, class, `this`,
|