@vertesia/build-tools 1.5.0-dev.20260722.120446Z → 1.5.0-dev.20260804.124748Z

package/src/core/transformers/skill.ts

@@ -6,7 +6,12 @@ import { existsSync } from 'node:fs';
 import path from 'node:path';
 import { z } from 'zod';
 import { parseFrontmatter } from '../parsers/frontmatter.js';
-import type { TransformerPreset } from '../types.js';
+import {
+    assertSkillMarkdown,
+    type PreprocessSkillMarkdownOptions,
+    preprocessSkillMarkdown,
+} from '../skill-markdown/preprocess.js';
+import type { TransformerPreset, TransformerRule, TransformResult } from '../types.js';
 import { discoverSkillAssets } from '../utils/asset-discovery.js';
 
 /**
@@ -78,6 +83,7 @@ const SkillFrontmatterSchema = z
         // Flat structure fields (legacy)
         keywords: z.array(z.string()).optional(),
         tools: z.array(z.string()).optional(),
+        supporting_tools: z.array(z.string()).optional(),
         data_patterns: z.array(z.string()).optional(),
         language: z.string().optional(),
         packages: z.array(z.string()).optional(),
@@ -234,9 +240,12 @@ function buildSkillDefinition(
         }
     }
 
-    // Tools unlocked by this skill (from frontmatter `tools:` key)
-    if (frontmatter.tools) {
-        skill.tools = frontmatter.tools;
+    // Tools unlocked by this skill. `supporting_tools` unlock identically — the split exists only
+    // so validation knows which grants the author owes the reader an explanation for. Merged here so
+    // no runtime path has to know about the distinction.
+    const unlocked = [...(frontmatter.tools ?? []), ...(frontmatter.supporting_tools ?? [])];
+    if (unlocked.length > 0) {
+        skill.tools = unlocked;
     }
 
     // Input schema from frontmatter
@@ -267,51 +276,106 @@ function buildSkillDefinition(
  * // Both are SkillDefinition objects
  * ```
  */
-export const skillTransformer: TransformerPreset = {
-    pattern: /(\.md\?skill$|\/SKILL\.md$)/,
-    schema: SkillDefinitionSchema,
-    transform: (content: string, filePath: string) => {
-        const { frontmatter, content: markdown } = parseFrontmatter(content);
-
-        // Validate frontmatter first to catch unknown properties
-        const frontmatterValidation = SkillFrontmatterSchema.safeParse(frontmatter);
-        if (!frontmatterValidation.success) {
-            const errors = frontmatterValidation.error.issues
-                .map((err) => {
-                    const pathStr = err.path.length > 0 ? err.path.join('.') : 'frontmatter';
-                    return `  - ${pathStr}: ${err.message}`;
-                })
-                .join('\n');
-            throw new Error(`Invalid frontmatter in ${filePath}:\n${errors}`);
-        }
+export interface SkillTransformerConfig {
+    /**
+     * Catalog used to resolve `{@tool …}` / `{@skill …}` and validate `tool=`-tagged examples.
+     *
+     * Supplied by the consuming build, never discovered here: a transformer that went looking
+     * for a registry would couple build-tools to whichever package happens to own one.
+     */
+    markdown?: PreprocessSkillMarkdownOptions;
+}
 
-        const validatedFrontmatter = frontmatterValidation.data;
+/**
+ * Render the skill body, failing closed when the semantic syntax is used without a catalog.
+ *
+ * Without this guard a package that writes `{@tool x}` but forgets to configure `skillCatalog`
+ * would ship the raw construct straight to the model — silently, and looking fine in review.
+ */
+function renderSkillMarkdown(markdown: string, filePath: string, options?: PreprocessSkillMarkdownOptions): string {
+    if (options) {
+        return assertSkillMarkdown(markdown, options, filePath).markdown;
+    }
+    const probe = preprocessSkillMarkdown(markdown, { tools: new Set(), skills: new Set() });
+    if (probe.references.length > 0 || probe.examples.length > 0 || probe.errors.length > 0) {
+        throw new Error(
+            `${filePath} uses {@tool …}, {@skill …} or a tool= example tag, but this build supplies no ` +
+                'skill catalog. Set "vertesia-build.skillCatalog" in package.json to a module exporting ' +
+                '{ tools, skills } (and optionally validateExample).',
+        );
+    }
+    return markdown;
+}
 
-        // Determine content type from frontmatter or file extension
-        const content_type: SkillContentType = validatedFrontmatter.content_type || 'md';
+/**
+ * Marks a rule as *this* preset rather than any rule that happens to share its pattern.
+ *
+ * The CLI swaps in a catalog-bound skill transformer; matching on `pattern.source` would also
+ * replace an unrelated custom transformer registered for the same files.
+ */
+const SKILL_TRANSFORMER = Symbol.for('@vertesia/build-tools:skillTransformer');
 
-        // Discover assets (scripts and widgets) in the skill directory
-        const assets = discoverSkillAssets(filePath);
+/** True when `rule` is a skill transformer produced by `createSkillTransformer`. */
+export function isSkillTransformer(rule: TransformerRule): boolean {
+    return (rule as unknown as Record<PropertyKey, unknown>)[SKILL_TRANSFORMER] === true;
+}
 
-        // Build skill definition using the same logic as parseSkillFile in tools-sdk
-        const skillData = buildSkillDefinition(
-            validatedFrontmatter,
-            markdown,
-            content_type,
-            assets.widgets,
-            assets.scripts,
-        );
+/** Build a skill transformer bound to a specific catalog. `skillTransformer` is the unbound one. */
+export function createSkillTransformer(config: SkillTransformerConfig = {}): TransformerPreset {
+    return {
+        [SKILL_TRANSFORMER]: true,
+        pattern: /(\.md\?skill$|\/SKILL\.md$)/,
+        schema: SkillDefinitionSchema,
+        transform: (content: string, filePath: string) => transformSkill(content, filePath, config),
+    } as TransformerPreset;
+}
 
-        // Check if properties.ts exists in the skill directory
-        const skillDir = path.dirname(filePath);
-        const propertiesPath = path.join(skillDir, 'properties.ts');
-        const hasProperties = existsSync(propertiesPath);
+export const skillTransformer: TransformerPreset = createSkillTransformer();
 
-        // If properties.ts exists, generate custom code with import and merge
-        // Rollup will handle transpiling properties.ts to properties.js
-        if (hasProperties) {
-            const skillDataJson = JSON.stringify(skillData, null, 2);
-            const code = `import properties from './properties.js';
+function transformSkill(content: string, filePath: string, config: SkillTransformerConfig): TransformResult {
+    const { frontmatter, content: rawMarkdown } = parseFrontmatter(content);
+    const markdown = renderSkillMarkdown(rawMarkdown, filePath, config.markdown);
+
+    // Validate frontmatter first to catch unknown properties
+    const frontmatterValidation = SkillFrontmatterSchema.safeParse(frontmatter);
+    if (!frontmatterValidation.success) {
+        const errors = frontmatterValidation.error.issues
+            .map((err) => {
+                const pathStr = err.path.length > 0 ? err.path.join('.') : 'frontmatter';
+                return `  - ${pathStr}: ${err.message}`;
+            })
+            .join('\n');
+        throw new Error(`Invalid frontmatter in ${filePath}:\n${errors}`);
+    }
+
+    const validatedFrontmatter = frontmatterValidation.data;
+
+    // Determine content type from frontmatter or file extension
+    const content_type: SkillContentType = validatedFrontmatter.content_type || 'md';
+
+    // Discover assets (scripts and widgets) in the skill directory
+    const assets = discoverSkillAssets(filePath);
+
+    // Build skill definition using the same logic as parseSkillFile in tools-sdk
+    const skillData = buildSkillDefinition(
+        validatedFrontmatter,
+        markdown,
+        content_type,
+        assets.widgets,
+        assets.scripts,
+    );
+
+    // Check if properties.ts exists in the skill directory
+    const skillDir = path.dirname(filePath);
+    const propertiesPath = path.join(skillDir, 'properties.ts');
+    const hasProperties = existsSync(propertiesPath);
+
+    // If properties.ts exists, generate custom code with import and merge.
+    // `tsc` has already emitted properties.js beside this chunk — the pipeline runs after it and
+    // never compiles TypeScript itself — so the generated module can import './properties.js'.
+    if (hasProperties) {
+        const skillDataJson = JSON.stringify(skillData, null, 2);
+        const code = `import properties from './properties.js';
 
 // Runtime validation for function properties
 if ('isEnabled' in properties && typeof properties.isEnabled !== 'function') {
@@ -322,18 +386,17 @@ const skill = ${skillDataJson};
 
 export default { ...skill, ...properties };
 `;
-            return {
-                data: skillData,
-                assets: assets.assetFiles,
-                widgets: assets.widgetMetadata,
-                code,
-            };
-        }
-
         return {
             data: skillData,
             assets: assets.assetFiles,
             widgets: assets.widgetMetadata,
+            code,
         };
-    },
-};
+    }
+
+    return {
+        data: skillData,
+        assets: assets.assetFiles,
+        widgets: assets.widgetMetadata,
+    };
+}

package/src/import-transform/detector.ts

@@ -1,15 +1,20 @@
 /**
- * Finds Vertesia query-style import specifiers inside a JavaScript source
- * file and maps each one to the transformer that should handle it.
+ * Finds Vertesia query-style import specifiers in a JavaScript source file and maps each one to
+ * the transformer that should handle it.
  *
- * The detector is regex-based and works on `tsc` output where import
- * specifiers are preserved literally. It identifies any string literal that
- * contains a known marker (`?skill`, `?raw`, …, or `/SKILL.md`) and then
- * matches the specifier against the configured transformer patterns.
+ * Detection uses a real ESM lexer (`es-module-lexer`) rather than a regex over string literals,
+ * because a regex models the wrong thing: it matches anything that *looks* like a specifier. A doc
+ * comment mentioning `` `?skill` ``, or an ordinary constant such as
+ * `const example = './example/SKILL.md'`, were both detected as imports — the former resolved to a
+ * directory and killed the build with `EISDIR`.
+ *
+ * Lexing the module handles comments, template literals, regex literals, static imports,
+ * re-exports (`export … from`) and dynamic `import()` in one pass, and still yields the source
+ * offsets the rewriter needs.
  */
 
+import { initSync, parse } from 'es-module-lexer';
 import type { TransformerRule } from '../core/types.js';
-import { QUERY_STRING_LITERAL } from './patterns.js';
 
 export interface ImportOccurrence {
     /** The transformer whose pattern matched the specifier. */
@@ -28,28 +33,75 @@ export interface ImportOccurrence {
     quote: "'" | '"' | '`';
 }
 
+/**
+ * `es-module-lexer` is WASM-backed and must be initialised before the first `parse`.
+ *
+ * Initialised synchronously and once, so `detectQueryImports` stays a plain function — making it
+ * async would push a promise through the scanner and every caller for no benefit.
+ */
+let initialised = false;
+function ensureInitialised(): void {
+    if (!initialised) {
+        initSync();
+        initialised = true;
+    }
+}
+
+const QUOTES = new Set(["'", '"', '`']);
+
+/**
+ * Locate the quotes around a specifier.
+ *
+ * The lexer reports the specifier *without* surrounding quotes for static imports but *with* them
+ * for dynamic ones, and the rewriter splices over the quoted range — so normalise here rather than
+ * making every caller know the difference.
+ */
+function quoteBounds(source: string, start: number, end: number): { quoteStart: number; quoteEnd: number } | undefined {
+    if (QUOTES.has(source[start - 1]) && QUOTES.has(source[end])) {
+        return { quoteStart: start - 1, quoteEnd: end + 1 };
+    }
+    if (QUOTES.has(source[start]) && QUOTES.has(source[end - 1])) {
+        return { quoteStart: start, quoteEnd: end };
+    }
+    // A dynamic import whose specifier is a computed expression: nothing to rewrite.
+    return undefined;
+}
+
+/**
+ * Throws if `content` cannot be lexed.
+ *
+ * Deliberately not caught: the scanner only reaches this for files that already contain a
+ * transformation marker, so a swallowed lexer failure would mean a real `?skill` import is left
+ * untransformed and the raw specifier ships to the runtime. A build that cannot read a file must
+ * stop, not quietly emit less. `transformImports` adds the file path to the message.
+ */
 export function detectQueryImports(content: string, transformers: TransformerRule[]): ImportOccurrence[] {
+    ensureInitialised();
+
+    const [imports] = parse(content);
+
     const occurrences: ImportOccurrence[] = [];
-    QUERY_STRING_LITERAL.lastIndex = 0;
-
-    let match: RegExpExecArray | null;
-    // biome-ignore lint/suspicious/noAssignInExpressions: regex .exec() loop
-    while ((match = QUERY_STRING_LITERAL.exec(content)) !== null) {
-        const quote = match[1] as "'" | '"' | '`';
-        const specifier = match[2];
-
-        for (const transformer of transformers) {
-            if (transformer.pattern.test(specifier)) {
-                occurrences.push({
-                    transformer,
-                    specifier,
-                    quoteStart: match.index,
-                    quoteEnd: match.index + match[0].length,
-                    quote,
-                });
-                break;
-            }
+    for (const entry of imports) {
+        const specifier = entry.n;
+        if (specifier === undefined) {
+            // `import.meta`, or a dynamic import with a non-literal specifier.
+            continue;
+        }
+        const bounds = quoteBounds(content, entry.s, entry.e);
+        if (!bounds) {
+            continue;
+        }
+        const transformer = transformers.find((rule) => rule.pattern.test(specifier));
+        if (!transformer) {
+            continue;
         }
+        occurrences.push({
+            transformer,
+            specifier,
+            quoteStart: bounds.quoteStart,
+            quoteEnd: bounds.quoteEnd,
+            quote: content[bounds.quoteStart] as "'" | '"' | '`',
+        });
     }
 
     return occurrences;

package/src/import-transform/index.ts

@@ -23,7 +23,7 @@ import type { AssetFile, TransformerRule } from '../core/types.js';
 import { copyAssets } from '../core/utils/asset-copy.js';
 import type { WidgetMetadata } from '../core/utils/asset-discovery.js';
 import { emitChunk } from './chunk-emitter.js';
-import { detectQueryImports } from './detector.js';
+import { detectQueryImports, type ImportOccurrence } from './detector.js';
 import { SNIFF_PATTERN } from './patterns.js';
 import { resolveImport } from './resolver.js';
 import { type ImportReplacement, writeRewrittenFile } from './rewriter.js';
@@ -94,7 +94,17 @@ export async function transformImports(opts: TransformImportsOptions): Promise<T
 
     while (workQueue.length > 0) {
         const file = workQueue.shift() as WorkItem;
-        const occurrences = detectQueryImports(file.content, transformers);
+
+        let occurrences: ImportOccurrence[];
+        try {
+            occurrences = detectQueryImports(file.content, transformers);
+        } catch (error) {
+            // Fail closed. This file reached the queue because it contains a transformation
+            // marker, so a lexer failure means a real import may go untransformed — the raw
+            // specifier would then ship to the runtime.
+            throw new Error(`Failed to parse imports in ${file.path}: ${(error as Error).message}`);
+        }
+
         if (occurrences.length === 0) {
             continue;
         }

package/src/import-transform/patterns.ts

@@ -1,7 +1,11 @@
 /**
- * Shared regex fragments used by the scanner, detector, and orchestrator to
- * identify Vertesia query-style imports. Keep these in sync with the
- * `pattern` field of each transformer in `core/transformers/`.
+ * Shared regex fragments used by the scanner and orchestrator to cheaply pre-filter files that
+ * could contain Vertesia query-style imports. Keep these in sync with the `pattern` field of each
+ * transformer in `core/transformers/`.
+ *
+ * These are a *sniff* only. Deciding what is actually an import is the detector's job, and it
+ * lexes the module rather than matching string literals — a literal-matching regex used to live
+ * here too, and it could not tell an import from a comment or a constant.
  */
 
 /**
@@ -23,12 +27,3 @@ const BARE_GROUP = BARE_FILENAMES.map((name) => `\\/${name.replace('.', '\\.')}`
  * Cheap regex check before invoking the more expensive detector.
  */
 export const SNIFF_PATTERN = new RegExp(`\\?(?:${QUERY_GROUP})\\b|(?:${BARE_GROUP})\\b`);
-
-/**
- * Matches a quoted string literal whose contents end with a query marker.
- * Capture group 1 is the quote character; capture group 2 is the specifier.
- */
-export const QUERY_STRING_LITERAL = new RegExp(
-    `(['"\`])([^'"\`]*?(?:\\?(?:${QUERY_GROUP})|${BARE_GROUP})[^'"\`]*)\\1`,
-    'g',
-);

package/src/index.ts

@@ -20,12 +20,13 @@
  *
  * @example
  * ```typescript
- * import { transformImports } from '@vertesia/build-tools';
+ * import { resolveTransformerNames, transformImports } from '@vertesia/build-tools';
  *
  * await transformImports({
  *   libDir: './lib',
  *   srcDir: './src',
- *   transformers: ['skill', 'raw'],
+ *   // Resolved rules, not names — use resolveTransformerNames to go from one to the other.
+ *   transformers: resolveTransformerNames(['skill', 'raw']),
  *   assetsDir: './dist',
  * });
  * ```
@@ -40,8 +41,39 @@ export {
 } from './core/compilers/widget.js';
 // Parsers
 export { type FrontmatterResult, parseFrontmatter } from './core/parsers/frontmatter.js';
+export {
+    loadSkillCatalog,
+    readPackageJson,
+} from './core/skill-markdown/load-catalog.js';
+// Skill markdown preprocessor: `{@tool …}` / `{@skill …}` resolution and tagged-example
+// validation. Pure — the consuming build supplies the catalog and the schema validator.
+export {
+    assertSkillMarkdown,
+    DEFAULT_SKILL_TOOL_PREFIX,
+    type PreprocessSkillMarkdownOptions,
+    type PreprocessSkillMarkdownResult,
+    preprocessSkillMarkdown,
+    type SkillExample,
+    type SkillReference,
+    type SkillReferenceKind,
+} from './core/skill-markdown/preprocess.js';
+export {
+    checkDispatchDescriptor,
+    createSchemaExampleValidator,
+    createSchemaFieldValidator,
+    type DispatchResolution,
+    nodesAtPath,
+    pairDispatchedInputs,
+    resolveDispatchedNames,
+    schemaNodeAtPath,
+    type ToolDispatchDescriptor,
+    type ToolSchemaEntry,
+    toolNamesAtPath,
+} from './core/skill-markdown/schema-validator.js';
 // Transformers (the pure transformation functions)
 export {
+    createSkillTransformer,
+    isSkillTransformer,
     type PromptContentType,
     type PromptDefinition,
     PromptDefinitionSchema,
@@ -54,6 +86,7 @@ export {
     type SkillDefinition,
     SkillDefinitionSchema,
     SkillPropertiesSchema,
+    type SkillTransformerConfig,
     skillCollectionTransformer,
     skillTransformer,
     TemplateType,

package/src/vite/dev-server.ts

@@ -33,6 +33,9 @@
 import { readFileSync } from 'node:fs';
 import path from 'node:path';
 import type { Plugin } from 'vite';
+import { loadSkillCatalog, readPackageJson, resolveSkillCatalogPath } from '../core/skill-markdown/load-catalog.js';
+import type { PreprocessSkillMarkdownOptions } from '../core/skill-markdown/preprocess.js';
+import { createSkillTransformer, isSkillTransformer } from '../core/transformers/skill.js';
 import type { TransformerRule } from '../core/types.js';
 import { BUILTIN_TRANSFORMER_NAMES, resolveTransformerNames } from '../import-transform/builtins.js';
 
@@ -43,16 +46,57 @@ export interface VertesiaDevServerPluginOptions {
      * Pass a subset to disable transformers you don't use.
      */
     transformers?: readonly string[];
+
+    /**
+     * Catalog used to resolve `{@tool …}` / `{@skill …}` and validate `tool=`-tagged examples.
+     *
+     * Defaults to the module named by `vertesia-build.skillCatalog` in the package.json of the
+     * Vite root, which is the same key the CLI reads — so dev, test and build agree without the
+     * setting being written twice. Pass a path to override it, or an object to supply a catalog
+     * directly (useful in tests).
+     */
+    skillCatalog?: string | PreprocessSkillMarkdownOptions | false;
 }
 
 export function vertesiaDevServerPlugin(options: VertesiaDevServerPluginOptions = {}): Plugin {
     const names = options.transformers ?? BUILTIN_TRANSFORMER_NAMES;
     const transformers = resolveTransformerNames([...names]);
 
+    // Resolved once, lazily: the catalog module is compiled output, so it may not exist when the
+    // config is first evaluated.
+    let root = process.cwd();
+
+    let boundPromise: Promise<TransformerRule | null> | undefined;
+
+    /** The catalog-bound skill transformer, or null when this package configures no catalog. */
+    const boundSkillTransformer = (): Promise<TransformerRule | null> => {
+        if (!boundPromise) {
+            boundPromise = (async () => {
+                if (options.skillCatalog === false) return null;
+                const markdown =
+                    typeof options.skillCatalog === 'object'
+                        ? options.skillCatalog
+                        : await (async () => {
+                              const modulePath =
+                                  typeof options.skillCatalog === 'string'
+                                      ? path.resolve(root, options.skillCatalog)
+                                      : resolveSkillCatalogPath(readPackageJson(root) ?? {}, root);
+                              return modulePath ? await loadSkillCatalog(modulePath) : null;
+                          })();
+                return markdown ? createSkillTransformer({ markdown }) : null;
+            })();
+        }
+        return boundPromise;
+    };
+
     return {
         name: 'vertesia-dev-server',
         enforce: 'pre',
 
+        configResolved(config: { root?: string }) {
+            if (config.root) root = config.root;
+        },
+
         /**
          * Map relative query-style imports to absolute paths so Vite's loader
          * sees a stable id keyed by the original file location. The query
@@ -87,13 +131,17 @@ export function vertesiaDevServerPlugin(options: VertesiaDevServerPluginOptions
             const matched = findMatchingTransformer(id, transformers);
             if (!matched) return null;
 
+            // Bind the skill transformer to the catalog on first use. Matched by identity rather
+            // than by pattern so a consumer's own transformer for the same files is left alone.
+            const active = isSkillTransformer(matched) ? ((await boundSkillTransformer()) ?? matched) : matched;
+
             const cleanId = stripQuery(id);
             try {
-                const content = matched.virtual ? '' : readFileSync(cleanId, 'utf-8');
-                const result = await matched.transform(content, cleanId);
+                const content = active.virtual ? '' : readFileSync(cleanId, 'utf-8');
+                const result = await active.transform(content, cleanId);
 
-                if (matched.schema) {
-                    const validation = matched.schema.safeParse(result.data);
+                if (active.schema) {
+                    const validation = active.schema.safeParse(result.data);
                     if (!validation.success) {
                         const errors = validation.error.issues
                             .map((err) => `  - ${err.path.join('.')}: ${err.message}`)