@vertesia/build-tools 0.80.0

package/lib/build-tools.js

@@ -0,0 +1,616 @@
+import { copyFileSync, mkdirSync, readFileSync, readdirSync, statSync } from 'node:fs';
+import path from 'node:path';
+import { rollup } from 'rollup';
+import { z } from 'zod';
+import matter from 'gray-matter';
+
+/**
+ * Utilities for copying asset files during build
+ */
+/**
+ * Ensure a directory exists, creating it recursively if needed
+ */
+function ensureDirectory(dirPath) {
+    try {
+        mkdirSync(dirPath, { recursive: true });
+    }
+    catch (error) {
+        // Ignore if directory already exists
+        if (error.code !== 'EEXIST') {
+            throw error;
+        }
+    }
+}
+/**
+ * Copy an asset file to its destination
+ *
+ * @param asset - Asset file information
+ * @param assetsRoot - Root directory for assets
+ */
+function copyAssetFile(asset, assetsRoot) {
+    const destPath = path.join(assetsRoot, asset.destPath);
+    const destDir = path.dirname(destPath);
+    // Ensure destination directory exists
+    ensureDirectory(destDir);
+    // Copy file
+    try {
+        copyFileSync(asset.sourcePath, destPath);
+    }
+    catch (error) {
+        throw new Error(`Failed to copy asset from ${asset.sourcePath} to ${destPath}: ${error instanceof Error ? error.message : String(error)}`);
+    }
+}
+/**
+ * Copy multiple asset files
+ *
+ * @param assets - Array of asset files to copy
+ * @param assetsRoot - Root directory for assets
+ * @returns Number of files copied
+ */
+function copyAssets(assets, assetsRoot) {
+    let copied = 0;
+    for (const asset of assets) {
+        copyAssetFile(asset, assetsRoot);
+        copied++;
+    }
+    return copied;
+}
+
+/**
+ * Widget compilation utility using Rollup
+ */
+/**
+ * Default external dependencies for widgets
+ */
+const DEFAULT_EXTERNALS = [
+    'react',
+    'react-dom',
+    'react/jsx-runtime',
+    'react/jsx-dev-runtime',
+    'react-dom/client'
+];
+/**
+ * Compile widgets using Rollup
+ *
+ * @param widgets - Array of widget metadata to compile
+ * @param outputDir - Directory to write compiled widgets
+ * @param config - Widget compilation configuration
+ * @returns Number of widgets compiled
+ */
+async function compileWidgets(widgets, outputDir, config = {}) {
+    if (widgets.length === 0) {
+        return 0;
+    }
+    const { external = DEFAULT_EXTERNALS, tsconfig = './tsconfig.json', typescript: typescriptOptions = {}, minify = false } = config;
+    // Build each widget separately to get individual bundles
+    const buildPromises = widgets.map(async (widget) => {
+        // Dynamically import plugins - use any to bypass TypeScript module resolution issues
+        const typescript = (await import('@rollup/plugin-typescript')).default;
+        const nodeResolve = (await import('@rollup/plugin-node-resolve')).default;
+        const commonjs = (await import('@rollup/plugin-commonjs')).default;
+        const plugins = [
+            typescript({
+                tsconfig,
+                declaration: false,
+                sourceMap: true,
+                ...typescriptOptions
+            }),
+            nodeResolve({
+                browser: true,
+                preferBuiltins: false,
+                extensions: ['.tsx', '.ts', '.jsx', '.js']
+            }),
+            commonjs()
+        ];
+        // Add minification if requested
+        if (minify) {
+            const { terser } = await import('rollup-plugin-terser');
+            plugins.push(terser({
+                compress: {
+                    drop_console: false
+                }
+            }));
+        }
+        const rollupConfig = {
+            input: widget.path,
+            output: {
+                file: path.join(outputDir, `${widget.name}.js`),
+                format: 'es',
+                sourcemap: true,
+                inlineDynamicImports: true
+            },
+            external,
+            plugins
+        };
+        const bundle = await rollup(rollupConfig);
+        await bundle.write(rollupConfig.output);
+        await bundle.close();
+    });
+    await Promise.all(buildPromises);
+    return widgets.length;
+}
+
+/**
+ * Core Rollup plugin implementation for transforming imports
+ */
+/**
+ * Creates a Rollup plugin that transforms imports based on configured rules
+ */
+function vertesiaImportPlugin(config) {
+    const { transformers, assetsDir = './dist', widgetConfig } = config;
+    if (!transformers || transformers.length === 0) {
+        throw new Error('vertesiaImportPlugin: At least one transformer must be configured');
+    }
+    // Track assets to copy and widgets to compile
+    const assetsToProcess = [];
+    const widgetsToCompile = [];
+    const shouldCopyAssets = assetsDir !== false;
+    const shouldCompileWidgets = widgetConfig !== undefined && assetsDir !== false;
+    return {
+        name: 'vertesia-import-plugin',
+        /**
+         * Resolve import IDs to handle pattern-based imports
+         */
+        resolveId(source, importer) {
+            // Check if any transformer pattern matches
+            for (const transformer of transformers) {
+                if (transformer.pattern.test(source)) {
+                    // Handle relative imports
+                    if (source.startsWith('.') && importer) {
+                        const cleanSource = source.replace(transformer.pattern, '');
+                        const resolved = path.resolve(path.dirname(importer), cleanSource);
+                        // Return with the pattern suffix to identify it in load
+                        const suffix = source.match(transformer.pattern)?.[0] || '';
+                        return resolved + suffix;
+                    }
+                    return source;
+                }
+            }
+            return null; // Let other plugins handle it
+        },
+        /**
+         * Load and transform the file content
+         */
+        async load(id) {
+            // Find matching transformer
+            let matchedTransformer;
+            let cleanId = id;
+            for (const transformer of transformers) {
+                if (transformer.pattern.test(id)) {
+                    matchedTransformer = transformer;
+                    // Remove query parameters to get actual file path
+                    // For example: '/path/file.md?skill' -> '/path/file.md'
+                    //              '/path/file.html?raw' -> '/path/file.html'
+                    const queryIndex = id.indexOf('?');
+                    cleanId = queryIndex >= 0 ? id.substring(0, queryIndex) : id;
+                    break;
+                }
+            }
+            if (!matchedTransformer) {
+                return null; // Not for us
+            }
+            try {
+                // Read file content
+                const content = readFileSync(cleanId, 'utf-8');
+                // Transform the content
+                const result = await matchedTransformer.transform(content, cleanId);
+                // Collect assets if any
+                if (result.assets && shouldCopyAssets) {
+                    assetsToProcess.push(...result.assets);
+                }
+                // Collect widgets if any
+                if (result.widgets && shouldCompileWidgets) {
+                    widgetsToCompile.push(...result.widgets);
+                }
+                // Validate if schema provided
+                if (matchedTransformer.schema) {
+                    const validation = matchedTransformer.schema.safeParse(result.data);
+                    if (!validation.success) {
+                        const errors = validation.error.errors
+                            .map((err) => `  - ${err.path.join('.')}: ${err.message}`)
+                            .join('\n');
+                        throw new Error(`Validation failed for ${id}:\n${errors}`);
+                    }
+                }
+                // Generate code
+                if (result.code) {
+                    // Custom code provided
+                    return result.code;
+                }
+                else {
+                    // Default: export data (escape if string, otherwise stringify as JSON)
+                    const imports = result.imports ? result.imports.join('\n') + '\n\n' : '';
+                    const dataJson = JSON.stringify(result.data, null, 2);
+                    return `${imports}export default ${dataJson};`;
+                }
+            }
+            catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                this.error(`Failed to transform ${id}: ${message}`);
+            }
+        },
+        /**
+         * Copy assets and compile widgets after all modules are loaded
+         */
+        async buildEnd() {
+            // Copy script assets
+            if (shouldCopyAssets && assetsToProcess.length > 0) {
+                try {
+                    const copied = copyAssets(assetsToProcess, assetsDir);
+                    console.log(`Copied ${copied} asset file(s) to ${assetsDir}`);
+                }
+                catch (error) {
+                    const message = error instanceof Error ? error.message : String(error);
+                    this.warn(`Failed to copy assets: ${message}`);
+                }
+            }
+            // Compile widgets
+            if (shouldCompileWidgets && widgetsToCompile.length > 0) {
+                try {
+                    const widgetsDir = config.widgetsDir || 'widgets';
+                    const outputDir = path.join(assetsDir, widgetsDir);
+                    console.log(`Compiling ${widgetsToCompile.length} widget(s)...`);
+                    const compiled = await compileWidgets(widgetsToCompile, outputDir, widgetConfig);
+                    console.log(`Compiled ${compiled} widget(s) to ${outputDir}`);
+                }
+                catch (error) {
+                    const message = error instanceof Error ? error.message : String(error);
+                    this.error(`Failed to compile widgets: ${message}`);
+                }
+            }
+        }
+    };
+}
+
+/**
+ * Frontmatter parser utility using gray-matter
+ */
+/**
+ * Parse YAML frontmatter from markdown content
+ *
+ * @param content - Raw markdown content with optional frontmatter
+ * @returns Parsed frontmatter and content
+ */
+function parseFrontmatter(content) {
+    const result = matter(content);
+    return {
+        frontmatter: result.data,
+        content: result.content,
+        original: content
+    };
+}
+
+/**
+ * Utilities for discovering asset files (scripts, widgets) in skill directories
+ */
+/**
+ * Check if a file exists and is a regular file
+ */
+function isFile(filePath) {
+    try {
+        return statSync(filePath).isFile();
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Get all files in a directory (non-recursive)
+ */
+function getFilesInDirectory(dirPath) {
+    try {
+        return readdirSync(dirPath).filter(file => {
+            const fullPath = path.join(dirPath, file);
+            return isFile(fullPath);
+        });
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Check if a file is a script file (.js or .py)
+ */
+function isScriptFile(fileName) {
+    return /\.(js|py)$/.test(fileName);
+}
+/**
+ * Check if a file is a widget file (.tsx)
+ */
+function isWidgetFile(fileName) {
+    return /\.tsx$/.test(fileName);
+}
+/**
+ * Extract widget name from .tsx file (remove extension)
+ */
+function getWidgetName(fileName) {
+    return fileName.replace(/\.tsx$/, '');
+}
+/**
+ * Discover assets (scripts and widgets) in a skill directory
+ *
+ * @param skillFilePath - Absolute path to the skill.md file
+ * @param options - Asset discovery options
+ * @returns Discovered assets and metadata
+ */
+function discoverSkillAssets(skillFilePath, options = {}) {
+    const skillDir = path.dirname(skillFilePath);
+    const files = getFilesInDirectory(skillDir);
+    const scripts = [];
+    const widgets = [];
+    const widgetMetadata = [];
+    const assetFiles = [];
+    const scriptsDir = options.scriptsDir || 'scripts';
+    for (const file of files) {
+        const fullPath = path.join(skillDir, file);
+        if (isScriptFile(file)) {
+            // Script file (.js or .py)
+            scripts.push(file);
+            assetFiles.push({
+                sourcePath: fullPath,
+                destPath: path.join(scriptsDir, file),
+                type: 'script'
+            });
+        }
+        else if (isWidgetFile(file)) {
+            // Widget file (.tsx)
+            const widgetName = getWidgetName(file);
+            widgets.push(widgetName);
+            widgetMetadata.push({
+                name: widgetName,
+                path: fullPath
+            });
+            // Note: We don't add widget .tsx files to assetFiles
+            // Widgets are compiled by the plugin if widgetConfig is provided
+        }
+    }
+    return {
+        scripts,
+        widgets,
+        widgetMetadata,
+        assetFiles
+    };
+}
+
+/**
+ * Skill transformer preset for markdown files with frontmatter
+ */
+/**
+ * Context triggers for auto-injection of skills (for frontmatter validation)
+ */
+const SkillContextTriggersFrontmatterSchema = z.object({
+    keywords: z.array(z.string()).optional(),
+    tool_names: z.array(z.string()).optional(),
+    data_patterns: z.array(z.string()).optional()
+}).strict();
+/**
+ * Context triggers for auto-injection of skills (for output validation)
+ */
+const SkillContextTriggersSchema = z.object({
+    keywords: z.array(z.string()).optional(),
+    tool_names: z.array(z.string()).optional(),
+    data_patterns: z.array(z.string()).optional()
+}).optional();
+/**
+ * Execution configuration for skills that need code execution (for frontmatter validation)
+ */
+const SkillExecutionFrontmatterSchema = z.object({
+    language: z.string(),
+    packages: z.array(z.string()).optional(),
+    system_packages: z.array(z.string()).optional(),
+    template: z.string().optional()
+}).strict();
+/**
+ * Execution configuration for skills that need code execution (for output validation)
+ */
+const SkillExecutionSchema = z.object({
+    language: z.string(),
+    packages: z.array(z.string()).optional(),
+    system_packages: z.array(z.string()).optional(),
+    template: z.string().optional()
+}).optional();
+/**
+ * Zod schema for skill frontmatter validation
+ * This validates the YAML frontmatter before transformation
+ * Supports both flat and nested structures
+ */
+const SkillFrontmatterSchema = z.object({
+    // Required fields
+    name: z.string().min(1, 'Skill name is required'),
+    description: z.string().min(1, 'Skill description is required'),
+    // Optional fields
+    title: z.string().optional(),
+    content_type: z.enum(['md', 'jst']).optional(),
+    // Flat structure fields (legacy)
+    keywords: z.array(z.string()).optional(),
+    tools: z.array(z.string()).optional(),
+    data_patterns: z.array(z.string()).optional(),
+    language: z.string().optional(),
+    packages: z.array(z.string()).optional(),
+    system_packages: z.array(z.string()).optional(),
+    // Nested structure fields
+    context_triggers: SkillContextTriggersFrontmatterSchema.optional(),
+    execution: SkillExecutionFrontmatterSchema.optional(),
+    related_tools: z.array(z.string()).optional(),
+    input_schema: z.object({
+        type: z.literal('object'),
+        properties: z.record(z.any()).optional(),
+        required: z.array(z.string()).optional()
+    }).optional(),
+    // Asset fields (auto-discovered but can be overridden)
+    scripts: z.array(z.string()).optional(),
+    widgets: z.array(z.string()).optional()
+}).strict();
+/**
+ * MUST be kept in sync with @vertesia/tools-sdk SkillDefinition
+ * Zod schema for skill definition
+ * This validates the structure of skill objects generated from markdown
+ * Matches the SkillDefinition interface from @vertesia/tools-sdk
+ */
+const SkillDefinitionSchema = z.object({
+    name: z.string().min(1, 'Skill name is required'),
+    title: z.string().optional(),
+    description: z.string().min(1, 'Skill description is required'),
+    instructions: z.string(),
+    content_type: z.enum(['md', 'jst']),
+    input_schema: z.object({
+        type: z.literal('object'),
+        properties: z.record(z.any()).optional(),
+        required: z.array(z.string()).optional()
+    }).optional(),
+    context_triggers: SkillContextTriggersSchema,
+    execution: SkillExecutionSchema,
+    related_tools: z.array(z.string()).optional(),
+    scripts: z.array(z.string()).optional(),
+    widgets: z.array(z.string()).optional()
+});
+/**
+ * Build a SkillDefinition from frontmatter and markdown content.
+ * This mirrors the logic in @vertesia/tools-sdk parseSkillFile function.
+ *
+ * Supports two frontmatter structures:
+ *
+ * 1. Flat structure (matches parseSkillFile in tools-sdk):
+ *    keywords: [...]
+ *    tools: [...]
+ *    language: python
+ *    packages: [...]
+ *
+ * 2. Nested structure (for more explicit YAML):
+ *    context_triggers:
+ *      keywords: [...]
+ *      tool_names: [...]
+ *    execution:
+ *      language: python
+ *      packages: [...]
+ *    related_tools: [...]
+ *
+ * @param frontmatter - Parsed frontmatter object
+ * @param instructions - Markdown content (body of the file)
+ * @param contentType - Content type ('md' or 'jst')
+ * @param widgets - Discovered widget names
+ * @param scripts - Discovered script names
+ * @returns Skill definition object
+ */
+function buildSkillDefinition(frontmatter, instructions, contentType, widgets, scripts) {
+    const skill = {
+        name: frontmatter.name,
+        title: frontmatter.title,
+        description: frontmatter.description,
+        instructions,
+        content_type: contentType,
+        widgets: widgets.length > 0 ? widgets : undefined,
+        scripts: scripts.length > 0 ? scripts : undefined,
+    };
+    // Build context triggers - support both flat and nested structure
+    // Nested: context_triggers: { keywords: [...], tool_names: [...] }
+    // Flat: keywords: [...], tools: [...]
+    const contextTriggers = frontmatter.context_triggers;
+    const hasNestedTriggers = contextTriggers && typeof contextTriggers === 'object';
+    const hasFlatTriggers = frontmatter.keywords || frontmatter.tools || frontmatter.data_patterns;
+    if (hasNestedTriggers || hasFlatTriggers) {
+        skill.context_triggers = {
+            keywords: hasNestedTriggers ? contextTriggers.keywords : frontmatter.keywords,
+            tool_names: hasNestedTriggers ? contextTriggers.tool_names : frontmatter.tools,
+            data_patterns: hasNestedTriggers ? contextTriggers.data_patterns : frontmatter.data_patterns,
+        };
+    }
+    // Build execution config - support both flat and nested structure
+    const execution = frontmatter.execution;
+    const hasNestedExecution = execution && typeof execution === 'object';
+    const hasFlatExecution = frontmatter.language;
+    if (hasNestedExecution || hasFlatExecution) {
+        skill.execution = {
+            language: hasNestedExecution ? execution.language : frontmatter.language,
+            packages: hasNestedExecution ? execution.packages : frontmatter.packages,
+            system_packages: hasNestedExecution ? execution.system_packages : frontmatter.system_packages,
+        };
+        // Extract code template from instructions if present
+        const codeBlockMatch = instructions.match(/```(?:python|javascript|typescript|js|ts|py)\n([\s\S]*?)```/);
+        if (codeBlockMatch) {
+            skill.execution.template = codeBlockMatch[1].trim();
+        }
+    }
+    // Related tools - support both direct field and from tools field
+    if (frontmatter.related_tools) {
+        skill.related_tools = frontmatter.related_tools;
+    }
+    else if (frontmatter.tools && !hasNestedTriggers) {
+        // If tools is not part of context_triggers, use it as related_tools
+        skill.related_tools = frontmatter.tools;
+    }
+    // Input schema from frontmatter
+    if (frontmatter.input_schema) {
+        skill.input_schema = frontmatter.input_schema;
+    }
+    return skill;
+}
+/**
+ * Skill transformer preset
+ * Transforms markdown files with ?skill suffix OR SKILL.md files into skill definition objects
+ *
+ * Matches:
+ * - Files with ?skill suffix: ./my-skill.md?skill
+ * - SKILL.md files: ./my-skill/SKILL.md
+ *
+ * @example
+ * ```typescript
+ * import skill1 from './my-skill.md?skill';
+ * import skill2 from './my-skill/SKILL.md';
+ * // Both are SkillDefinition objects
+ * ```
+ */
+const skillTransformer = {
+    pattern: /(\.md\?skill$|\/SKILL\.md$)/,
+    schema: SkillDefinitionSchema,
+    transform: (content, filePath) => {
+        const { frontmatter, content: markdown } = parseFrontmatter(content);
+        // Validate frontmatter first to catch unknown properties
+        const frontmatterValidation = SkillFrontmatterSchema.safeParse(frontmatter);
+        if (!frontmatterValidation.success) {
+            const errors = frontmatterValidation.error.errors
+                .map((err) => {
+                const path = err.path.length > 0 ? err.path.join('.') : 'frontmatter';
+                return `  - ${path}: ${err.message}`;
+            })
+                .join('\n');
+            throw new Error(`Invalid frontmatter in ${filePath}:\n${errors}`);
+        }
+        // Determine content type from frontmatter or file extension
+        const content_type = frontmatter.content_type || 'md';
+        // Discover assets (scripts and widgets) in the skill directory
+        const assets = discoverSkillAssets(filePath);
+        // Build skill definition using the same logic as parseSkillFile in tools-sdk
+        const skillData = buildSkillDefinition(frontmatter, markdown, content_type, assets.widgets, assets.scripts);
+        return {
+            data: skillData,
+            assets: assets.assetFiles,
+            widgets: assets.widgetMetadata
+        };
+    }
+};
+
+/**
+ * Raw transformer preset for importing file content as strings
+ */
+/**
+ * Raw transformer preset
+ * Transforms any file with ?raw suffix into a string export
+ *
+ * @example
+ * ```typescript
+ * import template from './template.html?raw';
+ * // template is a string containing the file content
+ * ```
+ */
+const rawTransformer = {
+    pattern: /\?raw$/,
+    transform: (content) => {
+        return {
+            data: content
+        };
+    }
+};
+
+export { SkillDefinitionSchema, parseFrontmatter, rawTransformer, skillTransformer, vertesiaImportPlugin };
+//# sourceMappingURL=build-tools.js.map