@vellumai/assistant 0.4.41 → 0.4.43

package/src/bundler/app-bundler.ts

@@ -9,12 +9,12 @@ import { createHash } from "node:crypto";
 import { createWriteStream, existsSync, readFileSync } from "node:fs";
 import { readFile, stat, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
-import { extname, join } from "node:path";
+import { join } from "node:path";
 
 import archiver from "archiver";
 import JSZip from "jszip";
 
-import { getApp, getAppsDir, isMultifileApp } from "../memory/app-store.js";
+import { getApp, getAppsDir } from "../memory/app-store.js";
 import { computeContentId } from "../util/content-id.js";
 import { getLogger } from "../util/logger.js";
 import { compileApp } from "./app-compiler.js";
@@ -28,147 +28,7 @@ const bundlerLog = getLogger("app-bundler");
 import { APP_VERSION } from "../version.js";
 const PACKAGE_VERSION = APP_VERSION;
 
-const HASH_DISPLAY_LENGTH = 12;
 const MAX_BUNDLE_SIZE_BYTES = 25 * 1024 * 1024; // 25 MB
-const ASSET_FETCH_TIMEOUT_MS = 10_000;
-
-interface FetchedAsset {
-  archivePath: string; // e.g. "assets/a1b2c3d4.png"
-  data: Buffer;
-}
-
-/**
- * Extract all remote (http/https) URLs from HTML content.
- * Looks in src=, href= on all elements except <a> tags, and CSS url() references.
- */
-export function extractRemoteUrls(html: string): string[] {
-  const urls = new Set<string>();
-
-  // Match src="..." attributes on any element
-  const srcRe = /\bsrc\s*=\s*(?:"([^"]*?)"|'([^']*?)'|([^\s>]+))/gi;
-  let m: RegExpExecArray | null;
-  while ((m = srcRe.exec(html)) != null) {
-    const url = m[1] ?? m[2] ?? m[3];
-    if (url && /^https?:\/\//i.test(url)) {
-      urls.add(url);
-    }
-  }
-
-  // Match href="..." on any element except navigation/resolution tags (not assets).
-  // Captures the tag name and href value so we can skip them.
-  const hrefRe =
-    /<(\w+)\b[^>]*?\bhref\s*=\s*(?:"([^"]*?)"|'([^']*?)'|([^\s>]+))[^>]*?\/?>/gi;
-  while ((m = hrefRe.exec(html)) != null) {
-    const tagName = m[1];
-    if (["a", "base", "area"].includes(tagName.toLowerCase())) continue;
-    const url = m[2] ?? m[3] ?? m[4];
-    if (url && /^https?:\/\//i.test(url)) {
-      urls.add(url);
-    }
-  }
-
-  // Match CSS url() references (inline styles and <style> blocks)
-  const urlRe = /url\(\s*(?:"([^"]*?)"|'([^']*?)'|([^)"'\s]+))\s*\)/gi;
-  while ((m = urlRe.exec(html)) != null) {
-    const url = m[1] ?? m[2] ?? m[3];
-    if (url && /^https?:\/\//i.test(url)) {
-      urls.add(url);
-    }
-  }
-
-  return [...urls];
-}
-
-/**
- * Derive a deterministic filename for a remote asset URL.
- * Uses a hash of the URL to avoid collisions, preserving the original extension.
- */
-function assetFilename(url: string): string {
-  const hash = createHash("sha256")
-    .update(url)
-    .digest("hex")
-    .slice(0, HASH_DISPLAY_LENGTH);
-  let ext = "";
-  try {
-    const parsed = new URL(url);
-    ext = extname(parsed.pathname);
-  } catch {
-    // no extension
-  }
-  // Fallback: if no extension or it's too long/weird, drop it
-  if (!ext || ext.length > 10 || !/^\.\w+$/.test(ext)) {
-    ext = "";
-  }
-  return `${hash}${ext}`;
-}
-
-/**
- * Fetch remote assets referenced in HTML, returning the fetched buffers
- * and the rewritten HTML with local asset paths. Assets that fail to fetch
- * are left with their original URLs.
- */
-export async function materializeAssets(
-  html: string,
-): Promise<{ rewrittenHtml: string; assets: FetchedAsset[] }> {
-  const urls = extractRemoteUrls(html);
-  if (urls.length === 0) {
-    return { rewrittenHtml: html, assets: [] };
-  }
-
-  const assets: FetchedAsset[] = [];
-  // Map from original URL to its local archive path
-  const urlMap = new Map<string, string>();
-
-  await Promise.all(
-    urls.map(async (url) => {
-      try {
-        const controller = new AbortController();
-        const timeout = setTimeout(
-          () => controller.abort(),
-          ASSET_FETCH_TIMEOUT_MS,
-        );
-        let buf: Buffer;
-        try {
-          const resp = await fetch(url, { signal: controller.signal });
-          if (!resp.ok) {
-            bundlerLog.warn(
-              { url, status: resp.status },
-              "Failed to fetch asset, keeping original URL",
-            );
-            return;
-          }
-          buf = Buffer.from(await resp.arrayBuffer());
-        } finally {
-          clearTimeout(timeout);
-        }
-        const filename = assetFilename(url);
-        const archivePath = `assets/${filename}`;
-        assets.push({ archivePath, data: buf });
-        urlMap.set(url, archivePath);
-      } catch (err) {
-        bundlerLog.warn(
-          { url, err },
-          "Failed to fetch asset, keeping original URL",
-        );
-      }
-    }),
-  );
-
-  // Rewrite URLs in HTML — replace each occurrence of the original URL with the local path.
-  // Sort by length descending so longer URLs are replaced first, preventing prefix collisions
-  // (e.g. "https://cdn/x" replacing part of "https://cdn/x/y.png").
-  const sortedEntries = [...urlMap.entries()].sort(
-    (a, b) => b[0].length - a[0].length,
-  );
-  let rewrittenHtml = html;
-  for (const [originalUrl, localPath] of sortedEntries) {
-    // Escape regex special chars in the URL
-    const escaped = originalUrl.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-    rewrittenHtml = rewrittenHtml.replace(new RegExp(escaped, "g"), localPath);
-  }
-
-  return { rewrittenHtml, assets };
-}
 
 export interface BundleResult {
   bundlePath: string;
@@ -195,15 +55,13 @@ export async function packageApp(
     throw new Error(`App not found: ${appId}`);
   }
 
-  const multifile = isMultifileApp(app);
-
   // Build manifest
   const createdBy = `vellum-assistant/${PACKAGE_VERSION}`;
   const version = app.version ?? "1.0.0";
   const contentId = computeContentId(app.name);
 
   const manifest: AppManifest = {
-    format_version: multifile ? 2 : 1,
+    format_version: 2,
     name: app.name,
     ...(app.description ? { description: app.description } : {}),
     ...(app.icon ? { icon: app.icon } : {}),
@@ -216,62 +74,34 @@ export async function packageApp(
     content_id: contentId,
   };
 
-  // For multifile apps, compile first then bundle the dist/ output.
-  // For legacy apps, materialize remote assets and bundle the HTML directly.
-  let rewrittenHtml = "";
-  let allAssets: FetchedAsset[] = [];
-  const rewrittenPages: Record<string, string> = {};
+  // Compile the app and bundle the dist/ output.
   const compiledFiles: { name: string; data: Buffer }[] = [];
 
-  if (multifile) {
-    const appDir = join(getAppsDir(), appId);
-    const compileResult = await compileApp(appDir);
-    if (!compileResult.ok) {
-      const messages = compileResult.errors
-        .map((e) => {
-          const loc = e.location
-            ? ` (${e.location.file}:${e.location.line}:${e.location.column})`
-            : "";
-          return `${e.text}${loc}`;
-        })
-        .join("\n");
-      throw new Error(`Compilation failed for app "${app.name}":\n${messages}`);
-    }
-
-    const distDir = join(appDir, "dist");
-    const indexHtml = await readFile(join(distDir, "index.html"), "utf-8");
-    const mainJs = await readFile(join(distDir, "main.js"));
-
-    compiledFiles.push({ name: "index.html", data: Buffer.from(indexHtml) });
-    compiledFiles.push({ name: "main.js", data: mainJs });
+  const appDir = join(getAppsDir(), appId);
+  const compileResult = await compileApp(appDir);
+  if (!compileResult.ok) {
+    const messages = compileResult.errors
+      .map((e) => {
+        const loc = e.location
+          ? ` (${e.location.file}:${e.location.line}:${e.location.column})`
+          : "";
+        return `${e.text}${loc}`;
+      })
+      .join("\n");
+    throw new Error(`Compilation failed for app "${app.name}":\n${messages}`);
+  }
 
-    // main.css is optional — only produced when the app imports CSS
-    const cssPath = join(distDir, "main.css");
-    if (existsSync(cssPath)) {
-      compiledFiles.push({ name: "main.css", data: await readFile(cssPath) });
-    }
-  } else {
-    // Legacy path: fetch remote assets and rewrite HTML
-    const materialized = await materializeAssets(app.htmlDefinition);
-    rewrittenHtml = materialized.rewrittenHtml;
-    const fetchedAssets = materialized.assets;
+  const distDir = join(appDir, "dist");
+  const indexHtml = await readFile(join(distDir, "index.html"), "utf-8");
+  const mainJs = await readFile(join(distDir, "main.js"));
 
-    // Also materialize assets in additional pages
-    const pageAssets: FetchedAsset[] = [];
-    if (app.pages) {
-      for (const [filename, content] of Object.entries(app.pages)) {
-        const result = await materializeAssets(content);
-        rewrittenPages[filename] = result.rewrittenHtml;
-        pageAssets.push(...result.assets);
-      }
-    }
+  compiledFiles.push({ name: "index.html", data: Buffer.from(indexHtml) });
+  compiledFiles.push({ name: "main.js", data: mainJs });
 
-    // Deduplicate assets by archive path
-    const allAssetsMap = new Map<string, FetchedAsset>();
-    for (const asset of [...fetchedAssets, ...pageAssets]) {
-      allAssetsMap.set(asset.archivePath, asset);
-    }
-    allAssets = [...allAssetsMap.values()];
+  // main.css is optional — only produced when the app imports CSS
+  const cssPath = join(distDir, "main.css");
+  if (existsSync(cssPath)) {
+    compiledFiles.push({ name: "main.css", data: await readFile(cssPath) });
   }
 
   // Create the zip archive
@@ -302,27 +132,9 @@ export async function packageApp(
     // Add manifest.json at root level
     archive.append(serializeManifest(manifest), { name: "manifest.json" });
 
-    if (multifile) {
-      // Add compiled dist/ files
-      for (const file of compiledFiles) {
-        archive.append(file.data, { name: file.name });
-      }
-    } else {
-      // Add index.html at root level
-      archive.append(rewrittenHtml, { name: "index.html" });
-
-      // Add additional pages alongside index.html (with rewritten asset URLs)
-      if (app.pages) {
-        for (const filename of Object.keys(app.pages)) {
-          const content = rewrittenPages[filename] ?? app.pages[filename];
-          archive.append(content, { name: filename });
-        }
-      }
-
-      // Add fetched remote assets
-      for (const asset of allAssets) {
-        archive.append(asset.data, { name: asset.archivePath });
-      }
+    // Add compiled dist/ files
+    for (const file of compiledFiles) {
+      archive.append(file.data, { name: file.name });
     }
 
     // Include app icon if one was generated

package/src/bundler/app-compiler.ts

@@ -1,19 +1,21 @@
 /**
- * esbuild wrapper for compiling multi-file TSX apps.
+ * Compiler for multi-file TSX apps.
  *
- * Compiles src/main.tsx → dist/main.js, copies index.html with
- * script/style tag injection, and returns structured diagnostics.
+ * Shells out to the esbuild CLI binary (JIT-downloaded on first use) to
+ * compile src/main.tsx -> dist/main.js, then copies index.html with
+ * script/style tag injection.
+ *
+ * This avoids importing esbuild's JS API (which caches its native binary
+ * path at module load time and breaks inside bun --compile's /$bunfs/).
  */
 
 import { existsSync, rmSync } from "node:fs";
 import { mkdir, readdir, readFile, writeFile } from "node:fs/promises";
-import { join, resolve } from "node:path";
-
-import { build, type Message, type Plugin } from "esbuild";
+import { dirname, join } from "node:path";
 
 import { getLogger } from "../util/logger.js";
+import { ensureCompilerTools } from "./compiler-tools.js";
 import {
-  ALLOWED_PACKAGES,
   getCacheDir,
   isBareImport,
   packageName,
@@ -34,19 +36,70 @@ export interface CompileResult {
   durationMs: number;
 }
 
-function mapDiagnostics(messages: Message[]): CompileDiagnostic[] {
-  return messages.map((msg) => ({
-    text: msg.text,
-    ...(msg.location
-      ? {
-          location: {
-            file: msg.location.file,
-            line: msg.location.line,
-            column: msg.location.column,
-          },
+/**
+ * Parse esbuild CLI stderr into structured diagnostics.
+ * esbuild outputs errors like:
+ *   ✘ [ERROR] Could not resolve "foo"
+ *       src/main.tsx:3:7:
+ */
+function parseEsbuildStderr(stderr: string): {
+  errors: CompileDiagnostic[];
+  warnings: CompileDiagnostic[];
+} {
+  const errors: CompileDiagnostic[] = [];
+  const warnings: CompileDiagnostic[] = [];
+  const lines = stderr.split("\n");
+
+  for (let i = 0; i < lines.length; i++) {
+    const errorMatch = lines[i].match(/✘ \[ERROR\] (.+)/);
+    const warnMatch = lines[i].match(/▲ \[WARNING\] (.+)/);
+
+    if (errorMatch || warnMatch) {
+      const text = (errorMatch ?? warnMatch)![1];
+      const diag: CompileDiagnostic = { text };
+
+      // Next non-empty line may have location: "    file:line:col:"
+      const locLine = lines[i + 1]?.trim();
+      if (locLine) {
+        const locMatch = locLine.match(/^(.+):(\d+):(\d+):?$/);
+        if (locMatch) {
+          diag.location = {
+            file: locMatch[1],
+            line: parseInt(locMatch[2], 10),
+            column: parseInt(locMatch[3], 10),
+          };
         }
-      : {}),
-  }));
+      }
+
+      if (errorMatch) errors.push(diag);
+      else warnings.push(diag);
+    }
+  }
+
+  return { errors, warnings };
+}
+
+/**
+ * Scan source files for bare import specifiers and pre-install any
+ * allowlisted packages into the shared cache so esbuild can resolve them.
+ */
+async function resolveAppImports(srcDir: string): Promise<void> {
+  const importRe = /(?:import|from)\s+["']([^"'.][^"']*)["']/g;
+  const seen = new Set<string>();
+
+  const files = await readdir(srcDir, { recursive: true });
+  for (const file of files) {
+    if (!/\.[jt]sx?$/.test(String(file))) continue;
+    const content = await readFile(join(srcDir, String(file)), "utf-8");
+    for (const match of content.matchAll(importRe)) {
+      const specifier = match[1];
+      if (!isBareImport(specifier)) continue;
+      const pkg = packageName(specifier);
+      if (seen.has(pkg)) continue;
+      seen.add(pkg);
+      await resolvePackage(pkg);
+    }
+  }
 }
 
 /**
@@ -68,95 +121,70 @@ export async function compileApp(appDir: string): Promise<CompileResult> {
   }
   await mkdir(distDir, { recursive: true });
 
-  // Resolve preact from the assistant's own node_modules so per-app
-  // directories don't need their own copy.
-  const preactDir = resolve(
-    import.meta.dirname ?? __dirname,
-    "../../node_modules/preact",
-  );
-
-  // Plugin that resolves bare third-party imports against the allowlist
-  const resolvePlugin: Plugin = {
-    name: "vellum-package-resolver",
-    setup(pluginBuild) {
-      pluginBuild.onResolve({ filter: /.*/ }, async (args) => {
-        // Only intercept bare specifiers (not relative, not preact/react aliases)
-        if (
-          args.kind !== "import-statement" &&
-          args.kind !== "dynamic-import"
-        ) {
-          return undefined;
-        }
-        if (!isBareImport(args.path)) return undefined;
-
-        const pkg = packageName(args.path);
-        const nodeModulesDir = await resolvePackage(pkg);
-
-        if (nodeModulesDir) {
-          // Let esbuild resolve normally — nodePaths will pick it up
-          return undefined;
-        }
-
-        // Not allowed — produce a clear error
-        return {
-          errors: [
-            {
-              text: `Package '${pkg}' is not in the allowed list. Allowed: ${ALLOWED_PACKAGES.join(", ")}`,
-            },
-          ],
-        };
-      });
-    },
-  };
-
-  const cacheNodeModules = join(getCacheDir(), "node_modules");
-
-  let result;
+  // JIT download esbuild binary + preact on first use
+  let tools;
   try {
-    result = await build({
-      entryPoints: [entryPoint],
-      bundle: true,
-      minify: true,
-      sourcemap: false,
-      outdir: distDir,
-      format: "esm",
-      target: ["es2022"],
-      jsx: "automatic",
-      jsxImportSource: "preact",
-      loader: {
-        ".tsx": "tsx",
-        ".ts": "ts",
-        ".jsx": "jsx",
-        ".js": "js",
-        ".css": "css",
-      },
-      alias: {
-        react: "preact/compat",
-        "react-dom": "preact/compat",
-      },
-      plugins: [resolvePlugin],
-      // Point esbuild at assistant's preact and at the shared package cache
-      nodePaths: [resolve(preactDir, ".."), cacheNodeModules],
-      logLevel: "silent",
-    });
-  } catch (err: unknown) {
-    // esbuild throws on hard failures (e.g. syntax errors) with .errors/.warnings
-    const esbuildErr = err as {
-      errors?: Message[];
-      warnings?: Message[];
-    };
+    tools = await ensureCompilerTools();
+  } catch (err) {
     const durationMs = Math.round(performance.now() - start);
-    const errors = mapDiagnostics(esbuildErr.errors ?? []);
-    const warnings = mapDiagnostics(esbuildErr.warnings ?? []);
-    log.info({ durationMs, errorCount: errors.length }, "Build failed");
-    return { ok: false, errors, warnings, durationMs };
+    const text = err instanceof Error ? err.message : String(err);
+    log.error({ err, durationMs }, "Failed to ensure compiler tools");
+    return {
+      ok: false,
+      errors: [{ text: `Compiler setup failed: ${text}` }],
+      warnings: [],
+      durationMs,
+    };
   }
 
-  const errors = mapDiagnostics(result.errors);
-  const warnings = mapDiagnostics(result.warnings);
+  // Scan source files for bare imports and JIT-install allowed packages
+  await resolveAppImports(srcDir);
 
-  if (errors.length > 0) {
+  // Build NODE_PATH: preact parent dir + shared package cache
+  const preactParent = dirname(tools.preactDir);
+  const cacheNodeModules = join(getCacheDir(), "node_modules");
+  const nodePath = [preactParent, cacheNodeModules]
+    .filter((p) => existsSync(p))
+    .join(":");
+
+  // Shell out to esbuild CLI
+  const args = [
+    entryPoint,
+    "--bundle",
+    "--minify",
+    `--outdir=${distDir}`,
+    "--format=esm",
+    "--target=es2022",
+    "--jsx=automatic",
+    "--jsx-import-source=preact",
+    "--alias:react=preact/compat",
+    "--alias:react-dom=preact/compat",
+    "--loader:.tsx=tsx",
+    "--loader:.ts=ts",
+    "--loader:.jsx=jsx",
+    "--loader:.js=js",
+    "--loader:.css=css",
+    "--log-level=warning",
+  ];
+
+  const proc = Bun.spawn({
+    cmd: [tools.esbuildBin, ...args],
+    cwd: appDir,
+    stdout: "pipe",
+    stderr: "pipe",
+    env: { ...process.env, NODE_PATH: nodePath },
+  });
+
+  await proc.exited;
+  const stderr = await new Response(proc.stderr).text();
+
+  if (proc.exitCode !== 0) {
     const durationMs = Math.round(performance.now() - start);
+    const { errors, warnings } = parseEsbuildStderr(stderr);
+    // If parsing found nothing, use raw stderr as the error
+    if (errors.length === 0 && stderr.trim()) {
+      errors.push({ text: stderr.trim() });
+    }
     log.info({ durationMs, errorCount: errors.length }, "Build failed");
     return { ok: false, errors, warnings, durationMs };
   }
@@ -191,5 +219,5 @@ export async function compileApp(appDir: string): Promise<CompileResult> {
 
   const durationMs = Math.round(performance.now() - start);
   log.info({ durationMs }, "Build succeeded");
-  return { ok: true, errors, warnings, durationMs };
+  return { ok: true, errors: [], warnings: [], durationMs };
 }