@vellumai/assistant 0.4.40 → 0.4.42

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.4.40",
+  "version": "0.4.42",
   "type": "module",
   "bin": {
     "assistant": "./src/index.ts"

package/src/bundler/app-compiler.ts

@@ -1,19 +1,21 @@
 /**
- * esbuild wrapper for compiling multi-file TSX apps.
+ * Compiler for multi-file TSX apps.
  *
- * Compiles src/main.tsx → dist/main.js, copies index.html with
- * script/style tag injection, and returns structured diagnostics.
+ * Shells out to the esbuild CLI binary (JIT-downloaded on first use) to
+ * compile src/main.tsx -> dist/main.js, then copies index.html with
+ * script/style tag injection.
+ *
+ * This avoids importing esbuild's JS API (which caches its native binary
+ * path at module load time and breaks inside bun --compile's /$bunfs/).
  */
 
 import { existsSync, rmSync } from "node:fs";
 import { mkdir, readdir, readFile, writeFile } from "node:fs/promises";
-import { join, resolve } from "node:path";
-
-import { build, type Message, type Plugin } from "esbuild";
+import { dirname, join } from "node:path";
 
 import { getLogger } from "../util/logger.js";
+import { ensureCompilerTools } from "./compiler-tools.js";
 import {
-  ALLOWED_PACKAGES,
   getCacheDir,
   isBareImport,
   packageName,
@@ -34,19 +36,70 @@ export interface CompileResult {
   durationMs: number;
 }
 
-function mapDiagnostics(messages: Message[]): CompileDiagnostic[] {
-  return messages.map((msg) => ({
-    text: msg.text,
-    ...(msg.location
-      ? {
-          location: {
-            file: msg.location.file,
-            line: msg.location.line,
-            column: msg.location.column,
-          },
+/**
+ * Parse esbuild CLI stderr into structured diagnostics.
+ * esbuild outputs errors like:
+ *   ✘ [ERROR] Could not resolve "foo"
+ *       src/main.tsx:3:7:
+ */
+function parseEsbuildStderr(stderr: string): {
+  errors: CompileDiagnostic[];
+  warnings: CompileDiagnostic[];
+} {
+  const errors: CompileDiagnostic[] = [];
+  const warnings: CompileDiagnostic[] = [];
+  const lines = stderr.split("\n");
+
+  for (let i = 0; i < lines.length; i++) {
+    const errorMatch = lines[i].match(/✘ \[ERROR\] (.+)/);
+    const warnMatch = lines[i].match(/▲ \[WARNING\] (.+)/);
+
+    if (errorMatch || warnMatch) {
+      const text = (errorMatch ?? warnMatch)![1];
+      const diag: CompileDiagnostic = { text };
+
+      // Next non-empty line may have location: "    file:line:col:"
+      const locLine = lines[i + 1]?.trim();
+      if (locLine) {
+        const locMatch = locLine.match(/^(.+):(\d+):(\d+):?$/);
+        if (locMatch) {
+          diag.location = {
+            file: locMatch[1],
+            line: parseInt(locMatch[2], 10),
+            column: parseInt(locMatch[3], 10),
+          };
         }
-      : {}),
-  }));
+      }
+
+      if (errorMatch) errors.push(diag);
+      else warnings.push(diag);
+    }
+  }
+
+  return { errors, warnings };
+}
+
+/**
+ * Scan source files for bare import specifiers and pre-install any
+ * allowlisted packages into the shared cache so esbuild can resolve them.
+ */
+async function resolveAppImports(srcDir: string): Promise<void> {
+  const importRe = /(?:import|from)\s+["']([^"'.][^"']*)["']/g;
+  const seen = new Set<string>();
+
+  const files = await readdir(srcDir, { recursive: true });
+  for (const file of files) {
+    if (!/\.[jt]sx?$/.test(String(file))) continue;
+    const content = await readFile(join(srcDir, String(file)), "utf-8");
+    for (const match of content.matchAll(importRe)) {
+      const specifier = match[1];
+      if (!isBareImport(specifier)) continue;
+      const pkg = packageName(specifier);
+      if (seen.has(pkg)) continue;
+      seen.add(pkg);
+      await resolvePackage(pkg);
+    }
+  }
 }
 
 /**
@@ -68,95 +121,70 @@ export async function compileApp(appDir: string): Promise<CompileResult> {
   }
   await mkdir(distDir, { recursive: true });
 
-  // Resolve preact from the assistant's own node_modules so per-app
-  // directories don't need their own copy.
-  const preactDir = resolve(
-    import.meta.dirname ?? __dirname,
-    "../../node_modules/preact",
-  );
-
-  // Plugin that resolves bare third-party imports against the allowlist
-  const resolvePlugin: Plugin = {
-    name: "vellum-package-resolver",
-    setup(pluginBuild) {
-      pluginBuild.onResolve({ filter: /.*/ }, async (args) => {
-        // Only intercept bare specifiers (not relative, not preact/react aliases)
-        if (
-          args.kind !== "import-statement" &&
-          args.kind !== "dynamic-import"
-        ) {
-          return undefined;
-        }
-        if (!isBareImport(args.path)) return undefined;
-
-        const pkg = packageName(args.path);
-        const nodeModulesDir = await resolvePackage(pkg);
-
-        if (nodeModulesDir) {
-          // Let esbuild resolve normally — nodePaths will pick it up
-          return undefined;
-        }
-
-        // Not allowed — produce a clear error
-        return {
-          errors: [
-            {
-              text: `Package '${pkg}' is not in the allowed list. Allowed: ${ALLOWED_PACKAGES.join(", ")}`,
-            },
-          ],
-        };
-      });
-    },
-  };
-
-  const cacheNodeModules = join(getCacheDir(), "node_modules");
-
-  let result;
+  // JIT download esbuild binary + preact on first use
+  let tools;
   try {
-    result = await build({
-      entryPoints: [entryPoint],
-      bundle: true,
-      minify: true,
-      sourcemap: false,
-      outdir: distDir,
-      format: "esm",
-      target: ["es2022"],
-      jsx: "automatic",
-      jsxImportSource: "preact",
-      loader: {
-        ".tsx": "tsx",
-        ".ts": "ts",
-        ".jsx": "jsx",
-        ".js": "js",
-        ".css": "css",
-      },
-      alias: {
-        react: "preact/compat",
-        "react-dom": "preact/compat",
-      },
-      plugins: [resolvePlugin],
-      // Point esbuild at assistant's preact and at the shared package cache
-      nodePaths: [resolve(preactDir, ".."), cacheNodeModules],
-      logLevel: "silent",
-    });
-  } catch (err: unknown) {
-    // esbuild throws on hard failures (e.g. syntax errors) with .errors/.warnings
-    const esbuildErr = err as {
-      errors?: Message[];
-      warnings?: Message[];
-    };
+    tools = await ensureCompilerTools();
+  } catch (err) {
     const durationMs = Math.round(performance.now() - start);
-    const errors = mapDiagnostics(esbuildErr.errors ?? []);
-    const warnings = mapDiagnostics(esbuildErr.warnings ?? []);
-    log.info({ durationMs, errorCount: errors.length }, "Build failed");
-    return { ok: false, errors, warnings, durationMs };
+    const text = err instanceof Error ? err.message : String(err);
+    log.error({ err, durationMs }, "Failed to ensure compiler tools");
+    return {
+      ok: false,
+      errors: [{ text: `Compiler setup failed: ${text}` }],
+      warnings: [],
+      durationMs,
+    };
   }
 
-  const errors = mapDiagnostics(result.errors);
-  const warnings = mapDiagnostics(result.warnings);
+  // Scan source files for bare imports and JIT-install allowed packages
+  await resolveAppImports(srcDir);
 
-  if (errors.length > 0) {
+  // Build NODE_PATH: preact parent dir + shared package cache
+  const preactParent = dirname(tools.preactDir);
+  const cacheNodeModules = join(getCacheDir(), "node_modules");
+  const nodePath = [preactParent, cacheNodeModules]
+    .filter((p) => existsSync(p))
+    .join(":");
+
+  // Shell out to esbuild CLI
+  const args = [
+    entryPoint,
+    "--bundle",
+    "--minify",
+    `--outdir=${distDir}`,
+    "--format=esm",
+    "--target=es2022",
+    "--jsx=automatic",
+    "--jsx-import-source=preact",
+    "--alias:react=preact/compat",
+    "--alias:react-dom=preact/compat",
+    "--loader:.tsx=tsx",
+    "--loader:.ts=ts",
+    "--loader:.jsx=jsx",
+    "--loader:.js=js",
+    "--loader:.css=css",
+    "--log-level=warning",
+  ];
+
+  const proc = Bun.spawn({
+    cmd: [tools.esbuildBin, ...args],
+    cwd: appDir,
+    stdout: "pipe",
+    stderr: "pipe",
+    env: { ...process.env, NODE_PATH: nodePath },
+  });
+
+  await proc.exited;
+  const stderr = await new Response(proc.stderr).text();
+
+  if (proc.exitCode !== 0) {
     const durationMs = Math.round(performance.now() - start);
+    const { errors, warnings } = parseEsbuildStderr(stderr);
+    // If parsing found nothing, use raw stderr as the error
+    if (errors.length === 0 && stderr.trim()) {
+      errors.push({ text: stderr.trim() });
+    }
     log.info({ durationMs, errorCount: errors.length }, "Build failed");
     return { ok: false, errors, warnings, durationMs };
   }
@@ -191,5 +219,5 @@ export async function compileApp(appDir: string): Promise<CompileResult> {
 
   const durationMs = Math.round(performance.now() - start);
   log.info({ durationMs }, "Build succeeded");
-  return { ok: true, errors, warnings, durationMs };
+  return { ok: true, errors: [], warnings: [], durationMs };
 }

package/src/bundler/compiler-tools.ts

@@ -0,0 +1,248 @@
+/**
+ * JIT download and cache of esbuild binary + preact for app compilation.
+ *
+ * Instead of shipping these in the .app bundle (~11 MB), we download them
+ * on first compile to ~/.vellum/workspace/compiler-tools/. Follows the
+ * same pattern as EmbeddingRuntimeManager.
+ */
+
+import {
+  chmodSync,
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { arch, homedir, platform } from "node:os";
+import { join } from "node:path";
+
+import { getLogger } from "../util/logger.js";
+import { PromiseGuard } from "../util/promise-guard.js";
+
+const log = getLogger("compiler-tools");
+
+// Pinned versions matching assistant/bun.lock
+const ESBUILD_VERSION = "0.24.2";
+const PREACT_VERSION = "10.28.4";
+
+const TOOLS_VERSION = `esbuild-${ESBUILD_VERSION}_preact-${PREACT_VERSION}`;
+
+export interface CompilerTools {
+  esbuildBin: string;
+  preactDir: string;
+}
+
+interface VersionManifest {
+  toolsVersion: string;
+  esbuildVersion: string;
+  preactVersion: string;
+  platform: string;
+  arch: string;
+  installedAt: string;
+}
+
+function getToolsDir(): string {
+  return join(homedir(), ".vellum", "workspace", "compiler-tools");
+}
+
+const installGuard = new PromiseGuard<void>();
+
+function npmTarballUrl(pkg: string, version: string): string {
+  const encoded = pkg.replace("/", "%2f");
+  const basename = pkg.startsWith("@") ? pkg.split("/")[1] : pkg;
+  return `https://registry.npmjs.org/${encoded}/-/${basename}-${version}.tgz`;
+}
+
+async function downloadAndExtract(
+  url: string,
+  targetDir: string,
+): Promise<void> {
+  log.info({ url, targetDir }, "Downloading npm package");
+
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new Error(
+      `Failed to download ${url}: ${response.status} ${response.statusText}`,
+    );
+  }
+
+  const tarball = await response.arrayBuffer();
+  mkdirSync(targetDir, { recursive: true });
+
+  const tmpTar = join(targetDir, `download-${Date.now()}.tgz`);
+  writeFileSync(tmpTar, Buffer.from(tarball));
+
+  try {
+    const proc = Bun.spawn({
+      cmd: ["tar", "xzf", tmpTar, "-C", targetDir, "--strip-components=1"],
+      stdout: "ignore",
+      stderr: "pipe",
+    });
+    await proc.exited;
+    if (proc.exitCode !== 0) {
+      const stderr = await new Response(proc.stderr).text();
+      throw new Error(`Failed to extract ${url}: ${stderr}`);
+    }
+  } finally {
+    try {
+      rmSync(tmpTar);
+    } catch {
+      /* ignore */
+    }
+  }
+}
+
+function readManifest(baseDir: string): VersionManifest | null {
+  const manifestPath = join(baseDir, "version.json");
+  if (!existsSync(manifestPath)) return null;
+  try {
+    return JSON.parse(readFileSync(manifestPath, "utf-8"));
+  } catch {
+    return null;
+  }
+}
+
+function isReady(baseDir: string): boolean {
+  const manifest = readManifest(baseDir);
+  if (!manifest || manifest.toolsVersion !== TOOLS_VERSION) return false;
+  return (
+    existsSync(join(baseDir, "bin", "esbuild")) &&
+    existsSync(join(baseDir, "node_modules", "preact"))
+  );
+}
+
+/**
+ * Ensure esbuild binary + preact are downloaded and cached.
+ * Safe to call concurrently — deduplicates via PromiseGuard.
+ */
+export async function ensureCompilerTools(): Promise<CompilerTools> {
+  const baseDir = getToolsDir();
+
+  if (!isReady(baseDir)) {
+    await installGuard.run(() => install(baseDir));
+    if (!isReady(baseDir)) {
+      installGuard.reset();
+      throw new Error("Compiler tools installation failed");
+    }
+  }
+
+  return {
+    esbuildBin: join(baseDir, "bin", "esbuild"),
+    preactDir: join(baseDir, "node_modules", "preact"),
+  };
+}
+
+async function install(baseDir: string): Promise<void> {
+  if (isReady(baseDir)) return;
+
+  const os = platform();
+  const cpu = arch();
+  log.info(
+    { os, cpu, toolsVersion: TOOLS_VERSION },
+    "Installing compiler tools",
+  );
+
+  mkdirSync(baseDir, { recursive: true });
+
+  // Lock file to prevent concurrent cross-process installs
+  const lockPath = join(baseDir, ".downloading");
+  if (existsSync(lockPath)) {
+    try {
+      const lockPid = parseInt(readFileSync(lockPath, "utf-8").trim(), 10);
+      if (!isNaN(lockPid) && lockPid !== process.pid) {
+        try {
+          process.kill(lockPid, 0);
+          log.info(
+            { lockPid },
+            "Another process is installing compiler tools, waiting",
+          );
+          // Wait up to 60s for the other process to finish
+          for (let i = 0; i < 60; i++) {
+            await new Promise((r) => setTimeout(r, 1000));
+            if (isReady(baseDir)) return;
+          }
+          log.warn("Timed out waiting for other installer, proceeding");
+        } catch {
+          log.info({ lockPid }, "Cleaning up stale compiler tools lock");
+        }
+      }
+    } catch {
+      // Can't read lock, proceed
+    }
+  }
+
+  writeFileSync(lockPath, String(process.pid));
+
+  const tmpDir = join(baseDir, `.installing-${Date.now()}`);
+  mkdirSync(tmpDir, { recursive: true });
+
+  try {
+    // Determine esbuild platform package name
+    const esbuildPlatform =
+      os === "darwin"
+        ? cpu === "arm64"
+          ? "darwin-arm64"
+          : "darwin-x64"
+        : cpu === "arm64"
+          ? "linux-arm64"
+          : "linux-x64";
+
+    // Download esbuild binary + preact in parallel
+    await Promise.all([
+      downloadAndExtract(
+        npmTarballUrl(`@esbuild/${esbuildPlatform}`, ESBUILD_VERSION),
+        join(tmpDir, "esbuild-pkg"),
+      ),
+      downloadAndExtract(
+        npmTarballUrl("preact", PREACT_VERSION),
+        join(tmpDir, "node_modules", "preact"),
+      ),
+    ]);
+
+    // Move esbuild binary to bin/
+    const esbuildBinSrc = join(tmpDir, "esbuild-pkg", "bin", "esbuild");
+    const binDir = join(tmpDir, "bin");
+    mkdirSync(binDir, { recursive: true });
+    const { renameSync } = await import("node:fs");
+    renameSync(esbuildBinSrc, join(binDir, "esbuild"));
+    chmodSync(join(binDir, "esbuild"), 0o755);
+    rmSync(join(tmpDir, "esbuild-pkg"), { recursive: true, force: true });
+
+    // Write version manifest
+    const manifest: VersionManifest = {
+      toolsVersion: TOOLS_VERSION,
+      esbuildVersion: ESBUILD_VERSION,
+      preactVersion: PREACT_VERSION,
+      platform: os,
+      arch: cpu,
+      installedAt: new Date().toISOString(),
+    };
+    writeFileSync(
+      join(tmpDir, "version.json"),
+      JSON.stringify(manifest, null, 2) + "\n",
+    );
+
+    // Atomic swap: clear old install, move new files in
+    const { readdirSync } = await import("node:fs");
+    for (const entry of readdirSync(baseDir)) {
+      if (entry.startsWith(".") || entry === tmpDir.split("/").pop()) continue;
+      rmSync(join(baseDir, entry), { recursive: true, force: true });
+    }
+    for (const entry of readdirSync(tmpDir)) {
+      renameSync(join(tmpDir, entry), join(baseDir, entry));
+    }
+
+    log.info({ toolsVersion: TOOLS_VERSION }, "Compiler tools installed");
+  } catch (err) {
+    log.error({ err }, "Failed to install compiler tools");
+    throw err;
+  } finally {
+    rmSync(tmpDir, { recursive: true, force: true });
+    try {
+      rmSync(lockPath);
+    } catch {
+      /* ignore */
+    }
+  }
+}