@covibes/zeroshot 2.0.0 → 3.0.0

package/src/isolation-manager.js

@@ -9,10 +9,23 @@
  */
 
 const { spawn, execSync } = require('child_process');
+const { Worker } = require('worker_threads');
 const path = require('path');
 const os = require('os');
 const fs = require('fs');
 
+/**
+ * Escape a string for safe use in shell commands
+ * Prevents shell injection when passing dynamic values to execSync with shell: true
+ * @param {string} str - String to escape
+ * @returns {string} Shell-escaped string
+ */
+function escapeShell(str) {
+  // Replace single quotes with escaped version and wrap in single quotes
+  // This is the safest approach for shell escaping
+  return `'${str.replace(/'/g, "'\\''")}'`;
+}
+
 const DEFAULT_IMAGE = 'zeroshot-cluster-base';
 
 class IsolationManager {
@@ -21,6 +34,7 @@ class IsolationManager {
     this.containers = new Map(); // clusterId -> containerId
     this.isolatedDirs = new Map(); // clusterId -> { path, originalDir }
     this.clusterConfigDirs = new Map(); // clusterId -> configDirPath
+    this.worktrees = new Map(); // clusterId -> { path, branch, repoRoot }
   }
 
   /**
@@ -52,7 +66,7 @@ class IsolationManager {
    * @param {boolean} [config.reuseExistingWorkspace=false] - If true, reuse existing isolated workspace (for resume)
    * @returns {Promise<string>} Container ID
    */
-  createContainer(clusterId, config) {
+  async createContainer(clusterId, config) {
     const image = config.image || this.image;
     let workDir = config.workDir || process.cwd();
     const containerName = `zeroshot-cluster-${clusterId}`;
@@ -86,7 +100,7 @@ class IsolationManager {
         workDir = isolatedPath;
       } else {
         // Fresh start: create new isolated copy
-        const isolatedDir = this._createIsolatedCopy(clusterId, workDir);
+        const isolatedDir = await this._createIsolatedCopy(clusterId, workDir);
         this.isolatedDirs = this.isolatedDirs || new Map();
         this.isolatedDirs.set(clusterId, {
           path: isolatedDir,
@@ -177,54 +191,118 @@ class IsolationManager {
 
           // Install dependencies if package.json exists
           // This enables e2e tests and other npm-based tools to run
+          // OPTIMIZATION: Use pre-baked deps when possible (30-40% faster startup)
+          // See: GitHub issue #20
           try {
             console.log(`[IsolationManager] Checking for package.json in ${workDir}...`);
             if (fs.existsSync(path.join(workDir, 'package.json'))) {
-              console.log(`[IsolationManager] Installing npm dependencies in container...`);
-
-              // Retry npm install with exponential backoff (network issues are common)
-              const maxRetries = 3;
-              const baseDelay = 2000; // 2 seconds
-              let installResult = null;
-
-              for (let attempt = 1; attempt <= maxRetries; attempt++) {
-                try {
-                  installResult = await this.execInContainer(
+              // Check if node_modules already exists in container (pre-baked or previous run)
+              const checkResult = await this.execInContainer(
+                clusterId,
+                ['sh', '-c', 'test -d node_modules && test -f node_modules/.package-lock.json && echo "exists"'],
+                {}
+              );
+
+              if (checkResult.code === 0 && checkResult.stdout.trim() === 'exists') {
+                console.log(`[IsolationManager] ✓ Dependencies already installed (skipping npm install)`);
+              } else {
+                // Check if npm is available in container
+                const npmCheck = await this.execInContainer(clusterId, ['which', 'npm'], {});
+                if (npmCheck.code !== 0) {
+                  console.log(`[IsolationManager] npm not available in container, skipping dependency install`);
+                } else {
+                  // Issue #20: Try to use pre-baked dependencies first
+                  // Check if pre-baked deps exist and can satisfy project requirements
+                  const preBakeCheck = await this.execInContainer(
                     clusterId,
-                    ['sh', '-c', 'npm_config_engine_strict=false npm install --no-audit --no-fund'],
+                    ['sh', '-c', 'test -d /pre-baked-deps/node_modules && echo "exists"'],
                     {}
                   );
 
-                  if (installResult.code === 0) {
-                    console.log(`[IsolationManager] ✓ Dependencies installed`);
-                    break; // Success - exit retry loop
-                  }
+                  if (preBakeCheck.code === 0 && preBakeCheck.stdout.trim() === 'exists') {
+                    console.log(`[IsolationManager] Checking if pre-baked deps satisfy requirements...`);
 
-                  // Failed - retry if not last attempt
-                  // Use stderr if available, otherwise stdout (npm writes some errors to stdout)
-                  const errorOutput = (installResult.stderr || installResult.stdout || '').slice(0, 500);
-                  if (attempt < maxRetries) {
-                    const delay = baseDelay * Math.pow(2, attempt - 1);
-                    console.warn(
-                      `[IsolationManager] ⚠️ npm install failed (attempt ${attempt}/${maxRetries}), retrying in ${delay}ms...`
-                    );
-                    console.warn(`[IsolationManager] Error: ${errorOutput}`);
-                    await new Promise((_resolve) => setTimeout(_resolve, delay));
-                  } else {
-                    console.warn(
-                      `[IsolationManager] ⚠️ npm install failed after ${maxRetries} attempts (non-fatal): ${errorOutput}`
+                    // Copy pre-baked deps, then run npm install to add any missing
+                    // This is faster than full npm install: copy is ~2s, npm install adds ~5-10s for missing
+                    const copyResult = await this.execInContainer(
+                      clusterId,
+                      ['sh', '-c', 'cp -rn /pre-baked-deps/node_modules . 2>/dev/null || true'],
+                      {}
                     );
-                  }
-                } catch (execErr) {
-                  if (attempt < maxRetries) {
-                    const delay = baseDelay * Math.pow(2, attempt - 1);
-                    console.warn(
-                      `[IsolationManager] ⚠️ npm install execution error (attempt ${attempt}/${maxRetries}), retrying in ${delay}ms...`
-                    );
-                    console.warn(`[IsolationManager] Error: ${execErr.message}`);
-                    await new Promise((_resolve) => setTimeout(_resolve, delay));
+
+                    if (copyResult.code === 0) {
+                      console.log(`[IsolationManager] ✓ Copied pre-baked dependencies`);
+
+                      // Run npm install to add any missing deps (much faster with pre-baked base)
+                      const installResult = await this.execInContainer(
+                        clusterId,
+                        ['sh', '-c', 'npm_config_engine_strict=false npm install --no-audit --no-fund --prefer-offline'],
+                        {}
+                      );
+
+                      if (installResult.code === 0) {
+                        console.log(`[IsolationManager] ✓ Dependencies installed (pre-baked + incremental)`);
+                      } else {
+                        // Fallback: full install (pre-baked copy may have caused issues)
+                        console.warn(`[IsolationManager] Incremental install failed, falling back to full install`);
+                        await this.execInContainer(
+                          clusterId,
+                          ['sh', '-c', 'rm -rf node_modules && npm_config_engine_strict=false npm install --no-audit --no-fund'],
+                          {}
+                        );
+                        console.log(`[IsolationManager] ✓ Dependencies installed (full fallback)`);
+                      }
+                    }
                   } else {
-                    throw execErr; // Re-throw on last attempt
+                    // No pre-baked deps, full npm install with retries
+                    console.log(`[IsolationManager] Installing npm dependencies in container...`);
+
+                    // Retry npm install with exponential backoff (network issues are common)
+                    const maxRetries = 3;
+                    const baseDelay = 2000; // 2 seconds
+                    let installResult = null;
+
+                    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+                      try {
+                        installResult = await this.execInContainer(
+                          clusterId,
+                          ['sh', '-c', 'npm_config_engine_strict=false npm install --no-audit --no-fund'],
+                          {}
+                        );
+
+                        if (installResult.code === 0) {
+                          console.log(`[IsolationManager] ✓ Dependencies installed`);
+                          break; // Success - exit retry loop
+                        }
+
+                        // Failed - retry if not last attempt
+                        // Use stderr if available, otherwise stdout (npm writes some errors to stdout)
+                        const errorOutput = (installResult.stderr || installResult.stdout || '').slice(0, 500);
+                        if (attempt < maxRetries) {
+                          const delay = baseDelay * Math.pow(2, attempt - 1);
+                          console.warn(
+                            `[IsolationManager] ⚠️ npm install failed (attempt ${attempt}/${maxRetries}), retrying in ${delay}ms...`
+                          );
+                          console.warn(`[IsolationManager] Error: ${errorOutput}`);
+                          await new Promise((_resolve) => setTimeout(_resolve, delay));
+                        } else {
+                          console.warn(
+                            `[IsolationManager] ⚠️ npm install failed after ${maxRetries} attempts (non-fatal): ${errorOutput}`
+                          );
+                        }
+                      } catch (execErr) {
+                        if (attempt < maxRetries) {
+                          const delay = baseDelay * Math.pow(2, attempt - 1);
+                          console.warn(
+                            `[IsolationManager] ⚠️ npm install execution error (attempt ${attempt}/${maxRetries}), retrying in ${delay}ms...`
+                          );
+                          console.warn(`[IsolationManager] Error: ${execErr.message}`);
+                          await new Promise((_resolve) => setTimeout(_resolve, delay));
+                        } else {
+                          throw execErr; // Re-throw on last attempt
+                        }
+                      }
+                    }
                   }
                 }
               }
@@ -447,9 +525,9 @@ class IsolationManager {
    * @private
    * @param {string} clusterId - Cluster ID
    * @param {string} sourceDir - Source directory to copy
-   * @returns {string} Path to isolated directory
+   * @returns {Promise<string>} Path to isolated directory
    */
-  _createIsolatedCopy(clusterId, sourceDir) {
+  async _createIsolatedCopy(clusterId, sourceDir) {
     const isolatedPath = path.join(os.tmpdir(), 'zeroshot-isolated', clusterId);
 
     // Clean up existing dir
@@ -461,7 +539,7 @@ class IsolationManager {
     fs.mkdirSync(isolatedPath, { recursive: true });
 
     // Copy files (excluding .git and common build artifacts)
-    this._copyDirExcluding(sourceDir, isolatedPath, [
+    await this._copyDirExcluding(sourceDir, isolatedPath, [
       '.git',
       'node_modules',
       '.next',
@@ -495,13 +573,15 @@ class IsolationManager {
       // No remote configured in source
     }
 
-    // Initialize fresh git repo
-    execSync('git init', { cwd: isolatedPath, stdio: 'pipe' });
+    // Initialize fresh git repo with all setup in a single batched command
+    // This reduces ~500ms overhead (5 execSync calls @ ~100ms each) to ~100ms (1 call)
+    // Issue #22: Batch git operations for 5-10% startup reduction
+    const branchName = `zeroshot/${clusterId}`;
 
-    // Add remote if source had one (needed for git push / PR creation)
-    // Inject gh token into URL for authentication inside container
+    // Build authenticated remote URL if source had one (needed for git push / PR creation)
+    let authRemoteUrl = null;
     if (remoteUrl) {
-      let authRemoteUrl = remoteUrl;
+      authRemoteUrl = remoteUrl;
       const token = this._getGhToken();
       if (token && remoteUrl.startsWith('https://github.com/')) {
         // Convert https://github.com/org/repo.git to https://x-access-token:TOKEN@github.com/org/repo.git
@@ -510,81 +590,182 @@ class IsolationManager {
           `https://x-access-token:${token}@github.com/`
         );
       }
-      execSync(`git remote add origin "${authRemoteUrl}"`, {
-        cwd: isolatedPath,
-        stdio: 'pipe',
-      });
     }
 
-    execSync('git add -A', { cwd: isolatedPath, stdio: 'pipe' });
-
-    try {
-      execSync('git commit -m "Initial commit (isolated copy)"', {
-        cwd: isolatedPath,
-        stdio: 'pipe',
-      });
-    } catch {
-      // May fail if nothing to commit (empty dir)
-    }
-
-    // Create feature branch for work
-    const branchName = `zeroshot/${clusterId}`;
-    execSync(`git checkout -b "${branchName}"`, {
+    // Batch all git operations into a single shell command
+    // Using --allow-empty on commit to handle edge case of empty directories
+    const gitCommands = [
+      'git init',
+      authRemoteUrl ? `git remote add origin ${escapeShell(authRemoteUrl)}` : null,
+      'git add -A',
+      'git commit -m "Initial commit (isolated copy)" --allow-empty',
+      `git checkout -b ${escapeShell(branchName)}`,
+    ]
+      .filter(Boolean)
+      .join(' && ');
+
+    execSync(gitCommands, {
       cwd: isolatedPath,
       stdio: 'pipe',
+      shell: '/bin/bash',
     });
 
     return isolatedPath;
   }
 
   /**
-   * Copy directory excluding certain paths
+   * Copy directory excluding certain paths using parallel worker threads
    * Supports exact matches and glob patterns (*.ext)
+   *
+   * Performance optimization for large repos (10k+ files):
+   * - Phase 1: Collect all files async (non-blocking traversal)
+   * - Phase 2: Create directory structure (must be sequential)
+   * - Phase 3: Copy files in parallel using worker threads
+   *
    * @private
+   * @param {string} src - Source directory
+   * @param {string} dest - Destination directory
+   * @param {string[]} exclude - Patterns to exclude
+   * @returns {Promise<void>}
    */
-  _copyDirExcluding(src, dest, exclude) {
-    const entries = fs.readdirSync(src, { withFileTypes: true });
-
-    for (const entry of entries) {
-      // Check exclusions (exact match or glob pattern)
-      const shouldExclude = exclude.some((pattern) => {
-        if (pattern.startsWith('*.')) {
-          return entry.name.endsWith(pattern.slice(1));
+  async _copyDirExcluding(src, dest, exclude) {
+    // Phase 1: Collect all files and directories
+    const files = [];
+    const directories = new Set();
+
+    const collectFiles = (currentSrc, relativePath = '') => {
+      let entries;
+      try {
+        entries = fs.readdirSync(currentSrc, { withFileTypes: true });
+      } catch (err) {
+        if (err.code === 'EACCES' || err.code === 'EPERM' || err.code === 'ENOENT') {
+          return;
         }
-        return entry.name === pattern;
-      });
-      if (shouldExclude) continue;
+        throw err;
+      }
+
+      for (const entry of entries) {
+        // Check exclusions (exact match or glob pattern)
+        const shouldExclude = exclude.some((pattern) => {
+          if (pattern.startsWith('*.')) {
+            return entry.name.endsWith(pattern.slice(1));
+          }
+          return entry.name === pattern;
+        });
+        if (shouldExclude) continue;
 
-      const srcPath = path.join(src, entry.name);
-      const destPath = path.join(dest, entry.name);
+        const srcPath = path.join(currentSrc, entry.name);
+        const relPath = relativePath ? path.join(relativePath, entry.name) : entry.name;
 
-      try {
-        // Handle symlinks: resolve to actual target and copy appropriately
-        // This avoids EISDIR errors when symlink points to directory
-        if (entry.isSymbolicLink()) {
-          // Get the actual target stats (follows the symlink)
-          const targetStats = fs.statSync(srcPath);
-          if (targetStats.isDirectory()) {
-            fs.mkdirSync(destPath, { recursive: true });
-            this._copyDirExcluding(srcPath, destPath, exclude);
+        try {
+          // Handle symlinks: resolve to actual target
+          if (entry.isSymbolicLink()) {
+            const targetStats = fs.statSync(srcPath);
+            if (targetStats.isDirectory()) {
+              directories.add(relPath);
+              collectFiles(srcPath, relPath);
+            } else {
+              files.push(relPath);
+              // Ensure parent directory is tracked
+              if (relativePath) directories.add(relativePath);
+            }
+          } else if (entry.isDirectory()) {
+            directories.add(relPath);
+            collectFiles(srcPath, relPath);
           } else {
-            fs.copyFileSync(srcPath, destPath);
+            files.push(relPath);
+            // Ensure parent directory is tracked
+            if (relativePath) directories.add(relativePath);
           }
-        } else if (entry.isDirectory()) {
-          fs.mkdirSync(destPath, { recursive: true });
-          this._copyDirExcluding(srcPath, destPath, exclude);
-        } else {
-          fs.copyFileSync(srcPath, destPath);
+        } catch (err) {
+          if (err.code === 'EACCES' || err.code === 'EPERM' || err.code === 'ENOENT') {
+            continue;
+          }
+          throw err;
         }
+      }
+    };
+
+    collectFiles(src);
+
+    // Phase 2: Create directory structure (sequential - must exist before file copy)
+    // Sort directories by depth to ensure parents are created before children
+    const sortedDirs = Array.from(directories).sort((a, b) => {
+      const depthA = a.split(path.sep).length;
+      const depthB = b.split(path.sep).length;
+      return depthA - depthB;
+    });
+
+    for (const dir of sortedDirs) {
+      const destDir = path.join(dest, dir);
+      try {
+        fs.mkdirSync(destDir, { recursive: true });
       } catch (err) {
-        // Skip files we can't copy (permission denied, broken symlinks, etc.)
-        // These are usually cache/temp files that aren't needed
-        if (err.code === 'EACCES' || err.code === 'EPERM' || err.code === 'ENOENT') {
-          continue;
+        if (err.code !== 'EEXIST') {
+          throw err;
+        }
+      }
+    }
+
+    // Phase 3: Copy files in parallel using worker threads
+    // For small file counts (<100), use synchronous copy (worker overhead not worth it)
+    if (files.length < 100) {
+      for (const relPath of files) {
+        const srcPath = path.join(src, relPath);
+        const destPath = path.join(dest, relPath);
+        try {
+          fs.copyFileSync(srcPath, destPath);
+        } catch (err) {
+          if (err.code !== 'EACCES' && err.code !== 'EPERM' && err.code !== 'ENOENT') {
+            throw err;
+          }
         }
-        throw err; // Re-throw other errors
       }
+      return;
+    }
+
+    // Use worker threads for larger file counts
+    const numWorkers = Math.min(4, os.cpus().length);
+    const chunkSize = Math.ceil(files.length / numWorkers);
+    const workerPath = path.join(__dirname, 'copy-worker.js');
+
+    // Split files into chunks for workers
+    const chunks = [];
+    for (let i = 0; i < files.length; i += chunkSize) {
+      chunks.push(files.slice(i, i + chunkSize));
     }
+
+    // Spawn workers and wait for completion
+    const workerPromises = chunks.map((chunk) => {
+      return new Promise((resolve, reject) => {
+        const worker = new Worker(workerPath, {
+          workerData: {
+            files: chunk,
+            sourceBase: src,
+            destBase: dest,
+          },
+        });
+
+        worker.on('message', (result) => {
+          resolve(result);
+        });
+
+        worker.on('error', (err) => {
+          reject(err);
+        });
+
+        worker.on('exit', (code) => {
+          if (code !== 0) {
+            reject(new Error(`Worker exited with code ${code}`));
+          }
+        });
+      });
+    });
+
+    // Wait for all workers to complete (proper async/await - no busy-wait!)
+    // FIX: Previous version used busy-wait which blocked the event loop,
+    // preventing worker thread messages from being processed (timeout bug)
+    await Promise.all(workerPromises);
   }
 
   /**
@@ -826,7 +1007,7 @@ class IsolationManager {
    */
   _isContainerRunning(containerId) {
     try {
-      const result = execSync(`docker inspect -f '{{.State.Running}}' ${containerId} 2>/dev/null`, {
+      const result = execSync(`docker inspect -f '{{.State.Running}}' ${escapeShell(containerId)} 2>/dev/null`, {
         encoding: 'utf8',
       });
       return result.trim() === 'true';
@@ -841,7 +1022,7 @@ class IsolationManager {
    */
   _removeContainerByName(name) {
     try {
-      execSync(`docker rm -f ${name} 2>/dev/null`, { encoding: 'utf8' });
+      execSync(`docker rm -f ${escapeShell(name)} 2>/dev/null`, { encoding: 'utf8' });
     } catch {
       // Ignore - container doesn't exist
     }
@@ -867,7 +1048,7 @@ class IsolationManager {
    */
   static imageExists(image = DEFAULT_IMAGE) {
     try {
-      execSync(`docker image inspect ${image} 2>/dev/null`, {
+      execSync(`docker image inspect ${escapeShell(image)} 2>/dev/null`, {
         encoding: 'utf8',
         stdio: 'pipe',
       });
@@ -900,7 +1081,7 @@ class IsolationManager {
       try {
         // CRITICAL: Run from repo root so build context includes package.json and src/
         // Use -f flag to specify Dockerfile location
-        execSync(`docker build -f docker/zeroshot-cluster/Dockerfile -t ${image} .`, {
+        execSync(`docker build -f docker/zeroshot-cluster/Dockerfile -t ${escapeShell(image)} .`, {
           cwd: repoRoot,
           encoding: 'utf8',
           stdio: 'inherit',
@@ -980,14 +1161,52 @@ class IsolationManager {
     }
   }
 
+  /**
+   * Create worktree-based isolation for a cluster (lightweight alternative to Docker)
+   * Creates a git worktree at /tmp/zeroshot-worktrees/{clusterId}
+   * @param {string} clusterId - Cluster ID
+   * @param {string} workDir - Original working directory (must be a git repo)
+   * @returns {{ path: string, branch: string, repoRoot: string }}
+   */
+  createWorktreeIsolation(clusterId, workDir) {
+    if (!this._isGitRepo(workDir)) {
+      throw new Error(`Worktree isolation requires a git repository. ${workDir} is not a git repo.`);
+    }
+
+    const worktreeInfo = this.createWorktree(clusterId, workDir);
+    this.worktrees.set(clusterId, worktreeInfo);
+
+    console.log(`[IsolationManager] Created worktree isolation at ${worktreeInfo.path}`);
+    console.log(`[IsolationManager] Branch: ${worktreeInfo.branch}`);
+
+    return worktreeInfo;
+  }
+
+  /**
+   * Clean up worktree isolation for a cluster
+   * @param {string} clusterId - Cluster ID
+   * @param {object} [options] - Cleanup options
+   * @param {boolean} [options.preserveBranch=true] - Keep the branch after removing worktree
+   */
+  cleanupWorktreeIsolation(clusterId, options = {}) {
+    const worktreeInfo = this.worktrees.get(clusterId);
+    if (!worktreeInfo) {
+      return; // No worktree to clean up
+    }
+
+    this.removeWorktree(worktreeInfo, options);
+    this.worktrees.delete(clusterId);
+
+    console.log(`[IsolationManager] Cleaned up worktree isolation for ${clusterId}`);
+  }
+
   /**
    * Create a git worktree for isolated work
-   * @private
    * @param {string} clusterId - Cluster ID (used as branch name)
    * @param {string} workDir - Original working directory
    * @returns {{ path: string, branch: string, repoRoot: string }}
    */
-  _createWorktree(clusterId, workDir) {
+  createWorktree(clusterId, workDir) {
     const repoRoot = this._getGitRoot(workDir);
     if (!repoRoot) {
       throw new Error(`Cannot find git root for ${workDir}`);
@@ -1043,10 +1262,11 @@ class IsolationManager {
 
   /**
    * Remove a git worktree
-   * @private
    * @param {{ path: string, branch: string, repoRoot: string }} worktreeInfo
+   * @param {object} [options] - Removal options
+   * @param {boolean} [options.deleteBranch=false] - Also delete the branch
    */
-  _removeWorktree(worktreeInfo) {
+  removeWorktree(worktreeInfo, _options = {}) {
     try {
       // Remove the worktree
       execSync(`git worktree remove --force "${worktreeInfo.path}" 2>/dev/null`, {