agentsys 5.0.0 → 5.0.2

package/plugins/deslop/lib/repo-map/runner.js

@@ -6,7 +6,7 @@
 
 'use strict';
 
-const { execFileSync, spawnSync } = require('child_process');
+const { execFileSync, spawnSync, spawn } = require('child_process');
 const path = require('path');
 const fs = require('fs');
 const fsPromises = require('fs').promises;
@@ -15,6 +15,7 @@ const crypto = require('crypto');
 const installer = require('./installer');
 const queries = require('./queries');
 const slopAnalyzers = require('../patterns/slop-analyzers');
+const { runWithConcurrency } = require('./concurrency');
 
 // Language file extensions mapping
 const LANGUAGE_EXTENSIONS = {
@@ -33,6 +34,7 @@ const EXCLUDE_DIRS = Array.from(new Set([
 ]));
 
 const AST_GREP_BATCH_SIZE = 100;
+const AST_GREP_CONCURRENCY = 4;
 const LANGUAGE_EXTENSION_SCAN_LIMIT = 500;
 const FILE_READ_BATCH_SIZE = 50; // Concurrent file reads
 
@@ -253,8 +255,12 @@ async function fullScan(basePath, languages, options = {}) {
           const pattern = typeof patternDef === 'string' ? patternDef : patternDef.pattern;
           if (!pattern) continue;
 
-          for (const chunk of chunks) {
-            const matches = runAstGrepPattern(cmd, pattern, sgLang, basePath, chunk);
+          const matchesByChunk = await runAstGrepPatternBatches(cmd, pattern, sgLang, basePath, chunks, {
+            onError: (error) => map.stats.errors.push(error),
+            concurrency: options.astGrepConcurrency
+          });
+
+          for (const matches of matchesByChunk) {
             for (const match of matches) {
               const matchedPath = normalizeMatchPath(match.file, basePath);
               if (!matchedPath) continue;
@@ -495,6 +501,127 @@ function chunkArray(items, size) {
   return chunks;
 }
 
+function truncatePattern(pattern, max = 120) {
+  if (typeof pattern !== 'string') return '';
+  if (pattern.length <= max) return pattern;
+  return `${pattern.slice(0, max - 3)}...`;
+}
+
+function buildAstGrepError({ reason, pattern, lang, filePaths, basePath, stderr }) {
+  const batchLabel = Array.isArray(filePaths) && filePaths.length === 1
+    ? normalizeMatchPath(filePaths[0], basePath)
+    : '[batch]';
+
+  const details = stderr && String(stderr).trim()
+    ? ` (${String(stderr).trim()})`
+    : '';
+
+  return {
+    file: batchLabel,
+    error: `ast-grep ${reason} for ${lang}${details}`,
+    pattern: truncatePattern(pattern)
+  };
+}
+
+function runAstGrepPatternAsync(cmd, pattern, lang, basePath, filePaths, options = {}) {
+  if (!pattern || !filePaths || filePaths.length === 0) {
+    return Promise.resolve([]);
+  }
+
+  return new Promise((resolve) => {
+    const child = spawn(cmd, [
+      'run',
+      '--pattern', pattern,
+      '--lang', lang,
+      '--json=stream',
+      ...filePaths
+    ], {
+      cwd: basePath,
+      windowsHide: true,
+      stdio: ['ignore', 'pipe', 'pipe']
+    });
+
+    let stdout = '';
+    let stderr = '';
+    let settled = false;
+
+    const timeoutHandle = setTimeout(() => {
+      if (settled) return;
+      settled = true;
+      child.kill();
+      if (typeof options.onError === 'function') {
+        options.onError(buildAstGrepError({
+          reason: 'timed out after 300000ms',
+          pattern,
+          lang,
+          filePaths,
+          basePath,
+          stderr
+        }));
+      }
+      resolve([]);
+    }, 300000);
+
+    child.stdout.on('data', (chunk) => {
+      stdout += chunk.toString();
+    });
+
+    child.stderr.on('data', (chunk) => {
+      stderr += chunk.toString();
+    });
+
+    child.on('error', (error) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timeoutHandle);
+      if (typeof options.onError === 'function') {
+        options.onError(buildAstGrepError({
+          reason: 'execution failed',
+          pattern,
+          lang,
+          filePaths,
+          basePath,
+          stderr: error.message
+        }));
+      }
+      resolve([]);
+    });
+
+    child.on('close', (code) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timeoutHandle);
+
+      if (typeof code === 'number' && code > 1) {
+        if (typeof options.onError === 'function') {
+          options.onError(buildAstGrepError({
+            reason: `returned exit code ${code}`,
+            pattern,
+            lang,
+            filePaths,
+            basePath,
+            stderr
+          }));
+        }
+        resolve([]);
+        return;
+      }
+
+      resolve(parseNdjson(stdout));
+    });
+  });
+}
+
+async function runAstGrepPatternBatches(cmd, pattern, lang, basePath, chunks, options = {}) {
+  const concurrency = Number.isFinite(options.concurrency)
+    ? Math.max(1, Math.floor(options.concurrency))
+    : AST_GREP_CONCURRENCY;
+
+  return runWithConcurrency(chunks, concurrency, async (chunk) => {
+    return runAstGrepPatternAsync(cmd, pattern, lang, basePath, chunk, options);
+  });
+}
+
 function normalizeMatchPath(matchFile, basePath) {
   if (!matchFile) return null;
   const absolutePath = path.isAbsolute(matchFile) ? matchFile : path.join(basePath, matchFile);
@@ -513,7 +640,7 @@ function addSymbolToMap(map, name, match, kind, extra = {}) {
   }
 }
 
-function runAstGrepPattern(cmd, pattern, lang, basePath, filePaths) {
+function runAstGrepPattern(cmd, pattern, lang, basePath, filePaths, options = {}) {
   if (!pattern || !filePaths || filePaths.length === 0) return [];
 
   try {
@@ -532,15 +659,45 @@ function runAstGrepPattern(cmd, pattern, lang, basePath, filePaths) {
     });
 
     if (result.error) {
+      if (typeof options.onError === 'function') {
+        options.onError(buildAstGrepError({
+          reason: 'execution failed',
+          pattern,
+          lang,
+          filePaths,
+          basePath,
+          stderr: result.error.message
+        }));
+      }
       return [];
     }
 
     if (typeof result.status === 'number' && result.status > 1) {
+      if (typeof options.onError === 'function') {
+        options.onError(buildAstGrepError({
+          reason: `returned exit code ${result.status}`,
+          pattern,
+          lang,
+          filePaths,
+          basePath,
+          stderr: result.stderr
+        }));
+      }
       return [];
     }
 
     return parseNdjson(result.stdout);
-  } catch {
+  } catch (error) {
+    if (typeof options.onError === 'function') {
+      options.onError(buildAstGrepError({
+        reason: 'threw an exception',
+        pattern,
+        lang,
+        filePaths,
+        basePath,
+        stderr: error.message
+      }));
+    }
     return [];
   }
 }
@@ -555,7 +712,7 @@ function runAstGrepPattern(cmd, pattern, lang, basePath, filePaths) {
  * @param {string} content - File content
  * @returns {Object} - Extracted symbols
  */
-function extractSymbols(cmd, file, language, langQueries, basePath, content) {
+function extractSymbols(cmd, file, language, langQueries, basePath, content, options = {}) {
   const symbols = {
     exports: [],
     functions: [],
@@ -588,7 +745,7 @@ function extractSymbols(cmd, file, language, langQueries, basePath, content) {
     if (!patterns) return;
     for (const patternDef of patterns) {
       const pattern = patternDef.pattern || patternDef;
-      const results = runAstGrep(cmd, file, pattern, sgLang, basePath);
+      const results = runAstGrep(cmd, file, pattern, sgLang, basePath, options);
       for (const match of results) {
         const names = extractNamesFromMatch(match, patternDef);
         for (const name of names) {
@@ -640,7 +797,7 @@ function extractSymbols(cmd, file, language, langQueries, basePath, content) {
  * @param {string} basePath - Repository root (for cwd)
  * @returns {Array} - Extracted imports
  */
-function extractImports(cmd, file, language, langQueries, basePath) {
+function extractImports(cmd, file, language, langQueries, basePath, options = {}) {
   const imports = [];
 
   if (!langQueries.imports) return imports;
@@ -650,7 +807,7 @@ function extractImports(cmd, file, language, langQueries, basePath) {
 
   for (const patternDef of langQueries.imports) {
     const pattern = patternDef.pattern || patternDef;
-    const results = runAstGrep(cmd, file, pattern, sgLang, basePath);
+    const results = runAstGrep(cmd, file, pattern, sgLang, basePath, options);
     for (const match of results) {
       const sourceResult = extractSourceFromMatch(match, patternDef);
       const sources = Array.isArray(sourceResult) ? sourceResult : [sourceResult];
@@ -680,7 +837,7 @@ function extractImports(cmd, file, language, langQueries, basePath) {
  * @param {string} basePath - Working directory
  * @returns {Array} - Match results
  */
-function runAstGrep(cmd, file, pattern, lang, basePath) {
+function runAstGrep(cmd, file, pattern, lang, basePath, options = {}) {
   try {
     const result = spawnSync(cmd, [
       'run',
@@ -697,16 +854,46 @@ function runAstGrep(cmd, file, pattern, lang, basePath) {
     });
 
     if (result.error) {
+      if (typeof options.onError === 'function') {
+        options.onError(buildAstGrepError({
+          reason: 'execution failed',
+          pattern,
+          lang,
+          filePaths: [file],
+          basePath,
+          stderr: result.error.message
+        }));
+      }
       return [];
     }
 
     // ast-grep exits with 1 when no matches
     if (typeof result.status === 'number' && result.status > 1) {
+      if (typeof options.onError === 'function') {
+        options.onError(buildAstGrepError({
+          reason: `returned exit code ${result.status}`,
+          pattern,
+          lang,
+          filePaths: [file],
+          basePath,
+          stderr: result.stderr
+        }));
+      }
       return [];
     }
 
     return parseNdjson(result.stdout);
-  } catch {
+  } catch (error) {
+    if (typeof options.onError === 'function') {
+      options.onError(buildAstGrepError({
+        reason: 'threw an exception',
+        pattern,
+        lang,
+        filePaths: [file],
+        basePath,
+        stderr: error.message
+      }));
+    }
     return [];
   }
 }
@@ -1070,7 +1257,7 @@ function detectProjectType(languages) {
  * @param {string} basePath - Repository root
  * @returns {Object|null} - File data or null if failed
  */
-function scanSingleFile(cmd, file, basePath) {
+function scanSingleFile(cmd, file, basePath, options = {}) {
   const ext = path.extname(file).toLowerCase();
 
   // Find language for this extension
@@ -1091,8 +1278,8 @@ function scanSingleFile(cmd, file, basePath) {
     const content = fs.readFileSync(file, 'utf8');
     const hash = crypto.createHash('sha256').update(content).digest('hex').slice(0, 16);
 
-    const symbols = extractSymbols(cmd, file, language, langQueries, basePath, content);
-    const imports = extractImports(cmd, file, language, langQueries, basePath);
+    const symbols = extractSymbols(cmd, file, language, langQueries, basePath, content, options);
+    const imports = extractImports(cmd, file, language, langQueries, basePath, options);
 
     return {
       hash,
@@ -1101,7 +1288,13 @@ function scanSingleFile(cmd, file, basePath) {
       symbols,
       imports
     };
-  } catch {
+  } catch (error) {
+    if (typeof options.onError === 'function') {
+      options.onError({
+        file: normalizeMatchPath(file, basePath) || file,
+        error: `Failed to scan file: ${error.message}`
+      });
+    }
     return null;
   }
 }
@@ -1114,7 +1307,7 @@ function scanSingleFile(cmd, file, basePath) {
  * @param {string} basePath - Repository root
  * @returns {Promise<Object|null>} - File data or null if failed
  */
-async function scanSingleFileAsync(cmd, file, basePath) {
+async function scanSingleFileAsync(cmd, file, basePath, options = {}) {
   const ext = path.extname(file).toLowerCase();
 
   // Find language for this extension
@@ -1135,8 +1328,8 @@ async function scanSingleFileAsync(cmd, file, basePath) {
     const content = await fsPromises.readFile(file, 'utf8');
     const hash = crypto.createHash('sha256').update(content).digest('hex').slice(0, 16);
 
-    const symbols = extractSymbols(cmd, file, language, langQueries, basePath, content);
-    const imports = extractImports(cmd, file, language, langQueries, basePath);
+    const symbols = extractSymbols(cmd, file, language, langQueries, basePath, content, options);
+    const imports = extractImports(cmd, file, language, langQueries, basePath, options);
 
     return {
       hash,
@@ -1145,7 +1338,13 @@ async function scanSingleFileAsync(cmd, file, basePath) {
       symbols,
       imports
     };
-  } catch {
+  } catch (error) {
+    if (typeof options.onError === 'function') {
+      options.onError({
+        file: normalizeMatchPath(file, basePath) || file,
+        error: `Failed to scan file: ${error.message}`
+      });
+    }
     return null;
   }
 }

package/plugins/deslop/lib/repo-map/updater.js

@@ -6,7 +6,6 @@
 
 'use strict';
 
-const fs = require('fs');
 const fsPromises = require('fs').promises;
 const path = require('path');
 const { execFileSync } = require('child_process');
@@ -14,6 +13,15 @@ const { execFileSync } = require('child_process');
 const runner = require('./runner');
 const cache = require('./cache');
 const installer = require('./installer');
+const { runWithConcurrency } = require('./concurrency');
+
+const SCAN_CONCURRENCY = 8;
+const SCANNABLE_EXTENSIONS = new Set(Object.values(runner.LANGUAGE_EXTENSIONS).flat());
+
+function isScannableFile(filePath) {
+  const ext = path.extname(filePath).toLowerCase();
+  return SCANNABLE_EXTENSIONS.has(ext);
+}
 
 /**
  * Perform incremental update based on git diff
@@ -47,6 +55,10 @@ async function incrementalUpdate(basePath, map) {
       needsFullRebuild: true
     };
   }
+  map.stats = map.stats || {};
+  if (!Array.isArray(map.stats.errors)) {
+    map.stats.errors = [];
+  }
   if (map.docs) {
     delete map.docs;
   }
@@ -118,19 +130,46 @@ async function incrementalUpdate(basePath, map) {
     })
   );
 
-  // Process files that exist
-  for (const { file, fullPath, exists } of existenceChecks) {
-    if (!exists) continue;
+  // Process files that exist with bounded concurrency
+  const scanTargets = existenceChecks.filter(({ file, exists }) => exists && isScannableFile(file));
+  const scanResults = await runWithConcurrency(scanTargets, SCAN_CONCURRENCY, async ({ file, fullPath }) => {
+    const astErrors = [];
+    const fileData = await runner.scanSingleFileAsync(installed.command, fullPath, basePath, {
+      onError: (error) => astErrors.push(error)
+    });
+    return { file, fileData, astErrors };
+  });
 
-    const fileData = await runner.scanSingleFileAsync(installed.command, fullPath, basePath);
-    if (fileData) {
-      map.files[file] = fileData;
-      if (fileData.imports && fileData.imports.length > 0) {
-        map.dependencies[file] = Array.from(new Set(fileData.imports.map(imp => imp.source)));
-      } else {
-        delete map.dependencies[file];
+  const scanFailures = [];
+  for (const result of scanResults) {
+    if (!result) continue;
+
+    if (result.astErrors.length > 0) {
+      map.stats.errors.push(...result.astErrors);
+    }
+
+    if (!result.fileData) {
+      if (result.astErrors.length > 0) {
+        scanFailures.push(result.file);
       }
+      continue;
     }
+
+    map.files[result.file] = result.fileData;
+    if (result.fileData.imports && result.fileData.imports.length > 0) {
+      map.dependencies[result.file] = Array.from(new Set(result.fileData.imports.map(imp => imp.source)));
+    } else {
+      delete map.dependencies[result.file];
+    }
+  }
+
+  if (scanFailures.length > 0) {
+    return {
+      success: false,
+      error: `Failed to rescan ${scanFailures.length} file(s) during incremental update`,
+      needsFullRebuild: true,
+      failedFiles: scanFailures
+    };
   }
 
   // Recalculate stats
@@ -163,6 +202,10 @@ async function incrementalUpdate(basePath, map) {
 async function updateWithoutGit(basePath, map, cmd) {
   const currentFiles = new Set();
   const languages = map.project?.languages || [];
+  map.stats = map.stats || {};
+  if (!Array.isArray(map.stats.errors)) {
+    map.stats.errors = [];
+  }
   if (map.docs) {
     delete map.docs;
   }
@@ -204,31 +247,76 @@ async function updateWithoutGit(basePath, map, cmd) {
   }
 
   // Process existing files for modifications (async file reads)
-  for (const file of filesToCheck) {
+  const checkResults = await runWithConcurrency(filesToCheck, SCAN_CONCURRENCY, async (file) => {
     const fullPath = path.join(basePath, file);
-    const fileData = await runner.scanSingleFileAsync(cmd, fullPath, basePath);
-    if (fileData && fileData.hash !== map.files[file].hash) {
-      map.files[file] = fileData;
-      if (fileData.imports && fileData.imports.length > 0) {
-        map.dependencies[file] = Array.from(new Set(fileData.imports.map(imp => imp.source)));
+    const astErrors = [];
+    const fileData = await runner.scanSingleFileAsync(cmd, fullPath, basePath, {
+      onError: (error) => astErrors.push(error)
+    });
+    return { file, fileData, astErrors };
+  });
+
+  const scanFailures = [];
+  for (const result of checkResults) {
+    if (!result) continue;
+
+    if (result.astErrors.length > 0) {
+      map.stats.errors.push(...result.astErrors);
+    }
+
+    if (!result.fileData) {
+      scanFailures.push(result.file);
+      continue;
+    }
+
+    if (result.fileData.hash !== map.files[result.file].hash) {
+      map.files[result.file] = result.fileData;
+      if (result.fileData.imports && result.fileData.imports.length > 0) {
+        map.dependencies[result.file] = Array.from(new Set(result.fileData.imports.map(imp => imp.source)));
       } else {
-        delete map.dependencies[file];
+        delete map.dependencies[result.file];
       }
-      changes.modified.push(file);
+      changes.modified.push(result.file);
     }
   }
 
   // Process new files (async file reads)
-  for (const file of currentFiles) {
+  const addedFiles = Array.from(currentFiles);
+  const addResults = await runWithConcurrency(addedFiles, SCAN_CONCURRENCY, async (file) => {
     const fullPath = path.join(basePath, file);
-    const fileData = await runner.scanSingleFileAsync(cmd, fullPath, basePath);
-    if (fileData) {
-      map.files[file] = fileData;
-      if (fileData.imports && fileData.imports.length > 0) {
-        map.dependencies[file] = Array.from(new Set(fileData.imports.map(imp => imp.source)));
-      }
-      changes.added.push(file);
+    const astErrors = [];
+    const fileData = await runner.scanSingleFileAsync(cmd, fullPath, basePath, {
+      onError: (error) => astErrors.push(error)
+    });
+    return { file, fileData, astErrors };
+  });
+
+  for (const result of addResults) {
+    if (!result) continue;
+
+    if (result.astErrors.length > 0) {
+      map.stats.errors.push(...result.astErrors);
     }
+
+    if (!result.fileData) {
+      scanFailures.push(result.file);
+      continue;
+    }
+
+    map.files[result.file] = result.fileData;
+    if (result.fileData.imports && result.fileData.imports.length > 0) {
+      map.dependencies[result.file] = Array.from(new Set(result.fileData.imports.map(imp => imp.source)));
+    }
+    changes.added.push(result.file);
+  }
+
+  if (scanFailures.length > 0) {
+    return {
+      success: false,
+      error: `Failed to rescan ${scanFailures.length} file(s) during non-git update`,
+      needsFullRebuild: true,
+      failedFiles: scanFailures
+    };
   }
 
   changes.total = changes.added.length + changes.modified.length + changes.deleted.length;