magector 1.7.0 → 1.7.2

package/README.md

@@ -318,11 +318,53 @@ The `describe` command and `magento_describe` MCP tool require an Anthropic API
 | `MAGECTOR_DB` | Path to index database | `./.magector/index.db` |
 | `MAGECTOR_BIN` | Path to magector-core binary | Auto-detected |
 | `MAGECTOR_MODELS` | Path to ONNX model directory | `~/.magector/models/` |
-| `MAGECTOR_INDEX_TIMEOUT` | Indexing timeout in milliseconds | `1800000` (30 min) |
-| `MAGECTOR_THREADS` | Max ONNX threads for embedding generation | Half of CPU cores |
-| `MAGECTOR_BATCH_SIZE` | Embedding batch size (higher = faster, more RAM) | `256` |
+| `MAGECTOR_INDEX_TIMEOUT` | Indexing wall-clock timeout in milliseconds. Override for very large codebases or CPU-constrained environments. | `14400000` (4 h) |
+| `MAGECTOR_THREADS` | Max ONNX intra-op + rayon parsing threads. Equivalent to the `--threads` CLI flag. | Half of CPU cores |
+| `OMP_NUM_THREADS` | Fallback thread limit if `MAGECTOR_THREADS` is not set (de facto standard for ONNX/OpenMP). | — |
+| `MAGECTOR_BATCH_SIZE` | Embedding batch size (higher = faster, more RAM). Equivalent to `--batch-size`. | `256` |
 | `ANTHROPIC_API_KEY` | API key for description generation (`describe` command) | — |
 
+### Constraining CPU usage during indexing
+
+Indexing a large enterprise codebase (~80K files) can saturate CPU during PHASE 2 (ONNX embedding generation). To keep a developer machine responsive while indexing, lower the thread count:
+
+```bash
+npx magector index --threads 2                  # use only 2 cores for both parsing and embedding
+MAGECTOR_THREADS=2 npx magector index           # equivalent via env var
+OMP_NUM_THREADS=2 npx magector index            # also honored as a fallback
+```
+
+The `--threads` flag and `MAGECTOR_THREADS` / `OMP_NUM_THREADS` env vars constrain **both** the rayon thread pool used by PHASE 1 (parallel AST parsing) and the ONNX intra-op thread pool used by PHASE 2 (embedding inference). The active thread source is logged at startup so you can verify it took effect:
+
+```
+INFO Rayon global pool: 2 threads (available: 16)
+INFO ONNX intra_threads: 2 (available: 16, source: --threads flag)
+```
+
+For very large or CPU-constrained runs, you may also need to extend the wall-clock timeout (default 4 hours):
+
+```bash
+MAGECTOR_INDEX_TIMEOUT=28800000 npx magector index --threads 2   # 8 h timeout, 2 threads
+```
+
+### Resume after timeout or interrupt
+
+Indexing writes a crash-safe checkpoint to disk every 50 batches (~12,800 files). If the process is killed or times out mid-run, **just re-run `npx magector index`** — it auto-resumes from the last checkpoint:
+
+```bash
+npx magector index
+# ♻️  Resuming from previous run: 38400 vectors across 12200 files already indexed
+# ✓ Found 79771 total files; 12200 already indexed, 67571 remaining to process
+```
+
+The indexer collects already-embedded file paths from the existing DB, filters them out of file discovery, preserves the existing HNSW state, and only parses/embeds the files that aren't in the DB yet. Partial resume also picks up new files added to the tree since the previous run.
+
+To force a full rebuild (e.g. after a schema change or if you want to discard stale vectors), pass `--force`:
+
+```bash
+npx magector index --force
+```
+
 ---
 
 ## MCP Server Tools

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "magector",
-  "version": "1.7.0",
+  "version": "1.7.2",
   "description": "Semantic code search for Magento 2 — index, search, MCP server",
   "type": "module",
   "main": "src/mcp-server.js",
@@ -37,10 +37,10 @@
     "ruvector": "^0.1.96"
   },
   "optionalDependencies": {
-    "@magector/cli-darwin-arm64": "1.7.0",
-    "@magector/cli-linux-x64": "1.7.0",
-    "@magector/cli-linux-arm64": "1.7.0",
-    "@magector/cli-win32-x64": "1.7.0"
+    "@magector/cli-darwin-arm64": "1.7.2",
+    "@magector/cli-linux-x64": "1.7.2",
+    "@magector/cli-linux-arm64": "1.7.2",
+    "@magector/cli-win32-x64": "1.7.2"
   },
   "keywords": [
     "magento",

package/src/cli.js

@@ -30,21 +30,37 @@ Usage:
   npx magector setup [path]      IDE setup only (no indexing)
   npx magector help              Show this help
 
-Options:
-  -l, --limit <n>    Number of search results (default: 10)
-  -f, --format <fmt> Output format: text, json (default: text)
+Search options:
+  -l, --limit <n>      Number of search results (default: 10)
+  -f, --format <fmt>   Output format: text, json (default: text)
+
+Index options:
+  --threads <n>        Max ONNX/rayon threads (default: half of CPU cores).
+                       Lower this on shared developer machines to keep the
+                       system responsive during indexing.
+  --batch-size <n>     Embedding batch size (default: 256). Higher = faster
+                       but more RAM.
+  --force              Discard any existing index and rebuild from scratch.
+                       Without --force, indexing auto-resumes from the last
+                       incremental save (written every ~50 batches).
 
 Environment Variables:
-  MAGENTO_ROOT     Path to Magento installation (default: cwd)
-  MAGECTOR_DB      Path to index database (default: ./.magector/index.db)
-  MAGECTOR_BIN     Path to magector-core binary
-  MAGECTOR_MODELS  Path to ONNX model directory
+  MAGENTO_ROOT             Path to Magento installation (default: cwd)
+  MAGECTOR_DB              Path to index database (default: ./.magector/index.db)
+  MAGECTOR_BIN             Path to magector-core binary
+  MAGECTOR_MODELS          Path to ONNX model directory
+  MAGECTOR_THREADS         Max threads (overridden by --threads)
+  MAGECTOR_BATCH_SIZE      Embedding batch size (overridden by --batch-size)
+  MAGECTOR_INDEX_TIMEOUT   Indexing wall-clock timeout in ms (default: 14400000 = 4h)
+  OMP_NUM_THREADS          Fallback thread limit if MAGECTOR_THREADS unset
 
 Examples:
   npx magector init /var/www/magento
   npx magector search "product price calculation"
   npx magector search "checkout controller" -l 20
   npx magector index
+  npx magector index --threads 4 --batch-size 128
+  MAGECTOR_INDEX_TIMEOUT=28800000 npx magector index   # 8h timeout
   npx magector mcp
 `);
 }
@@ -67,12 +83,16 @@ function parseArgs(argv) {
       opts.verbose = true;
     } else if (argv[i] === '--force') {
       opts.force = true;
+    } else if (argv[i] === '--threads') {
+      opts.threads = argv[++i];
+    } else if (argv[i] === '--batch-size') {
+      opts.batchSize = argv[++i];
     }
   }
   return opts;
 }
 
-async function runIndex(targetPath) {
+async function runIndex(targetPath, opts = {}) {
   const config = getConfig();
   const root = targetPath || config.magentoRoot;
   const binary = resolveBinary();
@@ -85,7 +105,9 @@ async function runIndex(targetPath) {
   const magectorDir = path.resolve(root, '.magector');
   mkdirSync(magectorDir, { recursive: true });
 
-  const indexTimeout = parseInt(process.env.MAGECTOR_INDEX_TIMEOUT, 10) || 1800000;
+  // Default 4 hours — generous enough for ~80K-file enterprise codebases under
+  // CPU constraint. Override via MAGECTOR_INDEX_TIMEOUT (milliseconds).
+  const indexTimeout = parseInt(process.env.MAGECTOR_INDEX_TIMEOUT, 10) || 14400000;
   try {
     const indexArgs = [
       'index',
@@ -93,6 +115,21 @@ async function runIndex(targetPath) {
       '-d', path.resolve(config.dbPath),
       '-c', modelPath
     ];
+    // Forward thread/batch limits to the Rust binary. The Rust side already
+    // honors MAGECTOR_THREADS / OMP_NUM_THREADS via env, but explicit flags
+    // give the user a CLI-level override and make the limit visible in logs.
+    if (opts.threads != null) {
+      indexArgs.push('--threads', String(opts.threads));
+    }
+    if (opts.batchSize != null) {
+      indexArgs.push('--batch-size', String(opts.batchSize));
+    }
+    // --force discards any existing partial index and rebuilds from scratch.
+    // Without it, the Rust indexer auto-resumes from the last incremental
+    // save on disk and only re-embeds files that aren't in the DB yet.
+    if (opts.force) {
+      indexArgs.push('--force');
+    }
     // Pass descriptions DB if it exists
     const descDbPath = path.resolve(root, '.magector', 'sqlite.db');
     if (existsSync(descDbPath)) {
@@ -106,7 +143,15 @@ async function runIndex(targetPath) {
       process.exit(err.status);
     }
     if (err.message && err.message.includes('ETIMEDOUT')) {
-      console.error(`Indexing timed out after ${indexTimeout / 1000}s. For large codebases, increase the timeout:\n  MAGECTOR_INDEX_TIMEOUT=3600000 npx magector index`);
+      console.error(
+        `Indexing timed out after ${indexTimeout / 1000}s.\n` +
+        `Partial progress was saved to disk every ~50 batches — re-run\n` +
+        `'npx magector index' to auto-resume from the last checkpoint.\n` +
+        `\n` +
+        `For large codebases or CPU-constrained environments, also consider:\n` +
+        `  MAGECTOR_INDEX_TIMEOUT=28800000 npx magector index    # 8 hours\n` +
+        `  npx magector index --threads 2                        # lower CPU usage`
+      );
     } else {
       console.error(`Indexing error: ${err.message}`);
     }
@@ -202,13 +247,22 @@ async function main() {
   await checkForUpdate(command, args);
 
   switch (command) {
-    case 'init':
-      await init(args[1]);
+    case 'init': {
+      const initArgv = args.slice(1);
+      const initTarget = initArgv.find(a => !a.startsWith('-'));
+      const initOpts = parseArgs(initArgv);
+      await init(initTarget, initOpts);
       break;
+    }
 
-    case 'index':
-      await runIndex(args[1]);
+    case 'index': {
+      // First non-flag arg after `index` is the path; everything else is options.
+      const indexArgv = args.slice(1);
+      const targetPath = indexArgv.find(a => !a.startsWith('-'));
+      const indexOpts = parseArgs(indexArgv);
+      await runIndex(targetPath, indexOpts);
       break;
+    }
 
     case 'search': {
       const query = args.slice(1).filter(a => !a.startsWith('-')).join(' ');

package/src/init.js

@@ -198,8 +198,13 @@ function updateGitignore(projectPath) {
 
 /**
  * Main init function.
+ *
+ * @param {string} projectPath  - Magento root (defaults to cwd).
+ * @param {object} [opts]
+ * @param {number|string} [opts.threads]   - Forwarded as `--threads` to magector-core.
+ * @param {number|string} [opts.batchSize] - Forwarded as `--batch-size`.
  */
-export async function init(projectPath) {
+export async function init(projectPath, opts = {}) {
   projectPath = path.resolve(projectPath || process.cwd());
   mkdirSync(path.join(projectPath, '.magector'), { recursive: true });
   const dbPath = path.join(projectPath, '.magector', 'index.db');
@@ -244,21 +249,39 @@ export async function init(projectPath) {
   // 4. Run indexing
   console.log('\nIndexing codebase...');
   const startTime = Date.now();
+  // Default 4 hours — generous enough for ~80K-file enterprise codebases under
+  // CPU constraint. Override via MAGECTOR_INDEX_TIMEOUT (milliseconds).
+  const initTimeout = parseInt(process.env.MAGECTOR_INDEX_TIMEOUT, 10) || 14400000;
   try {
-    execFileSync(binary, [
+    const indexArgs = [
       'index',
       '-m', projectPath,
       '-d', dbPath,
       '-c', modelPath
-    ], { timeout: parseInt(process.env.MAGECTOR_INDEX_TIMEOUT, 10) || 1800000, stdio: 'inherit' });
+    ];
+    if (opts.threads != null) {
+      indexArgs.push('--threads', String(opts.threads));
+    }
+    if (opts.batchSize != null) {
+      indexArgs.push('--batch-size', String(opts.batchSize));
+    }
+    if (opts.force) {
+      indexArgs.push('--force');
+    }
+    execFileSync(binary, indexArgs, { timeout: initTimeout, stdio: 'inherit' });
   } catch (err) {
     if (err.status) {
       console.error('Indexing failed.');
       process.exit(err.status);
     }
-    const initTimeout = parseInt(process.env.MAGECTOR_INDEX_TIMEOUT, 10) || 1800000;
     if (err.message && err.message.includes('ETIMEDOUT')) {
-      console.error(`Indexing timed out after ${initTimeout / 1000}s. For large codebases, increase the timeout:\n  MAGECTOR_INDEX_TIMEOUT=3600000 npx magector init ${projectPath}`);
+      console.error(
+        `Indexing timed out after ${initTimeout / 1000}s.\n` +
+        `For large codebases or CPU-constrained environments, increase the timeout:\n` +
+        `  MAGECTOR_INDEX_TIMEOUT=28800000 npx magector init ${projectPath}    # 8 hours\n` +
+        `Or reduce CPU usage:\n` +
+        `  npx magector init ${projectPath} --threads 2`
+      );
     } else {
       console.error(`Indexing error: ${err.message}`);
     }

package/src/mcp-server.js

@@ -552,13 +552,21 @@ function rustIndex(magentoRoot) {
   if (existsSync(descDbPath)) {
     indexArgs.push('--descriptions-db', descDbPath);
   }
-  const indexTimeout = parseInt(process.env.MAGECTOR_INDEX_TIMEOUT, 10) || 1800000;
+  // Default 4 hours, matching cli.js/init.js. Previously 1800000 (30 min),
+  // which was too short for ~80K-file enterprise Magento installs under CPU
+  // constraint and caused silent re-index loops via the MCP auto-index path.
+  const indexTimeout = parseInt(process.env.MAGECTOR_INDEX_TIMEOUT, 10) || 14400000;
   try {
     const result = execFileSync(config.rustBinary, indexArgs, { encoding: 'utf-8', timeout: indexTimeout, stdio: ['pipe', 'pipe', 'pipe'], env: rustEnv });
     return result;
   } catch (err) {
     if (err.message && err.message.includes('ETIMEDOUT')) {
-      throw new Error(`Indexing timed out after ${indexTimeout / 1000}s. Set MAGECTOR_INDEX_TIMEOUT=3600000 (or higher) in your environment to increase the limit.`);
+      throw new Error(
+        `Indexing timed out after ${indexTimeout / 1000}s. Partial progress was saved ` +
+        `to disk — the next indexing run will auto-resume from the last checkpoint. ` +
+        `To raise the timeout further, set MAGECTOR_INDEX_TIMEOUT (milliseconds) in the ` +
+        `MCP server env, e.g. MAGECTOR_INDEX_TIMEOUT=28800000 for 8 hours.`
+      );
     }
     throw err;
   }