magector 1.7.1 → 1.7.2

package/README.md

@@ -347,6 +347,24 @@ For very large or CPU-constrained runs, you may also need to extend the wall-clo
 MAGECTOR_INDEX_TIMEOUT=28800000 npx magector index --threads 2   # 8 h timeout, 2 threads
 ```
 
+### Resume after timeout or interrupt
+
+Indexing writes a crash-safe checkpoint to disk every 50 batches (~12,800 files). If the process is killed or times out mid-run, **just re-run `npx magector index`** — it auto-resumes from the last checkpoint:
+
+```bash
+npx magector index
+# ♻️  Resuming from previous run: 38400 vectors across 12200 files already indexed
+# ✓ Found 79771 total files; 12200 already indexed, 67571 remaining to process
+```
+
+The indexer collects already-embedded file paths from the existing DB, filters them out of file discovery, preserves the existing HNSW state, and only parses/embeds the files that aren't in the DB yet. Partial resume also picks up new files added to the tree since the previous run.
+
+To force a full rebuild (e.g. after a schema change or if you want to discard stale vectors), pass `--force`:
+
+```bash
+npx magector index --force
+```
+
 ---
 
 ## MCP Server Tools

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "magector",
-  "version": "1.7.1",
+  "version": "1.7.2",
   "description": "Semantic code search for Magento 2 — index, search, MCP server",
   "type": "module",
   "main": "src/mcp-server.js",
@@ -37,10 +37,10 @@
     "ruvector": "^0.1.96"
   },
   "optionalDependencies": {
-    "@magector/cli-darwin-arm64": "1.7.1",
-    "@magector/cli-linux-x64": "1.7.1",
-    "@magector/cli-linux-arm64": "1.7.1",
-    "@magector/cli-win32-x64": "1.7.1"
+    "@magector/cli-darwin-arm64": "1.7.2",
+    "@magector/cli-linux-x64": "1.7.2",
+    "@magector/cli-linux-arm64": "1.7.2",
+    "@magector/cli-win32-x64": "1.7.2"
   },
   "keywords": [
     "magento",

package/src/cli.js

@@ -40,7 +40,9 @@ Index options:
                        system responsive during indexing.
   --batch-size <n>     Embedding batch size (default: 256). Higher = faster
                        but more RAM.
-  --force              Force re-index even if index exists
+  --force              Discard any existing index and rebuild from scratch.
+                       Without --force, indexing auto-resumes from the last
+                       incremental save (written every ~50 batches).
 
 Environment Variables:
   MAGENTO_ROOT             Path to Magento installation (default: cwd)
@@ -122,6 +124,12 @@ async function runIndex(targetPath, opts = {}) {
     if (opts.batchSize != null) {
       indexArgs.push('--batch-size', String(opts.batchSize));
     }
+    // --force discards any existing partial index and rebuilds from scratch.
+    // Without it, the Rust indexer auto-resumes from the last incremental
+    // save on disk and only re-embeds files that aren't in the DB yet.
+    if (opts.force) {
+      indexArgs.push('--force');
+    }
     // Pass descriptions DB if it exists
     const descDbPath = path.resolve(root, '.magector', 'sqlite.db');
     if (existsSync(descDbPath)) {
@@ -137,10 +145,12 @@ async function runIndex(targetPath, opts = {}) {
     if (err.message && err.message.includes('ETIMEDOUT')) {
       console.error(
         `Indexing timed out after ${indexTimeout / 1000}s.\n` +
-        `For large codebases or CPU-constrained environments, increase the timeout:\n` +
+        `Partial progress was saved to disk every ~50 batches — re-run\n` +
+        `'npx magector index' to auto-resume from the last checkpoint.\n` +
+        `\n` +
+        `For large codebases or CPU-constrained environments, also consider:\n` +
         `  MAGECTOR_INDEX_TIMEOUT=28800000 npx magector index    # 8 hours\n` +
-        `Or reduce CPU usage with fewer threads:\n` +
-        `  npx magector index --threads 2`
+        `  npx magector index --threads 2                        # lower CPU usage`
       );
     } else {
       console.error(`Indexing error: ${err.message}`);

package/src/init.js

@@ -265,6 +265,9 @@ export async function init(projectPath, opts = {}) {
     if (opts.batchSize != null) {
       indexArgs.push('--batch-size', String(opts.batchSize));
     }
+    if (opts.force) {
+      indexArgs.push('--force');
+    }
     execFileSync(binary, indexArgs, { timeout: initTimeout, stdio: 'inherit' });
   } catch (err) {
     if (err.status) {

package/src/mcp-server.js

@@ -552,13 +552,21 @@ function rustIndex(magentoRoot) {
   if (existsSync(descDbPath)) {
     indexArgs.push('--descriptions-db', descDbPath);
   }
-  const indexTimeout = parseInt(process.env.MAGECTOR_INDEX_TIMEOUT, 10) || 1800000;
+  // Default 4 hours, matching cli.js/init.js. Previously 1800000 (30 min),
+  // which was too short for ~80K-file enterprise Magento installs under CPU
+  // constraint and caused silent re-index loops via the MCP auto-index path.
+  const indexTimeout = parseInt(process.env.MAGECTOR_INDEX_TIMEOUT, 10) || 14400000;
   try {
     const result = execFileSync(config.rustBinary, indexArgs, { encoding: 'utf-8', timeout: indexTimeout, stdio: ['pipe', 'pipe', 'pipe'], env: rustEnv });
     return result;
   } catch (err) {
     if (err.message && err.message.includes('ETIMEDOUT')) {
-      throw new Error(`Indexing timed out after ${indexTimeout / 1000}s. Set MAGECTOR_INDEX_TIMEOUT=3600000 (or higher) in your environment to increase the limit.`);
+      throw new Error(
+        `Indexing timed out after ${indexTimeout / 1000}s. Partial progress was saved ` +
+        `to disk — the next indexing run will auto-resume from the last checkpoint. ` +
+        `To raise the timeout further, set MAGECTOR_INDEX_TIMEOUT (milliseconds) in the ` +
+        `MCP server env, e.g. MAGECTOR_INDEX_TIMEOUT=28800000 for 8 hours.`
+      );
     }
     throw err;
   }