magector 2.16.0 → 2.16.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "magector",
-  "version": "2.16.0",
+  "version": "2.16.2",
   "description": "Semantic code search for Magento 2 — index, search, MCP server",
   "type": "module",
   "main": "src/mcp-server.js",
@@ -33,10 +33,10 @@
     "ruvector": "^0.1.96"
   },
   "optionalDependencies": {
-    "@magector/cli-darwin-arm64": "2.16.0",
-    "@magector/cli-linux-x64": "2.16.0",
-    "@magector/cli-linux-arm64": "2.16.0",
-    "@magector/cli-win32-x64": "2.16.0"
+    "@magector/cli-darwin-arm64": "2.16.2",
+    "@magector/cli-linux-x64": "2.16.2",
+    "@magector/cli-linux-arm64": "2.16.2",
+    "@magector/cli-win32-x64": "2.16.2"
   },
   "keywords": [
     "magento",

package/src/mcp-server.js

@@ -423,6 +423,13 @@ let reindexInProgress = false;
 let reindexProcess = null;
 let warmupInProgress = true; // true until checkDbFormat + serve process ready
 
+// Re-index progress tracking (updated from INDEX log lines)
+let reindexStartTime = null;
+let reindexPhase = 0;      // 0=init, 1=AST, 2=embeddings, 3=HNSW
+let reindexTotalFiles = 0;
+let reindexItemsToEmbed = 0;
+let reindexPhase2Start = null;
+
 /**
  * Check if the database file is compatible with the current binary.
  * Uses a cached result to avoid running stats (30-60s) on every startup.
@@ -557,6 +564,11 @@ function startBackgroundReindex() {
   }
 
   reindexInProgress = true;
+  reindexStartTime = Date.now();
+  reindexPhase = 0;
+  reindexTotalFiles = 0;
+  reindexItemsToEmbed = 0;
+  reindexPhase2Start = null;
 
   const hadExistingDb = existsSync(config.dbPath);
   logToFile('WARN', `Starting background re-index to temp path. Old DB ${hadExistingDb ? 'preserved for queries' : 'not found'}.`);
@@ -590,18 +602,31 @@ function startBackgroundReindex() {
   // entries arrive in large chunks instead of in real time.
   const indexStdout = createInterface({ input: reindexProcess.stdout });
   const indexStderr = createInterface({ input: reindexProcess.stderr });
+  const parseIndexProgress = (text) => {
+    const m = text.match(/Found (\d[\d,]+) files to index/);
+    if (m) reindexTotalFiles = parseInt(m[1].replace(/,/g, ''), 10);
+    if (text.includes('PHASE 1') || text.includes('AST analyzer')) reindexPhase = 1;
+    if (text.includes('PHASE 2') || text.includes('semantic embedding') || text.includes('Generating semantic')) {
+      if (reindexPhase < 2) { reindexPhase = 2; reindexPhase2Start = Date.now(); }
+    }
+    if (text.includes('PHASE 3') || text.includes('Building HNSW') || text.includes('HNSW')) reindexPhase = 3;
+    const em = text.match(/Items to embed: (\d[\d,]+)/);
+    if (em) reindexItemsToEmbed = parseInt(em[1].replace(/,/g, ''), 10);
+  };
   indexStdout.on('line', (line) => {
     const text = line.replace(/\x1b\[[0-9;]*m/g, '').trim();
-    if (text) logToFile('INDEX', text);
+    if (text) { logToFile('INDEX', text); parseIndexProgress(text); }
   });
   indexStderr.on('line', (line) => {
     const text = line.replace(/\x1b\[[0-9;]*m/g, '').trim();
-    if (text) logToFile('INDEX', text);
+    if (text) { logToFile('INDEX', text); parseIndexProgress(text); }
   });
 
   reindexProcess.on('exit', (code) => {
     reindexInProgress = false;
     reindexProcess = null;
+    reindexStartTime = null;
+    reindexPhase = 0;
     removeReindexPidFile();
     if (code === 0) {
       // Atomic swap: old → .bak, new → current
@@ -938,6 +963,10 @@ function tryConnectSocket() {
 let globalServeQuery = null;
 
 function serveQuery(command, params = {}, timeoutMs = 30000) {
+  if (!serveProcess || !serveReady) {
+    logToFile('WARN', `serveQuery(${command}): serve process not ready — returning error`);
+    return Promise.resolve({ ok: false, error: 'Serve process not ready' });
+  }
   return new Promise((resolve, reject) => {
     const id = serveNextId++;
     logToFile('QUERY', `[${id}] → ${command}(${JSON.stringify(params).slice(0, 200)})`);
@@ -3631,7 +3660,7 @@ server.setRequestHandler(ListToolsRequestSchema, async () => ({
   tools: [
     {
       name: 'magento_search',
-      description: 'Search Magento codebase semantically — find any PHP class, method, XML config, PHTML template, JS file, or GraphQL schema by describing what you need in natural language. Use this as a general-purpose search when no specialized tool fits. See also: magento_find_class, magento_find_method, magento_find_config for targeted searches.',
+      description: 'Search Magento codebase semantically — find any PHP class, method, XML config, PHTML template, JS file, or GraphQL schema by describing what you need in natural language. Use this as a general-purpose search when no specialized tool fits. Works best for Magento core and popular vendor modules. For small/custom project-specific modules (e.g. proprietary modules not widely known to the embedding model), use magento_grep instead — semantic search may return 0 results for these. See also: magento_find_class, magento_find_method, magento_find_config for targeted searches.',
       inputSchema: {
         type: 'object',
         properties: {
@@ -4278,7 +4307,7 @@ server.setRequestHandler(ListToolsRequestSchema, async () => ({
     },
     {
       name: 'magento_batch',
-      description: 'Execute multiple Magector tool calls in a single request to reduce MCP round-trip overhead. Each query runs in parallel and returns combined results. Use this when you need 2+ independent lookups (e.g., find a class AND its plugins AND its observers in one call instead of three).',
+      description: 'Execute multiple Magector tool calls in a single request to reduce MCP round-trip overhead. Each query runs in parallel and returns combined results. Use this when you need 2+ independent lookups (e.g., find a class AND its plugins AND its observers in one call instead of three). Supported tools: magento_search, magento_find_class, magento_find_method, magento_find_plugin, magento_find_observer, magento_find_config, magento_find_event_flow, magento_find_di_wiring, magento_find_callers, magento_find_preference, magento_find_fieldset, magento_module_structure, magento_trace_dependency, magento_impact_analysis, magento_grep, magento_read, magento_ast_search, magento_find_dataobject_issues, magento_find_null_risks.',
       inputSchema: {
         type: 'object',
         properties: {
@@ -4391,7 +4420,7 @@ server.setRequestHandler(ListToolsRequestSchema, async () => ({
     },
     {
       name: 'magento_find_null_risks',
-      description: 'Find method chains without null guards using the pre-built enrichment index. Returns all ->firstMethod()->secondMethod() calls where no null check (=== null, !== null, ?->, ??, isset, is_null) was detected in surrounding code. Requires magento_enrich to have been run first. 100× faster than grep — O(1) SQLite query vs O(n) file scan. Use firstMethod to filter (e.g., "getPayment" finds all ->getPayment()->anything() without null guard). ⚡ For multi-query workflows use magento_batch.',
+      description: 'Find method chains without null guards using the pre-built enrichment index. Returns all ->firstMethod()->secondMethod() calls where no null check (=== null, !== null, ?->, ??, isset, is_null) was detected in surrounding code. Requires magento_enrich to have been run first (magento_index triggers it automatically in the background). 100× faster than grep — O(1) SQLite query vs O(n) file scan. Use firstMethod to filter (e.g., "getPayment" finds all ->getPayment()->anything() without null guard). ⚡ For multi-query workflows use magento_batch.',
       inputSchema: {
         type: 'object',
         properties: {
@@ -4458,7 +4487,49 @@ server.setRequestHandler(ListToolsRequestSchema, async () => ({
   ]
 }));
 
-server.setRequestHandler(CallToolRequestSchema, async (request) => {
+/**
+ * Build a reindex warning string for tool responses during background re-index.
+ * Returns null when not re-indexing.
+ */
+function getReindexWarning() {
+  if (!reindexInProgress || !reindexStartTime) return null;
+  const elapsedSec = Math.round((Date.now() - reindexStartTime) / 1000);
+  const elapsedStr = elapsedSec >= 60
+    ? `${Math.floor(elapsedSec / 60)}m ${elapsedSec % 60}s`
+    : `${elapsedSec}s`;
+
+  let phaseStr, etaStr;
+  if (reindexPhase <= 0) {
+    phaseStr = 'initializing';
+    etaStr = 'est. ~40–70 min total';
+  } else if (reindexPhase === 1) {
+    const filesStr = reindexTotalFiles > 0 ? ` (${reindexTotalFiles.toLocaleString('en')} files)` : '';
+    phaseStr = `phase 1/3: AST parsing${filesStr}`;
+    etaStr = 'est. ~1–3 min for this phase, then ~40–60 min for embeddings';
+  } else if (reindexPhase === 2) {
+    const itemsStr = reindexItemsToEmbed > 0 ? ` (${reindexItemsToEmbed.toLocaleString('en')} items)` : '';
+    phaseStr = `phase 2/3: generating embeddings${itemsStr}`;
+    if (reindexPhase2Start && reindexItemsToEmbed > 0) {
+      // Empirical rate: ~87k items ≈ 45 min on 8-core ONNX. Scale linearly.
+      const estimatedTotalSec = Math.round((reindexItemsToEmbed / 87000) * 45 * 60);
+      const phase2Elapsed = (Date.now() - reindexPhase2Start) / 1000;
+      const remainingSec = Math.max(estimatedTotalSec - phase2Elapsed, 0);
+      etaStr = remainingSec > 60
+        ? `est. ~${Math.round(remainingSec / 60)} min remaining`
+        : 'almost done with embeddings';
+    } else {
+      etaStr = 'est. 30–60 min for this phase';
+    }
+  } else {
+    phaseStr = 'phase 3/3: building HNSW vector index';
+    etaStr = 'est. ~5–10 min remaining';
+  }
+
+  return `> ⏳ **Re-indexing in progress** — ${elapsedStr} elapsed, ${phaseStr}. ${etaStr}.\n` +
+         `> Results below use the **previous index** — valid, but may miss recently added files.\n\n`;
+}
+
+const _callToolHandler = async (request) => {
   const { name, arguments: args } = request.params;
   const reqStart = Date.now();
   logToFile('REQ', `${name}(${JSON.stringify(args || {})})`);
@@ -5200,43 +5271,63 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
       }
 
       case 'magento_module_structure': {
-        const raw = await rustSearchAsync(args.moduleName, 200);
-        // Support both app/code (Magento/Catalog/) and vendor (module-catalog/) paths
-        const modulePath = args.moduleName.replace('_', '/') + '/';
         const parts = args.moduleName.split('_');
+        // Support both app/code (Magento/Catalog/) and vendor (magento/module-catalog/) paths
+        const modulePath = args.moduleName.replace('_', '/') + '/';
         // Hyphenate camelCase for vendor path: OrderSplit → order-split
+        const vendorDir = parts.length === 2 ? parts[0].toLowerCase() : '';
         const vendorPath = parts.length === 2
           ? `module-${parts[1].replace(/([a-z])([A-Z])/g, '$1-$2').toLowerCase()}/`
           : '';
-        let results = raw.map(normalizeResult).filter(r => {
-          const p = r.path || '';
-          const mod = r.module || '';
-          // Exact module match or directory-level path match (trailing slash prevents Catalog matching CatalogRule)
-          return mod === args.moduleName ||
-            p.includes(modulePath) ||
-            (vendorPath && p.toLowerCase().includes(vendorPath));
-        });
-
-        // Filesystem fallback: if vector search found nothing, glob the module directory
-        if (results.length === 0 && config.magentoRoot && vendorPath) {
-          logToFile('INFO', `module_structure: vector search returned 0 results for "${args.moduleName}" — using filesystem fallback (glob ${vendorPath})`);
-          try {
-            const vendorGlob = `**/${vendorPath}**/*.{php,xml,phtml}`;
-            const files = await glob(vendorGlob, { cwd: config.magentoRoot, absolute: false, nodir: true });
-            for (const f of files.slice(0, 100)) {
-              const entry = { path: f, score: 0.5 };
-              if (f.includes('/Controller/')) entry.isController = true;
-              if (f.includes('/Model/')) entry.isModel = true;
-              if (f.includes('/Block/')) entry.isBlock = true;
-              if (f.includes('/Plugin/')) entry.isPlugin = true;
-              if (f.includes('/Observer/')) entry.isObserver = true;
-              if (f.endsWith('.xml')) entry.type = 'xml';
-              // Extract class name from path
-              const phpMatch = f.match(/\/([A-Z]\w+)\.php$/);
-              if (phpMatch) entry.className = phpMatch[1];
-              results.push(entry);
+        let results = [];
+
+        // Primary: filesystem-based (authoritative — avoids mixing cross-references from vector search)
+        if (config.magentoRoot) {
+          const fsGlobs = [];
+          if (parts.length === 2) {
+            // app/code/{Vendor}/{Module}/
+            fsGlobs.push(`app/code/${parts[0]}/${parts[1]}/**/*.{php,xml,phtml}`);
+            // vendor/{vendor-lower}/{module-lower}/ — vendor-specific to avoid false positives
+            if (vendorDir && vendorPath) {
+              fsGlobs.push(`vendor/${vendorDir}/${vendorPath}**/*.{php,xml,phtml}`);
             }
-          } catch {}
+          }
+          for (const globPattern of fsGlobs) {
+            try {
+              const files = await glob(globPattern, { cwd: config.magentoRoot, absolute: false, nodir: true });
+              if (files.length > 0) {
+                logToFile('INFO', `module_structure: filesystem found ${files.length} files for "${args.moduleName}" (${globPattern})`);
+                for (const f of files.slice(0, 100)) {
+                  const entry = { path: f, score: 1.0 };
+                  if (f.includes('/Controller/')) entry.isController = true;
+                  if (f.includes('/Model/')) entry.isModel = true;
+                  if (f.includes('/Block/')) entry.isBlock = true;
+                  if (f.includes('/Plugin/')) entry.isPlugin = true;
+                  if (f.includes('/Observer/')) entry.isObserver = true;
+                  if (f.endsWith('.xml')) entry.type = 'xml';
+                  const phpMatch = f.match(/\/([A-Z]\w+)\.php$/);
+                  if (phpMatch) entry.className = phpMatch[1];
+                  results.push(entry);
+                }
+                break; // Found in one location, stop
+              }
+            } catch {}
+          }
+        }
+
+        // Fallback: vector search with strict path/module filtering (only if filesystem found nothing)
+        if (results.length === 0) {
+          logToFile('INFO', `module_structure: filesystem found 0 files for "${args.moduleName}" — falling back to vector search`);
+          const raw = await rustSearchAsync(args.moduleName, 200);
+          results = raw.map(normalizeResult).filter(r => {
+            const p = r.path || '';
+            const mod = r.module || '';
+            // Exact module match or directory-level path match (trailing slash prevents Catalog matching CatalogRule)
+            return mod === args.moduleName ||
+              p.includes(modulePath) ||
+              // Vendor-specific path check to avoid matching other vendors' same-named modules
+              (vendorDir && vendorPath && p.toLowerCase().includes(`${vendorDir}/${vendorPath}`));
+          });
         }
 
         const structure = {
@@ -6182,6 +6273,21 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                 }
                 break;
               }
+              case 'magento_find_config': {
+                let cfgQuery = a.query;
+                if (a.configType && a.configType !== 'other') cfgQuery = `${a.configType}.xml xml config ${a.query}`;
+                const cfgRaw = await rustSearchAsync(cfgQuery, 100);
+                let cfgRes = cfgRaw.map(normalizeResult).filter(r =>
+                  r.type === 'xml' || r.path?.endsWith('.xml') || r.path?.includes('.xml')
+                );
+                if (a.configType && a.configType !== 'other') {
+                  const cfgTypeFile = `${a.configType}.xml`;
+                  const cfgTyped = cfgRes.filter(r => r.path?.includes(cfgTypeFile));
+                  if (cfgTyped.length >= 3) cfgRes = cfgTyped;
+                }
+                text = formatSearchResults(cfgRes.slice(0, 10));
+                break;
+              }
               case 'magento_trace_dependency': {
                 const dep = await traceDependency(a.className, a.direction || 'both');
                 text = `Preferences: ${dep.preferences.length}, Plugins: ${dep.plugins.length}, VirtualTypes: ${dep.virtualTypes.length}, Args: ${dep.argumentOverrides.length}\n`;
@@ -6271,35 +6377,52 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                 break;
               }
               case 'magento_module_structure': {
-                const raw = await rustSearchAsync(a.moduleName, 200);
-                const modulePath = a.moduleName.replace('_', '/') + '/';
                 const mParts = a.moduleName.split('_');
-                // Hyphenate camelCase for vendor path: OrderSplit → order-split
+                const modulePath = a.moduleName.replace('_', '/') + '/';
+                const mVendorDir = mParts.length === 2 ? mParts[0].toLowerCase() : '';
                 const vendorPath = mParts.length === 2
                   ? `module-${mParts[1].replace(/([a-z])([A-Z])/g, '$1-$2').toLowerCase()}/`
                   : '';
-                let res = raw.map(normalizeResult).filter(r => {
-                  const p = r.path || '';
-                  const mod = r.module || '';
-                  return mod === a.moduleName || p.includes(modulePath) || (vendorPath && p.toLowerCase().includes(vendorPath));
-                });
-                // Filesystem fallback
-                if (res.length === 0 && config.magentoRoot && vendorPath) {
-                  try {
-                    const vendorGlob = `**/${vendorPath}**/*.{php,xml,phtml}`;
-                    const files = await glob(vendorGlob, { cwd: config.magentoRoot, absolute: false, nodir: true });
-                    for (const f of files.slice(0, 100)) {
-                      const entry = { path: f, score: 0.5 };
-                      if (f.includes('/Controller/')) entry.isController = true;
-                      if (f.includes('/Model/')) entry.isModel = true;
-                      if (f.includes('/Plugin/')) entry.isPlugin = true;
-                      if (f.includes('/Observer/')) entry.isObserver = true;
-                      if (f.endsWith('.xml')) entry.type = 'xml';
-                      const phpMatch = f.match(/\/([A-Z]\w+)\.php$/);
-                      if (phpMatch) entry.className = phpMatch[1];
-                      res.push(entry);
+                let res = [];
+                // Primary: filesystem-based (avoids mixing cross-references)
+                if (config.magentoRoot) {
+                  const msGlobs = [];
+                  if (mParts.length === 2) {
+                    msGlobs.push(`app/code/${mParts[0]}/${mParts[1]}/**/*.{php,xml,phtml}`);
+                    if (mVendorDir && vendorPath) {
+                      msGlobs.push(`vendor/${mVendorDir}/${vendorPath}**/*.{php,xml,phtml}`);
                     }
-                  } catch {}
+                  }
+                  for (const gp of msGlobs) {
+                    try {
+                      const files = await glob(gp, { cwd: config.magentoRoot, absolute: false, nodir: true });
+                      if (files.length > 0) {
+                        for (const f of files.slice(0, 100)) {
+                          const entry = { path: f, score: 1.0 };
+                          if (f.includes('/Controller/')) entry.isController = true;
+                          if (f.includes('/Model/')) entry.isModel = true;
+                          if (f.includes('/Plugin/')) entry.isPlugin = true;
+                          if (f.includes('/Observer/')) entry.isObserver = true;
+                          if (f.endsWith('.xml')) entry.type = 'xml';
+                          const phpMatch = f.match(/\/([A-Z]\w+)\.php$/);
+                          if (phpMatch) entry.className = phpMatch[1];
+                          res.push(entry);
+                        }
+                        break;
+                      }
+                    } catch {}
+                  }
+                }
+                // Fallback: vector search with vendor-specific path filtering
+                if (res.length === 0) {
+                  const raw = await rustSearchAsync(a.moduleName, 200);
+                  res = raw.map(normalizeResult).filter(r => {
+                    const p = r.path || '';
+                    const mod = r.module || '';
+                    return mod === a.moduleName ||
+                      p.includes(modulePath) ||
+                      (mVendorDir && vendorPath && p.toLowerCase().includes(`${mVendorDir}/${vendorPath}`));
+                  });
                 }
                 text = `Module: ${a.moduleName} (${res.length} files)\n`;
                 const cats = { controllers: '/Controller/', models: '/Model/', plugins: '/Plugin/', observers: '/Observer/', api: '/Api/' };
@@ -6872,6 +6995,16 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
       serveQuery('feedback', { signals }).catch((err) => logToFile('WARN', `Feedback signal send failed: ${err.message}`));
     }
   }
+};
+
+server.setRequestHandler(CallToolRequestSchema, async (request) => {
+  const result = await _callToolHandler(request);
+  // Append re-index warning to non-error responses during background re-index
+  if (reindexInProgress && !result?.isError && result?.content?.[0]?.type === 'text') {
+    const warning = getReindexWarning();
+    if (warning) result.content[0].text = warning + result.content[0].text;
+  }
+  return result;
 });
 
 server.setRequestHandler(ListResourcesRequestSchema, async () => ({