@codeyam/codeyam-cli 0.1.0-staging.76566f9 → 0.1.0-staging.79ef713

package/analyzer-template/packages/ai/src/lib/dataStructure/equivalencyManagers/frameworks/JavascriptFrameworkManager.ts

@@ -72,7 +72,11 @@ export class JavascriptFrameworkManager implements EquivalencyManager {
   }
 
   finalize(scopeNode: ScopeNode, scopeDataStructure: ScopeDataStructure) {
-    cleanNonObjectFunctions(scopeNode, scopeDataStructure, true);
+    // Don't call cleanNonObjectFunctions with final=true here.
+    // The aggressive cleanup (transformKeyMapping, clearAttributes) removes method call
+    // evidence like .includes() before type inference can use it.
+    // Instead, defer final cleanup to toSerializable() when we're done with inference.
+    cleanNonObjectFunctions(scopeNode, scopeDataStructure, false);
     cleanKnownObjectFunctions(scopeNode, scopeDataStructure);
     this.handleUniformArrays(scopeNode, scopeDataStructure);
   }

package/analyzer-template/packages/ai/src/lib/dataStructure/equivalencyManagers/frameworks/MuiManager.ts

@@ -39,6 +39,203 @@ export class MuiManager implements EquivalencyManager {
 
     // Also check if this scope is inside a DataGrid callback and propagate row property accesses
     this.propagateFromChildScope(scopeNode, scopeDataStructure);
+
+    // NOTE: processGridApiContext, processCreateTheme, and processGridSelector
+    // are now called from finalize() instead of here.
+    // This is because external function calls are only added AFTER postProcess()
+    // runs (in trackFunctionCalls), so getExternalFunctionCallInfo() would return
+    // undefined during postProcess(). See Issue 23.
+  }
+
+  /**
+   * Handle useGridApiContext() calls by adding the expected apiRef structure to the schema.
+   *
+   * useGridApiContext() returns an apiRef that has internal state used by useGridSelector.
+   * When useGridSelector(apiRef, selector) is called, it accesses apiRef.current.state
+   * to extract data like filterModel, sortModel, etc.
+   *
+   * We add the state structure directly to the external function call's schema
+   * to ensure mocks include the necessary paths.
+   */
+  private processGridApiContext(
+    _scopeNode: ScopeNode,
+    scopeDataStructure: ScopeDataStructure,
+  ) {
+    // Find the external function call for useGridApiContext
+    const gridApiContextCall =
+      scopeDataStructure.getExternalFunctionCallInfo('useGridApiContext');
+
+    if (!gridApiContextCall) {
+      return;
+    }
+
+    // Add the state structure that useGridSelector needs to the external function call's schema
+    // useGridSelector reads from apiRef.current.state internally
+    // gridFilterModelSelector reads state.filter.filterModel
+    const basePath = 'useGridApiContext().functionCallReturnValue.current';
+    const statePaths: Record<string, string> = {
+      [`${basePath}.state`]: 'object',
+      [`${basePath}.state.filter`]: 'object',
+      [`${basePath}.state.filter.filterModel`]: 'object',
+      [`${basePath}.state.filter.filterModel.items`]: 'array',
+      // Also add common state paths that might be needed by other selectors
+      [`${basePath}.state.sorting`]: 'object',
+      [`${basePath}.state.sorting.sortModel`]: 'array',
+      [`${basePath}.state.pagination`]: 'object',
+      [`${basePath}.state.pagination.paginationModel`]: 'object',
+      [`${basePath}.state.pagination.paginationModel.page`]: 'number',
+      [`${basePath}.state.pagination.paginationModel.pageSize`]: 'number',
+      [`${basePath}.state.rows`]: 'object',
+      [`${basePath}.state.rows.dataRowIds`]: 'array',
+      [`${basePath}.state.columns`]: 'object',
+      [`${basePath}.state.columns.all`]: 'array',
+    };
+
+    // Add each path to the external function call's schema if it doesn't exist
+    for (const [path, type] of Object.entries(statePaths)) {
+      if (!gridApiContextCall.schema[path]) {
+        gridApiContextCall.schema[path] = type;
+      }
+    }
+  }
+
+  /**
+   * Handle createTheme() calls by adding required MUI theme properties to the schema.
+   *
+   * MUI's ThemeProvider and related components check for specific properties:
+   * - `'$$material' in theme` - validates it's a MUI theme object
+   * - `'colorSchemes' in theme` - checks for color scheme support (MUI v6+)
+   *
+   * Without these properties, ThemeProvider throws:
+   * "TypeError: Cannot use 'in' operator to search for '$$material' in object"
+   * "TypeError: Cannot use 'in' operator to search for 'colorSchemes' in true"
+   *
+   * We add these properties to the createTheme return value schema
+   * to ensure mocks include the necessary properties.
+   */
+  private processCreateTheme(
+    _scopeNode: ScopeNode,
+    scopeDataStructure: ScopeDataStructure,
+  ) {
+    // Find the external function call for createTheme
+    const createThemeCall =
+      scopeDataStructure.getExternalFunctionCallInfo('createTheme');
+
+    if (!createThemeCall) {
+      return;
+    }
+
+    // Use the actual call signature to construct the path
+    // createTheme can be called with various arguments:
+    // - createTheme()
+    // - createTheme(options)
+    // - createTheme(getTheme(false), { components: {} })
+    const callSignature = createThemeCall.callSignature;
+    const basePath = `${callSignature}.functionCallReturnValue`;
+
+    // Add required MUI theme properties
+    const themeProperties: Record<string, string> = {
+      [`${basePath}.$$material`]: 'boolean',
+      [`${basePath}.colorSchemes`]: 'object',
+    };
+
+    for (const [path, type] of Object.entries(themeProperties)) {
+      if (!createThemeCall.schema[path]) {
+        createThemeCall.schema[path] = type;
+      }
+    }
+  }
+
+  /**
+   * Handle useGridSelector(apiRef, selector) calls by adding return value schema.
+   *
+   * useGridSelector is a MUI X Data Grid hook that extracts data from the grid's
+   * internal state using a selector function. It's unique because:
+   * 1. It takes arguments (apiRef and selector), so it's not automatically mocked
+   * 2. But its return value depends on which selector is used
+   * 3. The selector reads from apiRef.current.state
+   *
+   * Common selectors and their return types:
+   * - gridFilterModelSelector -> { items: GridFilterItem[] }
+   * - gridSortModelSelector -> GridSortItem[]
+   * - gridPaginationModelSelector -> { page: number, pageSize: number }
+   * - gridRowCountSelector -> number
+   *
+   * We detect which selector is used from the call signature and add the
+   * appropriate return value schema.
+   */
+  private processGridSelector(
+    _scopeNode: ScopeNode,
+    scopeDataStructure: ScopeDataStructure,
+  ) {
+    // Find external function calls for useGridSelector
+    const gridSelectorCall =
+      scopeDataStructure.getExternalFunctionCallInfo('useGridSelector');
+
+    if (!gridSelectorCall) {
+      return;
+    }
+
+    const callSignature = gridSelectorCall.callSignature;
+    const basePath = `${callSignature}.functionCallReturnValue`;
+
+    // Determine which selector is being used and add appropriate schema
+    if (callSignature.includes('gridFilterModelSelector')) {
+      // gridFilterModelSelector returns { items: GridFilterItem[] }
+      const filterPaths: Record<string, string> = {
+        [basePath]: 'object',
+        [`${basePath}.items`]: 'array',
+        [`${basePath}.items[]`]: 'object',
+        [`${basePath}.items[].field`]: 'string',
+        [`${basePath}.items[].operator`]: 'string',
+        [`${basePath}.items[].value`]: 'array',
+        [`${basePath}.items[].value[]`]: 'string',
+      };
+
+      for (const [path, type] of Object.entries(filterPaths)) {
+        if (!gridSelectorCall.schema[path]) {
+          gridSelectorCall.schema[path] = type;
+        }
+      }
+    } else if (callSignature.includes('gridSortModelSelector')) {
+      // gridSortModelSelector returns GridSortItem[]
+      const sortPaths: Record<string, string> = {
+        [basePath]: 'array',
+        [`${basePath}[]`]: 'object',
+        [`${basePath}[].field`]: 'string',
+        [`${basePath}[].sort`]: 'string',
+      };
+
+      for (const [path, type] of Object.entries(sortPaths)) {
+        if (!gridSelectorCall.schema[path]) {
+          gridSelectorCall.schema[path] = type;
+        }
+      }
+    } else if (callSignature.includes('gridPaginationModelSelector')) {
+      // gridPaginationModelSelector returns { page: number, pageSize: number }
+      const paginationPaths: Record<string, string> = {
+        [basePath]: 'object',
+        [`${basePath}.page`]: 'number',
+        [`${basePath}.pageSize`]: 'number',
+      };
+
+      for (const [path, type] of Object.entries(paginationPaths)) {
+        if (!gridSelectorCall.schema[path]) {
+          gridSelectorCall.schema[path] = type;
+        }
+      }
+    } else if (callSignature.includes('gridRowCountSelector')) {
+      // gridRowCountSelector returns number
+      if (!gridSelectorCall.schema[basePath]) {
+        gridSelectorCall.schema[basePath] = 'number';
+      }
+    } else {
+      // Unknown selector - add generic object return type
+      // This ensures the hook is at least mocked with some structure
+      if (!gridSelectorCall.schema[basePath]) {
+        gridSelectorCall.schema[basePath] = 'object';
+      }
+    }
   }
 
   private processDataGridCallbacks(
@@ -49,50 +246,107 @@ export class MuiManager implements EquivalencyManager {
     for (const [path, equivalencies] of Object.entries(
       scopeNode.equivalencies,
     )) {
-      // Look for DataGrid().signature[0].columns equivalency (DataGrid, DataGridPro, DataGridPremium)
+      // Case 1: Look for DataGrid().signature[0].columns equivalency pointing to a columns variable
       const dataGridMatch = path.match(
         /^(DataGrid(?:Pro|Premium)?\(\))\.signature\[0\]\.columns$/,
       );
-      if (!dataGridMatch) continue;
-
-      const dataGridCall = dataGridMatch[1];
+      if (dataGridMatch) {
+        const dataGridCall = dataGridMatch[1];
 
-      // Find what columns is equivalent to (e.g., "columns" variable)
-      for (const equiv of equivalencies) {
-        const columnsVar = equiv.schemaPath;
+        // Find what columns is equivalent to (e.g., "columns" variable)
+        for (const equiv of equivalencies) {
+          const columnsVar = equiv.schemaPath;
 
-        // Find renderCell callbacks, following equivalency chains if needed
-        const callbackScopes = this.findRenderCellCallbacks(
-          columnsVar,
-          scopeNode,
-          new Set(),
-          scopeDataStructure,
-        );
+          // Find renderCell callbacks, following equivalency chains if needed
+          const callbackScopes = this.findRenderCellCallbacks(
+            columnsVar,
+            scopeNode,
+            new Set(),
+            scopeDataStructure,
+          );
 
-        for (const callbackScopeName of callbackScopes) {
-          // Get the callback scope node
-          const callbackScopeNode =
-            scopeDataStructure.getScopeOrFunctionCallInfo(callbackScopeName);
-          if (!callbackScopeNode) {
-            continue;
+          for (const callbackScopeName of callbackScopes) {
+            this.addRowEquivalencyToCallback(
+              callbackScopeName,
+              dataGridCall,
+              scopeNode,
+              scopeDataStructure,
+            );
           }
+        }
+        continue;
+      }
 
-          // Add equivalency: signature[0].row ≡ DataGrid().signature[0].rows[]
-          const callbackParamRowPath = 'signature[0].row';
-          const rowsArrayElementPath = `${dataGridCall}.signature[0].rows[]`;
+      // Case 2: Look for inline array props where DataGrid().signature[0].columns[N].renderCell
+      // directly points to a callback scope (e.g., DataGrid().signature[0].columns[0].renderCell → cyScope1())
+      // Also handles columns[].renderCell for normalized paths
+      const inlineRenderCellMatch = path.match(
+        /^(DataGrid(?:Pro|Premium)?\(\))\.signature\[0\]\.columns\[(?:\d+|)\]\.renderCell$/,
+      );
 
-          scopeDataStructure.addEquivalency(
-            callbackParamRowPath,
-            rowsArrayElementPath,
-            scopeNode.name,
-            callbackScopeNode as ScopeNode,
-            'MUI DataGrid renderCell params.row equivalency',
+      if (inlineRenderCellMatch) {
+        const dataGridCall = inlineRenderCellMatch[1];
+
+        for (const equiv of equivalencies) {
+          // Check if the equivalency points to a callback scope (cyScope*)
+          // Could be cyScope1() or cyScope1().functionCallReturnValue
+          const callbackMatch = equiv.schemaPath.match(
+            /^(cyScope[0-9A-Fa-f]+)\(\)(?:\.functionCallReturnValue)?$/,
           );
+          if (callbackMatch) {
+            // Try both naming conventions:
+            // 1. Just the cyScope name (e.g., "cyScope1")
+            // 2. Parent____cyScope name (e.g., "CustomersPage____cyScope1")
+            const callbackScopeName = callbackMatch[1];
+            const prefixedCallbackScopeName = `${scopeNode.name}____${callbackMatch[1]}`;
+            // Try unprefixed name first, then prefixed
+            this.addRowEquivalencyToCallback(
+              callbackScopeName,
+              dataGridCall,
+              scopeNode,
+              scopeDataStructure,
+            );
+            // Also try prefixed version in case it exists
+            this.addRowEquivalencyToCallback(
+              prefixedCallbackScopeName,
+              dataGridCall,
+              scopeNode,
+              scopeDataStructure,
+            );
+          }
         }
       }
     }
   }
 
+  /**
+   * Helper to add the signature[0].row → DataGrid().signature[0].rows[] equivalency
+   */
+  private addRowEquivalencyToCallback(
+    callbackScopeName: string,
+    dataGridCall: string,
+    scopeNode: ScopeNode,
+    scopeDataStructure: ScopeDataStructure,
+  ) {
+    const callbackScopeNode =
+      scopeDataStructure.getScopeOrFunctionCallInfo(callbackScopeName);
+    if (!callbackScopeNode) {
+      return;
+    }
+
+    // Add equivalency: signature[0].row ≡ DataGrid().signature[0].rows[]
+    const callbackParamRowPath = 'signature[0].row';
+    const rowsArrayElementPath = `${dataGridCall}.signature[0].rows[]`;
+
+    scopeDataStructure.addEquivalency(
+      callbackParamRowPath,
+      rowsArrayElementPath,
+      scopeNode.name,
+      callbackScopeNode as ScopeNode,
+      'MUI DataGrid renderCell params.row equivalency',
+    );
+  }
+
   /**
    * Process Autocomplete callbacks to link option parameters to options array.
    *
@@ -605,6 +859,14 @@ export class MuiManager implements EquivalencyManager {
   }
 
   finalize(_scopeNode: ScopeNode, scopeDataStructure: ScopeDataStructure) {
+    // Process external function call schemas ONCE (not per-scope) since they
+    // are global. These were moved from postProcess() because external function
+    // calls are only available AFTER postProcess() runs (in trackFunctionCalls).
+    // See Issue 23.
+    this.processGridApiContext(_scopeNode, scopeDataStructure);
+    this.processCreateTheme(_scopeNode, scopeDataStructure);
+    this.processGridSelector(_scopeNode, scopeDataStructure);
+
     // Iterate through all scopes (finalize is only called on root)
     for (const [scopeName, scopeNode] of Object.entries(
       scopeDataStructure.scopeNodes,
@@ -702,12 +964,11 @@ export class MuiManager implements EquivalencyManager {
     }
   }
 
-  internalFunctions: Set<string> = new Set([
-    'DataGrid',
-    'DataGridPro',
-    'DataGridPremium',
-    'Autocomplete',
-  ]);
+  // NOTE: We intentionally do NOT include DataGrid/Autocomplete in internalFunctions.
+  // While they're MUI components (not external data sources), we need them to remain
+  // in externalFunctionCalls so that data flow tracing works correctly.
+  // For example: DataGrid().signature[0].rows → data.items[] tracing
+  internalFunctions: Set<string> = new Set([]);
 
   toString() {
     return 'MuiManager';

package/analyzer-template/packages/ai/src/lib/dataStructure/equivalencyManagers/frameworks/ReactFrameworkManager.ts

@@ -239,10 +239,18 @@ export class ReactFrameworkManager implements EquivalencyManager {
             });
           }
 
+          // Get the initial value path (what signature[0] is equivalent to)
+          const initialValuePath = equivalentValues[0].schemaPath;
+          const initialValueScopeName = equivalentValues[0].scopeNodeName;
+
+          // Note: Callback scope tracing (for useState(() => { return prop; }))
+          // is handled in getEquivalentSignatureVariables() where the full
+          // equivalency database is available for deep tracing.
+
           scopeDataStructure.addEquivalency(
             relevantUseStateInfo.value,
-            equivalentValues[0].schemaPath,
-            scopeNode.name,
+            initialValuePath,
+            initialValueScopeName,
             scopeNode,
             'useState value equivalency',
           );

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/BatchSchemaProcessor.ts

@@ -41,6 +41,7 @@ export interface BatchProcessorStats {
 
 export class BatchSchemaProcessor {
   private workQueue: WorkItem[] = [];
+  private queueIndex = 0;
   private stats: BatchProcessorStats = {
     itemsProcessed: 0,
     itemsSkipped: 0, // Kept for API compatibility, always 0
@@ -71,17 +72,28 @@ export class BatchSchemaProcessor {
    * Check if there are more items to process.
    */
   hasWork(): boolean {
-    return this.workQueue.length > 0;
+    return this.queueIndex < this.workQueue.length;
   }
 
   /**
    * Get the next work item from the queue.
    */
   getNextWork(): WorkItem | undefined {
-    const item = this.workQueue.shift();
+    if (this.queueIndex >= this.workQueue.length) {
+      return undefined;
+    }
+
+    const item = this.workQueue[this.queueIndex++];
     if (item) {
       this.stats.itemsProcessed++;
     }
+
+    // Once we've consumed the queue, reset to avoid unbounded growth
+    if (this.queueIndex >= this.workQueue.length) {
+      this.workQueue = [];
+      this.queueIndex = 0;
+    }
+
     return item;
   }
 
@@ -122,6 +134,7 @@ export class BatchSchemaProcessor {
    */
   reset(): void {
     this.workQueue = [];
+    this.queueIndex = 0;
     this.stats = {
       itemsProcessed: 0,
       itemsSkipped: 0,
@@ -134,6 +147,6 @@ export class BatchSchemaProcessor {
    * Get current queue size (for debugging/monitoring).
    */
   getQueueSize(): number {
-    return this.workQueue.length;
+    return this.workQueue.length - this.queueIndex;
   }
 }

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/ScopeTreeManager.ts

@@ -117,9 +117,10 @@ export class ScopeTreeManager {
    */
   findNode(name: string): ScopeTreeNode | undefined {
     const queue: ScopeTreeNode[] = [this.tree];
+    let index = 0;
 
-    while (queue.length > 0) {
-      const node = queue.shift()!;
+    while (index < queue.length) {
+      const node = queue[index++]!;
       if (node.name === name) {
         return node;
       }
@@ -135,9 +136,10 @@ export class ScopeTreeManager {
   getAllNames(): string[] {
     const names: string[] = [];
     const queue: ScopeTreeNode[] = [this.tree];
+    let index = 0;
 
-    while (queue.length > 0) {
-      const node = queue.shift()!;
+    while (index < queue.length) {
+      const node = queue[index++]!;
       names.push(node.name);
       queue.push(...node.children);
     }

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/cleanKnownObjectFunctions.ts

@@ -35,6 +35,47 @@ const getParts = (path: string): string[] => {
   return parts;
 };
 
+function buildPrefixParts(parts: string[]): string[] {
+  if (parts.length === 0) return [];
+  const prefixes = new Array(parts.length);
+  let current = '';
+  for (let i = 0; i < parts.length; i++) {
+    const part = parts[i];
+    if (i === 0) {
+      current = part;
+    } else if (part.startsWith('[') || part.startsWith('(')) {
+      current += part;
+    } else {
+      current += `.${part}`;
+    }
+    prefixes[i] = current;
+  }
+  return prefixes;
+}
+
+/**
+ * Clear the module-level cache.
+ * Call this between entity analyses to prevent unbounded memory growth.
+ */
+export function clearCleanKnownObjectFunctionsCache(): {
+  count: number;
+  estimatedBytes: number;
+} {
+  let totalBytes = 0;
+  for (const [key, value] of partsCache) {
+    // Key is a string, value is string[] - estimate UTF-16 encoding (2 bytes per char)
+    totalBytes += key.length * 2;
+    for (const part of value) {
+      totalBytes += part.length * 2;
+    }
+    // Add overhead for array and Map entry (~50 bytes per entry)
+    totalBytes += 50;
+  }
+  const count = partsCache.size;
+  partsCache.clear();
+  return { count, estimatedBytes: totalBytes };
+}
+
 /** pre-compute the prototype-method sets once per module load */
 const protoMethodSets: Record<PrimitiveKind, Set<string>> = {
   array: new Set(
@@ -110,6 +151,7 @@ function scrub(
 
   for (const keyPath of sortedKeys) {
     const keyParts = getParts(keyPath);
+    const prefixParts = buildPrefixParts(keyParts);
 
     if (
       IGNORE_CLASSES_AND_OBJECTS.find(
@@ -122,7 +164,7 @@ function scrub(
 
     // walk back through the prefixes: foo.bar.baz.qux → foo.bar.baz → foo.bar …
     for (let cut = keyParts.length - 1; cut > 0; --cut) {
-      const prefix = joinParenthesesAndArrays(keyParts.slice(0, cut));
+      const prefix = prefixParts[cut - 1];
       const rootKind = rootTable.get(prefix);
       if (!rootKind) continue;
 
@@ -130,9 +172,18 @@ function scrub(
       const rawMethod = rawToken.split('(')[0];
 
       // Check if it's a prototype method (e.g., map, filter, toString)
+      // Only match when the token is actually a method reference/call:
+      // - Has parentheses (explicit call like map(), filter(x => x))
+      // - OR is typed as "function" (method reference like .split)
+      // Bare property names that happen to match method names
+      // (like .entries = "array", .values = "object") are preserved.
       if (protoMethodSets[rootKind].has(rawMethod)) {
-        onRemove(keyPath);
-        break;
+        const isMethodCall = rawToken.includes('(');
+        const isMethodReference = mapping[keyPath] === 'function';
+        if (isMethodCall || isMethodReference) {
+          onRemove(keyPath);
+          break;
+        }
       }
 
       // Check if it's a built-in property (e.g., length on arrays/strings)