@codeyam/codeyam-cli 0.1.0-staging.1669d45 → 0.1.0-staging.2a88920

package/analyzer-template/packages/ai/src/lib/dataStructure/equivalencyManagers/frameworks/MuiManager.ts

@@ -39,6 +39,203 @@ export class MuiManager implements EquivalencyManager {
 
     // Also check if this scope is inside a DataGrid callback and propagate row property accesses
     this.propagateFromChildScope(scopeNode, scopeDataStructure);
+
+    // NOTE: processGridApiContext, processCreateTheme, and processGridSelector
+    // are now called from finalize() instead of here.
+    // This is because external function calls are only added AFTER postProcess()
+    // runs (in trackFunctionCalls), so getExternalFunctionCallInfo() would return
+    // undefined during postProcess(). See Issue 23.
+  }
+
+  /**
+   * Handle useGridApiContext() calls by adding the expected apiRef structure to the schema.
+   *
+   * useGridApiContext() returns an apiRef that has internal state used by useGridSelector.
+   * When useGridSelector(apiRef, selector) is called, it accesses apiRef.current.state
+   * to extract data like filterModel, sortModel, etc.
+   *
+   * We add the state structure directly to the external function call's schema
+   * to ensure mocks include the necessary paths.
+   */
+  private processGridApiContext(
+    _scopeNode: ScopeNode,
+    scopeDataStructure: ScopeDataStructure,
+  ) {
+    // Find the external function call for useGridApiContext
+    const gridApiContextCall =
+      scopeDataStructure.getExternalFunctionCallInfo('useGridApiContext');
+
+    if (!gridApiContextCall) {
+      return;
+    }
+
+    // Add the state structure that useGridSelector needs to the external function call's schema
+    // useGridSelector reads from apiRef.current.state internally
+    // gridFilterModelSelector reads state.filter.filterModel
+    const basePath = 'useGridApiContext().functionCallReturnValue.current';
+    const statePaths: Record<string, string> = {
+      [`${basePath}.state`]: 'object',
+      [`${basePath}.state.filter`]: 'object',
+      [`${basePath}.state.filter.filterModel`]: 'object',
+      [`${basePath}.state.filter.filterModel.items`]: 'array',
+      // Also add common state paths that might be needed by other selectors
+      [`${basePath}.state.sorting`]: 'object',
+      [`${basePath}.state.sorting.sortModel`]: 'array',
+      [`${basePath}.state.pagination`]: 'object',
+      [`${basePath}.state.pagination.paginationModel`]: 'object',
+      [`${basePath}.state.pagination.paginationModel.page`]: 'number',
+      [`${basePath}.state.pagination.paginationModel.pageSize`]: 'number',
+      [`${basePath}.state.rows`]: 'object',
+      [`${basePath}.state.rows.dataRowIds`]: 'array',
+      [`${basePath}.state.columns`]: 'object',
+      [`${basePath}.state.columns.all`]: 'array',
+    };
+
+    // Add each path to the external function call's schema if it doesn't exist
+    for (const [path, type] of Object.entries(statePaths)) {
+      if (!gridApiContextCall.schema[path]) {
+        gridApiContextCall.schema[path] = type;
+      }
+    }
+  }
+
+  /**
+   * Handle createTheme() calls by adding required MUI theme properties to the schema.
+   *
+   * MUI's ThemeProvider and related components check for specific properties:
+   * - `'$$material' in theme` - validates it's a MUI theme object
+   * - `'colorSchemes' in theme` - checks for color scheme support (MUI v6+)
+   *
+   * Without these properties, ThemeProvider throws:
+   * "TypeError: Cannot use 'in' operator to search for '$$material' in object"
+   * "TypeError: Cannot use 'in' operator to search for 'colorSchemes' in true"
+   *
+   * We add these properties to the createTheme return value schema
+   * to ensure mocks include the necessary properties.
+   */
+  private processCreateTheme(
+    _scopeNode: ScopeNode,
+    scopeDataStructure: ScopeDataStructure,
+  ) {
+    // Find the external function call for createTheme
+    const createThemeCall =
+      scopeDataStructure.getExternalFunctionCallInfo('createTheme');
+
+    if (!createThemeCall) {
+      return;
+    }
+
+    // Use the actual call signature to construct the path
+    // createTheme can be called with various arguments:
+    // - createTheme()
+    // - createTheme(options)
+    // - createTheme(getTheme(false), { components: {} })
+    const callSignature = createThemeCall.callSignature;
+    const basePath = `${callSignature}.functionCallReturnValue`;
+
+    // Add required MUI theme properties
+    const themeProperties: Record<string, string> = {
+      [`${basePath}.$$material`]: 'boolean',
+      [`${basePath}.colorSchemes`]: 'object',
+    };
+
+    for (const [path, type] of Object.entries(themeProperties)) {
+      if (!createThemeCall.schema[path]) {
+        createThemeCall.schema[path] = type;
+      }
+    }
+  }
+
+  /**
+   * Handle useGridSelector(apiRef, selector) calls by adding return value schema.
+   *
+   * useGridSelector is a MUI X Data Grid hook that extracts data from the grid's
+   * internal state using a selector function. It's unique because:
+   * 1. It takes arguments (apiRef and selector), so it's not automatically mocked
+   * 2. But its return value depends on which selector is used
+   * 3. The selector reads from apiRef.current.state
+   *
+   * Common selectors and their return types:
+   * - gridFilterModelSelector -> { items: GridFilterItem[] }
+   * - gridSortModelSelector -> GridSortItem[]
+   * - gridPaginationModelSelector -> { page: number, pageSize: number }
+   * - gridRowCountSelector -> number
+   *
+   * We detect which selector is used from the call signature and add the
+   * appropriate return value schema.
+   */
+  private processGridSelector(
+    _scopeNode: ScopeNode,
+    scopeDataStructure: ScopeDataStructure,
+  ) {
+    // Find external function calls for useGridSelector
+    const gridSelectorCall =
+      scopeDataStructure.getExternalFunctionCallInfo('useGridSelector');
+
+    if (!gridSelectorCall) {
+      return;
+    }
+
+    const callSignature = gridSelectorCall.callSignature;
+    const basePath = `${callSignature}.functionCallReturnValue`;
+
+    // Determine which selector is being used and add appropriate schema
+    if (callSignature.includes('gridFilterModelSelector')) {
+      // gridFilterModelSelector returns { items: GridFilterItem[] }
+      const filterPaths: Record<string, string> = {
+        [basePath]: 'object',
+        [`${basePath}.items`]: 'array',
+        [`${basePath}.items[]`]: 'object',
+        [`${basePath}.items[].field`]: 'string',
+        [`${basePath}.items[].operator`]: 'string',
+        [`${basePath}.items[].value`]: 'array',
+        [`${basePath}.items[].value[]`]: 'string',
+      };
+
+      for (const [path, type] of Object.entries(filterPaths)) {
+        if (!gridSelectorCall.schema[path]) {
+          gridSelectorCall.schema[path] = type;
+        }
+      }
+    } else if (callSignature.includes('gridSortModelSelector')) {
+      // gridSortModelSelector returns GridSortItem[]
+      const sortPaths: Record<string, string> = {
+        [basePath]: 'array',
+        [`${basePath}[]`]: 'object',
+        [`${basePath}[].field`]: 'string',
+        [`${basePath}[].sort`]: 'string',
+      };
+
+      for (const [path, type] of Object.entries(sortPaths)) {
+        if (!gridSelectorCall.schema[path]) {
+          gridSelectorCall.schema[path] = type;
+        }
+      }
+    } else if (callSignature.includes('gridPaginationModelSelector')) {
+      // gridPaginationModelSelector returns { page: number, pageSize: number }
+      const paginationPaths: Record<string, string> = {
+        [basePath]: 'object',
+        [`${basePath}.page`]: 'number',
+        [`${basePath}.pageSize`]: 'number',
+      };
+
+      for (const [path, type] of Object.entries(paginationPaths)) {
+        if (!gridSelectorCall.schema[path]) {
+          gridSelectorCall.schema[path] = type;
+        }
+      }
+    } else if (callSignature.includes('gridRowCountSelector')) {
+      // gridRowCountSelector returns number
+      if (!gridSelectorCall.schema[basePath]) {
+        gridSelectorCall.schema[basePath] = 'number';
+      }
+    } else {
+      // Unknown selector - add generic object return type
+      // This ensures the hook is at least mocked with some structure
+      if (!gridSelectorCall.schema[basePath]) {
+        gridSelectorCall.schema[basePath] = 'object';
+      }
+    }
   }
 
   private processDataGridCallbacks(
@@ -662,6 +859,14 @@ export class MuiManager implements EquivalencyManager {
   }
 
   finalize(_scopeNode: ScopeNode, scopeDataStructure: ScopeDataStructure) {
+    // Process external function call schemas ONCE (not per-scope) since they
+    // are global. These were moved from postProcess() because external function
+    // calls are only available AFTER postProcess() runs (in trackFunctionCalls).
+    // See Issue 23.
+    this.processGridApiContext(_scopeNode, scopeDataStructure);
+    this.processCreateTheme(_scopeNode, scopeDataStructure);
+    this.processGridSelector(_scopeNode, scopeDataStructure);
+
     // Iterate through all scopes (finalize is only called on root)
     for (const [scopeName, scopeNode] of Object.entries(
       scopeDataStructure.scopeNodes,

package/analyzer-template/packages/ai/src/lib/dataStructure/equivalencyManagers/frameworks/ReactFrameworkManager.ts

@@ -239,10 +239,18 @@ export class ReactFrameworkManager implements EquivalencyManager {
             });
           }
 
+          // Get the initial value path (what signature[0] is equivalent to)
+          const initialValuePath = equivalentValues[0].schemaPath;
+          const initialValueScopeName = equivalentValues[0].scopeNodeName;
+
+          // Note: Callback scope tracing (for useState(() => { return prop; }))
+          // is handled in getEquivalentSignatureVariables() where the full
+          // equivalency database is available for deep tracing.
+
           scopeDataStructure.addEquivalency(
             relevantUseStateInfo.value,
-            equivalentValues[0].schemaPath,
-            scopeNode.name,
+            initialValuePath,
+            initialValueScopeName,
             scopeNode,
             'useState value equivalency',
           );

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/BatchSchemaProcessor.ts

@@ -41,6 +41,7 @@ export interface BatchProcessorStats {
 
 export class BatchSchemaProcessor {
   private workQueue: WorkItem[] = [];
+  private queueIndex = 0;
   private stats: BatchProcessorStats = {
     itemsProcessed: 0,
     itemsSkipped: 0, // Kept for API compatibility, always 0
@@ -71,17 +72,28 @@ export class BatchSchemaProcessor {
    * Check if there are more items to process.
    */
   hasWork(): boolean {
-    return this.workQueue.length > 0;
+    return this.queueIndex < this.workQueue.length;
   }
 
   /**
    * Get the next work item from the queue.
    */
   getNextWork(): WorkItem | undefined {
-    const item = this.workQueue.shift();
+    if (this.queueIndex >= this.workQueue.length) {
+      return undefined;
+    }
+
+    const item = this.workQueue[this.queueIndex++];
     if (item) {
       this.stats.itemsProcessed++;
     }
+
+    // Once we've consumed the queue, reset to avoid unbounded growth
+    if (this.queueIndex >= this.workQueue.length) {
+      this.workQueue = [];
+      this.queueIndex = 0;
+    }
+
     return item;
   }
 
@@ -122,6 +134,7 @@ export class BatchSchemaProcessor {
    */
   reset(): void {
     this.workQueue = [];
+    this.queueIndex = 0;
     this.stats = {
       itemsProcessed: 0,
       itemsSkipped: 0,
@@ -134,6 +147,6 @@ export class BatchSchemaProcessor {
    * Get current queue size (for debugging/monitoring).
    */
   getQueueSize(): number {
-    return this.workQueue.length;
+    return this.workQueue.length - this.queueIndex;
   }
 }

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/ScopeTreeManager.ts

@@ -117,9 +117,10 @@ export class ScopeTreeManager {
    */
   findNode(name: string): ScopeTreeNode | undefined {
     const queue: ScopeTreeNode[] = [this.tree];
+    let index = 0;
 
-    while (queue.length > 0) {
-      const node = queue.shift()!;
+    while (index < queue.length) {
+      const node = queue[index++]!;
       if (node.name === name) {
         return node;
       }
@@ -135,9 +136,10 @@ export class ScopeTreeManager {
   getAllNames(): string[] {
     const names: string[] = [];
     const queue: ScopeTreeNode[] = [this.tree];
+    let index = 0;
 
-    while (queue.length > 0) {
-      const node = queue.shift()!;
+    while (index < queue.length) {
+      const node = queue[index++]!;
       names.push(node.name);
       queue.push(...node.children);
     }

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/cleanKnownObjectFunctions.ts

@@ -35,6 +35,47 @@ const getParts = (path: string): string[] => {
   return parts;
 };
 
+function buildPrefixParts(parts: string[]): string[] {
+  if (parts.length === 0) return [];
+  const prefixes = new Array(parts.length);
+  let current = '';
+  for (let i = 0; i < parts.length; i++) {
+    const part = parts[i];
+    if (i === 0) {
+      current = part;
+    } else if (part.startsWith('[') || part.startsWith('(')) {
+      current += part;
+    } else {
+      current += `.${part}`;
+    }
+    prefixes[i] = current;
+  }
+  return prefixes;
+}
+
+/**
+ * Clear the module-level cache.
+ * Call this between entity analyses to prevent unbounded memory growth.
+ */
+export function clearCleanKnownObjectFunctionsCache(): {
+  count: number;
+  estimatedBytes: number;
+} {
+  let totalBytes = 0;
+  for (const [key, value] of partsCache) {
+    // Key is a string, value is string[] - estimate UTF-16 encoding (2 bytes per char)
+    totalBytes += key.length * 2;
+    for (const part of value) {
+      totalBytes += part.length * 2;
+    }
+    // Add overhead for array and Map entry (~50 bytes per entry)
+    totalBytes += 50;
+  }
+  const count = partsCache.size;
+  partsCache.clear();
+  return { count, estimatedBytes: totalBytes };
+}
+
 /** pre-compute the prototype-method sets once per module load */
 const protoMethodSets: Record<PrimitiveKind, Set<string>> = {
   array: new Set(
@@ -110,6 +151,7 @@ function scrub(
 
   for (const keyPath of sortedKeys) {
     const keyParts = getParts(keyPath);
+    const prefixParts = buildPrefixParts(keyParts);
 
     if (
       IGNORE_CLASSES_AND_OBJECTS.find(
@@ -122,7 +164,7 @@ function scrub(
 
     // walk back through the prefixes: foo.bar.baz.qux → foo.bar.baz → foo.bar …
     for (let cut = keyParts.length - 1; cut > 0; --cut) {
-      const prefix = joinParenthesesAndArrays(keyParts.slice(0, cut));
+      const prefix = prefixParts[cut - 1];
       const rootKind = rootTable.get(prefix);
       if (!rootKind) continue;
 

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/cleanNonObjectFunctions.ts

@@ -119,6 +119,17 @@ const stringMethodsSet = new Set([
 ]);
 export const allMethodsSet = new Set([...arrayMethodsSet, ...stringMethodsSet]);
 
+// Methods that exist in both stringMethodsSet and arrayMethodsSet.
+// When ONLY these methods are called on an object, we can't determine if it's a string or array.
+// Don't infer 'string' type from these alone - leave as 'unknown' for other type resolution.
+const stringArrayAmbiguousMethodsSet = new Set([
+  'includes', // exists in both String.includes() and Array.includes()
+  'indexOf', // exists in both String.indexOf() and Array.indexOf()
+  'lastIndexOf', // exists in both String.lastIndexOf() and Array.lastIndexOf()
+  'concat', // exists in both String.concat() and Array.concat()
+  'slice', // exists in both String.slice() and Array.slice()
+]);
+
 const signatureKeyWord = 'signature';
 const returnValueKeyWord = 'functionCallReturnValue';
 
@@ -140,21 +151,63 @@ function cachedSplit(str: string): string[] {
   return parts;
 }
 
+/**
+ * Clear the module-level cache.
+ * Call this between entity analyses to prevent unbounded memory growth.
+ */
+export function clearCleanNonObjectFunctionsCache(): {
+  count: number;
+  estimatedBytes: number;
+} {
+  let totalBytes = 0;
+  for (const [key, value] of splitCache) {
+    // Key is a string, value is string[] - estimate UTF-16 encoding (2 bytes per char)
+    totalBytes += key.length * 2;
+    for (const part of value) {
+      totalBytes += part.length * 2;
+    }
+    // Add overhead for array and Map entry (~50 bytes per entry)
+    totalBytes += 50;
+  }
+  const count = splitCache.size;
+  splitCache.clear();
+  return { count, estimatedBytes: totalBytes };
+}
+
 // --- Prefix Index for O(1) lookup in subPathIsFromKnownObject ---
 // Maps parent prefix -> array of last parts (method names without parens)
 type PrefixIndex = Map<string, string[]>;
 
+function buildPrefixPartsWithRoot(parts: string[]): string[] {
+  const prefixes = new Array(parts.length + 1);
+  prefixes[0] = '';
+  let current = '';
+  for (let i = 0; i < parts.length; i++) {
+    const part = parts[i];
+    if (i === 0) {
+      current = part;
+    } else if (part.startsWith('[') || part.startsWith('(')) {
+      current += part;
+    } else {
+      current += `.${part}`;
+    }
+    prefixes[i + 1] = current;
+  }
+  return prefixes;
+}
+
 function buildPrefixIndex(allPaths: string[]): PrefixIndex {
   const index: PrefixIndex = new Map();
 
   for (const path of allPaths) {
     const parts = cachedSplit(path);
     if (parts.length < 1) continue;
+    const prefixes = buildPrefixPartsWithRoot(parts);
 
     // For each possible prefix length, add the last part to that prefix's entry
     // This allows lookup at any depth
     for (let prefixLen = 0; prefixLen < parts.length; prefixLen++) {
-      const prefix = joinParenthesesAndArrays(parts.slice(0, prefixLen));
+      const prefix = prefixes[prefixLen];
       const lastPart = parts[prefixLen]?.split('(')?.[0];
       if (lastPart) {
         let existing = index.get(prefix);
@@ -697,7 +750,14 @@ function handleFunctionMapping(
         allProperties.every((prop) => stringMethodsSet.has(getMethodName(prop)))
       ) {
         if (allProperties.every((prop) => isValidStringMethod(prop))) {
-          keyMapping[previousPath] = 'string';
+          // Don't infer string if ALL methods are ambiguous between string and array
+          // (like includes, indexOf, concat, slice - these exist on both types)
+          const hasOnlyStringArrayAmbiguousMethods = allProperties.every(
+            (prop) => stringArrayAmbiguousMethodsSet.has(getMethodName(prop)),
+          );
+          if (!hasOnlyStringArrayAmbiguousMethods) {
+            keyMapping[previousPath] = 'string';
+          }
         } else {
           return false;
         }
@@ -718,19 +778,36 @@ function handleFunctionMapping(
 
     // Don't add [] element for ambiguous methods without other array evidence
     // Also don't add [] if the path was already identified as a string type
+    // Also don't add [] if the method is ambiguous between string and array (like includes)
     const methodName = getMethodName(lastPart);
     const isAmbiguousMethod = ambiguousArrayMethodsSet.has(methodName);
+    const isStringArrayAmbiguous =
+      stringArrayAmbiguousMethodsSet.has(methodName);
     const isStringType = keyMapping[previousPath] === 'string';
     if (
       !isValidFunctionCall(previousPart) &&
       arrayMethodsSet.has(methodName) &&
       !keyMapping[previousPath + '[]'] &&
       !isAmbiguousMethod &&
+      !isStringArrayAmbiguous &&
       !isStringType
     ) {
       keyMapping[previousPath + '[]'] = 'unknown';
     }
 
+    // For primitive-returning methods (like includes, indexOf, some, every),
+    // also delete the .functionCallReturnValue children since the return value
+    // is just a primitive (boolean/number) and not useful data.
+    // For other methods (like filter, map), keep children for transformation.
+    const isPrimitiveReturning = primitiveReturningMethodsSet.has(methodName);
+    if (isPrimitiveReturning) {
+      for (const otherPath of Object.keys(keyMapping)) {
+        if (otherPath.startsWith(path + '.')) {
+          delete keyMapping[otherPath];
+        }
+      }
+    }
+
     delete keyMapping[path];
     return true;
   }
@@ -956,14 +1033,34 @@ function clearAttributes(
     }
   }
 
+  // Track paths that have primitive-returning method calls (like includes, some, every)
+  // These need cleanup even if we can't determine the base type
+  const pathsWithPrimitiveReturningMethods = new Set<string>();
+
   for (const key in functionCallsOnObjects) {
     const functionCalls = functionCallsOnObjects[key];
+
+    // Check if any method calls are primitive-returning
+    const hasPrimitiveReturningMethods = functionCalls.some((f) =>
+      primitiveReturningMethodsSet.has(getMethodName(f)),
+    );
+    if (hasPrimitiveReturningMethods) {
+      pathsWithPrimitiveReturningMethods.add(key);
+    }
+
     if (
       functionCalls.every(
         (f) => stringMethodsSet.has(getMethodName(f)) && isValidStringMethod(f),
       )
     ) {
-      mapping[key] = 'string';
+      // Don't infer string if ALL methods are ambiguous between string and array
+      // (like includes, indexOf, concat, slice - these exist on both types)
+      const hasOnlyStringArrayAmbiguousMethods = functionCalls.every((f) =>
+        stringArrayAmbiguousMethodsSet.has(getMethodName(f)),
+      );
+      if (!hasOnlyStringArrayAmbiguousMethods) {
+        mapping[key] = 'string';
+      }
     } else if (
       functionCalls.every(
         (f) => arrayMethodsSet.has(getMethodName(f)) && isValidArrayMethod(f),
@@ -1000,29 +1097,39 @@ function clearAttributes(
     }
   }
 
+  // Also add paths with primitive-returning method calls (like includes, some, every)
+  // to knownBasePaths so their method call paths get cleaned up, even if we couldn't
+  // determine the base type (e.g., when only ambiguous methods like includes are used)
+  for (const path of pathsWithPrimitiveReturningMethods) {
+    knownBasePaths.add(path);
+  }
+
+  // OPTIMIZATION: Instead of O(keys × basePaths), use O(keys × keyDepth) where keyDepth is small
+  // For each key, find all its prefixes and check if any is a known basePath (O(1) Set lookup)
   for (const key in mapping) {
     const keyParts = cachedSplit(key);
-    for (const basePath of knownBasePaths) {
-      const basePathParts = cachedSplit(basePath);
-      if (keyParts.length <= basePathParts.length) continue;
-      if (
-        joinParenthesesAndArrays(keyParts.slice(0, basePathParts.length)) !==
-        basePath
-      )
-        continue;
+    if (keyParts.length <= 1) continue; // No possible parent basePath
+
+    // Check each prefix of the key to see if it's a known basePath
+    // Start from shortest prefix (most likely to match) for early exit
+    for (let prefixLen = 1; prefixLen < keyParts.length; prefixLen++) {
+      const prefix = joinParenthesesAndArrays(keyParts.slice(0, prefixLen));
+      if (!knownBasePaths.has(prefix)) continue;
+
+      // Found a matching basePath - now check the conditions
+      const basePath = prefix;
+
       // Preserve functionCallReturnValue for non-string types, UNLESS
       // the method is a primitive-returning method (like some, every, includes, indexOf)
       // whose return values are not mockable
-      const methodPart = keyParts[basePathParts.length];
+      const methodPart = keyParts[prefixLen];
       const methodName = methodPart?.split('(')[0];
       const isPrimitiveReturning = primitiveReturningMethodsSet.has(
         methodName ?? '',
       );
       if (
         mapping[basePath] !== 'string' &&
-        keyParts[basePathParts.length + 1]?.startsWith(
-          'functionCallReturnValue',
-        ) &&
+        keyParts[prefixLen + 1]?.startsWith('functionCallReturnValue') &&
         !isPrimitiveReturning
       )
         continue;