@codeyam/codeyam-cli 0.1.0-staging.1669d45 → 0.1.0-staging.25a2014

package/analyzer-template/packages/ai/src/lib/resolvePathToControllable.ts

@@ -336,6 +336,7 @@ function findControllablePrefix(
     // Check direct match in attributesMap
     if (prefix in attributesMap) {
       const type = attributesMap[prefix];
+
       // For objects, arrays, or unknown types, nested property access is controllable
       if (
         isArrayType(type) ||
@@ -375,16 +376,6 @@ function findControllableBaseForDerivedPath(
   path: string,
   attributesMap: Record<string, string>,
 ): string | null {
-  // Use cleanPathOfNonTransformingFunctions to strip non-transforming methods
-  // like .filter(), .sort(), .slice() etc. This handles all method calls generically
-  // without hardcoding specific method names.
-  const cleanedPath = cleanPathOfNonTransformingFunctions(path);
-
-  // If the path wasn't changed, there are no non-transforming functions to strip
-  if (cleanedPath === path) {
-    return null;
-  }
-
   // Helper to check if a path is in attributesMap
   const checkInAttributesMap = (pathToCheck: string): string | null => {
     // Direct match
@@ -401,21 +392,121 @@ function findControllableBaseForDerivedPath(
     return null;
   };
 
-  // Check the cleaned path directly
+  // Use cleanPathOfNonTransformingFunctions to strip non-transforming methods
+  // like .filter(), .sort(), .slice() etc. This handles all method calls generically
+  // without hardcoding specific method names.
+  const cleanedPath = cleanPathOfNonTransformingFunctions(path);
+
+  // Check the cleaned path directly (even if unchanged, later steps may find something)
   const directResult = checkInAttributesMap(cleanedPath);
   if (directResult) {
     return directResult;
   }
 
-  // Extract the short path (after functionCallReturnValue) for attributesMap lookup
-  const functionReturnMatch = cleanedPath.match(
+  // Fix 38: For controllability purposes, strip ALL derivation operations
+  // including transforming methods like .split(...) and array indices.
+  // The key insight is: if we control the base, we control all derived values.
+  //
+  // Pattern: base.split('...').filter(...)[N] → find base
+  // We progressively strip: [N] → filter(...) → split(...) → until we find something controllable
+
+  // Methods that derive new values (should be stripped for controllability)
+  const derivingMethods = [
+    'split',
+    'match',
+    'filter',
+    'map',
+    'find',
+    'findIndex',
+    'slice',
+    'substring',
+    'substr',
+    'trim',
+    'trimStart',
+    'trimEnd',
+    'toLowerCase',
+    'toUpperCase',
+    'replace',
+    'replaceAll',
+    'sort',
+    'reverse',
+    'flat',
+    'flatMap',
+  ];
+
+  // Build regex to match method calls: .methodName(...)
+  // Use a simpler approach: progressively strip trailing derivations
+  let workingPath = cleanedPath;
+
+  // Keep stripping derivations until we find something controllable or can't strip anymore
+  // Max 20 iterations to prevent infinite loops
+  for (let iteration = 0; iteration < 20; iteration++) {
+    // Try current path
+    const result = checkInAttributesMap(workingPath);
+    if (result) {
+      return result;
+    }
+
+    // Also check via functionCallReturnValue extraction
+    // Note: We return workingPath (the FULL path) if the short path is found,
+    // because the full path is more meaningful for execution flows
+    const functionReturnMatch = workingPath.match(
+      /\.functionCallReturnValue\.(.+)$/,
+    );
+    if (functionReturnMatch) {
+      const shortPath = functionReturnMatch[1];
+      const shortResult = checkInAttributesMap(shortPath);
+      if (shortResult) {
+        // Return the full path, not just the short result
+        // This ensures execution flows have meaningful paths like
+        // "useParams().functionCallReturnValue.*" instead of just "*"
+        return workingPath;
+      }
+    }
+
+    // Try to strip trailing array index: path[0] → path
+    const arrayIndexMatch = workingPath.match(/^(.+)\[\d+\]$/);
+    if (arrayIndexMatch) {
+      workingPath = arrayIndexMatch[1];
+      continue;
+    }
+
+    // Try to strip trailing method call: path.method(...) → path
+    let strippedMethod = false;
+    for (const method of derivingMethods) {
+      // Match .method(...) at the end, handling nested parentheses
+      const methodRegex = new RegExp(
+        `\\.${method}\\([^()]*(?:\\([^()]*\\)[^()]*)*\\)$`,
+      );
+      if (methodRegex.test(workingPath)) {
+        workingPath = workingPath.replace(methodRegex, '');
+        strippedMethod = true;
+        break;
+      }
+    }
+
+    if (!strippedMethod) {
+      // Can't strip anymore
+      break;
+    }
+  }
+
+  // Final check after stripping
+  const finalResult = checkInAttributesMap(workingPath);
+  if (finalResult) {
+    return finalResult;
+  }
+
+  // Also check via functionCallReturnValue extraction for final path
+  const finalFunctionReturnMatch = workingPath.match(
     /\.functionCallReturnValue\.(.+)$/,
   );
-  if (functionReturnMatch) {
-    const shortPath = functionReturnMatch[1];
+  if (finalFunctionReturnMatch) {
+    const shortPath = finalFunctionReturnMatch[1];
     const shortResult = checkInAttributesMap(shortPath);
     if (shortResult) {
-      return shortResult;
+      // Return the full path, not just the short result
+      return workingPath;
     }
   }
 
@@ -434,7 +525,7 @@ function findControllableBaseForDerivedPath(
 export default function resolvePathToControllable(
   localPath: string,
   attributesMap: Record<string, string>,
-  equivalentSignatureVariables: Record<string, string>,
+  equivalentSignatureVariables: Record<string, string | string[]>,
   fullToShortPathMap: Record<string, string>,
 ): PathResolutionResult {
   const chain: string[] = [localPath];
@@ -509,7 +600,11 @@ export default function resolvePathToControllable(
 
   // 4. Equivalent variable resolution
   const localVarName = extractLocalVariableName(localPath);
-  const dataSourceBase = equivalentSignatureVariables[localVarName];
+  // Handle array case (OR expressions) - use first element if array
+  const rawDataSourceBase = equivalentSignatureVariables[localVarName];
+  const dataSourceBase = Array.isArray(rawDataSourceBase)
+    ? rawDataSourceBase[0]
+    : rawDataSourceBase;
 
   if (dataSourceBase) {
     chain.push(`equivalent var: ${localVarName} → ${dataSourceBase}`);
@@ -535,18 +630,26 @@ export default function resolvePathToControllable(
         baseVarName in equivalentSignatureVariables &&
         baseVarName !== localVarName
       ) {
-        const baseDataSource = equivalentSignatureVariables[baseVarName];
-        chain.push(`transitive resolution: ${baseVarName} → ${baseDataSource}`);
-        // Append the array access suffix to the resolved base
-        if (baseDataSource.endsWith('()')) {
-          fullResolvedPath =
-            baseDataSource + '.functionCallReturnValue' + accessSuffix;
-        } else if (baseDataSource.endsWith('.functionCallReturnValue')) {
-          fullResolvedPath = baseDataSource + accessSuffix;
-        } else {
-          fullResolvedPath = baseDataSource + accessSuffix;
+        // Handle array case (OR expressions) - use first element if array
+        const rawBaseDataSource = equivalentSignatureVariables[baseVarName];
+        const baseDataSource = Array.isArray(rawBaseDataSource)
+          ? rawBaseDataSource[0]
+          : rawBaseDataSource;
+        if (baseDataSource) {
+          chain.push(
+            `transitive resolution: ${baseVarName} → ${baseDataSource}`,
+          );
+          // Append the array access suffix to the resolved base
+          if (baseDataSource.endsWith('()')) {
+            fullResolvedPath =
+              baseDataSource + '.functionCallReturnValue' + accessSuffix;
+          } else if (baseDataSource.endsWith('.functionCallReturnValue')) {
+            fullResolvedPath = baseDataSource + accessSuffix;
+          } else {
+            fullResolvedPath = baseDataSource + accessSuffix;
+          }
+          chain.push(`transitively resolved: ${fullResolvedPath}`);
         }
-        chain.push(`transitively resolved: ${fullResolvedPath}`);
       }
     }
 
@@ -593,10 +696,11 @@ export default function resolvePathToControllable(
       };
     }
 
-    // Fix 34: Handle derived paths from controllable bases
+    // Fix 34/38: Handle derived paths from controllable bases
     // Paths like "useParams().functionCallReturnValue.*.split('/').filter(Boolean)[0]"
     // are derived from "useParams().functionCallReturnValue.*" via string methods.
-    // If the base path is controllable, the derived path should also be controllable.
+    // If the base path is controllable, return the BASE path (not the derived path)
+    // because that's what we need to mock in the data structure.
     const derivedBaseResult = findControllableBaseForDerivedPath(
       fullResolvedPath,
       attributesMap,
@@ -604,7 +708,9 @@ export default function resolvePathToControllable(
     if (derivedBaseResult) {
       chain.push(`derived from controllable base: ${derivedBaseResult}`);
       return {
-        resolvedPath: fullResolvedPath,
+        // Return the controllable BASE path, not the derived path
+        // This ensures execution flows contain paths that exist in attributesMap
+        resolvedPath: derivedBaseResult,
         resolutionChain: chain,
         isControllable: true,
       };
@@ -632,21 +738,48 @@ export default function resolvePathToControllable(
   // 5. Fix 35: Check for controllable prefix on the local path directly
   // This handles cases where the localPath itself (before any resolution)
   // has a controllable prefix in attributesMap
-  const localPrefixResult = findControllablePrefix(
-    localPath,
-    attributesMap,
-    fullToShortPathMap,
+  //
+  // IMPORTANT: Only do prefix matching if:
+  // a) The root variable is known in equivalentSignatureVariables, OR
+  // b) The path looks like an already-translated path (contains .functionCallReturnValue.)
+  //
+  // If the root variable is NOT known AND it's not an already-translated path, it's likely
+  // from an internal scope (callback, async function) and shouldn't be matched
+  // against similarly-named variables from different scopes.
+  // For example, "data.status" from inside `await response.json()` should NOT
+  // match the "data" from a useFetcher hook.
+  const localVarNameForPrefix = extractLocalVariableName(localPath);
+  const isKnownVariable = localVarNameForPrefix in equivalentSignatureVariables;
+
+  // Fix 35b: Also allow prefix matching for already-translated paths from child components
+  // These paths start with hook patterns like useLoaderData<...>().functionCallReturnValue
+  // and should be checked for controllable prefixes even if the hook itself isn't
+  // in equivalentSignatureVariables (because it's a fully resolved path, not a local variable)
+  const isAlreadyTranslatedPath = localPath.includes(
+    '.functionCallReturnValue.',
   );
-  if (localPrefixResult) {
-    chain.push(`nested property of controllable prefix (direct)`);
-    return {
-      resolvedPath: localPath,
-      resolutionChain: chain,
-      isControllable: true,
-    };
+
+  if (isKnownVariable || isAlreadyTranslatedPath) {
+    const localPrefixResult = findControllablePrefix(
+      localPath,
+      attributesMap,
+      fullToShortPathMap,
+    );
+    if (localPrefixResult) {
+      chain.push(`nested property of controllable prefix (direct)`);
+      return {
+        resolvedPath: localPath,
+        resolutionChain: chain,
+        isControllable: true,
+      };
+    }
+  } else {
+    chain.push(
+      `root variable '${localVarNameForPrefix}' not in equivalentSignatureVariables - skipping prefix match`,
+    );
   }
 
-  // 5b. Fix 36: Check for derived paths on localPath directly
+  // 5b. Fix 36/38: Check for derived paths on localPath directly
   // When localPath contains derivation methods like .filter(cyScope()), check if
   // the base path (before .filter()) is controllable
   const localDerivedResult = findControllableBaseForDerivedPath(
@@ -658,7 +791,8 @@ export default function resolvePathToControllable(
       `derived from controllable base (direct): ${localDerivedResult}`,
     );
     return {
-      resolvedPath: localPath,
+      // Return the controllable BASE path, not the derived path
+      resolvedPath: localDerivedResult,
       resolutionChain: chain,
       isControllable: true,
     };

package/analyzer-template/packages/ai/src/lib/worker/SerializableDataStructure.ts

@@ -13,6 +13,7 @@ import {
 import type {
   ConditionalEffect,
   CompoundConditional,
+  JsxRenderingUsage,
 } from '../astScopes/types';
 
 export interface SerializableScopeVariable {
@@ -62,7 +63,7 @@ export interface EnrichedConditionalUsage {
   /** For comparison conditions, the literal values being compared against */
   comparedValues?: string[];
   /** Where this conditional usage occurs */
-  location: 'if' | 'ternary' | 'logical-and' | 'switch';
+  location: 'if' | 'ternary' | 'logical-and' | 'switch' | 'unconditional';
   /**
    * The traced source data path in the format "scopeName.path"
    * e.g., "useParams().functionCallReturnValue['*']"
@@ -101,7 +102,8 @@ export interface EnrichedConditionalUsage {
       | 'and'
       | 'arrayIncludes'
       | 'arraySome'
-      | 'arrayEvery';
+      | 'arrayEvery'
+      | 'arrayLength';
     /** For comparison operations, the value being compared to */
     comparedValue?: string;
   };
@@ -118,7 +120,8 @@ export interface SerializableDataStructure {
   functionResults: Record<string, SerializableFunctionResult>;
 
   // Equivalent signature variables for root scope
-  equivalentSignatureVariables: Record<string, string>;
+  // Values can be arrays for OR expressions where a variable maps to multiple sources
+  equivalentSignatureVariables: Record<string, string | string[]>;
 
   environmentVariables: string[];
 
@@ -154,6 +157,15 @@ export interface SerializableDataStructure {
    * This allows us to merge child component execution flows with parent gating conditions.
    */
   childBoundaryGatingConditions?: Record<string, EnrichedConditionalUsage[]>;
+
+  /**
+   * JSX rendering usages - tracks arrays rendered via .map() and strings interpolated in JSX.
+   * Used to generate execution flows for array sizes and text lengths without explicit conditionals.
+   *
+   * Example:
+   * {items.map(i => <Item {...i} />)} produces an array-map usage for 'items'
+   */
+  jsxRenderingUsages?: JsxRenderingUsage[];
 }
 
 /**
@@ -285,7 +297,7 @@ export function getSourceEquivalencies(
 
 export function getEquivalentSignatureVariables(
   dataStructure: SerializableDataStructure,
-): Record<string, string> {
+): Record<string, string | string[]> {
   return dataStructure.equivalentSignatureVariables;
 }
 
@@ -329,3 +341,13 @@ export function getChildBoundaryGatingConditions(
 ): Record<string, EnrichedConditionalUsage[]> {
   return dataStructure.childBoundaryGatingConditions ?? {};
 }
+
+/**
+ * Get JSX rendering usages (arrays rendered via .map() and strings interpolated in JSX).
+ * Returns the array of JSX rendering usages, or empty array if not available.
+ */
+export function getJsxRenderingUsages(
+  dataStructure: SerializableDataStructure,
+): JsxRenderingUsage[] {
+  return dataStructure.jsxRenderingUsages ?? [];
+}

package/analyzer-template/packages/ai/src/lib/worker/analyzeScopeWorker.ts

@@ -9,13 +9,14 @@ import { Entity } from '~codeyam/types';
 import { FileAnalyzer, ProjectAnalyzer } from '~codeyam/analyze';
 import { analyzeScopeLocal } from '../analyzeScope';
 import {
-  ScopeInfo,
   resetScopeDataStructureMetrics,
+  ScopeInfo,
 } from '../dataStructure/ScopeDataStructure';
 import { SerializableDataStructure } from './SerializableDataStructure';
 import { AI } from '~codeyam/ai';
 import { setThreadContext } from '~codeyam/utils';
 import { threadId } from 'worker_threads';
+import v8 from 'v8';
 
 // Set thread context for logging
 setThreadContext(`<worker-${threadId}>`);
@@ -45,6 +46,80 @@ let projectAnalyzerKey: string | null = null;
 let fileAnalyzer: FileAnalyzer | null = null;
 let fileAnalyzerPath: string | null = null;
 
+// Memory tracking
+let entitiesProcessed = 0;
+let lastMemoryLogAt = 0;
+const MEMORY_LOG_INTERVAL = 100; // Log every N entities
+const MEMORY_LOG_TIME_INTERVAL = 60_000; // Or every 60 seconds
+
+/**
+ * Get memory stats for logging
+ */
+function getWorkerMemoryStats() {
+  const mem = process.memoryUsage();
+  const heapStats = v8.getHeapStatistics();
+  return {
+    rss: `${Math.round(mem.rss / 1024 / 1024)}MB`,
+    heapUsed: `${Math.round(mem.heapUsed / 1024 / 1024)}MB`,
+    heapTotal: `${Math.round(mem.heapTotal / 1024 / 1024)}MB`,
+    external: `${Math.round(mem.external / 1024 / 1024)}MB`,
+    // V8 heap stats give more detail
+    v8TotalHeapSize: `${Math.round(heapStats.total_heap_size / 1024 / 1024)}MB`,
+    v8UsedHeapSize: `${Math.round(heapStats.used_heap_size / 1024 / 1024)}MB`,
+    v8HeapSizeLimit: `${Math.round(heapStats.heap_size_limit / 1024 / 1024)}MB`,
+    v8MallocedMemory: `${Math.round(heapStats.malloced_memory / 1024 / 1024)}MB`,
+  };
+}
+
+/**
+ * Get ts.Program stats for understanding TypeScript memory usage
+ */
+function getTsProgramStats(pa: ProjectAnalyzer) {
+  const sourceFiles = pa.program.getSourceFiles();
+  const projectSourceFiles = sourceFiles.filter(
+    (sf) => !sf.fileName.includes('node_modules'),
+  );
+  const nodeModulesFiles = sourceFiles.filter((sf) =>
+    sf.fileName.includes('node_modules'),
+  );
+
+  // Estimate AST size by sampling text length (rough proxy)
+  let totalProjectTextLength = 0;
+  for (const sf of projectSourceFiles) {
+    totalProjectTextLength += sf.getFullText().length;
+  }
+
+  return {
+    totalSourceFiles: sourceFiles.length,
+    projectSourceFiles: projectSourceFiles.length,
+    nodeModulesFiles: nodeModulesFiles.length,
+    projectTextSizeMB: `${(totalProjectTextLength / 1024 / 1024).toFixed(1)}MB`,
+    // Note: Actual AST memory is typically 10-20x the source text size
+    estimatedAstSizeMB: `${((totalProjectTextLength * 15) / 1024 / 1024).toFixed(0)}MB`,
+  };
+}
+
+/**
+ * Write a heap snapshot for detailed profiling.
+ * The snapshot can be loaded in Chrome DevTools for analysis.
+ *
+ * TODO: Once we understand memory usage, change this back to env-var controlled
+ */
+const HEAP_SNAPSHOT_MODE: 'off' | 'once' | 'periodic' = 'off';
+
+function maybeWriteHeapSnapshot(label: string) {
+  if (HEAP_SNAPSHOT_MODE === 'off') return;
+
+  try {
+    const filename = v8.writeHeapSnapshot();
+    console.log(
+      `CodeYam Worker: Heap snapshot written (${label}): ${filename}`,
+    );
+  } catch (error) {
+    console.error('CodeYam Worker: Failed to write heap snapshot:', error);
+  }
+}
+
 /**
  * Main worker function called by Piscina
  */
@@ -61,8 +136,25 @@ export default async function runAnalyzeScope(
     // Create or reuse ProjectAnalyzer
     const paKey = `${dirPath}`;
     if (!projectAnalyzer || projectAnalyzerKey !== paKey) {
-      projectAnalyzer = await ProjectAnalyzer.from({ project, dirPath });
+      const memBefore = process.memoryUsage();
+      console.log('CodeYam Worker: Creating ProjectAnalyzer (ts.Program)...', {
+        memBefore: getWorkerMemoryStats(),
+      });
+
+      projectAnalyzer = ProjectAnalyzer.from({ project, dirPath });
       projectAnalyzerKey = paKey;
+
+      const memAfter = process.memoryUsage();
+      const tsProgramStats = getTsProgramStats(projectAnalyzer);
+      console.log('CodeYam Worker: ProjectAnalyzer created', {
+        memAfter: getWorkerMemoryStats(),
+        rssDelta: `+${Math.round((memAfter.rss - memBefore.rss) / 1024 / 1024)}MB`,
+        heapDelta: `+${Math.round((memAfter.heapUsed - memBefore.heapUsed) / 1024 / 1024)}MB`,
+        tsProgramStats,
+      });
+
+      // Optionally write heap snapshot after ts.Program creation
+      maybeWriteHeapSnapshot('after-ts-program');
     }
 
     // Create or reuse FileAnalyzer
@@ -95,6 +187,26 @@ export default async function runAnalyzeScope(
       );
     }
 
+    // Periodic memory logging
+    entitiesProcessed++;
+    const now = Date.now();
+    const shouldLogMemory =
+      entitiesProcessed % MEMORY_LOG_INTERVAL === 0 ||
+      now - lastMemoryLogAt > MEMORY_LOG_TIME_INTERVAL;
+
+    if (shouldLogMemory) {
+      lastMemoryLogAt = now;
+      console.log('CodeYam Worker: Memory stats', {
+        entitiesProcessed,
+        ...getWorkerMemoryStats(),
+      });
+
+      // Optionally write periodic heap snapshots
+      if (HEAP_SNAPSHOT_MODE === 'periodic' && entitiesProcessed % 500 === 0) {
+        maybeWriteHeapSnapshot(`after-${entitiesProcessed}-entities`);
+      }
+    }
+
     return { dataStructure: dataStructure.toSerializable() };
   } catch (error) {
     console.error(

package/analyzer-template/packages/analyze/index.ts

@@ -41,6 +41,8 @@ export { default as mergeValidatedDataStructures } from './src/lib/files/scenari
 
 export { discoverDirectDependencies } from './src/lib/files/analyze/dependencyResolver';
 
+export { transformationTracer } from './src/lib/files/scenarios/TransformationTracer';
+
 export { default as getAnalysisError } from './src/lib/utils/getAnalysisError';
 export { default as measureAndReportExecutionTime } from './src/lib/utils/measureAndReportExecutionTime';