@codeyam/codeyam-cli 0.1.0-staging.c90f8c9 → 0.1.0-staging.c9dc00c

package/analyzer-template/packages/ai/src/lib/mergeStatements.ts

@@ -7,58 +7,12 @@ import {
 
 export interface Statement {
   structure: { [key: string]: string };
-  equivalentVariables: { [key: string]: string };
+  // Supports multiple equivalencies per key for OR expressions (e.g., x = a || b)
+  equivalentVariables: { [key: string]: string | string[] };
   llmCall?: LlmCall;
 }
 
 export default function mergeStatements(statements: Statement[]) {
-  // Debug: Log input statements to understand what we're merging
-  const hasStateStructure = statements.some((s) =>
-    Object.keys(s.structure || {}).some(
-      (k) => k === 'state' || k.includes('state'),
-    ),
-  );
-  const hasStateEquiv = statements.some((s) =>
-    Object.keys(s.equivalentVariables || {}).some(
-      (k) => k === 'state' || k.includes('state'),
-    ),
-  );
-  if (hasStateStructure || hasStateEquiv) {
-    console.log(
-      `[UNION-TYPE] mergeStatements INPUT: ${statements.length} statements`,
-    );
-    statements.forEach((s, i) => {
-      const stateInStructure = Object.entries(s.structure || {}).filter(
-        ([k]) => k === 'state' || k.includes('state'),
-      );
-      const stateInEquiv = Object.entries(s.equivalentVariables || {}).filter(
-        ([k]) => k === 'state' || k.includes('state'),
-      );
-      const sigInEquiv = Object.entries(s.equivalentVariables || {}).filter(
-        ([k, v]) => k === 'state' || v === 'signature[0]',
-      );
-      if (
-        stateInStructure.length > 0 ||
-        stateInEquiv.length > 0 ||
-        sigInEquiv.length > 0
-      ) {
-        console.log(`[UNION-TYPE]   Statement ${i}:`);
-        if (stateInStructure.length > 0)
-          console.log(
-            `[UNION-TYPE]     structure: ${JSON.stringify(Object.fromEntries(stateInStructure))}`,
-          );
-        if (stateInEquiv.length > 0)
-          console.log(
-            `[UNION-TYPE]     equivalentVariables (state): ${JSON.stringify(Object.fromEntries(stateInEquiv))}`,
-          );
-        if (sigInEquiv.length > 0)
-          console.log(
-            `[UNION-TYPE]     equivalentVariables (sig): ${JSON.stringify(Object.fromEntries(sigInEquiv))}`,
-          );
-      }
-    });
-  }
-
   // This should be handled by the AST Analyzer but it currently has a bug
   const ensureFunctionCallReturnValue = (path: string, value: string) => {
     if (path.endsWith(')') && value !== 'function') {
@@ -172,12 +126,54 @@ export default function mergeStatements(statements: Statement[]) {
   // settingsData → fetcher.data.data uses fetcher::cyDuplicateKey1::.data.data
   const activeRemappings: Record<string, string> = {};
 
+  // Helper to normalize value to array and process
+  const normalizeValue = (value: string | string[]): string[] =>
+    Array.isArray(value) ? value : [value];
+
   const equivalentVariables = statements.reduce((acc: any, result) => {
-    for (const [key, value] of Object.entries(
+    for (const [key, rawValue] of Object.entries(
       result.equivalentVariables ?? {},
     )) {
+      // Handle arrays from AST analyzer (OR expressions like x = a || b)
+      // These should be preserved as arrays to track all sources
+      if (Array.isArray(rawValue)) {
+        const keyOptions = deBinaryPath(key);
+        const validValues = rawValue.filter(
+          (v): v is string => typeof v === 'string' && v.length > 0,
+        );
+        if (validValues.length === 0) continue;
+
+        // Process each array value through deBinaryPath
+        const allValueOptions: string[] = [];
+        for (const value of validValues) {
+          const valueOptions = deBinaryPath(value);
+          allValueOptions.push(...valueOptions);
+        }
+
+        // Remove duplicates while preserving order
+        const uniqueValueOptions = [...new Set(allValueOptions)];
+        const finalValue =
+          uniqueValueOptions.length === 1
+            ? uniqueValueOptions[0]
+            : uniqueValueOptions;
+
+        delete result.equivalentVariables[key];
+        if (keyOptions.length >= 2) {
+          for (const keyOption of keyOptions) {
+            result.equivalentVariables[keyOption] = finalValue;
+          }
+        } else {
+          result.equivalentVariables[keyOptions[0]] = finalValue;
+        }
+        continue;
+      }
+
+      // Original logic for string values with potential binary expressions in the string
+      const firstValue = rawValue;
+      if (!firstValue) continue;
+
       const keyOptions = deBinaryPath(key);
-      const valueOptions = deBinaryPath(value);
+      const valueOptions = deBinaryPath(firstValue);
       delete result.equivalentVariables[key];
       if (keyOptions.length === 2) {
         if (valueOptions.length === 2) {
@@ -199,27 +195,38 @@ export default function mergeStatements(statements: Statement[]) {
       }
     }
 
-    for (const [key, value] of Object.entries(
+    for (const [key, rawValue] of Object.entries(
       result.equivalentVariables ?? {},
     )) {
+      // Normalize to array for consistent handling
+      const values = normalizeValue(rawValue);
+
       // Apply any active remappings to the VALUE, but ONLY for property paths
       // e.g., if fetcher was remapped to fetcher::cyDuplicateKey1::,
       // then "fetcher.data.data" should become "fetcher::cyDuplicateKey1::.data.data"
       // However, we should NOT remap standalone variable references like "description"
       // because that would lose type information from the original variable.
-      let remappedValue = value;
-      for (const [originalKey, remappedKey] of Object.entries(
-        activeRemappings,
-      )) {
-        // Only remap if the value is a property path (contains a dot after the key)
-        // This preserves type information for simple variable references
-        const keyRegex = new RegExp(
-          `^${originalKey.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?=\\.)`,
-        );
-        if (keyRegex.test(remappedValue)) {
-          remappedValue = remappedValue.replace(keyRegex, remappedKey);
+      const remappedValues = values.map((value) => {
+        let remappedValue = value;
+        for (const [originalKey, remappedKey] of Object.entries(
+          activeRemappings,
+        )) {
+          // Only remap if the value is a property path (contains a dot after the key)
+          // This preserves type information for simple variable references
+          const keyRegex = new RegExp(
+            `^${originalKey.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?=\\.)`,
+          );
+          if (keyRegex.test(remappedValue)) {
+            remappedValue = remappedValue.replace(keyRegex, remappedKey);
+          }
         }
-      }
+        return remappedValue;
+      });
+
+      // Use array if multiple values, string if single
+      const remappedValue =
+        remappedValues.length === 1 ? remappedValues[0] : remappedValues;
+      const firstRemappedValue = remappedValues[0];
 
       if (acc[key]) {
         if (acc[key] === remappedValue) continue;
@@ -247,45 +254,32 @@ export default function mergeStatements(statements: Statement[]) {
   // If we have equivalentVariables[A] = B, and structure[A] is more specific than structure[B],
   // then update structure[B] to match structure[A]
   // This handles cases like: "state" -> "signature[0]" where "state" has a union type
-
-  // Debug: Log union types in merged structure
-  const unionTypesInStructure = Object.entries(structure).filter(
-    ([_, v]) => typeof v === 'string' && v.includes("'") && v.includes(' | '),
-  );
-  if (unionTypesInStructure.length > 0) {
-    console.log(
-      `[UNION-TYPE] mergeStatements: Found ${unionTypesInStructure.length} union types in merged structure`,
-    );
-    unionTypesInStructure.forEach(([k, v]) =>
-      console.log(`[UNION-TYPE]   structure["${k}"] = ${v}`),
-    );
-    console.log(
-      `[UNION-TYPE] mergeStatements: equivalentVariables = ${JSON.stringify(equivalentVariables)}`,
-    );
-  }
-
-  for (const [source, target] of Object.entries(equivalentVariables) as [
-    string,
+  for (const [source, rawTarget] of Object.entries(equivalentVariables) as [
     string,
+    string | string[],
   ][]) {
+    // Normalize to array for consistent handling
+    const targets = Array.isArray(rawTarget) ? rawTarget : [rawTarget];
     const sourceType = structure[source];
-    const targetType = structure[target];
 
-    // Check if source has a more specific type (union type vs generic type)
-    if (sourceType) {
-      const sourceIsUnion =
-        sourceType.includes("'") && sourceType.includes(' | ');
+    for (const target of targets) {
+      if (typeof target !== 'string') continue;
 
-      if (sourceIsUnion) {
-        // Target doesn't exist or is generic - propagate the union type
-        const targetIsGeneric =
-          !targetType || targetType === 'string' || targetType === 'unknown';
+      const targetType = structure[target];
 
-        if (targetIsGeneric) {
-          console.log(
-            `[UNION-TYPE] mergeStatements: Propagating "${source}" (${sourceType}) -> "${target}"`,
-          );
-          structure[target] = sourceType;
+      // Check if source has a more specific type (union type vs generic type)
+      if (sourceType) {
+        const sourceIsUnion =
+          sourceType.includes("'") && sourceType.includes(' | ');
+
+        if (sourceIsUnion) {
+          // Target doesn't exist or is generic - propagate the union type
+          const targetIsGeneric =
+            !targetType || targetType === 'string' || targetType === 'unknown';
+
+          if (targetIsGeneric) {
+            structure[target] = sourceType;
+          }
         }
       }
     }

package/analyzer-template/packages/ai/src/lib/promptGenerators/gatherAttributesMap.ts

@@ -139,16 +139,19 @@ export function gatherAttributesMap(
       // Use merged type if available, otherwise fall back to isolated type
       const typeValue =
         mergedSignatureSchema[key] ?? isolatedSignatureSchema[key];
-      const equivalentSignatureVariable =
-        equivalentSignatureVariables[keyParts[0]];
-      const equivalentSignatureVariableParts = splitOutsideParenthesesAndArrays(
-        equivalentSignatureVariable,
-      );
+      // Handle array case (OR expressions) - use first element if array
+      const rawEquivalent = equivalentSignatureVariables[keyParts[0]];
+      const equivalentSignatureVariable = Array.isArray(rawEquivalent)
+        ? rawEquivalent[0]
+        : rawEquivalent;
+      const equivalentSignatureVariableParts = equivalentSignatureVariable
+        ? splitOutsideParenthesesAndArrays(equivalentSignatureVariable)
+        : [];
       if (
         equivalentSignatureVariable &&
-        !equivalentSignatureVariableParts[0].includes('(')
+        !equivalentSignatureVariableParts[0]?.includes('(')
       ) {
-        const equivalentKey = equivalentSignatureVariables[keyParts[0]];
+        const equivalentKey = equivalentSignatureVariable;
         const equivalentPath = joinParenthesesAndArrays([
           equivalentKey,
           ...keyParts.slice(1),

package/analyzer-template/packages/ai/src/lib/resolvePathToControllable.ts

@@ -525,7 +525,7 @@ function findControllableBaseForDerivedPath(
 export default function resolvePathToControllable(
   localPath: string,
   attributesMap: Record<string, string>,
-  equivalentSignatureVariables: Record<string, string>,
+  equivalentSignatureVariables: Record<string, string | string[]>,
   fullToShortPathMap: Record<string, string>,
 ): PathResolutionResult {
   const chain: string[] = [localPath];
@@ -600,7 +600,11 @@ export default function resolvePathToControllable(
 
   // 4. Equivalent variable resolution
   const localVarName = extractLocalVariableName(localPath);
-  const dataSourceBase = equivalentSignatureVariables[localVarName];
+  // Handle array case (OR expressions) - use first element if array
+  const rawDataSourceBase = equivalentSignatureVariables[localVarName];
+  const dataSourceBase = Array.isArray(rawDataSourceBase)
+    ? rawDataSourceBase[0]
+    : rawDataSourceBase;
 
   if (dataSourceBase) {
     chain.push(`equivalent var: ${localVarName} → ${dataSourceBase}`);
@@ -626,18 +630,26 @@ export default function resolvePathToControllable(
         baseVarName in equivalentSignatureVariables &&
         baseVarName !== localVarName
       ) {
-        const baseDataSource = equivalentSignatureVariables[baseVarName];
-        chain.push(`transitive resolution: ${baseVarName} → ${baseDataSource}`);
-        // Append the array access suffix to the resolved base
-        if (baseDataSource.endsWith('()')) {
-          fullResolvedPath =
-            baseDataSource + '.functionCallReturnValue' + accessSuffix;
-        } else if (baseDataSource.endsWith('.functionCallReturnValue')) {
-          fullResolvedPath = baseDataSource + accessSuffix;
-        } else {
-          fullResolvedPath = baseDataSource + accessSuffix;
+        // Handle array case (OR expressions) - use first element if array
+        const rawBaseDataSource = equivalentSignatureVariables[baseVarName];
+        const baseDataSource = Array.isArray(rawBaseDataSource)
+          ? rawBaseDataSource[0]
+          : rawBaseDataSource;
+        if (baseDataSource) {
+          chain.push(
+            `transitive resolution: ${baseVarName} → ${baseDataSource}`,
+          );
+          // Append the array access suffix to the resolved base
+          if (baseDataSource.endsWith('()')) {
+            fullResolvedPath =
+              baseDataSource + '.functionCallReturnValue' + accessSuffix;
+          } else if (baseDataSource.endsWith('.functionCallReturnValue')) {
+            fullResolvedPath = baseDataSource + accessSuffix;
+          } else {
+            fullResolvedPath = baseDataSource + accessSuffix;
+          }
+          chain.push(`transitively resolved: ${fullResolvedPath}`);
         }
-        chain.push(`transitively resolved: ${fullResolvedPath}`);
       }
     }
 

package/analyzer-template/packages/ai/src/lib/worker/SerializableDataStructure.ts

@@ -63,7 +63,7 @@ export interface EnrichedConditionalUsage {
   /** For comparison conditions, the literal values being compared against */
   comparedValues?: string[];
   /** Where this conditional usage occurs */
-  location: 'if' | 'ternary' | 'logical-and' | 'switch';
+  location: 'if' | 'ternary' | 'logical-and' | 'switch' | 'unconditional';
   /**
    * The traced source data path in the format "scopeName.path"
    * e.g., "useParams().functionCallReturnValue['*']"
@@ -120,7 +120,8 @@ export interface SerializableDataStructure {
   functionResults: Record<string, SerializableFunctionResult>;
 
   // Equivalent signature variables for root scope
-  equivalentSignatureVariables: Record<string, string>;
+  // Values can be arrays for OR expressions where a variable maps to multiple sources
+  equivalentSignatureVariables: Record<string, string | string[]>;
 
   environmentVariables: string[];
 
@@ -296,7 +297,7 @@ export function getSourceEquivalencies(
 
 export function getEquivalentSignatureVariables(
   dataStructure: SerializableDataStructure,
-): Record<string, string> {
+): Record<string, string | string[]> {
   return dataStructure.equivalentSignatureVariables;
 }
 

package/analyzer-template/packages/ai/src/lib/worker/analyzeScopeWorker.ts

@@ -9,13 +9,14 @@ import { Entity } from '~codeyam/types';
 import { FileAnalyzer, ProjectAnalyzer } from '~codeyam/analyze';
 import { analyzeScopeLocal } from '../analyzeScope';
 import {
-  ScopeInfo,
   resetScopeDataStructureMetrics,
+  ScopeInfo,
 } from '../dataStructure/ScopeDataStructure';
 import { SerializableDataStructure } from './SerializableDataStructure';
 import { AI } from '~codeyam/ai';
 import { setThreadContext } from '~codeyam/utils';
 import { threadId } from 'worker_threads';
+import v8 from 'v8';
 
 // Set thread context for logging
 setThreadContext(`<worker-${threadId}>`);
@@ -45,6 +46,80 @@ let projectAnalyzerKey: string | null = null;
 let fileAnalyzer: FileAnalyzer | null = null;
 let fileAnalyzerPath: string | null = null;
 
+// Memory tracking
+let entitiesProcessed = 0;
+let lastMemoryLogAt = 0;
+const MEMORY_LOG_INTERVAL = 100; // Log every N entities
+const MEMORY_LOG_TIME_INTERVAL = 60_000; // Or every 60 seconds
+
+/**
+ * Get memory stats for logging
+ */
+function getWorkerMemoryStats() {
+  const mem = process.memoryUsage();
+  const heapStats = v8.getHeapStatistics();
+  return {
+    rss: `${Math.round(mem.rss / 1024 / 1024)}MB`,
+    heapUsed: `${Math.round(mem.heapUsed / 1024 / 1024)}MB`,
+    heapTotal: `${Math.round(mem.heapTotal / 1024 / 1024)}MB`,
+    external: `${Math.round(mem.external / 1024 / 1024)}MB`,
+    // V8 heap stats give more detail
+    v8TotalHeapSize: `${Math.round(heapStats.total_heap_size / 1024 / 1024)}MB`,
+    v8UsedHeapSize: `${Math.round(heapStats.used_heap_size / 1024 / 1024)}MB`,
+    v8HeapSizeLimit: `${Math.round(heapStats.heap_size_limit / 1024 / 1024)}MB`,
+    v8MallocedMemory: `${Math.round(heapStats.malloced_memory / 1024 / 1024)}MB`,
+  };
+}
+
+/**
+ * Get ts.Program stats for understanding TypeScript memory usage
+ */
+function getTsProgramStats(pa: ProjectAnalyzer) {
+  const sourceFiles = pa.program.getSourceFiles();
+  const projectSourceFiles = sourceFiles.filter(
+    (sf) => !sf.fileName.includes('node_modules'),
+  );
+  const nodeModulesFiles = sourceFiles.filter((sf) =>
+    sf.fileName.includes('node_modules'),
+  );
+
+  // Estimate AST size by sampling text length (rough proxy)
+  let totalProjectTextLength = 0;
+  for (const sf of projectSourceFiles) {
+    totalProjectTextLength += sf.getFullText().length;
+  }
+
+  return {
+    totalSourceFiles: sourceFiles.length,
+    projectSourceFiles: projectSourceFiles.length,
+    nodeModulesFiles: nodeModulesFiles.length,
+    projectTextSizeMB: `${(totalProjectTextLength / 1024 / 1024).toFixed(1)}MB`,
+    // Note: Actual AST memory is typically 10-20x the source text size
+    estimatedAstSizeMB: `${((totalProjectTextLength * 15) / 1024 / 1024).toFixed(0)}MB`,
+  };
+}
+
+/**
+ * Write a heap snapshot for detailed profiling.
+ * The snapshot can be loaded in Chrome DevTools for analysis.
+ *
+ * TODO: Once we understand memory usage, change this back to env-var controlled
+ */
+const HEAP_SNAPSHOT_MODE: 'off' | 'once' | 'periodic' = 'off';
+
+function maybeWriteHeapSnapshot(label: string) {
+  if (HEAP_SNAPSHOT_MODE === 'off') return;
+
+  try {
+    const filename = v8.writeHeapSnapshot();
+    console.log(
+      `CodeYam Worker: Heap snapshot written (${label}): ${filename}`,
+    );
+  } catch (error) {
+    console.error('CodeYam Worker: Failed to write heap snapshot:', error);
+  }
+}
+
 /**
  * Main worker function called by Piscina
  */
@@ -61,8 +136,25 @@ export default async function runAnalyzeScope(
     // Create or reuse ProjectAnalyzer
     const paKey = `${dirPath}`;
     if (!projectAnalyzer || projectAnalyzerKey !== paKey) {
-      projectAnalyzer = await ProjectAnalyzer.from({ project, dirPath });
+      const memBefore = process.memoryUsage();
+      console.log('CodeYam Worker: Creating ProjectAnalyzer (ts.Program)...', {
+        memBefore: getWorkerMemoryStats(),
+      });
+
+      projectAnalyzer = ProjectAnalyzer.from({ project, dirPath });
       projectAnalyzerKey = paKey;
+
+      const memAfter = process.memoryUsage();
+      const tsProgramStats = getTsProgramStats(projectAnalyzer);
+      console.log('CodeYam Worker: ProjectAnalyzer created', {
+        memAfter: getWorkerMemoryStats(),
+        rssDelta: `+${Math.round((memAfter.rss - memBefore.rss) / 1024 / 1024)}MB`,
+        heapDelta: `+${Math.round((memAfter.heapUsed - memBefore.heapUsed) / 1024 / 1024)}MB`,
+        tsProgramStats,
+      });
+
+      // Optionally write heap snapshot after ts.Program creation
+      maybeWriteHeapSnapshot('after-ts-program');
     }
 
     // Create or reuse FileAnalyzer
@@ -95,6 +187,26 @@ export default async function runAnalyzeScope(
       );
     }
 
+    // Periodic memory logging
+    entitiesProcessed++;
+    const now = Date.now();
+    const shouldLogMemory =
+      entitiesProcessed % MEMORY_LOG_INTERVAL === 0 ||
+      now - lastMemoryLogAt > MEMORY_LOG_TIME_INTERVAL;
+
+    if (shouldLogMemory) {
+      lastMemoryLogAt = now;
+      console.log('CodeYam Worker: Memory stats', {
+        entitiesProcessed,
+        ...getWorkerMemoryStats(),
+      });
+
+      // Optionally write periodic heap snapshots
+      if (HEAP_SNAPSHOT_MODE === 'periodic' && entitiesProcessed % 500 === 0) {
+        maybeWriteHeapSnapshot(`after-${entitiesProcessed}-entities`);
+      }
+    }
+
     return { dataStructure: dataStructure.toSerializable() };
   } catch (error) {
     console.error(

package/analyzer-template/packages/analyze/index.ts

@@ -41,6 +41,8 @@ export { default as mergeValidatedDataStructures } from './src/lib/files/scenari
 
 export { discoverDirectDependencies } from './src/lib/files/analyze/dependencyResolver';
 
+export { transformationTracer } from './src/lib/files/scenarios/TransformationTracer';
+
 export { default as getAnalysisError } from './src/lib/utils/getAnalysisError';
 export { default as measureAndReportExecutionTime } from './src/lib/utils/measureAndReportExecutionTime';