@codeyam/codeyam-cli 0.1.0-staging.ad31e3e → 0.1.0-staging.ad44cd3

package/analyzer-template/packages/ai/src/lib/worker/SerializableDataStructure.ts

@@ -10,6 +10,11 @@ import {
   fuzzyMatchFunctionName,
   findFuzzyMatchingKey,
 } from './fuzzyMatchFunctionName';
+import type {
+  ConditionalEffect,
+  CompoundConditional,
+  JsxRenderingUsage,
+} from '../astScopes/types';
 
 export interface SerializableScopeVariable {
   scopeNodeName: string;
@@ -22,6 +27,17 @@ export interface SerializableFunctionCallInfo {
   callScope: string;
   schema?: Record<string, string>;
   equivalencies?: Record<string, SerializableScopeVariable[]>;
+  /** All call signatures merged into this entry (e.g., logger.info, logger.debug) */
+  allCallSignatures?: string[];
+  /** Variable names that receive the return value of each call */
+  receivingVariableNames?: string[];
+  /** Maps each call signature to the variable that receives its return value */
+  callSignatureToVariable?: Record<string, string>;
+  /**
+   * Stores individual return value schemas per receiving variable, BEFORE merging.
+   * Key is the receiving variable name (e.g., "userFetcher", "reportFetcher").
+   */
+  perVariableSchemas?: Record<string, Record<string, string>>;
 }
 
 export interface SerializableFunctionResult {
@@ -47,13 +63,50 @@ export interface EnrichedConditionalUsage {
   /** For comparison conditions, the literal values being compared against */
   comparedValues?: string[];
   /** Where this conditional usage occurs */
-  location: 'if' | 'ternary' | 'logical-and' | 'switch';
+  location: 'if' | 'ternary' | 'logical-and' | 'switch' | 'unconditional';
   /**
    * The traced source data path in the format "scopeName.path"
    * e.g., "useParams().functionCallReturnValue['*']"
    * This connects the local variable to the original data attribute.
    */
   sourceDataPath?: string;
+  /** Whether this condition is negated (e.g., !foo) */
+  isNegated?: boolean;
+  /**
+   * Whether this conditional controls JSX rendering.
+   * True when the conditional appears in a JSX expression like {cond && <Component />}
+   */
+  controlsJsxRendering?: boolean;
+  /** Source location information */
+  sourceLocation?: {
+    lineNumber: number;
+    column: number;
+    codeSnippet: string;
+  };
+  /**
+   * Information about how this variable was derived from other variables.
+   * Used to trace derived boolean variables back to their data sources.
+   */
+  derivedFrom?: {
+    /** For single-source derivations (notNull, equals, etc.) */
+    sourcePath?: string;
+    /** For multi-source derivations (or, and) */
+    sourcePaths?: string[];
+    /** The operation that derived this variable */
+    operation:
+      | 'notNull'
+      | 'isNull'
+      | 'equals'
+      | 'notEquals'
+      | 'or'
+      | 'and'
+      | 'arrayIncludes'
+      | 'arraySome'
+      | 'arrayEvery'
+      | 'arrayLength';
+    /** For comparison operations, the value being compared to */
+    comparedValue?: string;
+  };
 }
 
 export interface SerializableDataStructure {
@@ -67,7 +120,8 @@ export interface SerializableDataStructure {
   functionResults: Record<string, SerializableFunctionResult>;
 
   // Equivalent signature variables for root scope
-  equivalentSignatureVariables: Record<string, string>;
+  // Values can be arrays for OR expressions where a variable maps to multiple sources
+  equivalentSignatureVariables: Record<string, string | string[]>;
 
   environmentVariables: string[];
 
@@ -77,6 +131,41 @@ export interface SerializableDataStructure {
    * the traced source data path.
    */
   conditionalUsages?: Record<string, EnrichedConditionalUsage[]>;
+
+  /**
+   * Conditional effects tracking.
+   * Tracks what setter calls happen inside conditionals (if, switch, ternary).
+   * Used to auto-generate execution flows from static analysis.
+   */
+  conditionalEffects?: ConditionalEffect[];
+
+  /**
+   * Compound conditionals grouped by chain (all conditions must be true together).
+   * Used to generate execution flows that satisfy ALL conditions in compound expressions like:
+   *   (hasNewerVersion || isFromDifferentVersion) && !isAnalyzing && !isQueued
+   */
+  compoundConditionals?: CompoundConditional[];
+
+  /**
+   * Gating conditions for child component boundaries.
+   * Maps child component name to the conditions that must be true for that child to render.
+   *
+   * Example:
+   * {hasAnalysis && <ScenarioViewer />} would produce:
+   * { ScenarioViewer: [{ path: 'hasAnalysis', conditionType: 'truthiness', ... }] }
+   *
+   * This allows us to merge child component execution flows with parent gating conditions.
+   */
+  childBoundaryGatingConditions?: Record<string, EnrichedConditionalUsage[]>;
+
+  /**
+   * JSX rendering usages - tracks arrays rendered via .map() and strings interpolated in JSX.
+   * Used to generate execution flows for array sizes and text lengths without explicit conditionals.
+   *
+   * Example:
+   * {items.map(i => <Item {...i} />)} produces an array-map usage for 'items'
+   */
+  jsxRenderingUsages?: JsxRenderingUsage[];
 }
 
 /**
@@ -208,7 +297,7 @@ export function getSourceEquivalencies(
 
 export function getEquivalentSignatureVariables(
   dataStructure: SerializableDataStructure,
-): Record<string, string> {
+): Record<string, string | string[]> {
   return dataStructure.equivalentSignatureVariables;
 }
 
@@ -221,3 +310,44 @@ export function getConditionalUsages(
 ): Record<string, EnrichedConditionalUsage[]> {
   return dataStructure.conditionalUsages ?? {};
 }
+
+/**
+ * Get conditional effects (setter calls inside conditionals).
+ * Returns the array of conditional effects, or empty array if not available.
+ */
+export function getConditionalEffects(
+  dataStructure: SerializableDataStructure,
+): ConditionalEffect[] {
+  return dataStructure.conditionalEffects ?? [];
+}
+
+/**
+ * Get compound conditionals (grouped conditions that must all be true).
+ * Returns the array of compound conditionals, or empty array if not available.
+ */
+export function getCompoundConditionals(
+  dataStructure: SerializableDataStructure,
+): CompoundConditional[] {
+  return dataStructure.compoundConditionals ?? [];
+}
+
+/**
+ * Get child boundary gating conditions.
+ * Returns the mapping of child component names to their gating conditions,
+ * or empty object if not available.
+ */
+export function getChildBoundaryGatingConditions(
+  dataStructure: SerializableDataStructure,
+): Record<string, EnrichedConditionalUsage[]> {
+  return dataStructure.childBoundaryGatingConditions ?? {};
+}
+
+/**
+ * Get JSX rendering usages (arrays rendered via .map() and strings interpolated in JSX).
+ * Returns the array of JSX rendering usages, or empty array if not available.
+ */
+export function getJsxRenderingUsages(
+  dataStructure: SerializableDataStructure,
+): JsxRenderingUsage[] {
+  return dataStructure.jsxRenderingUsages ?? [];
+}

package/analyzer-template/packages/ai/src/lib/worker/analyzeScopeWorker.ts

@@ -8,11 +8,15 @@
 import { Entity } from '~codeyam/types';
 import { FileAnalyzer, ProjectAnalyzer } from '~codeyam/analyze';
 import { analyzeScopeLocal } from '../analyzeScope';
-import { ScopeInfo } from '../dataStructure/ScopeDataStructure';
+import {
+  resetScopeDataStructureMetrics,
+  ScopeInfo,
+} from '../dataStructure/ScopeDataStructure';
 import { SerializableDataStructure } from './SerializableDataStructure';
 import { AI } from '~codeyam/ai';
 import { setThreadContext } from '~codeyam/utils';
 import { threadId } from 'worker_threads';
+import v8 from 'v8';
 
 // Set thread context for logging
 setThreadContext(`<worker-${threadId}>`);
@@ -42,20 +46,115 @@ let projectAnalyzerKey: string | null = null;
 let fileAnalyzer: FileAnalyzer | null = null;
 let fileAnalyzerPath: string | null = null;
 
+// Memory tracking
+let entitiesProcessed = 0;
+let lastMemoryLogAt = 0;
+const MEMORY_LOG_INTERVAL = 100; // Log every N entities
+const MEMORY_LOG_TIME_INTERVAL = 60_000; // Or every 60 seconds
+
+/**
+ * Get memory stats for logging
+ */
+function getWorkerMemoryStats() {
+  const mem = process.memoryUsage();
+  const heapStats = v8.getHeapStatistics();
+  return {
+    rss: `${Math.round(mem.rss / 1024 / 1024)}MB`,
+    heapUsed: `${Math.round(mem.heapUsed / 1024 / 1024)}MB`,
+    heapTotal: `${Math.round(mem.heapTotal / 1024 / 1024)}MB`,
+    external: `${Math.round(mem.external / 1024 / 1024)}MB`,
+    // V8 heap stats give more detail
+    v8TotalHeapSize: `${Math.round(heapStats.total_heap_size / 1024 / 1024)}MB`,
+    v8UsedHeapSize: `${Math.round(heapStats.used_heap_size / 1024 / 1024)}MB`,
+    v8HeapSizeLimit: `${Math.round(heapStats.heap_size_limit / 1024 / 1024)}MB`,
+    v8MallocedMemory: `${Math.round(heapStats.malloced_memory / 1024 / 1024)}MB`,
+  };
+}
+
+/**
+ * Get ts.Program stats for understanding TypeScript memory usage
+ */
+function getTsProgramStats(pa: ProjectAnalyzer) {
+  const sourceFiles = pa.program.getSourceFiles();
+  const projectSourceFiles = sourceFiles.filter(
+    (sf) => !sf.fileName.includes('node_modules'),
+  );
+  const nodeModulesFiles = sourceFiles.filter((sf) =>
+    sf.fileName.includes('node_modules'),
+  );
+
+  // Estimate AST size by sampling text length (rough proxy)
+  let totalProjectTextLength = 0;
+  for (const sf of projectSourceFiles) {
+    totalProjectTextLength += sf.getFullText().length;
+  }
+
+  return {
+    totalSourceFiles: sourceFiles.length,
+    projectSourceFiles: projectSourceFiles.length,
+    nodeModulesFiles: nodeModulesFiles.length,
+    projectTextSizeMB: `${(totalProjectTextLength / 1024 / 1024).toFixed(1)}MB`,
+    // Note: Actual AST memory is typically 10-20x the source text size
+    estimatedAstSizeMB: `${((totalProjectTextLength * 15) / 1024 / 1024).toFixed(0)}MB`,
+  };
+}
+
+/**
+ * Write a heap snapshot for detailed profiling.
+ * The snapshot can be loaded in Chrome DevTools for analysis.
+ *
+ * TODO: Once we understand memory usage, change this back to env-var controlled
+ */
+const HEAP_SNAPSHOT_MODE: 'off' | 'once' | 'periodic' = 'off';
+
+function maybeWriteHeapSnapshot(label: string) {
+  if (HEAP_SNAPSHOT_MODE === 'off') return;
+
+  try {
+    const filename = v8.writeHeapSnapshot();
+    console.log(
+      `CodeYam Worker: Heap snapshot written (${label}): ${filename}`,
+    );
+  } catch (error) {
+    console.error('CodeYam Worker: Failed to write heap snapshot:', error);
+  }
+}
+
 /**
  * Main worker function called by Piscina
  */
 export default async function runAnalyzeScope(
   input: AnalyzeScopeWorkerInput,
 ): Promise<AnalyzeScopeWorkerOutput> {
+  // Reset metrics counters at the start of each entity analysis to prevent
+  // false "infinite loop" detection when processing multiple entities
+  resetScopeDataStructureMetrics();
+
   try {
     const { entity, project, dirPath, filePath, file, scope, model } = input;
 
     // Create or reuse ProjectAnalyzer
     const paKey = `${dirPath}`;
     if (!projectAnalyzer || projectAnalyzerKey !== paKey) {
-      projectAnalyzer = await ProjectAnalyzer.from({ project, dirPath });
+      const memBefore = process.memoryUsage();
+      console.log('CodeYam Worker: Creating ProjectAnalyzer (ts.Program)...', {
+        memBefore: getWorkerMemoryStats(),
+      });
+
+      projectAnalyzer = ProjectAnalyzer.from({ project, dirPath });
       projectAnalyzerKey = paKey;
+
+      const memAfter = process.memoryUsage();
+      const tsProgramStats = getTsProgramStats(projectAnalyzer);
+      console.log('CodeYam Worker: ProjectAnalyzer created', {
+        memAfter: getWorkerMemoryStats(),
+        rssDelta: `+${Math.round((memAfter.rss - memBefore.rss) / 1024 / 1024)}MB`,
+        heapDelta: `+${Math.round((memAfter.heapUsed - memBefore.heapUsed) / 1024 / 1024)}MB`,
+        tsProgramStats,
+      });
+
+      // Optionally write heap snapshot after ts.Program creation
+      maybeWriteHeapSnapshot('after-ts-program');
     }
 
     // Create or reuse FileAnalyzer
@@ -88,6 +187,26 @@ export default async function runAnalyzeScope(
       );
     }
 
+    // Periodic memory logging
+    entitiesProcessed++;
+    const now = Date.now();
+    const shouldLogMemory =
+      entitiesProcessed % MEMORY_LOG_INTERVAL === 0 ||
+      now - lastMemoryLogAt > MEMORY_LOG_TIME_INTERVAL;
+
+    if (shouldLogMemory) {
+      lastMemoryLogAt = now;
+      console.log('CodeYam Worker: Memory stats', {
+        entitiesProcessed,
+        ...getWorkerMemoryStats(),
+      });
+
+      // Optionally write periodic heap snapshots
+      if (HEAP_SNAPSHOT_MODE === 'periodic' && entitiesProcessed % 500 === 0) {
+        maybeWriteHeapSnapshot(`after-${entitiesProcessed}-entities`);
+      }
+    }
+
     return { dataStructure: dataStructure.toSerializable() };
   } catch (error) {
     console.error(

package/analyzer-template/packages/analyze/index.ts

@@ -41,6 +41,8 @@ export { default as mergeValidatedDataStructures } from './src/lib/files/scenari
 
 export { discoverDirectDependencies } from './src/lib/files/analyze/dependencyResolver';
 
+export { transformationTracer } from './src/lib/files/scenarios/TransformationTracer';
+
 export { default as getAnalysisError } from './src/lib/utils/getAnalysisError';
 export { default as measureAndReportExecutionTime } from './src/lib/utils/measureAndReportExecutionTime';