@codeyam/codeyam-cli 0.1.0-staging.eb21b2f → 0.1.0-staging.f3b710d

package/analyzer-template/packages/ai/src/lib/generateExecutionFlows.ts

@@ -129,19 +129,6 @@ export default function generateExecutionFlows({
     fullToShortPathMap[fullPath] = shortPath;
   }
 
-  console.log(
-    `[generateExecutionFlows] ${entity.name}: attributesMap has ${Object.keys(resolvedAttributesMap).length} keys, fullToShortPathMap has ${Object.keys(fullToShortPathMap).length} entries`,
-  );
-  console.log(
-    `[generateExecutionFlows] ${entity.name}: attributesMap keys: [${Object.keys(resolvedAttributesMap).join(', ')}]`,
-  );
-  console.log(
-    `[generateExecutionFlows] ${entity.name}: fullToShortPathMap: ${JSON.stringify(fullToShortPathMap)}`,
-  );
-  console.log(
-    `[generateExecutionFlows] ${entity.name}: equivalentSignatureVariables: ${JSON.stringify(equivalentSignatureVariables)}`,
-  );
-
   // Get pre-computed conditional data from metadata
   const conditionalUsages: Record<string, ConditionalUsage[]> =
     entity.metadata?.isolatedDataStructure?.conditionalUsages ?? {};
@@ -186,26 +173,6 @@ export default function generateExecutionFlows({
     }
   }
 
-  console.log(
-    `[generateExecutionFlows] ${entity.name}: conditionalUsages keys: [${Object.keys(conditionalUsages).join(', ')}]`,
-  );
-  console.log(
-    `[generateExecutionFlows] ${entity.name}: conditionalUsages sourceDataPaths: ${JSON.stringify(
-      Object.fromEntries(
-        Object.entries(conditionalUsages).map(([k, v]) => [
-          k,
-          (v as any[]).map((u: any) => u.sourceDataPath ?? '(none)'),
-        ]),
-      ),
-    )}`,
-  );
-  console.log(
-    `[generateExecutionFlows] ${entity.name}: compoundConditionals count: ${compoundConditionals.length}`,
-  );
-  console.log(
-    `[generateExecutionFlows] ${entity.name}: sourceEquivalencies keys count: ${Object.keys(mergedDataStructure.sourceEquivalencies ?? {}).length}`,
-  );
-
   const conditionalsBasedFlows = generateExecutionFlowsFromConditionals({
     conditionalUsages,
     compoundConditionals,

package/analyzer-template/packages/analyze/index.ts

@@ -36,7 +36,10 @@ export { getAllEntityNodes } from './src/lib/asts/sourceFiles/getAllEntityNodes'
 export { default as fileAnalyzerFromCode } from './src/lib/files/fileAnalyzerFromCode';
 
 export { default as isolateDataStructure } from './src/lib/files/scenarios/isolateDataStructure';
-export { default as mergeInDependentDataStructure } from './src/lib/files/scenarios/mergeInDependentDataStructure';
+export {
+  default as mergeInDependentDataStructure,
+  DataStructureTimeoutError,
+} from './src/lib/files/scenarios/mergeInDependentDataStructure';
 export { default as mergeValidatedDataStructures } from './src/lib/files/scenarios/mergeValidatedDataStructures';
 
 export { discoverDirectDependencies } from './src/lib/files/analyze/dependencyResolver';

package/analyzer-template/packages/analyze/src/lib/ProjectAnalyzer.ts

@@ -46,6 +46,12 @@ export class ProjectAnalyzer {
       lib.asts.createProgramFromTsConfigPath(configPath, excludePatterns),
     );
 
+    if (this.programs.length === 0) {
+      throw new Error(
+        'No TypeScript programs could be created — no tsconfig.json found',
+      );
+    }
+
     // Primary program is the first one (usually root) for backward compatibility
     this.program = this.programs[0];
     this.typeChecker = this.program.getTypeChecker();
@@ -334,17 +340,23 @@ export class ProjectAnalyzer {
    * Refresh the TypeScript program to pick up file changes
    */
   refreshProgram(): void {
-    // Memory logging: Capture state before refresh
-    const memBefore = process.memoryUsage();
-
     // Apply the same unapprovedPaths filtering when refreshing
     const excludePatterns =
       this.project.metadata?.unapprovedPaths?.filter(Boolean) || [];
 
-    // Refresh all programs
-    this.programs = this.tsConfigPaths.map((configPath) =>
-      lib.asts.createProgramFromTsConfigPath(configPath, excludePatterns),
-    );
+    // Release old programs and caches BEFORE creating new ones to reduce peak memory.
+    // Without this, both old (~1.7GB) and new (~1.7GB) programs coexist during creation,
+    // causing OOM on monorepos with multiple tsconfigs.
+    this.programs = [];
+    this.sourceFileByPath.clear();
+    this.typeCheckerByPath.clear();
+
+    // Create programs one at a time so each old program can be GC'd before the next
+    for (const configPath of this.tsConfigPaths) {
+      this.programs.push(
+        lib.asts.createProgramFromTsConfigPath(configPath, excludePatterns),
+      );
+    }
 
     // Update primary program for backward compatibility
     this.program = this.programs[0];

package/analyzer-template/packages/analyze/src/lib/asts/index.ts

@@ -95,8 +95,13 @@ export function createProgramFromTsConfigPath(
     throw new Error('Could not parse tsconfig.json');
   }
 
-  // Filter file names based on exclude patterns to reduce memory usage
-  let fileNames = parsedCommandLine.fileNames;
+  // Always filter out node_modules from root files to prevent OOM on large projects.
+  // This only removes them as root compilation targets — TypeScript's module resolution
+  // still follows imports into node_modules for type info when processing source files.
+  let fileNames = parsedCommandLine.fileNames.filter(
+    (f) => !f.includes('/node_modules/'),
+  );
+
   if (excludePatterns && excludePatterns.length > 0) {
     const regexPatterns = excludePatterns
       .map((pattern) => {

package/analyzer-template/packages/analyze/src/lib/asts/nodes/getNodeType.ts

@@ -39,6 +39,7 @@ export function getNodeType(node: ts.Node, sourceFile: ts.SourceFile) {
     case 'StringLiteral':
     case 'NumericLiteral':
     case 'ObjectLiteralExpression':
+    case 'ArrayLiteralExpression':
       nodeType = 'data';
       break;
     case 'TypeAliasDeclaration':

package/analyzer-template/packages/analyze/src/lib/files/analyze/analyzeEntities/prepareDataStructures.ts

@@ -289,18 +289,44 @@ export default async function prepareEntityDataStructures(
     skipWorkerPool();
   }
 
+  awsLogDebugLevel(
+    1,
+    `Preparing data structures for ${entities.length} entities (${entitiesNeedingGeneration} need generation)...`,
+  );
+  const startTime = Date.now();
+  let completedCount = 0;
+
+  // Wrap processEntity to track progress
+  const processEntityWithProgress = async (entity: Entity) => {
+    await processEntity(entity);
+    completedCount++;
+    // Emit progress every 10 entities
+    if (completedCount % 10 === 0) {
+      awsLogDebugLevel(
+        1,
+        `Preparing data structures... (${completedCount}/${entities.length})`,
+      );
+    }
+  };
+
   const sequential =
     DEBUG_SEQUENTIAL_EXECUTION || options.sequentialDataStructurePrep;
 
   if (sequential) {
     // Sequential execution for debugging - process one entity at a time
     for (const entity of entities) {
-      await processEntity(entity);
+      await processEntityWithProgress(entity);
     }
   } else {
     // Parallel execution for speed
-    await Promise.all(entities.map(processEntity));
+    await Promise.all(entities.map(processEntityWithProgress));
   }
+
+  const elapsed = Date.now() - startTime;
+  awsLogDebugLevel(
+    1,
+    `Data structure preparation complete: ${entities.length} entities in ${elapsed}ms`,
+  );
 }
 
 /**

package/analyzer-template/packages/analyze/src/lib/files/analyze/analyzeEntities.ts

@@ -8,13 +8,11 @@ import {
 import { transformationTracer } from '../scenarios/TransformationTracer';
 import { getFileByPathSafe } from '../../utils/getFileByPath';
 import trackEntityCircularDependencies from './trackEntityCircularDependencies';
-import validateDependencyAnalyses from './validateDependencyAnalyses';
 import extractClassMethods from './analyzeEntities/extractClassMethods';
 import prepareDataStructures from './analyzeEntities/prepareDataStructures';
 import { strategies, asyncComplex } from './analyzeEntities/strategies';
 import {
   updateCommitMetadata,
-  updateFreshAnalysisMetadata,
   updateFreshAnalysisStatus,
   upsertAnalyses,
   upsertEntities,
@@ -263,40 +261,11 @@ export default async function analyzeEntities({
     }
   }
 
-  awsLog(`CodeYam: analyzeEntities: Running patch-up dependency validation`);
-  for (const analysis of analyses) {
-    if (!analysis.entity) {
-      throw new Error(
-        `CodeYam: analyzeEntities: Internal error: Analysis entity is missing for ${analysis.filePath}:${analysis.entityName}.`,
-      );
-    }
-
-    // Validate dependencies, which modifies analysis.metadata.dependentAnalyses
-    validateDependencyAnalyses({ analysis, entity: analysis.entity, context });
-
-    // Carefully update only the metadata field
-    if (analysis.id) {
-      const updatedMetadata = await updateFreshAnalysisMetadata(
-        analysis.id,
-        (metadata) => {
-          metadata.dependentAnalyses =
-            analysis.metadata?.dependentAnalyses || [];
-        },
-      );
-
-      if (updatedMetadata) {
-        analysis.metadata = updatedMetadata;
-      } else {
-        console.warn(
-          `CodeYam: analyzeEntities: Failed to update metadata for analysis ${analysis.id}`,
-        );
-      }
-    } else {
-      console.warn(
-        `CodeYam: analyzeEntities: Analysis ${analysis.filePath}:${analysis.entityName} has no ID, skipping metadata update`,
-      );
-    }
-  }
+  // NOTE: patch-up dependency validation loop was removed here.
+  // analysis.metadata.dependentAnalyses is already populated by validateDependencyAnalyses
+  // inside analyzeEntity() (Call #1) and persisted via upsertAnalysesWithScenarios at the
+  // end of analyzeEntity, then again by upsertAnalyses in analyzeOneEntityKey. Re-validating
+  // every analysis from scratch was redundant work — especially costly for large projects.
 
   const newlyReady = await updateReadyAnalyses();
   if (newlyReady.length > 0) {
@@ -310,7 +279,15 @@ export default async function analyzeEntities({
   );
 
   // Flush transformation trace if tracing is enabled
-  transformationTracer.flush();
+  // Wrapped in try/catch because trace data can exceed V8's max string length
+  // for large projects, and we must not crash the analysis process over tracing.
+  try {
+    transformationTracer.flush();
+  } catch (e) {
+    console.warn(
+      `CodeYam Warning: Failed to flush transformation trace: ${e.message}`,
+    );
+  }
 
   return analyses;
 

package/analyzer-template/packages/analyze/src/lib/files/analyze/dependencyResolver.ts

@@ -100,12 +100,6 @@ export function getCandidatesReadyForCapture(
       continue;
     }
 
-    // Skip notExported entities - they can't be imported into the simulation
-    // environment and will always fail capture. Don't mark them as ready.
-    if (candidate.entity?.metadata?.notExported) {
-      continue;
-    }
-
     // Get all transitive dependencies
     const candidateKey = `${candidate.filePath}:${candidate.entityName}`;
     const { missingDependencies } = getTransitiveDependencies(

package/analyzer-template/packages/analyze/src/lib/files/analyze/findOrCreateEntity.ts

@@ -94,12 +94,15 @@ export default async function findOrCreateEntity({
   try {
     code = fileAnalyzer.getEntityCode(localEntityName);
   } catch (e) {
-    console.log('CodeYam Error: could not get entity code', {
-      filePath: file.path,
-      localEntityName,
-      entityName,
-      error: e,
-    });
+    console.log(
+      `CodeYam Error: Could not extract code for "${localEntityName}" from ${file.path} — export not found in file, may have been moved or renamed during refactoring`,
+      {
+        filePath: file.path,
+        localEntityName,
+        entityName,
+        error: String(e),
+      },
+    );
     return null;
   }
   const sha = generateSha(file.path, entityName, code);
@@ -108,6 +111,7 @@ export default async function findOrCreateEntity({
     entity = await loadEntity({
       projectId: project.id,
       sha,
+      silent: true, // Entity not existing yet is the normal path — we create it below
     });
   }
 
@@ -138,6 +142,18 @@ export default async function findOrCreateEntity({
           break;
         }
       }
+
+      // Mark previous versions as superseded
+      const entitiesToSupersede = previousEntities.filter(
+        (e) => !e.metadata?.isSuperseded,
+      );
+      if (entitiesToSupersede.length > 0) {
+        for (const prev of entitiesToSupersede) {
+          prev.metadata = prev.metadata || {};
+          prev.metadata.isSuperseded = true;
+        }
+        await upsertEntities(entitiesToSupersede);
+      }
     }
 
     entity = {

package/analyzer-template/packages/analyze/src/lib/files/analyze/gatherEntityMap.ts

@@ -111,18 +111,15 @@ export default async function gatherEntityMap({
       });
 
       if (!importedEntity) {
-        console.log(
-          `CodeYam Error: Entity not found while gathering entity map: ${importedFile.path}: ${importedExportName}`,
-          {
-            entityName: entity.name,
-            entityFilePath: entity.filePath,
-            importedFilePath: importedFile.path,
-            importedExportName: importedExportName,
-          },
-        );
-        throw new Error(
-          `Entity not found while gathering entity map: ${importedFile.path}: ${importedExport.name}`,
-        );
+        const msg = [
+          `Entity not found: "${importedExportName}" in ${importedFile.path}`,
+          `  While analyzing dependencies of: ${entity.name} (${entity.filePath})`,
+          `  This usually means "${importedExportName}" was moved or renamed but`,
+          `  ${entity.filePath} still has stale import metadata referencing the old location.`,
+          `  Fix: Update or re-analyze ${entity.filePath} so its importedExports reflect current code.`,
+        ].join('\n');
+        console.log(`CodeYam Error: ${msg}`);
+        throw new Error(msg);
       }
 
       // Add newly created entity to context

package/analyzer-template/packages/analyze/src/lib/files/analyze/trackEntityCircularDependencies.ts

@@ -1,5 +1,6 @@
 import type { Entity } from '~codeyam/types';
 import { AnalysisContext } from '../../analysisContext';
+import { awsLogDebugLevel } from '~codeyam/utils';
 
 /**
  * Optimized version that processes multiple root entities in a single traversal.
@@ -16,6 +17,8 @@ export default function trackEntityCircularDependencies(
   try {
     const allEntities = new Set<Entity>();
     const globalVisited = new Set<string>(); // Track by SHA to avoid duplicate traversals
+    let circularCount = 0;
+    const startTime = Date.now();
 
     // Process each root entity, but skip if already processed
     for (const rootEntity of rootEntities) {
@@ -32,8 +35,26 @@ export default function trackEntityCircularDependencies(
         globalVisited,
         allEntities,
       );
+
+      // Emit periodic progress for large projects
+      if (allEntities.size % 100 === 0 && allEntities.size > 0) {
+        awsLogDebugLevel(
+          1,
+          `Mapping dependencies... (${allEntities.size} entities discovered)`,
+        );
+      }
     }
 
+    const elapsed = Date.now() - startTime;
+    // Count circular deps
+    for (const entity of allEntities) {
+      if (entity.metadata?.hasCircularDependency) circularCount++;
+    }
+    awsLogDebugLevel(
+      1,
+      `Dependency mapping complete: ${allEntities.size} entities, ${circularCount} circular, ${elapsed}ms`,
+    );
+
     return allEntities;
   } catch (e) {
     console.log(

package/analyzer-template/packages/analyze/src/lib/files/analyze/validateDependencyAnalyses.ts

@@ -6,6 +6,20 @@ import type {
   ReadonlyEntity,
 } from '~codeyam/types';
 import { ImmutableAnalysisContext } from '../../analysisContext';
+import { awsLogDebugLevel } from '~codeyam/utils';
+
+/**
+ * Tracks statistics across the recursive dependency validation.
+ * Shared by all recursive calls to provide a complete picture.
+ */
+interface ValidationStats {
+  totalVisited: number;
+  skippedDuplicate: number;
+  skippedMissing: number;
+  skippedTrivial: number;
+  maxDepth: number;
+  startTime: number;
+}
 
 export default function validateDependencyAnalyses({
   analysis,
@@ -13,12 +27,18 @@ export default function validateDependencyAnalyses({
   entity,
   context,
   dependentAnalyses,
+  _depth = 0,
+  _stats,
 }: {
   analysis?: Analysis;
   readonlyAnalysis?: ReadonlyAnalysis;
   entity: ReadonlyEntity;
   context: ImmutableAnalysisContext;
   dependentAnalyses?: ReadonlyAnalysisMap;
+  /** Recursion depth (internal, do not set) */
+  _depth?: number;
+  /** Shared stats tracker (internal, do not set) */
+  _stats?: ValidationStats;
 }): { dependentAnalyses: ReadonlyAnalysisMap } {
   if (!readonlyAnalysis) readonlyAnalysis = analysis;
   if (!readonlyAnalysis) {
@@ -32,6 +52,28 @@ export default function validateDependencyAnalyses({
     );
   }
 
+  // Initialize stats on the root call
+  const isRootCall = _depth === 0;
+  if (!_stats) {
+    _stats = {
+      totalVisited: 0,
+      skippedDuplicate: 0,
+      skippedMissing: 0,
+      skippedTrivial: 0,
+      maxDepth: 0,
+      startTime: Date.now(),
+    };
+  }
+  if (_depth > _stats.maxDepth) _stats.maxDepth = _depth;
+
+  const importCount = entity.metadata?.importedExports?.length ?? 0;
+  if (isRootCall && importCount > 0) {
+    awsLogDebugLevel(
+      2,
+      `validateDeps: ${entity.name} has ${importCount} imports to validate`,
+    );
+  }
+
   if (
     dependentAnalyses &&
     dependentAnalyses[readonlyAnalysis.filePath]?.[
@@ -45,6 +87,12 @@ export default function validateDependencyAnalyses({
     })
   ) {
     // already processed this analysis
+    if (isRootCall) {
+      awsLogDebugLevel(
+        2,
+        `validateDeps: ${entity.name} already fully processed (cache hit)`,
+      );
+    }
     return { dependentAnalyses };
   }
 
@@ -69,9 +117,20 @@ export default function validateDependencyAnalyses({
 
     // if the entity has duplicate dependencies, handle gracefully
     if (dependentAnalyses[dependencyFilePath]?.[dependencyName]) {
+      _stats.skippedDuplicate++;
       continue;
     }
 
+    _stats.totalVisited++;
+
+    // Emit periodic progress so the spinner stays alive for large trees
+    if (_stats.totalVisited % 50 === 0) {
+      awsLogDebugLevel(
+        1,
+        `Validating dependencies... (${_stats.totalVisited} resolved, depth ${_depth})`,
+      );
+    }
+
     // Check if dependency analysis exists in context
     const dependentAnalysis = context.getAnalysis(
       dependencyFilePath,
@@ -86,21 +145,20 @@ export default function validateDependencyAnalyses({
       );
 
       if (!dependentEntity) {
-        // console.warn(
-        //   `CodeYam: validateDependencyAnalyses: Missing dependent entity ${dependencyFilePath}:${dependencyName} for ${entity.filePath}:${entity.name}. This is likely a bug.`,
-        // );
+        _stats.skippedMissing++;
         continue;
       }
 
       // ignore trivial missing dependencies
-      if (!['library', 'visual'].includes(dependentEntity.entityType)) continue;
+      if (!['library', 'visual'].includes(dependentEntity.entityType)) {
+        _stats.skippedTrivial++;
+        continue;
+      }
 
       if (!dependentEntity?.metadata?.hasCircularDependency) {
         // TODO: This no longer seems to happen thanks to fixes elsewhere, but leave the warning
-        // console.warn(
-        //   `CodeYam: validateDependencyAnalyses: Missing analysis for dependency ${entity.filePath}:${entity.name} -> ${dependencyFilePath}:${dependencyName}, should be marked as having circular dependencies.`,
-        // );
       }
+      _stats.skippedMissing++;
       continue;
     }
 
@@ -136,9 +194,6 @@ export default function validateDependencyAnalyses({
     if (analysis) {
       // Track dependency relationship in analysis metadata
       if (!analysis.metadata.dependentAnalyses.includes(dependentAnalysis.id)) {
-        // console.log(
-        //   `CodeYam: validateDependencyAnalyses: Establishing dependency ${analysis.filePath}:${analysis.entityName} -> ${dependentAnalysis.filePath}:${dependentAnalysis.entityName}.`,
-        // );
         analysis.metadata.dependentAnalyses.push(dependentAnalysis.id);
       }
     }
@@ -152,9 +207,26 @@ export default function validateDependencyAnalyses({
         entity: dependentEntity,
         context: context,
         dependentAnalyses,
+        _depth: _depth + 1,
+        _stats,
       });
     }
   }
 
+  // Log summary on root call completion
+  if (isRootCall) {
+    const elapsed = Date.now() - _stats.startTime;
+    const totalDeps = Object.values(dependentAnalyses).reduce(
+      (sum, fileMap) => sum + Object.keys(fileMap).length,
+      0,
+    );
+    awsLogDebugLevel(
+      1,
+      `Dependency validation complete for ${entity.name}: ${totalDeps} dependencies resolved, ` +
+        `${_stats.skippedDuplicate} duplicates skipped, ${_stats.skippedMissing} missing, ` +
+        `${_stats.skippedTrivial} trivial, max depth ${_stats.maxDepth}, ${elapsed}ms`,
+    );
+  }
+
   return { dependentAnalyses };
 }

package/analyzer-template/packages/analyze/src/lib/files/analyzeChange.ts

@@ -91,6 +91,10 @@ export default async function analyzeChange({
     // Structure phase: CPU-intensive, serialize via semaphore if provided
     if (!steps || steps.includes(Step.Structure)) {
       if (structureSemaphore) {
+        awsLogDebugLevel(
+          1,
+          `Waiting for structure semaphore for ${entity.name}...`,
+        );
         await structureSemaphore.acquire();
       }
       try {

package/analyzer-template/packages/analyze/src/lib/files/analyzeInitial.ts

@@ -58,6 +58,10 @@ export default async function analyzeInitial({
     // Structure phase: CPU-intensive, serialize via semaphore if provided
     if (!steps || steps.includes(Step.Structure)) {
       if (structureSemaphore) {
+        awsLogDebugLevel(
+          1,
+          `Waiting for structure semaphore for ${entity.name}...`,
+        );
         await structureSemaphore.acquire();
       }
       try {

package/analyzer-template/packages/analyze/src/lib/files/analyzeNextRoute.ts

@@ -3,6 +3,8 @@ import { ProjectAnalyzer } from '../ProjectAnalyzer';
 import { generateSha } from '~codeyam/database';
 import { isNextRoute } from '~codeyam/utils';
 
+const warnedLayoutPaths = new Set<string>();
+
 export default function analyzeNextRoute(
   file: File,
   projectAnalyzer: ProjectAnalyzer,
@@ -56,9 +58,12 @@ export default function analyzeNextRoute(
 
           // Ensure entities array exists before trying to use it
           if (!layout.entities) {
-            console.log(
-              `CodeYam Warning: layout.entities not populated for ${layout.path}, skipping implicit dependency`,
-            );
+            if (!warnedLayoutPaths.has(layout.path)) {
+              warnedLayoutPaths.add(layout.path);
+              console.log(
+                `CodeYam Warning: layout.entities not populated for ${layout.path}, skipping implicit dependency`,
+              );
+            }
             return;
           }