@codeyam/codeyam-cli 0.1.0-staging.323686 → 0.1.0-staging.39719f5

package/analyzer-template/packages/analyze/src/lib/asts/sourceFiles/getResolvedModule.ts

@@ -2,21 +2,44 @@ import ts from 'typescript';
 import path from 'path';
 import fs from 'fs';
 
+/**
+ * Cache for nearest tsconfig lookups (keyed by directory).
+ * Speeds up repeated lookups during module resolution.
+ */
+const nearestTsConfigCache = new Map<string, string | undefined>();
+
 /**
  * Find the nearest tsconfig.json by walking up the directory tree from the source file
  */
 function findNearestTsConfig(sourceFileName: string): string | undefined {
   let currentDir = path.dirname(sourceFileName);
   const root = path.parse(currentDir).root;
+  const visited: string[] = [];
 
   while (currentDir !== root) {
+    const cached = nearestTsConfigCache.get(currentDir);
+    if (cached !== undefined || nearestTsConfigCache.has(currentDir)) {
+      for (const dir of visited) {
+        nearestTsConfigCache.set(dir, cached);
+      }
+      return cached;
+    }
+
+    visited.push(currentDir);
     const tsConfigPath = path.join(currentDir, 'tsconfig.json');
     if (fs.existsSync(tsConfigPath)) {
+      for (const dir of visited) {
+        nearestTsConfigCache.set(dir, tsConfigPath);
+      }
       return tsConfigPath;
     }
     currentDir = path.dirname(currentDir);
   }
 
+  for (const dir of visited) {
+    nearestTsConfigCache.set(dir, undefined);
+  }
+
   return undefined;
 }
 
@@ -25,6 +48,14 @@ function findNearestTsConfig(sourceFileName: string): string | undefined {
  */
 const tsConfigCache = new Map<string, ts.CompilerOptions>();
 
+/**
+ * Cache resolved module lookups by program for fast repeat access.
+ */
+const resolvedModuleCacheByProgram = new WeakMap<
+  ts.Program,
+  Map<string, ts.ResolvedModuleFull | null>
+>();
+
 /**
  * Get compiler options from a tsconfig file
  */
@@ -62,6 +93,19 @@ export function getResolvedModule(
   // Try to find the nearest tsconfig for this source file
   const nearestTsConfig = findNearestTsConfig(sourceFile.fileName);
 
+  const cacheForProgram =
+    resolvedModuleCacheByProgram.get(program) ??
+    new Map<string, ts.ResolvedModuleFull | null>();
+  if (!resolvedModuleCacheByProgram.has(program)) {
+    resolvedModuleCacheByProgram.set(program, cacheForProgram);
+  }
+
+  const cacheKey = `${nearestTsConfig ?? ''}|${sourceFile.fileName}|${moduleName}`;
+  if (cacheForProgram.has(cacheKey)) {
+    const cached = cacheForProgram.get(cacheKey);
+    return cached ?? undefined;
+  }
+
   // Start with the program's compiler options
   let compilerOptions = program.getCompilerOptions();
 
@@ -90,6 +134,7 @@ export function getResolvedModule(
 
   // If TypeScript's resolution succeeded, return it
   if (result.resolvedModule) {
+    cacheForProgram.set(cacheKey, result.resolvedModule);
     return result.resolvedModule;
   }
 
@@ -105,14 +150,17 @@ export function getResolvedModule(
     for (const ext of extensions) {
       const testPath = absolutePath + ext;
       if (fs.existsSync(testPath)) {
-        return {
+        const resolved = {
           resolvedFileName: testPath,
           extension: ext.includes('/') ? path.extname(ext) : ext,
           isExternalLibraryImport: false,
         } as ts.ResolvedModuleFull;
+        cacheForProgram.set(cacheKey, resolved);
+        return resolved;
       }
     }
   }
 
+  cacheForProgram.set(cacheKey, null);
   return undefined;
 }

package/analyzer-template/packages/analyze/src/lib/asts/sourceFiles/getSourceFilesForAllImports.ts

@@ -21,5 +21,6 @@ export function getSourceFilesForAllImports(
 
   visit(sourceFile); // Start visiting nodes from the root source file
 
-  return getSourceFilesForImports(moduleNames, sourceFile, program);
+  const uniqueModuleNames = Array.from(new Set(moduleNames));
+  return getSourceFilesForImports(uniqueModuleNames, sourceFile, program);
 }

package/analyzer-template/packages/analyze/src/lib/files/analyze/analyzeEntities/prepareDataStructures.ts

@@ -18,6 +18,7 @@ import {
   getCompoundConditionals,
   getChildBoundaryGatingConditions,
   getJsxRenderingUsages,
+  skipWorkerPool,
 } from '~codeyam/ai';
 import type { AnalysisOptions } from '../analyzeEntities';
 
@@ -271,18 +272,61 @@ export default async function prepareEntityDataStructures(
   };
 
   const entities = Array.from(allEntities);
+
+  // The worker thread creates its own ProjectAnalyzer (ts.Program) on first use,
+  // which takes 50-70s. For small entity counts the overhead far exceeds the actual
+  // processing time (<2s). Skip the worker and run on the main thread instead,
+  // reusing the existing ProjectAnalyzer.
+  const entitiesNeedingGeneration = entities.filter(
+    (e) =>
+      !e.metadata?.isolatedDataStructure || options.force || options.forceAll,
+  ).length;
+  const WORKER_ENTITY_THRESHOLD = 500;
+  if (entitiesNeedingGeneration < WORKER_ENTITY_THRESHOLD) {
+    console.log(
+      `CodeYam: Skipping worker thread for DS prep (${entitiesNeedingGeneration} entities < ${WORKER_ENTITY_THRESHOLD} threshold)`,
+    );
+    skipWorkerPool();
+  }
+
+  awsLogDebugLevel(
+    1,
+    `Preparing data structures for ${entities.length} entities (${entitiesNeedingGeneration} need generation)...`,
+  );
+  const startTime = Date.now();
+  let completedCount = 0;
+
+  // Wrap processEntity to track progress
+  const processEntityWithProgress = async (entity: Entity) => {
+    await processEntity(entity);
+    completedCount++;
+    // Emit progress every 10 entities
+    if (completedCount % 10 === 0) {
+      awsLogDebugLevel(
+        1,
+        `Preparing data structures... (${completedCount}/${entities.length})`,
+      );
+    }
+  };
+
   const sequential =
     DEBUG_SEQUENTIAL_EXECUTION || options.sequentialDataStructurePrep;
 
   if (sequential) {
     // Sequential execution for debugging - process one entity at a time
     for (const entity of entities) {
-      await processEntity(entity);
+      await processEntityWithProgress(entity);
     }
   } else {
     // Parallel execution for speed
-    await Promise.all(entities.map(processEntity));
+    await Promise.all(entities.map(processEntityWithProgress));
   }
+
+  const elapsed = Date.now() - startTime;
+  awsLogDebugLevel(
+    1,
+    `Data structure preparation complete: ${entities.length} entities in ${elapsed}ms`,
+  );
 }
 
 /**
@@ -336,11 +380,23 @@ async function getEntityDataStructureAndFunctionCalls({
         functionName: importedExport.name,
       });
 
-      // If neither signature nor return value found, this function doesn't exist in the analysis.
-      // We need to check BOTH because hooks like useParams() have no signature (no arguments)
-      // but DO have a return value that needs to be mocked.
+      // If neither signature nor return value found, check if this function has matching
+      // external function calls. External functions (like useLoaderData from react-router)
+      // don't have entries in functionResults, so getReturnValue returns null for them.
+      // But their return values ARE traced in externalFunctionCalls via perVariableSchemas.
+      // We need to check for matching calls before skipping.
       if (!signatureSchema && !returnValueSchema) {
-        return acc;
+        const hasMatchingExternalCall =
+          dataStructure.externalFunctionCalls?.some(
+            (fc) =>
+              fc.name === importedExport.name ||
+              fc.name.startsWith(importedExport.name + '<'),
+          );
+
+        // Only skip if there are no matching external function calls
+        if (!hasMatchingExternalCall) {
+          return acc;
+        }
       }
 
       // Clone schemas before deduplication to avoid mutating shared references
@@ -543,6 +599,42 @@ async function getEntityDataStructureAndFunctionCalls({
     },
   );
 
+  // Add same-file child components to dependencySchemas.
+  // Same-file components (e.g., AgentCard defined in the same file as AgentTranscriptsPage)
+  // appear in externalFunctionCalls and functionResults but NOT in importedExports or
+  // nodeModuleImports. Without this, their signatures are missing from dependencySchemas
+  // and enrichArrayTypesFromChildSignatures can't find child component field accesses.
+  const allDependencyNames = new Set(allDependencies.map((d) => d.name));
+
+  for (const efc of dataStructure.externalFunctionCalls) {
+    // Skip if already covered by importedExports or nodeModuleImports
+    if (allDependencyNames.has(efc.name)) continue;
+
+    const signatureSchema = getFunctionSignature(dataStructure, {
+      functionName: efc.name,
+    });
+
+    const returnValueSchema = getReturnValue(dataStructure, {
+      functionName: efc.name,
+    });
+
+    // Only add if we have signature or return value data
+    if (!signatureSchema && !returnValueSchema) continue;
+
+    const clonedSignatureSchema = { ...(signatureSchema ?? {}) };
+    const clonedReturnValueSchema = { ...(returnValueSchema ?? {}) };
+
+    const entityFilePath = entity.filePath;
+    dependencySchemas[entityFilePath] ||= {};
+    dependencySchemas[entityFilePath][efc.name] = {
+      signatureSchema: deduplicateFunctionSchemas(clonedSignatureSchema),
+      returnValueSchema: deduplicateFunctionSchemas(clonedReturnValueSchema),
+      usageEquivalencies: getUsageEquivalencies(dataStructure, efc.name) ?? {},
+      sourceEquivalencies:
+        getSourceEquivalencies(dataStructure, efc.name) ?? {},
+    };
+  }
+
   const rawReturnValue =
     getReturnValue(dataStructure, { fillInUnknowns: false }) ?? {};
 
@@ -570,11 +662,13 @@ async function getEntityDataStructureAndFunctionCalls({
   // Get JSX rendering usages for array size and text length flow generation
   const jsxRenderingUsages = getJsxRenderingUsages(dataStructure);
 
+  const equivalentSignatureVariables =
+    getEquivalentSignatureVariables(dataStructure);
+
   const isolatedDataStructure = {
     signatureSchema: deduplicateFunctionSchemas(clonedRawSignature),
     returnValueSchema: deduplicateFunctionSchemas(clonedRawReturnValue),
-    equivalentSignatureVariables:
-      getEquivalentSignatureVariables(dataStructure),
+    equivalentSignatureVariables,
     dependencySchemas,
     environmentVariables: dataStructure.environmentVariables,
     conditionalUsages:
@@ -856,9 +950,21 @@ export function determineMockedDependencies(entity: Entity) {
       // A schema with only "EntityName(): function" just indicates the entity is callable,
       // it doesn't mean there's actual return data to mock
       const hasReturnValue = hasActualReturnValueData(entityReturnValueSchema);
-      const complexReturnValue = Object.values(
-        entityReturnValueSchema ?? {},
-      ).some((value) => value === 'object' || value === 'array');
+      // Check for complex return types (object/array), accounting for nullable types
+      // e.g., 'object', 'object | null', 'array | undefined' should all count as complex.
+      // Also detect when functionCallReturnValue has nested child properties — this
+      // proves it's a structured type even if its declared type is still 'unknown'.
+      const returnValueKeys = Object.keys(entityReturnValueSchema ?? {});
+      const complexReturnValue =
+        Object.values(entityReturnValueSchema ?? {}).some((value) => {
+          const baseType = value?.split(' | ')[0];
+          return baseType === 'object' || baseType === 'array';
+        }) ||
+        returnValueKeys.some(
+          (key) =>
+            key.includes('functionCallReturnValue.') ||
+            key.includes('functionCallReturnValue['),
+        );
 
       // Check if this is a validation function that should always be mocked
       const isValidationFn = isValidationFunction(entityName);

package/analyzer-template/packages/analyze/src/lib/files/analyze/analyzeEntities.ts

@@ -5,14 +5,14 @@ import {
   measureAndReportExecutionTime,
   ProjectAnalyzer,
 } from '~codeyam/analyze';
+import { transformationTracer } from '../scenarios/TransformationTracer';
+import { getFileByPathSafe } from '../../utils/getFileByPath';
 import trackEntityCircularDependencies from './trackEntityCircularDependencies';
-import validateDependencyAnalyses from './validateDependencyAnalyses';
 import extractClassMethods from './analyzeEntities/extractClassMethods';
 import prepareDataStructures from './analyzeEntities/prepareDataStructures';
 import { strategies, asyncComplex } from './analyzeEntities/strategies';
 import {
   updateCommitMetadata,
-  updateFreshAnalysisMetadata,
   updateFreshAnalysisStatus,
   upsertAnalyses,
   upsertEntities,
@@ -261,40 +261,11 @@ export default async function analyzeEntities({
     }
   }
 
-  awsLog(`CodeYam: analyzeEntities: Running patch-up dependency validation`);
-  for (const analysis of analyses) {
-    if (!analysis.entity) {
-      throw new Error(
-        `CodeYam: analyzeEntities: Internal error: Analysis entity is missing for ${analysis.filePath}:${analysis.entityName}.`,
-      );
-    }
-
-    // Validate dependencies, which modifies analysis.metadata.dependentAnalyses
-    validateDependencyAnalyses({ analysis, entity: analysis.entity, context });
-
-    // Carefully update only the metadata field
-    if (analysis.id) {
-      const updatedMetadata = await updateFreshAnalysisMetadata(
-        analysis.id,
-        (metadata) => {
-          metadata.dependentAnalyses =
-            analysis.metadata?.dependentAnalyses || [];
-        },
-      );
-
-      if (updatedMetadata) {
-        analysis.metadata = updatedMetadata;
-      } else {
-        console.warn(
-          `CodeYam: analyzeEntities: Failed to update metadata for analysis ${analysis.id}`,
-        );
-      }
-    } else {
-      console.warn(
-        `CodeYam: analyzeEntities: Analysis ${analysis.filePath}:${analysis.entityName} has no ID, skipping metadata update`,
-      );
-    }
-  }
+  // NOTE: patch-up dependency validation loop was removed here.
+  // analysis.metadata.dependentAnalyses is already populated by validateDependencyAnalyses
+  // inside analyzeEntity() (Call #1) and persisted via upsertAnalysesWithScenarios at the
+  // end of analyzeEntity, then again by upsertAnalyses in analyzeOneEntityKey. Re-validating
+  // every analysis from scratch was redundant work — especially costly for large projects.
 
   const newlyReady = await updateReadyAnalyses();
   if (newlyReady.length > 0) {
@@ -306,6 +277,18 @@ export default async function analyzeEntities({
   awsLog(
     `CodeYam: analyzeEntities: Analysis completed with ${analyses.length} analyses`,
   );
+
+  // Flush transformation trace if tracing is enabled
+  // Wrapped in try/catch because trace data can exceed V8's max string length
+  // for large projects, and we must not crash the analysis process over tracing.
+  try {
+    transformationTracer.flush();
+  } catch (e) {
+    console.warn(
+      `CodeYam Warning: Failed to flush transformation trace: ${e.message}`,
+    );
+  }
+
   return analyses;
 
   // PHASE 3: Pure graph traversal
@@ -388,10 +371,19 @@ export default async function analyzeEntities({
             // Circular dependency detected in path - skip
             continue;
           }
-          // If already visited via another traversal path, skip to avoid deadlock.
-          // Its subtree has been fully explored; any leaves are already in leafKeys.
-          // Blocking on it would just propagate the block and cause stuck analysis.
+          // If already visited via another traversal path, check if it's a leaf.
+          // If it's a leaf, we should still wait for it to complete.
+          // Only skip if we've already explored it and it's NOT a leaf (meaning it's
+          // blocked on something else - continuing to traverse would cause a deadlock).
           if (visited.has(dependentEntityKey)) {
+            // If the visited entity is a leaf, we should wait for it to complete
+            // before processing this entity. Don't skip it!
+            if (leafKeys.has(dependentEntityKey)) {
+              // It's a leaf that hasn't completed yet - wait for it
+              isLeaf = false;
+              continue;
+            }
+            // If it's not a leaf, it's blocked on something else - skip to avoid deadlock
             continue;
           }
           isLeaf = false;
@@ -426,7 +418,7 @@ export default async function analyzeEntities({
     }
 
     const file =
-      entity.file || project.files.find((f) => f.path === entity.filePath);
+      entity.file || getFileByPathSafe(projectAnalyzer, entity.filePath);
     if (!file) {
       awsLog(
         `CodeYam: analyzeEntities: File not found for entity: ${entityKey}`,

package/analyzer-template/packages/analyze/src/lib/files/analyze/dependencyResolver.ts

@@ -100,12 +100,6 @@ export function getCandidatesReadyForCapture(
       continue;
     }
 
-    // Skip notExported entities - they can't be imported into the simulation
-    // environment and will always fail capture. Don't mark them as ready.
-    if (candidate.entity?.metadata?.notExported) {
-      continue;
-    }
-
     // Get all transitive dependencies
     const candidateKey = `${candidate.filePath}:${candidate.entityName}`;
     const { missingDependencies } = getTransitiveDependencies(

package/analyzer-template/packages/analyze/src/lib/files/analyze/findOrCreateEntity.ts

@@ -94,12 +94,15 @@ export default async function findOrCreateEntity({
   try {
     code = fileAnalyzer.getEntityCode(localEntityName);
   } catch (e) {
-    console.log('CodeYam Error: could not get entity code', {
-      filePath: file.path,
-      localEntityName,
-      entityName,
-      error: e,
-    });
+    console.log(
+      `CodeYam Error: Could not extract code for "${localEntityName}" from ${file.path} — export not found in file, may have been moved or renamed during refactoring`,
+      {
+        filePath: file.path,
+        localEntityName,
+        entityName,
+        error: String(e),
+      },
+    );
     return null;
   }
   const sha = generateSha(file.path, entityName, code);
@@ -108,6 +111,7 @@ export default async function findOrCreateEntity({
     entity = await loadEntity({
       projectId: project.id,
       sha,
+      silent: true, // Entity not existing yet is the normal path — we create it below
     });
   }
 
@@ -138,6 +142,18 @@ export default async function findOrCreateEntity({
           break;
         }
       }
+
+      // Mark previous versions as superseded
+      const entitiesToSupersede = previousEntities.filter(
+        (e) => !e.metadata?.isSuperseded,
+      );
+      if (entitiesToSupersede.length > 0) {
+        for (const prev of entitiesToSupersede) {
+          prev.metadata = prev.metadata || {};
+          prev.metadata.isSuperseded = true;
+        }
+        await upsertEntities(entitiesToSupersede);
+      }
     }
 
     entity = {

package/analyzer-template/packages/analyze/src/lib/files/analyze/gatherEntityMap.ts

@@ -1,6 +1,7 @@
 import { Entity } from '~codeyam/types';
 import findOrCreateEntity from './findOrCreateEntity';
 import { ProjectAnalyzer } from '../../ProjectAnalyzer';
+import { getFileByPathSafe } from '../../utils/getFileByPath';
 import { AnalysisContext } from '../../analysisContext';
 
 // Track files we've already warned about to avoid duplicate warnings
@@ -71,8 +72,9 @@ export default async function gatherEntityMap({
     //   `  → imported export ${importedExportFilePath}:${importedExportName}`,
     // );
 
-    const importedFile = project.files.find(
-      (file) => file.path === importedExportFilePath,
+    const importedFile = getFileByPathSafe(
+      projectAnalyzer,
+      importedExportFilePath,
     );
 
     if (!importedFile) {
@@ -109,18 +111,15 @@ export default async function gatherEntityMap({
       });
 
       if (!importedEntity) {
-        console.log(
-          `CodeYam Error: Entity not found while gathering entity map: ${importedFile.path}: ${importedExportName}`,
-          {
-            entityName: entity.name,
-            entityFilePath: entity.filePath,
-            importedFilePath: importedFile.path,
-            importedExportName: importedExportName,
-          },
-        );
-        throw new Error(
-          `Entity not found while gathering entity map: ${importedFile.path}: ${importedExport.name}`,
-        );
+        const msg = [
+          `Entity not found: "${importedExportName}" in ${importedFile.path}`,
+          `  While analyzing dependencies of: ${entity.name} (${entity.filePath})`,
+          `  This usually means "${importedExportName}" was moved or renamed but`,
+          `  ${entity.filePath} still has stale import metadata referencing the old location.`,
+          `  Fix: Update or re-analyze ${entity.filePath} so its importedExports reflect current code.`,
+        ].join('\n');
+        console.log(`CodeYam Error: ${msg}`);
+        throw new Error(msg);
       }
 
       // Add newly created entity to context

package/analyzer-template/packages/analyze/src/lib/files/analyze/trackEntityCircularDependencies.ts

@@ -1,5 +1,6 @@
 import type { Entity } from '~codeyam/types';
 import { AnalysisContext } from '../../analysisContext';
+import { awsLogDebugLevel } from '~codeyam/utils';
 
 /**
  * Optimized version that processes multiple root entities in a single traversal.
@@ -16,6 +17,8 @@ export default function trackEntityCircularDependencies(
   try {
     const allEntities = new Set<Entity>();
     const globalVisited = new Set<string>(); // Track by SHA to avoid duplicate traversals
+    let circularCount = 0;
+    const startTime = Date.now();
 
     // Process each root entity, but skip if already processed
     for (const rootEntity of rootEntities) {
@@ -32,8 +35,26 @@ export default function trackEntityCircularDependencies(
         globalVisited,
         allEntities,
       );
+
+      // Emit periodic progress for large projects
+      if (allEntities.size % 100 === 0 && allEntities.size > 0) {
+        awsLogDebugLevel(
+          1,
+          `Mapping dependencies... (${allEntities.size} entities discovered)`,
+        );
+      }
     }
 
+    const elapsed = Date.now() - startTime;
+    // Count circular deps
+    for (const entity of allEntities) {
+      if (entity.metadata?.hasCircularDependency) circularCount++;
+    }
+    awsLogDebugLevel(
+      1,
+      `Dependency mapping complete: ${allEntities.size} entities, ${circularCount} circular, ${elapsed}ms`,
+    );
+
     return allEntities;
   } catch (e) {
     console.log(