@codeyam/codeyam-cli 0.1.0-staging.e38f7bd → 0.1.0-staging.f777668

package/analyzer-template/packages/analyze/src/lib/files/analyze/analyzeEntities.ts

@@ -5,6 +5,8 @@ import {
   measureAndReportExecutionTime,
   ProjectAnalyzer,
 } from '~codeyam/analyze';
+import { transformationTracer } from '../scenarios/TransformationTracer';
+import { getFileByPathSafe } from '../../utils/getFileByPath';
 import trackEntityCircularDependencies from './trackEntityCircularDependencies';
 import validateDependencyAnalyses from './validateDependencyAnalyses';
 import extractClassMethods from './analyzeEntities/extractClassMethods';
@@ -38,6 +40,10 @@ export interface AnalysisOptions {
   forceAll?: boolean;
   updateProgress?: (detail: string) => void;
   modelString?: string;
+  /** Process data structure preparation sequentially instead of in parallel (for debugging) */
+  sequentialDataStructurePrep?: boolean;
+  /** Called after data structures are prepared, signaling orchestration can begin */
+  onAnalysisProducing?: () => void;
 }
 
 interface AnalyzeEntitiesArgs {
@@ -113,6 +119,14 @@ export default async function analyzeEntities({
     'Data structure preparation',
   );
 
+  // Signal that analysis can now produce captures
+  // This allows orchestration to start polling for ready analyses
+  awsLogDebugLevel(
+    1,
+    'Data structures prepared, analysis can now produce captures',
+  );
+  options.onAnalysisProducing?.();
+
   // Persist entities so isolatedDataStructure is available in future runs.
   // This ensures stable dependencyAnalyzedTreeSha across multiple analyzeEntities calls.
   const entitiesWithDataStructure = Array.from(allEntities).filter(
@@ -294,6 +308,18 @@ export default async function analyzeEntities({
   awsLog(
     `CodeYam: analyzeEntities: Analysis completed with ${analyses.length} analyses`,
   );
+
+  // Flush transformation trace if tracing is enabled
+  // Wrapped in try/catch because trace data can exceed V8's max string length
+  // for large projects, and we must not crash the analysis process over tracing.
+  try {
+    transformationTracer.flush();
+  } catch (e) {
+    console.warn(
+      `CodeYam Warning: Failed to flush transformation trace: ${e.message}`,
+    );
+  }
+
   return analyses;
 
   // PHASE 3: Pure graph traversal
@@ -376,6 +402,21 @@ export default async function analyzeEntities({
             // Circular dependency detected in path - skip
             continue;
           }
+          // If already visited via another traversal path, check if it's a leaf.
+          // If it's a leaf, we should still wait for it to complete.
+          // Only skip if we've already explored it and it's NOT a leaf (meaning it's
+          // blocked on something else - continuing to traverse would cause a deadlock).
+          if (visited.has(dependentEntityKey)) {
+            // If the visited entity is a leaf, we should wait for it to complete
+            // before processing this entity. Don't skip it!
+            if (leafKeys.has(dependentEntityKey)) {
+              // It's a leaf that hasn't completed yet - wait for it
+              isLeaf = false;
+              continue;
+            }
+            // If it's not a leaf, it's blocked on something else - skip to avoid deadlock
+            continue;
+          }
           isLeaf = false;
           visitEntity(dependentEntity, [...path, dependentEntityKey]);
         }
@@ -408,7 +449,7 @@ export default async function analyzeEntities({
     }
 
     const file =
-      entity.file || project.files.find((f) => f.path === entity.filePath);
+      entity.file || getFileByPathSafe(projectAnalyzer, entity.filePath);
     if (!file) {
       awsLog(
         `CodeYam: analyzeEntities: File not found for entity: ${entityKey}`,
@@ -499,7 +540,23 @@ export default async function analyzeEntities({
 
     const successfullyUpdated: Analysis[] = [];
 
+    console.log('CodeYam DEBUG: updateReadyAnalyses - newlyReady analyses:', {
+      count: newlyReady.length,
+      analyses: newlyReady.map((a) => ({
+        id: a.id,
+        filePath: a.filePath,
+        entityName: a.entityName,
+        scenarioCount: a.scenarios?.length ?? 0,
+      })),
+    });
+
     for (const analysis of newlyReady) {
+      console.log('CodeYam DEBUG: Setting readyToBeCaptured=true for:', {
+        id: analysis.id,
+        filePath: analysis.filePath,
+        entityName: analysis.entityName,
+      });
+
       const updatedStatus = await updateFreshAnalysisStatus(
         analysis.id!,
         (status) => {
@@ -511,6 +568,10 @@ export default async function analyzeEntities({
       if (updatedStatus) {
         analysis.status = updatedStatus;
         successfullyUpdated.push(analysis);
+        console.log('CodeYam DEBUG: Successfully marked readyToBeCaptured:', {
+          id: analysis.id,
+          entityName: analysis.entityName,
+        });
       } else {
         console.warn(
           `CodeYam: updateReadyAnalyses: Failed to update status for analysis ${analysis.id}`,

package/analyzer-template/packages/analyze/src/lib/files/analyze/findOrCreateEntity.ts

@@ -138,6 +138,18 @@ export default async function findOrCreateEntity({
           break;
         }
       }
+
+      // Mark previous versions as superseded
+      const entitiesToSupersede = previousEntities.filter(
+        (e) => !e.metadata?.isSuperseded,
+      );
+      if (entitiesToSupersede.length > 0) {
+        for (const prev of entitiesToSupersede) {
+          prev.metadata = prev.metadata || {};
+          prev.metadata.isSuperseded = true;
+        }
+        await upsertEntities(entitiesToSupersede);
+      }
     }
 
     entity = {
@@ -161,6 +173,9 @@ export default async function findOrCreateEntity({
   const defaultExport = fileAnalyzer.isDefaultExport(entity.name);
   entity.metadata.namedExport = !defaultExport;
 
+  // Detect if the entity is an async function/component (Server Component in Next.js)
+  entity.metadata.isAsync = fileAnalyzer.isEntityAsync(localEntityName);
+
   if (defaultExport) {
     const allEntityNodes = fileAnalyzer.getAllEntityNodes();
     entity.metadata.exportAlias = Object.keys(allEntityNodes).find(

package/analyzer-template/packages/analyze/src/lib/files/analyze/gatherEntityMap.ts

@@ -1,6 +1,7 @@
 import { Entity } from '~codeyam/types';
 import findOrCreateEntity from './findOrCreateEntity';
 import { ProjectAnalyzer } from '../../ProjectAnalyzer';
+import { getFileByPathSafe } from '../../utils/getFileByPath';
 import { AnalysisContext } from '../../analysisContext';
 
 // Track files we've already warned about to avoid duplicate warnings
@@ -71,8 +72,9 @@ export default async function gatherEntityMap({
     //   `  → imported export ${importedExportFilePath}:${importedExportName}`,
     // );
 
-    const importedFile = project.files.find(
-      (file) => file.path === importedExportFilePath,
+    const importedFile = getFileByPathSafe(
+      projectAnalyzer,
+      importedExportFilePath,
     );
 
     if (!importedFile) {

package/analyzer-template/packages/analyze/src/lib/files/analyze/validateDependencyAnalyses.ts

@@ -1,5 +1,6 @@
 import type {
   Analysis,
+  Entity,
   ReadonlyAnalysisMap,
   ReadonlyAnalysis,
   ReadonlyEntity,
@@ -61,13 +62,13 @@ export default function validateDependencyAnalyses({
   for (const dependency of entity.metadata?.importedExports ?? []) {
     const dependencyFilePath =
       dependency.resolvedFilePath ?? dependency.filePath;
-    const dependencyName = dependency.resolvedName ?? dependency.name;
+    // Use the entity/import name, not the export name (resolvedName).
+    // For default exports, resolvedName is 'default' but the entity is stored
+    // under its actual identifier name (e.g., 'QuoteDetailsDrawer').
+    const dependencyName = dependency.name;
 
     // if the entity has duplicate dependencies, handle gracefully
     if (dependentAnalyses[dependencyFilePath]?.[dependencyName]) {
-      // console.log(
-      //   `CodeYam: validateDependencyAnalyses: Entity ${entity.filePath}:${entity.name} has duplicate dependency ${dependencyFilePath}:${dependencyName}. Skipping...`,
-      // );
       continue;
     }
 
@@ -103,6 +104,23 @@ export default function validateDependencyAnalyses({
       continue;
     }
 
+    // Ensure the entity is attached to the analysis.
+    // The entity might not be attached if the analysis was loaded from the database
+    // without its entity being in the context's entity map.
+    // This is critical for generateDataStructure to merge child component tRPC calls.
+    let dependentEntity: ReadonlyEntity | undefined = dependentAnalysis.entity;
+    if (!dependentEntity) {
+      // Try to get the entity from the context
+      dependentEntity = context.getEntity(dependencyFilePath, dependencyName);
+      if (dependentEntity) {
+        // Attach the entity to the analysis (mutating, but necessary for downstream use)
+        // We need the double cast because ReadonlyEntity has readonly arrays that aren't
+        // assignable to Entity's mutable arrays.
+        (dependentAnalysis as unknown as Analysis).entity =
+          dependentEntity as unknown as Entity;
+      }
+    }
+
     // Store the dependent analysis
     dependentAnalyses[dependencyFilePath] ||= {};
     dependentAnalyses[dependencyFilePath][dependencyName] = dependentAnalysis;
@@ -125,12 +143,17 @@ export default function validateDependencyAnalyses({
       }
     }
 
-    validateDependencyAnalyses({
-      readonlyAnalysis: dependentAnalysis,
-      entity: dependentAnalysis.entity,
-      context: context,
-      dependentAnalyses,
-    });
+    // Only recurse if we have the entity. Without the entity, we can't process
+    // the dependency's own dependencies, but we can still use this dependency's
+    // analysis data (mergedDataStructure, etc.)
+    if (dependentEntity) {
+      validateDependencyAnalyses({
+        readonlyAnalysis: dependentAnalysis,
+        entity: dependentEntity,
+        context: context,
+        dependentAnalyses,
+      });
+    }
   }
 
   return { dependentAnalyses };

package/analyzer-template/packages/analyze/src/lib/files/analyzeChange.ts

@@ -7,7 +7,7 @@ import type {
 import generateDataStructure from './scenarios/generateDataStructure';
 import generateChangesScenarioData from './scenarios/generateChangesScenarioData';
 import generateChangesScenarios from './scenarios/generateChangesScenarios';
-import generateKeyAttributes from './scenarios/generateKeyAttributes';
+import generateExecutionFlows from './scenarios/generateExecutionFlows';
 import { upsertAnalysesWithScenarios } from '~codeyam/database';
 import { Step } from './enums/steps';
 import recordStep from './recordStep';
@@ -22,6 +22,7 @@ import {
   pushAnalysisError,
   Semaphore,
 } from '~codeyam/utils';
+import deepEqual from '../utils/deepEqual';
 
 export interface AnalyzeChangeArgs {
   previousAnalysis: Analysis;
@@ -115,33 +116,48 @@ export default async function analyzeChange({
       }
     }
 
-    // KeyAttributes phase: I/O-bound LLM call, can run in parallel with other entities' Structure
-    if (!steps || steps.includes(Step.KeyAttributes)) {
-      awsLogDebugLevel(1, `Determining key attributes for ${entity.name}...`);
+    // ExecutionFlows phase: I/O-bound LLM call, can run in parallel with other entities' Structure
+    if (!steps || steps.includes(Step.ExecutionFlows)) {
+      awsLogDebugLevel(1, `Generating execution flows for ${entity.name}...`);
 
-      const { step: keyAttributesStep } = await recordStep(
-        'generateKeyAttributes',
+      const { step: executionFlowsStep } = await recordStep(
+        'generateExecutionFlows',
         () =>
-          generateKeyAttributes({
+          generateExecutionFlows({
             entity,
             analysis,
-            dependentAnalyses,
             model,
             updateProgress,
+            dependentAnalyses,
           }),
       );
-      analysis.status.steps.push(keyAttributesStep);
+      analysis.status.steps.push(executionFlowsStep);
     }
 
-    if (
-      analysis.metadata.scenariosDataStructure ===
-      previousAnalysis.metadata.scenariosDataStructure
-    ) {
+    const dataStructureUnchanged = deepEqual(
+      analysis.metadata.scenariosDataStructure,
+      previousAnalysis.metadata.scenariosDataStructure,
+    );
+
+    const executionFlowsUnchanged = deepEqual(
+      analysis.metadata.executionFlows,
+      previousAnalysis.metadata.executionFlows,
+    );
+
+    if (dataStructureUnchanged && executionFlowsUnchanged) {
       awsLogDebugLevel(
         1,
-        `Data structure unchanged, reusing previous scenarios for ${entity.name}...`,
+        `Data structure and execution flows unchanged, reusing previous scenarios for ${entity.name}...`,
       );
-      analysis.scenarios = previousAnalysis.scenarios;
+      // Reuse previous scenarios but mark them for recapture if entity code changed
+      analysis.scenarios = previousAnalysis.scenarios.map((scenario) => ({
+        ...scenario,
+        metadata: {
+          ...scenario.metadata,
+          // Only mark for recapture if entity code actually changed
+          needsRecapture: entityChanged,
+        },
+      }));
     } else {
       if (!steps || steps.includes(Step.Scenarios)) {
         awsLogDebugLevel(1, `Generating scenarios for ${entity.name}...`);

package/analyzer-template/packages/analyze/src/lib/files/analyzeEntity.ts

@@ -230,7 +230,7 @@ export default async function analyzeEntity({
 
       // For partial analyses, skip Step.Structure to reuse existing mergedDataStructure
       if (isPartialAnalysis && !steps) {
-        steps = [Step.KeyAttributes, Step.Scenarios, Step.Data];
+        steps = [Step.ExecutionFlows, Step.Scenarios, Step.Data];
         awsLog(
           'CodeYam: continuing from partial analysis, skipping Structure step',
           {
@@ -271,13 +271,17 @@ export default async function analyzeEntity({
         });
       }
 
+      // Return early if analysis is already complete for its purpose:
+      // - Full analysis: has analyzedTreeSha
+      // - Dependency analysis: has executionFlows (Structure + ExecutionFlows already ran)
+      const isCompleteDependencyAnalysis =
+        analyzeAsDependency && !!existingAnalysis.metadata?.executionFlows;
+
       if (
         !force &&
-        !!existingAnalysis.analyzedTreeSha &&
-        existingAnalysis.dependencyAnalyzedTreeSha === dependencyAnalyzedTreeSha //&&
-        // (existingAnalysis.status?.finishedAt ||
-        //   existingAnalysis.status?.readyToBeCaptured ||
-        //   branchCommitSha)
+        existingAnalysis.dependencyAnalyzedTreeSha ===
+          dependencyAnalyzedTreeSha &&
+        (!!existingAnalysis.analyzedTreeSha || isCompleteDependencyAnalysis)
       ) {
         awsLogDebugLevel(1, `Using existing analysis for ${entity.name}...`);
         awsLog('Returning existing analysis', {
@@ -375,7 +379,7 @@ export default async function analyzeEntity({
         }
 
         if (!steps || steps?.includes(Step.Scenarios)) {
-          delete existingAnalysis.metadata.keyAttributes;
+          delete existingAnalysis.metadata.executionFlows;
         }
 
         if (!steps || steps?.includes(Step.Data)) {

package/analyzer-template/packages/analyze/src/lib/files/analyzeInitial.ts

@@ -18,7 +18,7 @@ import {
   pushAnalysisError,
   Semaphore,
 } from '~codeyam/utils';
-import generateKeyAttributes from './scenarios/generateKeyAttributes';
+import generateExecutionFlows from './scenarios/generateExecutionFlows';
 
 export interface AnalyzeInitialArgs {
   analysis: Analysis;
@@ -83,30 +83,30 @@ export default async function analyzeInitial({
       }
     }
 
-    // Short-circuit for test mocking: only generate data structure, skip key attributes and scenarios
+    // Short-circuit for test mocking: only generate data structure, skip execution flows and scenarios
     if (process.env.ONLY_DATA_STRUCTURE === 'true') {
       awsLog(
-        'CodeYam: Skipping key attributes and scenario generation (ONLY_DATA_STRUCTURE=true)',
+        'CodeYam: Skipping execution flows and scenario generation (ONLY_DATA_STRUCTURE=true)',
       );
       return { analysis };
     }
 
-    // KeyAttributes phase: I/O-bound LLM call, can run in parallel with other entities' Structure
-    if (!steps || steps.includes(Step.KeyAttributes)) {
-      awsLogDebugLevel(1, `Determining key attributes for ${entity.name}...`);
+    // ExecutionFlows phase: I/O-bound LLM call, can run in parallel with other entities' Structure
+    if (!steps || steps.includes(Step.ExecutionFlows)) {
+      awsLogDebugLevel(1, `Generating execution flows for ${entity.name}...`);
 
-      const { step: dataStructureStep } = await recordStep(
-        'generateKeyAttributes',
+      const { step: executionFlowsStep } = await recordStep(
+        'generateExecutionFlows',
         () =>
-          generateKeyAttributes({
+          generateExecutionFlows({
             entity,
             analysis,
-            dependentAnalyses,
             model,
             updateProgress,
+            dependentAnalyses,
           }),
       );
-      analysis.status.steps.push(dataStructureStep);
+      analysis.status.steps.push(executionFlowsStep);
     }
 
     if (analyzeAsDependency) {
@@ -176,7 +176,6 @@ export default async function analyzeInitial({
       analysis.status.steps.push(mockDataStep);
     }
   } catch (error) {
-    console.log(error);
     pushAnalysisError({
       status: analysis.status,
       error,

package/analyzer-template/packages/analyze/src/lib/files/analyzeRemixRoute.ts

@@ -1,5 +1,6 @@
 import { Entity, EntityType, File, ProjectFramework } from '~codeyam/types';
 import { ProjectAnalyzer } from '../ProjectAnalyzer';
+import { getFileByPathSafe } from '../utils/getFileByPath';
 import { generateSha } from '~codeyam/database';
 import { isRemixRoute } from '~codeyam/utils';
 
@@ -88,11 +89,9 @@ export default function analyzeRemixRoute(
   // Find root.tsx for this route's webapp and add as implicit dependency
   // This ensures root.tsx dependencies (like useLoaderData in AppContent) get analyzed for mocking
   const routeAppDir = file.path.split('/routes/')[0];
-  const rootFile = projectAnalyzer.project.files.find(
-    (f) =>
-      f.path === `${routeAppDir}/root.tsx` ||
-      f.path === `${routeAppDir}/root.jsx`,
-  );
+  const rootFile =
+    getFileByPathSafe(projectAnalyzer, `${routeAppDir}/root.tsx`) ??
+    getFileByPathSafe(projectAnalyzer, `${routeAppDir}/root.jsx`);
 
   const rootDependencies: Entity['metadata']['importedExports'] = [];
   if (rootFile) {

package/analyzer-template/packages/analyze/src/lib/files/enums/steps.ts

@@ -12,5 +12,5 @@ export enum Step {
   // CodeQuality = 'codeQuality',
   Embed = 'embed',
   CodeExplanation = 'codeExplanation',
-  KeyAttributes = 'keyAttributes',
+  ExecutionFlows = 'executionFlows',
 }

package/analyzer-template/packages/analyze/src/lib/files/getImportedExports.ts

@@ -1,6 +1,7 @@
 import { Entity, File, Project } from '~codeyam/types';
 import { FileAnalyzer } from '../FileAnalyzer';
 import { ProjectAnalyzer } from '../ProjectAnalyzer';
+import { getFileByPathSafe } from '../utils/getFileByPath';
 import type { FunctionDependenciesMap } from '../asts/sourceFiles/getPseudoFile';
 // import { generateSha } from '~codeyam/database';
 
@@ -63,9 +64,7 @@ export default function getImportedExports(
           imports[filePath].resolvedPath ??
           filePath;
 
-        let importedExportFile = project.files.find(
-          (f) => f.path === searchPath,
-        );
+        let importedExportFile = getFileByPathSafe(projectAnalyzer, searchPath);
 
         // Track the actual path we found (may differ from searchPath after dist->src mapping)
         let actualResolvedPath = imports[filePath].resolvedPath;
@@ -84,7 +83,7 @@ export default function getImportedExports(
             const srcPath = `${packagePath}/src/${relativePath}`
               .replace(/\.d\.ts$/, '.ts')
               .replace(/\.js$/, '.ts');
-            importedExportFile = project.files.find((f) => f.path === srcPath);
+            importedExportFile = getFileByPathSafe(projectAnalyzer, srcPath);
             if (importedExportFile) {
               actualResolvedPath = srcPath;
               console.log(
@@ -97,7 +96,7 @@ export default function getImportedExports(
           if (!importedExportFile) {
             const srcPath = searchPath.replace('/dist/', '/src/');
             // Try exact match first
-            importedExportFile = project.files.find((f) => f.path === srcPath);
+            importedExportFile = getFileByPathSafe(projectAnalyzer, srcPath);
             if (importedExportFile) {
               actualResolvedPath = srcPath;
               console.log(
@@ -114,13 +113,16 @@ export default function getImportedExports(
             const basePath = srcPath
               .replace(/\.d\.ts$/, '')
               .replace(/\.js$/, '');
-            importedExportFile = project.files.find(
-              (f) =>
-                f.path === basePath + '.ts' ||
-                f.path === basePath + '.tsx' ||
-                (f.path.startsWith(basePath.replace('/index', '/')) &&
-                  !f.path.includes('/dist/')),
-            );
+            importedExportFile =
+              getFileByPathSafe(projectAnalyzer, basePath + '.ts') ??
+              getFileByPathSafe(projectAnalyzer, basePath + '.tsx');
+            if (!importedExportFile) {
+              importedExportFile = project.files.find(
+                (f) =>
+                  f.path.startsWith(basePath.replace('/index', '/')) &&
+                  !f.path.includes('/dist/'),
+              );
+            }
             if (importedExportFile) {
               actualResolvedPath = importedExportFile.path;
               console.log(
@@ -204,6 +206,13 @@ export default function getImportedExports(
           }
           const resolvedName = importedExportInfo.exportName;
 
+          // When a default export is re-exported as named (e.g., `export { default as X }`),
+          // isExportDefault will be false (it's exported as named from the barrel),
+          // but resolvedName will be 'default'. In this case, we need resolvedIsDefault
+          // to be true so that writeScenarioComponents adds the named re-export.
+          const resolvedIsDefault =
+            importedExportInfo.isExportDefault || resolvedName === 'default';
+
           return {
             fileId: importedExportFile.id,
             filePath: actualResolvedPath,
@@ -213,7 +222,7 @@ export default function getImportedExports(
                 ? undefined
                 : resolvedFilePath,
             resolvedName: resolvedName === name ? undefined : resolvedName,
-            resolvedIsDefault: importedExportInfo.isExportDefault,
+            resolvedIsDefault,
             entityType,
             name,
             code,