@codeyam/codeyam-cli 0.1.0-staging.8aea589 → 0.1.0-staging.8df382d

package/analyzer-template/packages/analyze/src/lib/files/analyze/analyzeEntities.ts

@@ -5,6 +5,8 @@ import {
   measureAndReportExecutionTime,
   ProjectAnalyzer,
 } from '~codeyam/analyze';
+import { transformationTracer } from '../scenarios/TransformationTracer';
+import { getFileByPathSafe } from '../../utils/getFileByPath';
 import trackEntityCircularDependencies from './trackEntityCircularDependencies';
 import validateDependencyAnalyses from './validateDependencyAnalyses';
 import extractClassMethods from './analyzeEntities/extractClassMethods';
@@ -16,9 +18,9 @@ import {
   updateFreshAnalysisStatus,
   upsertAnalyses,
   upsertEntities,
-} from '~codeyam/supabase';
+} from '~codeyam/database';
 import { getCandidatesReadyForCapture as getAnalysesReadyForCapture } from './dependencyResolver';
-import { awsLog, awsLogDebugLevel } from '~codeyam/utils';
+import { awsLog, awsLogDebugLevel, Semaphore } from '~codeyam/utils';
 
 const MAX_CONCURRENT_ANALYSES = 100; // fairly arbitrary, but more isn't necessarily better
 
@@ -38,6 +40,10 @@ export interface AnalysisOptions {
   forceAll?: boolean;
   updateProgress?: (detail: string) => void;
   modelString?: string;
+  /** Process data structure preparation sequentially instead of in parallel (for debugging) */
+  sequentialDataStructurePrep?: boolean;
+  /** Called after data structures are prepared, signaling orchestration can begin */
+  onAnalysisProducing?: () => void;
 }
 
 interface AnalyzeEntitiesArgs {
@@ -113,6 +119,14 @@ export default async function analyzeEntities({
     'Data structure preparation',
   );
 
+  // Signal that analysis can now produce captures
+  // This allows orchestration to start polling for ready analyses
+  awsLogDebugLevel(
+    1,
+    'Data structures prepared, analysis can now produce captures',
+  );
+  options.onAnalysisProducing?.();
+
   // Persist entities so isolatedDataStructure is available in future runs.
   // This ensures stable dependencyAnalyzedTreeSha across multiple analyzeEntities calls.
   const entitiesWithDataStructure = Array.from(allEntities).filter(
@@ -133,6 +147,10 @@ export default async function analyzeEntities({
   const markedReady = new Set<string>(); // Analyses we've attempted to mark ready
   const analyses: Analysis[] = [];
 
+  // Semaphore to serialize CPU-intensive Structure phase while allowing
+  // I/O-bound KeyAttributes LLM calls to run in parallel
+  const structureSemaphore = new Semaphore(1);
+
   const analysesAlreadyCompletedAtStart =
     commit?.metadata?.currentRun?.analysesCompleted ?? 0;
 
@@ -290,6 +308,10 @@ export default async function analyzeEntities({
   awsLog(
     `CodeYam: analyzeEntities: Analysis completed with ${analyses.length} analyses`,
   );
+
+  // Flush transformation trace if tracing is enabled
+  transformationTracer.flush();
+
   return analyses;
 
   // PHASE 3: Pure graph traversal
@@ -372,6 +394,21 @@ export default async function analyzeEntities({
             // Circular dependency detected in path - skip
             continue;
           }
+          // If already visited via another traversal path, check if it's a leaf.
+          // If it's a leaf, we should still wait for it to complete.
+          // Only skip if we've already explored it and it's NOT a leaf (meaning it's
+          // blocked on something else - continuing to traverse would cause a deadlock).
+          if (visited.has(dependentEntityKey)) {
+            // If the visited entity is a leaf, we should wait for it to complete
+            // before processing this entity. Don't skip it!
+            if (leafKeys.has(dependentEntityKey)) {
+              // It's a leaf that hasn't completed yet - wait for it
+              isLeaf = false;
+              continue;
+            }
+            // If it's not a leaf, it's blocked on something else - skip to avoid deadlock
+            continue;
+          }
           isLeaf = false;
           visitEntity(dependentEntity, [...path, dependentEntityKey]);
         }
@@ -404,7 +441,7 @@ export default async function analyzeEntities({
     }
 
     const file =
-      entity.file || project.files.find((f) => f.path === entity.filePath);
+      entity.file || getFileByPathSafe(projectAnalyzer, entity.filePath);
     if (!file) {
       awsLog(
         `CodeYam: analyzeEntities: File not found for entity: ${entityKey}`,
@@ -442,6 +479,7 @@ export default async function analyzeEntities({
           analyzeAsDependency: !allRootEntities.some(
             (e) => e.filePath === entity.filePath && e.name === entity.name,
           ),
+          structureSemaphore,
           ...options,
           force,
         }),
@@ -494,7 +532,23 @@ export default async function analyzeEntities({
 
     const successfullyUpdated: Analysis[] = [];
 
+    console.log('CodeYam DEBUG: updateReadyAnalyses - newlyReady analyses:', {
+      count: newlyReady.length,
+      analyses: newlyReady.map((a) => ({
+        id: a.id,
+        filePath: a.filePath,
+        entityName: a.entityName,
+        scenarioCount: a.scenarios?.length ?? 0,
+      })),
+    });
+
     for (const analysis of newlyReady) {
+      console.log('CodeYam DEBUG: Setting readyToBeCaptured=true for:', {
+        id: analysis.id,
+        filePath: analysis.filePath,
+        entityName: analysis.entityName,
+      });
+
       const updatedStatus = await updateFreshAnalysisStatus(
         analysis.id!,
         (status) => {
@@ -506,6 +560,10 @@ export default async function analyzeEntities({
       if (updatedStatus) {
         analysis.status = updatedStatus;
         successfullyUpdated.push(analysis);
+        console.log('CodeYam DEBUG: Successfully marked readyToBeCaptured:', {
+          id: analysis.id,
+          entityName: analysis.entityName,
+        });
       } else {
         console.warn(
           `CodeYam: updateReadyAnalyses: Failed to update status for analysis ${analysis.id}`,

package/analyzer-template/packages/analyze/src/lib/files/analyze/dependencyResolver.ts

@@ -100,6 +100,12 @@ export function getCandidatesReadyForCapture(
       continue;
     }
 
+    // Skip notExported entities - they can't be imported into the simulation
+    // environment and will always fail capture. Don't mark them as ready.
+    if (candidate.entity?.metadata?.notExported) {
+      continue;
+    }
+
     // Get all transitive dependencies
     const candidateKey = `${candidate.filePath}:${candidate.entityName}`;
     const { missingDependencies } = getTransitiveDependencies(

package/analyzer-template/packages/analyze/src/lib/files/analyze/findOrCreateEntity.ts

@@ -8,7 +8,7 @@ import {
   updateEntityBranch,
   upsertEntities,
   upsertEntityBranches,
-} from '~codeyam/supabase';
+} from '~codeyam/database';
 import getImportedExports from '../getImportedExports';
 import getNodeModuleImports from '../getNodeModuleImports';
 import { AnalysisContext } from '../../analysisContext';
@@ -161,6 +161,9 @@ export default async function findOrCreateEntity({
   const defaultExport = fileAnalyzer.isDefaultExport(entity.name);
   entity.metadata.namedExport = !defaultExport;
 
+  // Detect if the entity is an async function/component (Server Component in Next.js)
+  entity.metadata.isAsync = fileAnalyzer.isEntityAsync(localEntityName);
+
   if (defaultExport) {
     const allEntityNodes = fileAnalyzer.getAllEntityNodes();
     entity.metadata.exportAlias = Object.keys(allEntityNodes).find(

package/analyzer-template/packages/analyze/src/lib/files/analyze/findPreviousAnalysis.ts

@@ -1,4 +1,4 @@
-import { loadMostRecentPreviousAnalysis } from '~codeyam/supabase';
+import { loadMostRecentPreviousAnalysis } from '~codeyam/database';
 import { Commit, Entity, File, Project } from '~codeyam/types';
 
 export interface FindValidExistingAnalysisArgs {

package/analyzer-template/packages/analyze/src/lib/files/analyze/findValidExistingAnalysis.ts

@@ -1,4 +1,4 @@
-import { loadAnalysis } from '~codeyam/supabase';
+import { loadAnalysis } from '~codeyam/database';
 import { Analysis, Project } from '~codeyam/types';
 
 export interface FindValidExistingAnalysisArgs {

package/analyzer-template/packages/analyze/src/lib/files/analyze/gatherEntityMap.ts

@@ -1,8 +1,12 @@
 import { Entity } from '~codeyam/types';
 import findOrCreateEntity from './findOrCreateEntity';
 import { ProjectAnalyzer } from '../../ProjectAnalyzer';
+import { getFileByPathSafe } from '../../utils/getFileByPath';
 import { AnalysisContext } from '../../analysisContext';
 
+// Track files we've already warned about to avoid duplicate warnings
+const warnedMissingFiles = new Set<string>();
+
 interface EntityWithGatheredDependencies extends Entity {
   dependenciesGathered?: boolean;
 }
@@ -68,19 +72,25 @@ export default async function gatherEntityMap({
     //   `  → imported export ${importedExportFilePath}:${importedExportName}`,
     // );
 
-    const importedFile = project.files.find(
-      (file) => file.path === importedExportFilePath,
+    const importedFile = getFileByPathSafe(
+      projectAnalyzer,
+      importedExportFilePath,
     );
 
     if (!importedFile) {
-      console.log(
-        `CodeYam Error: File not found while gathering entity map: ${importedExportFilePath}`,
-        entity.filePath,
-        entity.name,
-      );
-      throw new Error(
-        `File not found while gathering entity map: ${importedExportFilePath}`,
-      );
+      // Skip dependencies when the file is not found in project.files.
+      // This commonly happens when a universal mock intercepts the import:
+      // 1. TypeScript resolves @formbricks/types/errors to the mock file
+      // 2. The original packages/types/errors.ts is never imported
+      // 3. So it's not in getSourceFiles() and thus not in project.files
+      // 4. Since the mock handles the import, we don't need to analyze the original
+      if (!warnedMissingFiles.has(importedExportFilePath)) {
+        warnedMissingFiles.add(importedExportFilePath);
+        console.log(
+          `CodeYam Warning: Skipping dependency - file not in project.files (likely mocked): ${importedExportFilePath}`,
+        );
+      }
+      continue;
     }
 
     let importedEntity = context.getEntity(

package/analyzer-template/packages/analyze/src/lib/files/analyze/setActiveAnalysisBranches.ts

@@ -2,7 +2,7 @@ import {
   loadAnalysisBranches,
   loadCommitBranches,
   upsertAnalysisBranches,
-} from '~codeyam/supabase';
+} from '~codeyam/database';
 import { Analysis, AnalysisBranch, Commit, Entity } from '~codeyam/types';
 import { awsLog } from '~codeyam/utils';
 

package/analyzer-template/packages/analyze/src/lib/files/analyze/trackEntityCircularDependencies.ts

@@ -99,19 +99,11 @@ function traverseEntity(
     );
 
     if (!dependentEntity) {
-      console.log(
-        'CodeYam Error: trackEntityCircularDependencies: Dependent entity not found',
-        {
-          entityFilePath: entity.filePath,
-          entityName: entity.name,
-          dependencyFilePath,
-          dependencyName,
-          fileEntities: context.getEntityNamesForFile(dependencyFilePath),
-        },
-      );
-      throw new Error(
-        'trackEntityCircularDependencies: Dependent entity not found',
-      );
+      // Skip dependencies when the entity is not found in context.
+      // This commonly happens when a universal mock intercepts the import:
+      // - gatherEntityMap skips adding the entity because the file isn't in project.files
+      // - The mock handles the import at runtime, so we don't need to track it
+      continue;
     }
 
     // Recursively traverse (will skip if already visited)

package/analyzer-template/packages/analyze/src/lib/files/analyze/validateDependencyAnalyses.ts

@@ -1,5 +1,6 @@
 import type {
   Analysis,
+  Entity,
   ReadonlyAnalysisMap,
   ReadonlyAnalysis,
   ReadonlyEntity,
@@ -61,13 +62,13 @@ export default function validateDependencyAnalyses({
   for (const dependency of entity.metadata?.importedExports ?? []) {
     const dependencyFilePath =
       dependency.resolvedFilePath ?? dependency.filePath;
-    const dependencyName = dependency.resolvedName ?? dependency.name;
+    // Use the entity/import name, not the export name (resolvedName).
+    // For default exports, resolvedName is 'default' but the entity is stored
+    // under its actual identifier name (e.g., 'QuoteDetailsDrawer').
+    const dependencyName = dependency.name;
 
     // if the entity has duplicate dependencies, handle gracefully
     if (dependentAnalyses[dependencyFilePath]?.[dependencyName]) {
-      // console.log(
-      //   `CodeYam: validateDependencyAnalyses: Entity ${entity.filePath}:${entity.name} has duplicate dependency ${dependencyFilePath}:${dependencyName}. Skipping...`,
-      // );
       continue;
     }
 
@@ -103,6 +104,23 @@ export default function validateDependencyAnalyses({
       continue;
     }
 
+    // Ensure the entity is attached to the analysis.
+    // The entity might not be attached if the analysis was loaded from the database
+    // without its entity being in the context's entity map.
+    // This is critical for generateDataStructure to merge child component tRPC calls.
+    let dependentEntity: ReadonlyEntity | undefined = dependentAnalysis.entity;
+    if (!dependentEntity) {
+      // Try to get the entity from the context
+      dependentEntity = context.getEntity(dependencyFilePath, dependencyName);
+      if (dependentEntity) {
+        // Attach the entity to the analysis (mutating, but necessary for downstream use)
+        // We need the double cast because ReadonlyEntity has readonly arrays that aren't
+        // assignable to Entity's mutable arrays.
+        (dependentAnalysis as unknown as Analysis).entity =
+          dependentEntity as unknown as Entity;
+      }
+    }
+
     // Store the dependent analysis
     dependentAnalyses[dependencyFilePath] ||= {};
     dependentAnalyses[dependencyFilePath][dependencyName] = dependentAnalysis;
@@ -125,12 +143,17 @@ export default function validateDependencyAnalyses({
       }
     }
 
-    validateDependencyAnalyses({
-      readonlyAnalysis: dependentAnalysis,
-      entity: dependentAnalysis.entity,
-      context: context,
-      dependentAnalyses,
-    });
+    // Only recurse if we have the entity. Without the entity, we can't process
+    // the dependency's own dependencies, but we can still use this dependency's
+    // analysis data (mergedDataStructure, etc.)
+    if (dependentEntity) {
+      validateDependencyAnalyses({
+        readonlyAnalysis: dependentAnalysis,
+        entity: dependentEntity,
+        context: context,
+        dependentAnalyses,
+      });
+    }
   }
 
   return { dependentAnalyses };

package/analyzer-template/packages/analyze/src/lib/files/analyzeChange.ts

@@ -7,8 +7,8 @@ import type {
 import generateDataStructure from './scenarios/generateDataStructure';
 import generateChangesScenarioData from './scenarios/generateChangesScenarioData';
 import generateChangesScenarios from './scenarios/generateChangesScenarios';
-import generateKeyAttributes from './scenarios/generateKeyAttributes';
-import { upsertAnalysesWithScenarios } from '~codeyam/supabase';
+import generateExecutionFlows from './scenarios/generateExecutionFlows';
+import { upsertAnalysesWithScenarios } from '~codeyam/database';
 import { Step } from './enums/steps';
 import recordStep from './recordStep';
 import { FileAnalyzer } from '..';
@@ -16,7 +16,13 @@ import { AI } from '~codeyam/ai';
 import extractDiffLines from './extractDiffLines';
 import analyzeCodeChange from './analyzeCodeChange';
 import relevantDiffPart from './relevantDiffPart';
-import { awsLog, awsLogDebugLevel, pushAnalysisError } from '~codeyam/utils';
+import {
+  awsLog,
+  awsLogDebugLevel,
+  pushAnalysisError,
+  Semaphore,
+} from '~codeyam/utils';
+import deepEqual from '../utils/deepEqual';
 
 export interface AnalyzeChangeArgs {
   previousAnalysis: Analysis;
@@ -29,6 +35,8 @@ export interface AnalyzeChangeArgs {
   steps?: string[];
   updateProgress?: (detail: string) => void;
   model: AI.Model;
+  /** Semaphore to serialize CPU-intensive Structure phase */
+  structureSemaphore?: Semaphore;
 }
 
 export default async function analyzeChange({
@@ -42,6 +50,7 @@ export default async function analyzeChange({
   steps,
   updateProgress,
   model,
+  structureSemaphore,
 }: AnalyzeChangeArgs) {
   analysis.entity = entity;
   awsLog('CodeYam: Analyzing change entity', {
@@ -79,50 +88,76 @@ export default async function analyzeChange({
       });
     }
 
+    // Structure phase: CPU-intensive, serialize via semaphore if provided
     if (!steps || steps.includes(Step.Structure)) {
-      awsLogDebugLevel(
-        1,
-        `Merging dependent data structures for ${entity.name}...`,
-      );
+      if (structureSemaphore) {
+        await structureSemaphore.acquire();
+      }
+      try {
+        awsLogDebugLevel(
+          1,
+          `Merging dependent data structures for ${entity.name}...`,
+        );
 
-      const { step: dataStructureStep } = await recordStep(
-        'generateDataStructure',
-        () =>
-          generateDataStructure({
-            entity,
-            dependentAnalyses,
-            analysis,
-          }),
-      );
-      analysis.status.steps.push(dataStructureStep);
+        const { step: dataStructureStep } = await recordStep(
+          'generateDataStructure',
+          () =>
+            generateDataStructure({
+              entity,
+              dependentAnalyses,
+              analysis,
+            }),
+        );
+        analysis.status.steps.push(dataStructureStep);
+      } finally {
+        if (structureSemaphore) {
+          structureSemaphore.release();
+        }
+      }
     }
 
-    if (!steps || steps.includes(Step.KeyAttributes)) {
-      awsLogDebugLevel(1, `Determining key attributes for ${entity.name}...`);
+    // ExecutionFlows phase: I/O-bound LLM call, can run in parallel with other entities' Structure
+    if (!steps || steps.includes(Step.ExecutionFlows)) {
+      awsLogDebugLevel(1, `Generating execution flows for ${entity.name}...`);
 
-      const { step: keyAttributesStep } = await recordStep(
-        'generateKeyAttributes',
+      const { step: executionFlowsStep } = await recordStep(
+        'generateExecutionFlows',
         () =>
-          generateKeyAttributes({
+          generateExecutionFlows({
             entity,
             analysis,
-            dependentAnalyses,
             model,
             updateProgress,
+            dependentAnalyses,
           }),
       );
-      analysis.status.steps.push(keyAttributesStep);
+      analysis.status.steps.push(executionFlowsStep);
     }
 
-    if (
-      analysis.metadata.scenariosDataStructure ===
-      previousAnalysis.metadata.scenariosDataStructure
-    ) {
+    const dataStructureUnchanged = deepEqual(
+      analysis.metadata.scenariosDataStructure,
+      previousAnalysis.metadata.scenariosDataStructure,
+    );
+
+    const executionFlowsUnchanged = deepEqual(
+      analysis.metadata.executionFlows,
+      previousAnalysis.metadata.executionFlows,
+    );
+
+    if (dataStructureUnchanged && executionFlowsUnchanged) {
       awsLogDebugLevel(
         1,
-        `Data structure unchanged, reusing previous scenarios for ${entity.name}...`,
+        `Data structure and execution flows unchanged, reusing previous scenarios for ${entity.name}...`,
       );
-      analysis.scenarios = previousAnalysis.scenarios;
+      // Reuse previous scenarios but mark them for recapture if entity code changed
+      analysis.scenarios = previousAnalysis.scenarios.map((scenario) => ({
+        ...scenario,
+        metadata: {
+          ...scenario.metadata,
+          // Only mark for recapture if entity code actually changed
+          needsRecapture: entityChanged,
+        },
+      }));
     } else {
       if (!steps || steps.includes(Step.Scenarios)) {
         awsLogDebugLevel(1, `Generating scenarios for ${entity.name}...`);

package/analyzer-template/packages/analyze/src/lib/files/analyzeEntity.ts

@@ -2,7 +2,7 @@ import {
   deleteScenarios,
   upsertAnalysesWithScenarios,
   upsertEntities,
-} from '~codeyam/supabase';
+} from '~codeyam/database';
 import {
   AI,
   generateChangesEntityDocumentation,
@@ -31,7 +31,12 @@ import relevantDiffPart from './relevantDiffPart';
 import { Step } from './enums/steps';
 import { ImmutableAnalysisContext } from '../analysisContext';
 import generateAnalyzedTreeSha from './analyze/generateAnalyzedTreeSha';
-import { awsLog, awsLogDebugLevel, pushAnalysisError } from '~codeyam/utils';
+import {
+  awsLog,
+  awsLogDebugLevel,
+  pushAnalysisError,
+  Semaphore,
+} from '~codeyam/utils';
 
 export interface AnalyzeEntityArgs {
   project: Project;
@@ -50,6 +55,8 @@ export interface AnalyzeEntityArgs {
   forceAll?: boolean;
   updateProgress?: (detail: string) => void;
   modelString?: string;
+  /** Semaphore to serialize CPU-intensive Structure phase */
+  structureSemaphore?: Semaphore;
 }
 
 /**
@@ -77,6 +84,7 @@ export default async function analyzeEntity({
   forceAll,
   updateProgress,
   modelString,
+  structureSemaphore,
 }: AnalyzeEntityArgs): Promise<{
   analysis: Analysis;
   dependentAnalyses?: ReadonlyAnalysisMap;
@@ -222,7 +230,7 @@ export default async function analyzeEntity({
 
       // For partial analyses, skip Step.Structure to reuse existing mergedDataStructure
       if (isPartialAnalysis && !steps) {
-        steps = [Step.KeyAttributes, Step.Scenarios, Step.Data];
+        steps = [Step.ExecutionFlows, Step.Scenarios, Step.Data];
         awsLog(
           'CodeYam: continuing from partial analysis, skipping Structure step',
           {
@@ -263,13 +271,17 @@ export default async function analyzeEntity({
         });
       }
 
+      // Return early if analysis is already complete for its purpose:
+      // - Full analysis: has analyzedTreeSha
+      // - Dependency analysis: has executionFlows (Structure + ExecutionFlows already ran)
+      const isCompleteDependencyAnalysis =
+        analyzeAsDependency && !!existingAnalysis.metadata?.executionFlows;
+
       if (
         !force &&
-        !!existingAnalysis.analyzedTreeSha &&
-        existingAnalysis.dependencyAnalyzedTreeSha === dependencyAnalyzedTreeSha //&&
-        // (existingAnalysis.status?.finishedAt ||
-        //   existingAnalysis.status?.readyToBeCaptured ||
-        //   branchCommitSha)
+        existingAnalysis.dependencyAnalyzedTreeSha ===
+          dependencyAnalyzedTreeSha &&
+        (!!existingAnalysis.analyzedTreeSha || isCompleteDependencyAnalysis)
       ) {
         awsLogDebugLevel(1, `Using existing analysis for ${entity.name}...`);
         awsLog('Returning existing analysis', {
@@ -367,7 +379,7 @@ export default async function analyzeEntity({
         }
 
         if (!steps || steps?.includes(Step.Scenarios)) {
-          delete existingAnalysis.metadata.keyAttributes;
+          delete existingAnalysis.metadata.executionFlows;
         }
 
         if (!steps || steps?.includes(Step.Data)) {
@@ -476,6 +488,7 @@ export default async function analyzeEntity({
             steps,
             updateProgress,
             model,
+            structureSemaphore,
           })
         : analyzeInitial({
             analysis,
@@ -485,6 +498,7 @@ export default async function analyzeEntity({
             steps,
             updateProgress,
             model,
+            structureSemaphore,
           }),
     ]);
 
@@ -498,8 +512,12 @@ export default async function analyzeEntity({
     analysis = analysisResult.analysis;
 
     analysis.completedAt = new Date().toISOString();
-    // Don't set analyzedTreeSha when ONLY_DATA_STRUCTURE=true, so full analysis can re-analyze later
-    if (process.env.ONLY_DATA_STRUCTURE !== 'true') {
+    // Don't set analyzedTreeSha when:
+    // 1. ONLY_DATA_STRUCTURE=true - so full analysis can re-analyze later
+    // 2. analyzeAsDependency=true - dependency analyses only generate Structure+KeyAttributes,
+    //    not scenarios. Leaving analyzedTreeSha unset allows partial analysis detection to work
+    //    when the entity is later analyzed directly.
+    if (process.env.ONLY_DATA_STRUCTURE !== 'true' && !analyzeAsDependency) {
       analysis.analyzedTreeSha = generateAnalyzedTreeSha(analysis);
     }
   } catch (error) {