@codeyam/codeyam-cli 0.1.0-staging.1 → 0.1.0-staging.1669d45

package/analyzer-template/packages/analyze/src/lib/files/analyze/analyzeEntities.ts

@@ -16,9 +16,9 @@ import {
   updateFreshAnalysisStatus,
   upsertAnalyses,
   upsertEntities,
-} from '~codeyam/supabase';
+} from '~codeyam/database';
 import { getCandidatesReadyForCapture as getAnalysesReadyForCapture } from './dependencyResolver';
-import { awsLog, awsLogDebugLevel } from '~codeyam/utils';
+import { awsLog, awsLogDebugLevel, Semaphore } from '~codeyam/utils';
 
 const MAX_CONCURRENT_ANALYSES = 100; // fairly arbitrary, but more isn't necessarily better
 
@@ -38,6 +38,10 @@ export interface AnalysisOptions {
   forceAll?: boolean;
   updateProgress?: (detail: string) => void;
   modelString?: string;
+  /** Process data structure preparation sequentially instead of in parallel (for debugging) */
+  sequentialDataStructurePrep?: boolean;
+  /** Called after data structures are prepared, signaling orchestration can begin */
+  onAnalysisProducing?: () => void;
 }
 
 interface AnalyzeEntitiesArgs {
@@ -113,6 +117,14 @@ export default async function analyzeEntities({
     'Data structure preparation',
   );
 
+  // Signal that analysis can now produce captures
+  // This allows orchestration to start polling for ready analyses
+  awsLogDebugLevel(
+    1,
+    'Data structures prepared, analysis can now produce captures',
+  );
+  options.onAnalysisProducing?.();
+
   // Persist entities so isolatedDataStructure is available in future runs.
   // This ensures stable dependencyAnalyzedTreeSha across multiple analyzeEntities calls.
   const entitiesWithDataStructure = Array.from(allEntities).filter(
@@ -133,6 +145,10 @@ export default async function analyzeEntities({
   const markedReady = new Set<string>(); // Analyses we've attempted to mark ready
   const analyses: Analysis[] = [];
 
+  // Semaphore to serialize CPU-intensive Structure phase while allowing
+  // I/O-bound KeyAttributes LLM calls to run in parallel
+  const structureSemaphore = new Semaphore(1);
+
   const analysesAlreadyCompletedAtStart =
     commit?.metadata?.currentRun?.analysesCompleted ?? 0;
 
@@ -372,6 +388,12 @@ export default async function analyzeEntities({
             // Circular dependency detected in path - skip
             continue;
           }
+          // If already visited via another traversal path, skip to avoid deadlock.
+          // Its subtree has been fully explored; any leaves are already in leafKeys.
+          // Blocking on it would just propagate the block and cause stuck analysis.
+          if (visited.has(dependentEntityKey)) {
+            continue;
+          }
           isLeaf = false;
           visitEntity(dependentEntity, [...path, dependentEntityKey]);
         }
@@ -442,6 +464,7 @@ export default async function analyzeEntities({
           analyzeAsDependency: !allRootEntities.some(
             (e) => e.filePath === entity.filePath && e.name === entity.name,
           ),
+          structureSemaphore,
           ...options,
           force,
         }),
@@ -494,7 +517,23 @@ export default async function analyzeEntities({
 
     const successfullyUpdated: Analysis[] = [];
 
+    console.log('CodeYam DEBUG: updateReadyAnalyses - newlyReady analyses:', {
+      count: newlyReady.length,
+      analyses: newlyReady.map((a) => ({
+        id: a.id,
+        filePath: a.filePath,
+        entityName: a.entityName,
+        scenarioCount: a.scenarios?.length ?? 0,
+      })),
+    });
+
     for (const analysis of newlyReady) {
+      console.log('CodeYam DEBUG: Setting readyToBeCaptured=true for:', {
+        id: analysis.id,
+        filePath: analysis.filePath,
+        entityName: analysis.entityName,
+      });
+
       const updatedStatus = await updateFreshAnalysisStatus(
         analysis.id!,
         (status) => {
@@ -506,6 +545,10 @@ export default async function analyzeEntities({
       if (updatedStatus) {
         analysis.status = updatedStatus;
         successfullyUpdated.push(analysis);
+        console.log('CodeYam DEBUG: Successfully marked readyToBeCaptured:', {
+          id: analysis.id,
+          entityName: analysis.entityName,
+        });
       } else {
         console.warn(
           `CodeYam: updateReadyAnalyses: Failed to update status for analysis ${analysis.id}`,

package/analyzer-template/packages/analyze/src/lib/files/analyze/findOrCreateEntity.ts

@@ -8,7 +8,7 @@ import {
   updateEntityBranch,
   upsertEntities,
   upsertEntityBranches,
-} from '~codeyam/supabase';
+} from '~codeyam/database';
 import getImportedExports from '../getImportedExports';
 import getNodeModuleImports from '../getNodeModuleImports';
 import { AnalysisContext } from '../../analysisContext';
@@ -161,6 +161,9 @@ export default async function findOrCreateEntity({
   const defaultExport = fileAnalyzer.isDefaultExport(entity.name);
   entity.metadata.namedExport = !defaultExport;
 
+  // Detect if the entity is an async function/component (Server Component in Next.js)
+  entity.metadata.isAsync = fileAnalyzer.isEntityAsync(localEntityName);
+
   if (defaultExport) {
     const allEntityNodes = fileAnalyzer.getAllEntityNodes();
     entity.metadata.exportAlias = Object.keys(allEntityNodes).find(

package/analyzer-template/packages/analyze/src/lib/files/analyze/findPreviousAnalysis.ts

@@ -1,4 +1,4 @@
-import { loadMostRecentPreviousAnalysis } from '~codeyam/supabase';
+import { loadMostRecentPreviousAnalysis } from '~codeyam/database';
 import { Commit, Entity, File, Project } from '~codeyam/types';
 
 export interface FindValidExistingAnalysisArgs {

package/analyzer-template/packages/analyze/src/lib/files/analyze/findValidExistingAnalysis.ts

@@ -1,4 +1,4 @@
-import { loadAnalysis } from '~codeyam/supabase';
+import { loadAnalysis } from '~codeyam/database';
 import { Analysis, Project } from '~codeyam/types';
 
 export interface FindValidExistingAnalysisArgs {

package/analyzer-template/packages/analyze/src/lib/files/analyze/gatherEntityMap.ts

@@ -3,6 +3,9 @@ import findOrCreateEntity from './findOrCreateEntity';
 import { ProjectAnalyzer } from '../../ProjectAnalyzer';
 import { AnalysisContext } from '../../analysisContext';
 
+// Track files we've already warned about to avoid duplicate warnings
+const warnedMissingFiles = new Set<string>();
+
 interface EntityWithGatheredDependencies extends Entity {
   dependenciesGathered?: boolean;
 }
@@ -73,14 +76,19 @@ export default async function gatherEntityMap({
     );
 
     if (!importedFile) {
-      console.log(
-        `CodeYam Error: File not found while gathering entity map: ${importedExportFilePath}`,
-        entity.filePath,
-        entity.name,
-      );
-      throw new Error(
-        `File not found while gathering entity map: ${importedExportFilePath}`,
-      );
+      // Skip dependencies when the file is not found in project.files.
+      // This commonly happens when a universal mock intercepts the import:
+      // 1. TypeScript resolves @formbricks/types/errors to the mock file
+      // 2. The original packages/types/errors.ts is never imported
+      // 3. So it's not in getSourceFiles() and thus not in project.files
+      // 4. Since the mock handles the import, we don't need to analyze the original
+      if (!warnedMissingFiles.has(importedExportFilePath)) {
+        warnedMissingFiles.add(importedExportFilePath);
+        console.log(
+          `CodeYam Warning: Skipping dependency - file not in project.files (likely mocked): ${importedExportFilePath}`,
+        );
+      }
+      continue;
     }
 
     let importedEntity = context.getEntity(

package/analyzer-template/packages/analyze/src/lib/files/analyze/setActiveAnalysisBranches.ts

@@ -2,7 +2,7 @@ import {
   loadAnalysisBranches,
   loadCommitBranches,
   upsertAnalysisBranches,
-} from '~codeyam/supabase';
+} from '~codeyam/database';
 import { Analysis, AnalysisBranch, Commit, Entity } from '~codeyam/types';
 import { awsLog } from '~codeyam/utils';
 

package/analyzer-template/packages/analyze/src/lib/files/analyze/trackEntityCircularDependencies.ts

@@ -99,19 +99,11 @@ function traverseEntity(
     );
 
     if (!dependentEntity) {
-      console.log(
-        'CodeYam Error: trackEntityCircularDependencies: Dependent entity not found',
-        {
-          entityFilePath: entity.filePath,
-          entityName: entity.name,
-          dependencyFilePath,
-          dependencyName,
-          fileEntities: context.getEntityNamesForFile(dependencyFilePath),
-        },
-      );
-      throw new Error(
-        'trackEntityCircularDependencies: Dependent entity not found',
-      );
+      // Skip dependencies when the entity is not found in context.
+      // This commonly happens when a universal mock intercepts the import:
+      // - gatherEntityMap skips adding the entity because the file isn't in project.files
+      // - The mock handles the import at runtime, so we don't need to track it
+      continue;
     }
 
     // Recursively traverse (will skip if already visited)

package/analyzer-template/packages/analyze/src/lib/files/analyzeChange.ts

@@ -7,8 +7,8 @@ import type {
 import generateDataStructure from './scenarios/generateDataStructure';
 import generateChangesScenarioData from './scenarios/generateChangesScenarioData';
 import generateChangesScenarios from './scenarios/generateChangesScenarios';
-import generateKeyAttributes from './scenarios/generateKeyAttributes';
-import { upsertAnalysesWithScenarios } from '~codeyam/supabase';
+import generateExecutionFlows from './scenarios/generateExecutionFlows';
+import { upsertAnalysesWithScenarios } from '~codeyam/database';
 import { Step } from './enums/steps';
 import recordStep from './recordStep';
 import { FileAnalyzer } from '..';
@@ -16,7 +16,13 @@ import { AI } from '~codeyam/ai';
 import extractDiffLines from './extractDiffLines';
 import analyzeCodeChange from './analyzeCodeChange';
 import relevantDiffPart from './relevantDiffPart';
-import { awsLog, awsLogDebugLevel, pushAnalysisError } from '~codeyam/utils';
+import {
+  awsLog,
+  awsLogDebugLevel,
+  pushAnalysisError,
+  Semaphore,
+} from '~codeyam/utils';
+import deepEqual from '../utils/deepEqual';
 
 export interface AnalyzeChangeArgs {
   previousAnalysis: Analysis;
@@ -29,6 +35,8 @@ export interface AnalyzeChangeArgs {
   steps?: string[];
   updateProgress?: (detail: string) => void;
   model: AI.Model;
+  /** Semaphore to serialize CPU-intensive Structure phase */
+  structureSemaphore?: Semaphore;
 }
 
 export default async function analyzeChange({
@@ -42,6 +50,7 @@ export default async function analyzeChange({
   steps,
   updateProgress,
   model,
+  structureSemaphore,
 }: AnalyzeChangeArgs) {
   analysis.entity = entity;
   awsLog('CodeYam: Analyzing change entity', {
@@ -79,50 +88,76 @@ export default async function analyzeChange({
       });
     }
 
+    // Structure phase: CPU-intensive, serialize via semaphore if provided
     if (!steps || steps.includes(Step.Structure)) {
-      awsLogDebugLevel(
-        1,
-        `Merging dependent data structures for ${entity.name}...`,
-      );
+      if (structureSemaphore) {
+        await structureSemaphore.acquire();
+      }
+      try {
+        awsLogDebugLevel(
+          1,
+          `Merging dependent data structures for ${entity.name}...`,
+        );
 
-      const { step: dataStructureStep } = await recordStep(
-        'generateDataStructure',
-        () =>
-          generateDataStructure({
-            entity,
-            dependentAnalyses,
-            analysis,
-          }),
-      );
-      analysis.status.steps.push(dataStructureStep);
+        const { step: dataStructureStep } = await recordStep(
+          'generateDataStructure',
+          () =>
+            generateDataStructure({
+              entity,
+              dependentAnalyses,
+              analysis,
+            }),
+        );
+        analysis.status.steps.push(dataStructureStep);
+      } finally {
+        if (structureSemaphore) {
+          structureSemaphore.release();
+        }
+      }
     }
 
-    if (!steps || steps.includes(Step.KeyAttributes)) {
-      awsLogDebugLevel(1, `Determining key attributes for ${entity.name}...`);
+    // ExecutionFlows phase: I/O-bound LLM call, can run in parallel with other entities' Structure
+    if (!steps || steps.includes(Step.ExecutionFlows)) {
+      awsLogDebugLevel(1, `Generating execution flows for ${entity.name}...`);
 
-      const { step: keyAttributesStep } = await recordStep(
-        'generateKeyAttributes',
+      const { step: executionFlowsStep } = await recordStep(
+        'generateExecutionFlows',
         () =>
-          generateKeyAttributes({
+          generateExecutionFlows({
             entity,
             analysis,
-            dependentAnalyses,
             model,
             updateProgress,
+            dependentAnalyses,
           }),
       );
-      analysis.status.steps.push(keyAttributesStep);
+      analysis.status.steps.push(executionFlowsStep);
     }
 
-    if (
-      analysis.metadata.scenariosDataStructure ===
-      previousAnalysis.metadata.scenariosDataStructure
-    ) {
+    const dataStructureUnchanged = deepEqual(
+      analysis.metadata.scenariosDataStructure,
+      previousAnalysis.metadata.scenariosDataStructure,
+    );
+
+    const executionFlowsUnchanged = deepEqual(
+      analysis.metadata.executionFlows,
+      previousAnalysis.metadata.executionFlows,
+    );
+
+    if (dataStructureUnchanged && executionFlowsUnchanged) {
       awsLogDebugLevel(
         1,
-        `Data structure unchanged, reusing previous scenarios for ${entity.name}...`,
+        `Data structure and execution flows unchanged, reusing previous scenarios for ${entity.name}...`,
       );
-      analysis.scenarios = previousAnalysis.scenarios;
+      // Reuse previous scenarios but mark them for recapture if entity code changed
+      analysis.scenarios = previousAnalysis.scenarios.map((scenario) => ({
+        ...scenario,
+        metadata: {
+          ...scenario.metadata,
+          // Only mark for recapture if entity code actually changed
+          needsRecapture: entityChanged,
+        },
+      }));
     } else {
       if (!steps || steps.includes(Step.Scenarios)) {
         awsLogDebugLevel(1, `Generating scenarios for ${entity.name}...`);

package/analyzer-template/packages/analyze/src/lib/files/analyzeEntity.ts

@@ -2,7 +2,7 @@ import {
   deleteScenarios,
   upsertAnalysesWithScenarios,
   upsertEntities,
-} from '~codeyam/supabase';
+} from '~codeyam/database';
 import {
   AI,
   generateChangesEntityDocumentation,
@@ -31,7 +31,12 @@ import relevantDiffPart from './relevantDiffPart';
 import { Step } from './enums/steps';
 import { ImmutableAnalysisContext } from '../analysisContext';
 import generateAnalyzedTreeSha from './analyze/generateAnalyzedTreeSha';
-import { awsLog, awsLogDebugLevel, pushAnalysisError } from '~codeyam/utils';
+import {
+  awsLog,
+  awsLogDebugLevel,
+  pushAnalysisError,
+  Semaphore,
+} from '~codeyam/utils';
 
 export interface AnalyzeEntityArgs {
   project: Project;
@@ -50,6 +55,8 @@ export interface AnalyzeEntityArgs {
   forceAll?: boolean;
   updateProgress?: (detail: string) => void;
   modelString?: string;
+  /** Semaphore to serialize CPU-intensive Structure phase */
+  structureSemaphore?: Semaphore;
 }
 
 /**
@@ -77,6 +84,7 @@ export default async function analyzeEntity({
   forceAll,
   updateProgress,
   modelString,
+  structureSemaphore,
 }: AnalyzeEntityArgs): Promise<{
   analysis: Analysis;
   dependentAnalyses?: ReadonlyAnalysisMap;
@@ -222,7 +230,7 @@ export default async function analyzeEntity({
 
       // For partial analyses, skip Step.Structure to reuse existing mergedDataStructure
       if (isPartialAnalysis && !steps) {
-        steps = [Step.KeyAttributes, Step.Scenarios, Step.Data];
+        steps = [Step.ExecutionFlows, Step.Scenarios, Step.Data];
         awsLog(
           'CodeYam: continuing from partial analysis, skipping Structure step',
           {
@@ -263,13 +271,17 @@ export default async function analyzeEntity({
         });
       }
 
+      // Return early if analysis is already complete for its purpose:
+      // - Full analysis: has analyzedTreeSha
+      // - Dependency analysis: has executionFlows (Structure + ExecutionFlows already ran)
+      const isCompleteDependencyAnalysis =
+        analyzeAsDependency && !!existingAnalysis.metadata?.executionFlows;
+
       if (
         !force &&
-        !!existingAnalysis.analyzedTreeSha &&
-        existingAnalysis.dependencyAnalyzedTreeSha === dependencyAnalyzedTreeSha //&&
-        // (existingAnalysis.status?.finishedAt ||
-        //   existingAnalysis.status?.readyToBeCaptured ||
-        //   branchCommitSha)
+        existingAnalysis.dependencyAnalyzedTreeSha ===
+          dependencyAnalyzedTreeSha &&
+        (!!existingAnalysis.analyzedTreeSha || isCompleteDependencyAnalysis)
       ) {
         awsLogDebugLevel(1, `Using existing analysis for ${entity.name}...`);
         awsLog('Returning existing analysis', {
@@ -367,7 +379,7 @@ export default async function analyzeEntity({
         }
 
         if (!steps || steps?.includes(Step.Scenarios)) {
-          delete existingAnalysis.metadata.keyAttributes;
+          delete existingAnalysis.metadata.executionFlows;
         }
 
         if (!steps || steps?.includes(Step.Data)) {
@@ -476,6 +488,7 @@ export default async function analyzeEntity({
             steps,
             updateProgress,
             model,
+            structureSemaphore,
           })
         : analyzeInitial({
             analysis,
@@ -485,6 +498,7 @@ export default async function analyzeEntity({
             steps,
             updateProgress,
             model,
+            structureSemaphore,
           }),
     ]);
 
@@ -498,8 +512,12 @@ export default async function analyzeEntity({
     analysis = analysisResult.analysis;
 
     analysis.completedAt = new Date().toISOString();
-    // Don't set analyzedTreeSha when ONLY_DATA_STRUCTURE=true, so full analysis can re-analyze later
-    if (process.env.ONLY_DATA_STRUCTURE !== 'true') {
+    // Don't set analyzedTreeSha when:
+    // 1. ONLY_DATA_STRUCTURE=true - so full analysis can re-analyze later
+    // 2. analyzeAsDependency=true - dependency analyses only generate Structure+KeyAttributes,
+    //    not scenarios. Leaving analyzedTreeSha unset allows partial analysis detection to work
+    //    when the entity is later analyzed directly.
+    if (process.env.ONLY_DATA_STRUCTURE !== 'true' && !analyzeAsDependency) {
       analysis.analyzedTreeSha = generateAnalyzedTreeSha(analysis);
     }
   } catch (error) {

package/analyzer-template/packages/analyze/src/lib/files/analyzeInitial.ts

@@ -11,9 +11,14 @@ import {
   type Entity,
   type ReadonlyAnalysisMap,
 } from '~codeyam/types';
-import { upsertAnalysesWithScenarios } from '~codeyam/supabase';
-import { awsLog, awsLogDebugLevel, pushAnalysisError } from '~codeyam/utils';
-import generateKeyAttributes from './scenarios/generateKeyAttributes';
+import { upsertAnalysesWithScenarios } from '~codeyam/database';
+import {
+  awsLog,
+  awsLogDebugLevel,
+  pushAnalysisError,
+  Semaphore,
+} from '~codeyam/utils';
+import generateExecutionFlows from './scenarios/generateExecutionFlows';
 
 export interface AnalyzeInitialArgs {
   analysis: Analysis;
@@ -23,6 +28,8 @@ export interface AnalyzeInitialArgs {
   steps?: string[];
   updateProgress?: (detail: string) => void;
   model: AI.Model;
+  /** Semaphore to serialize CPU-intensive Structure phase */
+  structureSemaphore?: Semaphore;
 }
 
 export default async function analyzeInitial({
@@ -33,6 +40,7 @@ export default async function analyzeInitial({
   steps,
   updateProgress,
   model,
+  structureSemaphore,
 }: AnalyzeInitialArgs) {
   analysis.entity = entity;
   awsLog('CodeYam: Analyzing initial entity', {
@@ -47,47 +55,58 @@ export default async function analyzeInitial({
   });
 
   try {
+    // Structure phase: CPU-intensive, serialize via semaphore if provided
     if (!steps || steps.includes(Step.Structure)) {
-      awsLogDebugLevel(
-        1,
-        `Merging dependent data structures for ${entity.name}...`,
-      );
-
-      const { step: dataStructureStep } = await recordStep(
-        'generateDataStructure',
-        () =>
-          generateDataStructure({
-            entity,
-            dependentAnalyses,
-            analysis,
-          }),
-      );
-      analysis.status.steps.push(dataStructureStep);
+      if (structureSemaphore) {
+        await structureSemaphore.acquire();
+      }
+      try {
+        awsLogDebugLevel(
+          1,
+          `Merging dependent data structures for ${entity.name}...`,
+        );
+
+        const { step: dataStructureStep } = await recordStep(
+          'generateDataStructure',
+          () =>
+            generateDataStructure({
+              entity,
+              dependentAnalyses,
+              analysis,
+            }),
+        );
+        analysis.status.steps.push(dataStructureStep);
+      } finally {
+        if (structureSemaphore) {
+          structureSemaphore.release();
+        }
+      }
     }
 
-    // Short-circuit for test mocking: only generate data structure, skip key attributes and scenarios
+    // Short-circuit for test mocking: only generate data structure, skip execution flows and scenarios
     if (process.env.ONLY_DATA_STRUCTURE === 'true') {
       awsLog(
-        'CodeYam: Skipping key attributes and scenario generation (ONLY_DATA_STRUCTURE=true)',
+        'CodeYam: Skipping execution flows and scenario generation (ONLY_DATA_STRUCTURE=true)',
       );
       return { analysis };
     }
 
-    if (!steps || steps.includes(Step.KeyAttributes)) {
-      awsLogDebugLevel(1, `Determining key attributes for ${entity.name}...`);
+    // ExecutionFlows phase: I/O-bound LLM call, can run in parallel with other entities' Structure
+    if (!steps || steps.includes(Step.ExecutionFlows)) {
+      awsLogDebugLevel(1, `Generating execution flows for ${entity.name}...`);
 
-      const { step: dataStructureStep } = await recordStep(
-        'generateKeyAttributes',
+      const { step: executionFlowsStep } = await recordStep(
+        'generateExecutionFlows',
         () =>
-          generateKeyAttributes({
+          generateExecutionFlows({
             entity,
             analysis,
-            dependentAnalyses,
             model,
             updateProgress,
+            dependentAnalyses,
           }),
       );
-      analysis.status.steps.push(dataStructureStep);
+      analysis.status.steps.push(executionFlowsStep);
     }
 
     if (analyzeAsDependency) {
@@ -157,7 +176,6 @@ export default async function analyzeInitial({
       analysis.status.steps.push(mockDataStep);
     }
   } catch (error) {
-    console.log(error);
     pushAnalysisError({
       status: analysis.status,
       error,

package/analyzer-template/packages/analyze/src/lib/files/analyzeNextRoute.ts

@@ -1,6 +1,6 @@
 import { Entity, EntityType, File, ProjectFramework } from '~codeyam/types';
 import { ProjectAnalyzer } from '../ProjectAnalyzer';
-import { generateSha } from '~codeyam/supabase';
+import { generateSha } from '~codeyam/database';
 import { isNextRoute } from '~codeyam/utils';
 
 export default function analyzeNextRoute(