@codeyam/codeyam-cli 0.1.0-bleeding-edge.8afd3ee

package/analyzer-template/project/analyzeFileEntities.ts

@@ -0,0 +1,282 @@
+import {
+  AnalysisContext,
+  analyzeEntities,
+  findOrCreateEntity,
+  gatherEntityMap,
+  ProjectAnalyzer,
+  setImportedExports,
+} from '~codeyam/analyze';
+import { Commit, Entity, Project } from '~codeyam/types';
+import checkApprovedPath from './checkApprovedPath';
+import {
+  loadEntity,
+  updateCommitMetadata,
+  upsertCommits,
+} from '~codeyam/supabase';
+import { analyzeFileMocks } from './mocks/analyzeFileMock';
+import { LazyFileStore } from './LazyFileStore';
+
+interface AnalyzeFileEntitiesArgs {
+  project: Project;
+  projectAnalyzer: ProjectAnalyzer;
+  commit?: Commit;
+  branchId: string;
+  entityShas: string[];
+  scenarioIds?: string[];
+  analyzeMockName?: string;
+  steps: string[];
+  context: AnalysisContext;
+  fileStore?: LazyFileStore;
+}
+
+export default async function analyzeFileEntities({
+  project,
+  projectAnalyzer,
+  commit,
+  branchId,
+  entityShas,
+  scenarioIds,
+  analyzeMockName,
+  steps,
+  context,
+  fileStore,
+}: AnalyzeFileEntitiesArgs) {
+  // Enable baseline mode for E2E tests to skip expensive importedBy tracking
+  // This eliminates thousands of DB operations that aren't needed for simulation/capture
+  if (process.env.CODEYAM_E2E_BASELINE_MODE === 'true') {
+    context.setBaselineMode(true);
+  }
+
+  // Create a basic entity map for the requested entity SHAs
+  // Use fileStore if available for iterating (entities should already be loaded for relevant files)
+  const filesToIterate = fileStore
+    ? Array.from(fileStore.values())
+    : project.files;
+  const basicEntityMap = filesToIterate.reduce(
+    (acc, file) => {
+      const fileEntities = (file.entities || []).filter((entity) => {
+        return entityShas.includes(entity.sha);
+      });
+      for (const entity of fileEntities) {
+        entity.file = file;
+        acc[entity.sha] = entity;
+      }
+      return acc;
+    },
+    {} as { [key: string]: Entity },
+  );
+
+  console.log(
+    `CodeYan: analyzeFileEntities: Gathering dependencies for ${entityShas.length} entity SHAs`,
+  );
+
+  // Initial entity discovery and setup
+  const entities: Entity[] = [];
+  for (let ii = 0; ii < entityShas.length; ii++) {
+    const entitySha = entityShas[ii];
+    let basicEntity = basicEntityMap[entitySha];
+
+    // If entity SHA not found, try to find it by looking up the entity in the database
+    // This handles cases where the code has changed and the SHA is different from what's in the database
+    if (!basicEntity) {
+      console.log(
+        `CodeYam Warning: Entity SHA not found in generated entities: ${entitySha}`,
+      );
+
+      // Try to load the entity from database to get its name and file path
+      const dbEntity = await loadEntity({
+        projectId: project.id,
+        sha: entitySha,
+      });
+
+      if (dbEntity && dbEntity.filePath) {
+        // Find the file by path - use fileStore for O(1) lookup when available
+        let file = fileStore
+          ? fileStore.getByPath(dbEntity.filePath)
+          : project.files.find((f) => f.path === dbEntity.filePath);
+
+        // Ensure entities are loaded for this file
+        if (file && fileStore && !fileStore.isEntitiesLoaded(file.path)) {
+          file = await fileStore.ensureEntities(file.path);
+        }
+
+        if (file) {
+          // Find entity by name in the current source code
+          const matchingEntity = file.entities.find(
+            (e) => e.name === dbEntity.name,
+          );
+
+          if (matchingEntity) {
+            console.log(
+              `CodeYam: Found entity by name from DB lookup: ${dbEntity.name} at ${dbEntity.filePath} (old SHA: ${entitySha}, new SHA: ${matchingEntity.sha})`,
+            );
+            matchingEntity.file = file;
+            basicEntity = matchingEntity;
+            // Update the entityShas array to use the current SHA
+            entityShas[ii] = matchingEntity.sha;
+          }
+        }
+      }
+    }
+
+    if (!basicEntity) {
+      // This should be very rare now that we have early validation in prep.ts
+      // If we get here, it means the validation was skipped or there's a race condition
+      const errorMessage = [
+        `\n${'━'.repeat(80)}`,
+        `❌ ENTITY NOT FOUND - This should have been caught earlier!`,
+        `${'━'.repeat(80)}`,
+        ``,
+        `Entity SHA: ${entitySha}`,
+        `Short SHA: ${entitySha.substring(0, 12)}...`,
+        ``,
+        `This error should have been caught by early validation in prep.ts.`,
+        `If you're seeing this, either:`,
+        `  1. Early validation was skipped (shouldn't happen for E2E tests)`,
+        `  2. There's a race condition with entity enumeration`,
+        `  3. The entity code changed between validation and analysis`,
+        ``,
+        `🔧 HOW TO FIX:`,
+        `  For E2E tests:`,
+        `    1. Ensure codeyam-db.sqlite3 has the entity`,
+        `    2. Entity will be looked up by name+path (no SHA needed)`,
+        `    3. Run 'cd background && pnpm sync-test-db' to sync from local db`,
+        ``,
+        `  For production use:`,
+        `    1. Check if entity code changed recently`,
+        `    2. Verify the entity SHA is from the correct commit`,
+        ``,
+        `${'━'.repeat(80)}`,
+      ].join('\n');
+
+      console.error(errorMessage);
+      throw new Error(
+        `Entity not found for SHA: ${entitySha.substring(0, 12)}... - See detailed error message above`,
+      );
+    }
+
+    const { name, file } = basicEntity;
+
+    const approvedPath = checkApprovedPath({ project, path: file.path });
+    if (!approvedPath) {
+      console.log(
+        'CodeYam: Skipping analysis, unapproved path :>> ',
+        JSON.stringify({ path: file.path }),
+      );
+      commit.metadata ||= {};
+      commit.metadata.skippedFiles ||= [];
+      commit.metadata.skippedFiles.push(file.path);
+      continue;
+    }
+
+    const fileAnalyzer = projectAnalyzer.getFileAnalyzer(file);
+    const entity = await findOrCreateEntity({
+      file,
+      branchId,
+      commitId: commit?.id,
+      entityName: name,
+      fileAnalyzer,
+      projectAnalyzer,
+      context,
+    });
+
+    if (!entity) {
+      console.log(
+        'CodeYam Error: findOrCreateEntity not created',
+        file.path,
+        name,
+      );
+      continue;
+    }
+
+    entities.push(entity);
+    context.addEntity(entity);
+
+    if (
+      commit &&
+      (entities.length % 10 === 0 || ii === entityShas.length - 1)
+    ) {
+      // TODO: is this going to clobber the commit.metadata tweaks above?
+      //       maybe updateCommitMetadata() should have a version that takes
+      //       current commit as argument rather than load it from the DB?
+      commit.metadata = await updateCommitMetadata({
+        commitId: commit.id,
+        runStatusUpdate: {
+          entityCount: entities.length,
+        },
+      });
+    }
+
+    // Gather dependencies for this entity
+    await gatherEntityMap({
+      entity,
+      commitId: commit?.id,
+      branchId,
+      context,
+      projectAnalyzer,
+    });
+  }
+
+  console.log(
+    `CodeYan: analyzeFileEntities: Establishing bidirectional mapping of ${entities.length} entities`,
+  );
+
+  // Establish bidirectional relationships between entities
+  await setImportedExports({
+    entities,
+    commit,
+    context,
+    projectAnalyzer,
+    fileStore,
+  });
+
+  console.log(
+    `CodeYam: analyzeFileEntities: Starting analysis of ${entities.length} anchor entities`,
+  );
+
+  if (analyzeMockName) {
+    // mocked-out analyzeEntities()
+    for (const entity of entities) {
+      const { file } = entity;
+
+      const analyzeFn = analyzeFileMocks[analyzeMockName];
+      const { analysisMap, entityMap } = await analyzeFn();
+
+      for (const entityName of Object.keys(analysisMap[file.path])) {
+        analysisMap[file.path][entityName].fileId = file.id;
+      }
+      context.mergeAnalysisMap(analysisMap);
+      context.mergeEntityMap(entityMap);
+    }
+  } else {
+    // normal execution path
+    await analyzeEntities({
+      rootEntities: entities,
+      context,
+      project,
+      projectAnalyzer,
+      commit,
+      options: {
+        branchId,
+        scenarioIds,
+        steps,
+        fresh: !!process.env.FRESH,
+        force: !!process.env.FORCE,
+        forceAll: !!process.env.FORCE_ALL,
+      },
+    });
+  }
+
+  console.log(
+    `CodeYam: analyzeFileEntities: Analysis completed for ${entities.length} entities`,
+  );
+
+  // Update commit if needed
+  if (commit) {
+    await upsertCommits({ projectId: project.id, commits: [commit] });
+  }
+
+  return {
+    entities,
+  };
+}

package/analyzer-template/project/analyzeReadyToBeCaptured.ts

@@ -0,0 +1,51 @@
+import { AnalysisContext, ProjectAnalyzer } from '~codeyam/analyze';
+import { Analysis, Commit, Project } from '~codeyam/types';
+import loadReadyToBeCaptured from './loadReadyToBeCaptured';
+import { awsLog } from '~codeyam/utils';
+import { LazyFileStore } from './LazyFileStore';
+
+export interface AnalyzeReadyToBeCapturedArgs {
+  project: Project;
+  projectAnalyzer: ProjectAnalyzer;
+  branchCommit?: Commit | null;
+  analysisIds?: string[];
+  context: AnalysisContext;
+  fileStore?: LazyFileStore;
+}
+
+export interface AnalyzeReadyToBeCapturedResult {
+  projectAnalyzer: ProjectAnalyzer;
+  readyToBeCaptured: Analysis[];
+  context: AnalysisContext;
+}
+
+export default async function analyzeReadyToBeCaptured({
+  project,
+  projectAnalyzer,
+  branchCommit,
+  analysisIds,
+  context,
+  fileStore,
+}: AnalyzeReadyToBeCapturedArgs): Promise<AnalyzeReadyToBeCapturedResult> {
+  awsLog('CodeYam: Analyze - readyToBeCaptured path');
+
+  const { readyToBeCaptured: loadedReadyToBeCaptured } =
+    await loadReadyToBeCaptured({
+      project,
+      branchCommitSha: branchCommit?.sha,
+      analysisIds,
+      context,
+      fileStore,
+    });
+
+  if (!loadedReadyToBeCaptured || loadedReadyToBeCaptured.length === 0) {
+    awsLog('No analyses ready to be captured, exiting');
+    process.exit(0);
+  }
+
+  return {
+    projectAnalyzer,
+    readyToBeCaptured: loadedReadyToBeCaptured,
+    context,
+  };
+}

package/analyzer-template/project/analyzeRegularCommit.ts

@@ -0,0 +1,276 @@
+import { AnalysisContext, ProjectAnalyzer } from '~codeyam/analyze';
+import {
+  Analysis,
+  BackgroundJob,
+  Branch,
+  Commit,
+  Entity,
+  Project,
+} from '~codeyam/types';
+import analyzeFileEntities from './analyzeFileEntities';
+import analyzeAndGatherIndirectEntities from './analyzeAndGatherIndirectEntities';
+import {
+  loadAnalyses,
+  loadAnalysisBranches,
+  updateBackgroundJobProgress,
+  updateCommitMetadata,
+  upsertAnalysesWithScenarios,
+  upsertAnalysisBranches,
+} from '~codeyam/supabase';
+import { awsLog, awsLogDebugLevel } from '~codeyam/utils';
+import { LazyFileStore } from './LazyFileStore';
+
+export interface AnalyzeRegularCommitArgs {
+  project: Project;
+  projectAnalyzer: ProjectAnalyzer;
+  commit: Commit;
+  branch: Branch;
+  branchCommit?: Commit | null;
+  analysisIds?: string[];
+  scenarioIds?: string[];
+  entityShas?: string[];
+  context: AnalysisContext;
+  steps?: string[];
+  analyzeMockName?: string;
+  isLocalMode: boolean;
+  backgroundJob?: BackgroundJob;
+  desiredAnalyses?: Analysis[];
+  fileStore?: LazyFileStore;
+}
+
+export interface AnalyzeRegularCommitResult {
+  projectAnalyzer: ProjectAnalyzer;
+  readyToBeCaptured: Analysis[];
+  context: AnalysisContext;
+}
+
+async function entitiesReadyToBeCaptured(
+  entities: Entity[],
+  entityShas: string[],
+  context: AnalysisContext,
+  analysisIds: string[],
+) {
+  const readyToBeCaptured: Analysis[] = [];
+  for (const entity of entities) {
+    let analysis = context.getAnalysis(
+      entity.filePath,
+      entity.name,
+    ) as Analysis;
+    if (
+      !analysis?.status?.readyToBeCaptured &&
+      !analysisIds?.includes(analysis.id)
+    )
+      continue;
+
+    // Due to circular dependencies, double check if any indirect entities are not properly labeled
+    if (!entityShas?.includes(entity.sha) && !analysis.indirect) {
+      analysis.indirect = true;
+      analysis = (
+        await upsertAnalysesWithScenarios({
+          analysesToUpsert: [analysis],
+        })
+      )[0];
+
+      context.addAnalysis(analysis, entity.filePath, entity.name);
+    }
+
+    analysis.entity = entity;
+    readyToBeCaptured.push(analysis);
+  }
+
+  return readyToBeCaptured;
+}
+
+export default async function analyzeRegularCommit({
+  project,
+  projectAnalyzer,
+  commit,
+  branch,
+  branchCommit,
+  analysisIds,
+  scenarioIds,
+  entityShas = [],
+  context,
+  steps,
+  analyzeMockName,
+  isLocalMode,
+  backgroundJob,
+  desiredAnalyses = [],
+  fileStore,
+}: AnalyzeRegularCommitArgs): Promise<AnalyzeRegularCommitResult> {
+  awsLog('CodeYam: Analyze - commit path');
+
+  const stepName = 'analyze-files';
+
+  let fileCount: number | undefined = undefined;
+
+  // Determine entityShas if not provided
+  if (!entityShas || entityShas.length === 0) {
+    if (analysisIds) {
+      const loadedAnalyses = await loadAnalyses({
+        projectId: project.id,
+        ids: analysisIds,
+      });
+      entityShas = loadedAnalyses.map((a) => a.entitySha);
+    } else if (!commit) {
+      throw new Error('Unimplemented code path.');
+    } else {
+      // for commit analysis, we actually do know how many files were involved
+      fileCount = commit.files.length;
+
+      // Load entities for commit files (lazy loading)
+      const commitFilePaths = commit.files.map((f) => f.fileName);
+      if (fileStore) {
+        await fileStore.ensureEntitiesBatch(commitFilePaths);
+      }
+
+      entityShas = commit.files
+        .flatMap((commitFile) => {
+          // Use fileStore for O(1) lookup when available
+          const file = fileStore
+            ? fileStore.getByPath(commitFile.fileName)
+            : project.files.find((f) => f.path === commitFile.fileName);
+          return file?.entities?.map((entity) => entity.sha);
+        })
+        .filter(Boolean);
+
+      const existingAnalysisBranches = await loadAnalysisBranches({
+        projectId: project.id,
+        branchId: branch.id,
+        filePaths: commit.files.map((f) => f.fileName),
+      });
+
+      for (const existingAnalysisBranch of existingAnalysisBranches) {
+        existingAnalysisBranch.active = false;
+      }
+
+      await upsertAnalysisBranches(existingAnalysisBranches);
+    }
+  }
+
+  // deduplicate entity SHAs
+  entityShas = Array.from(new Set(entityShas));
+
+  const entityCount = entityShas.length;
+  if (commit) {
+    commit.metadata = await updateCommitMetadata({
+      commitId: commit.id,
+      runStatusUpdate: {
+        entityCount,
+        fileCount,
+        filesCompleted: fileCount,
+      },
+    });
+  }
+
+  const { entities: fileEntities } = await analyzeFileEntities({
+    project,
+    projectAnalyzer,
+    commit,
+    branchId: branch.id,
+    entityShas,
+    scenarioIds,
+    analyzeMockName,
+    steps,
+    context,
+    fileStore,
+  });
+  let entities = fileEntities;
+
+  // Conditionally analyze indirect entities
+  const shouldSkipIndirect =
+    branchCommit ||
+    analysisIds ||
+    isLocalMode ||
+    analyzeMockName ||
+    entities.length === 0;
+
+  if (shouldSkipIndirect) {
+    if (branchCommit) {
+      console.log(`CodeYam: Skipping indirect analysis for branch commit`);
+    } else if (analysisIds) {
+      console.log(
+        `CodeYam: Skipping indirect analysis for specific analysisIds`,
+      );
+    } else if (isLocalMode) {
+      console.log(`CodeYam: Skipping indirect analysis for local mode`);
+    } else if (analyzeMockName) {
+      console.log(`CodeYam: Skipping indirect analysis for mock analysis`);
+    }
+  } else {
+    awsLog(`CodeYam: Analyze - indirect entities`);
+    const { entities: withIndirectEntities } =
+      await analyzeAndGatherIndirectEntities({
+        entities,
+        project,
+        projectAnalyzer,
+        commit,
+        context,
+        steps,
+      });
+
+    entities = withIndirectEntities;
+  }
+
+  const readyToBeCaptured = await entitiesReadyToBeCaptured(
+    entities,
+    entityShas,
+    context,
+    analysisIds,
+  );
+
+  awsLogDebugLevel(
+    1,
+    `Analysis complete, ${readyToBeCaptured.length} entities ready to be captured`,
+  );
+
+  awsLog(
+    'CodeYam: readyToBeCaptured',
+    readyToBeCaptured.map((analysis) => ({
+      path: analysis.entity.filePath,
+      entityName: analysis.entity.name,
+      entityType: analysis.entity.entityType,
+      analysisId: analysis.id,
+      scenarios: (analysis.scenarios ?? []).map((s) => s.name),
+      indirect: analysis.indirect,
+    })),
+  );
+
+  if (commit) {
+    const now = new Date().toISOString();
+    commit.metadata = await updateCommitMetadata({
+      commitId: commit.id,
+      runStatusUpdate: {
+        analysisCompletedAt: now,
+      },
+      metadataUpdate: {
+        entitiesAnalyzedAt: now,
+      },
+      // In local mode (CLI), let job.js handle archiving after BOTH analysis AND capture complete
+      // In cloud mode (Docker), archive immediately after analysis completes
+      archiveCurrentRun: !isLocalMode,
+    });
+  }
+
+  if (backgroundJob) {
+    await updateBackgroundJobProgress(backgroundJob, {
+      initiated: backgroundJob.progress.initiated,
+      steps: [
+        {
+          name: stepName,
+          initiated:
+            backgroundJob.progress.steps.find((s) => s.name === stepName)
+              ?.initiated ?? new Date().toISOString(),
+          completed: new Date().toISOString(),
+        },
+      ],
+      completed: new Date().toISOString(),
+    });
+  }
+
+  return {
+    projectAnalyzer,
+    readyToBeCaptured,
+    context,
+  };
+}