@codeyam/codeyam-cli 0.1.0-bleeding-edge.8afd3ee

package/analyzer-template/project/ScenarioIterator.ts

@@ -0,0 +1,133 @@
+import { TaskCompleteType } from './controller/startController';
+
+export interface ScenarioWithId {
+  id: string;
+  name: string;
+}
+
+/**
+ * Manages scenario iteration state for multi-scenario capture.
+ *
+ * This class handles the coordination between:
+ * - Server errors (which advance to the next scenario)
+ * - taskComplete events from capture (which should only advance if they match the current scenario)
+ *
+ * The key invariant is that stale taskComplete events (from scenarios that already
+ * advanced due to errors) are ignored to prevent skipping scenarios.
+ */
+export class ScenarioIterator {
+  private scenarios: ScenarioWithId[];
+  private currentIndex = 0;
+  private currentResolve: (() => void) | null = null;
+  private _taskCompletion: TaskCompleteType | undefined;
+  private _failedScenarios: string[] = [];
+  private pollingInterval: NodeJS.Timeout | undefined;
+
+  constructor(scenarios: ScenarioWithId[]) {
+    this.scenarios = scenarios;
+  }
+
+  get currentScenario(): ScenarioWithId | undefined {
+    return this.scenarios[this.currentIndex];
+  }
+
+  get currentScenarioIndex(): number {
+    return this.currentIndex;
+  }
+
+  get taskCompletion(): TaskCompleteType | undefined {
+    return this._taskCompletion;
+  }
+
+  get failedScenarios(): string[] {
+    return this._failedScenarios;
+  }
+
+  get isLastScenario(): boolean {
+    return this.currentIndex === this.scenarios.length - 1;
+  }
+
+  get totalScenarios(): number {
+    return this.scenarios.length;
+  }
+
+  setPollingInterval(interval: NodeJS.Timeout): void {
+    this.pollingInterval = interval;
+  }
+
+  clearPollingInterval(): void {
+    if (this.pollingInterval) {
+      clearInterval(this.pollingInterval);
+      this.pollingInterval = undefined;
+    }
+  }
+
+  /**
+   * Handle taskComplete event from capture process.
+   * Returns true if the event was handled, false if it was ignored (stale event).
+   */
+  handleTaskComplete(
+    completeType: TaskCompleteType,
+    receivedScenarioId?: string,
+  ): boolean {
+    const current = this.currentScenario;
+
+    // VALIDATE: If scenarioId was provided, it must match the current scenario
+    // Ignore stale taskComplete events from scenarios that already advanced due to errors
+    if (receivedScenarioId && current && receivedScenarioId !== current.id) {
+      return false;
+    }
+
+    this.clearPollingInterval();
+    this._taskCompletion = completeType;
+
+    if (this.currentResolve) {
+      this.currentResolve();
+      this.currentResolve = null;
+    }
+    return true;
+  }
+
+  /**
+   * Handle server error for current scenario.
+   * Marks the scenario as failed and resolves the wait.
+   */
+  handleServerError(scenarioName: string): void {
+    this._failedScenarios.push(scenarioName);
+
+    if (this.currentResolve) {
+      this.currentResolve();
+      this.currentResolve = null;
+    }
+  }
+
+  /**
+   * Check if a scenario has failed.
+   */
+  hasScenarioFailed(scenarioName: string): boolean {
+    return this._failedScenarios.includes(scenarioName);
+  }
+
+  /**
+   * Wait for the current scenario to complete.
+   */
+  waitForCompletion(): Promise<void> {
+    return new Promise<void>((resolve) => {
+      this.currentResolve = resolve;
+    });
+  }
+
+  /**
+   * Advance to the next scenario. Call this after waitForCompletion resolves.
+   */
+  advanceToNext(): void {
+    this.currentIndex++;
+  }
+
+  /**
+   * Reset state for a new scenario iteration.
+   */
+  resetForScenario(): void {
+    this._taskCompletion = undefined;
+  }
+}

package/analyzer-template/project/analyzeAndGatherIndirectEntities.ts

@@ -0,0 +1,131 @@
+import {
+  ProjectAnalyzer,
+  AnalysisContext,
+  analyzeEntities,
+  findOrCreateEntity,
+  gatherEntityMap,
+} from '~codeyam/analyze';
+import { Commit, Entity, Project } from '~codeyam/types';
+import { LazyFileStore } from './LazyFileStore';
+
+interface AnalyzeAndGatherIndirectEntitiesArgs {
+  entities: Entity[];
+  project: Project;
+  projectAnalyzer: ProjectAnalyzer;
+  commit: Commit;
+  context: AnalysisContext;
+  steps: string[];
+  fileStore?: LazyFileStore;
+}
+
+export default async function analyzeAndGatherIndirectEntities({
+  entities,
+  project,
+  projectAnalyzer,
+  commit,
+  context,
+  steps,
+  fileStore,
+}: AnalyzeAndGatherIndirectEntitiesArgs) {
+  const indirectEntities: Entity[] = [];
+
+  for (const entity of entities) {
+    if (!['visual', 'library'].includes(entity.entityType)) continue;
+
+    if (!entity.metadata.importedBy) {
+      continue;
+    }
+
+    for (const path of Object.keys(entity.metadata.importedBy)) {
+      for (const entityName of Object.keys(entity.metadata.importedBy[path])) {
+        // Use fileStore for O(1) lookup when available
+        let importedFile = fileStore
+          ? fileStore.getByPath(path)
+          : project.files.find((f) => f.path === path);
+
+        // Ensure content is loaded for the file (needed for FileAnalyzer)
+        if (importedFile && fileStore && !fileStore.isContentLoaded(path)) {
+          importedFile = await fileStore.ensureContent(path);
+        }
+
+        if (!importedFile) {
+          console.log(`CodeYam Warning: Imported by file not found: ${path}`);
+          continue;
+        }
+
+        const importedFileAnalyzer =
+          projectAnalyzer.getFileAnalyzer(importedFile);
+
+        console.log('CodeYam: Initial indirect entity', path, entityName);
+        const importedBy = await findOrCreateEntity({
+          file: importedFile,
+          commitId: commit.id,
+          branchId: commit.branchId,
+          fileAnalyzer: importedFileAnalyzer,
+          entityName,
+          projectAnalyzer,
+          context,
+        });
+
+        if (!importedBy) {
+          console.log(`CodeYam Warning: Imported by entity not found: ${path}`);
+          continue;
+        }
+
+        context.addEntity(importedBy);
+        indirectEntities.push(importedBy);
+
+        await gatherEntityMap({
+          entity: importedBy,
+          commitId: commit.id,
+          branchId: commit.branchId,
+          context,
+          projectAnalyzer,
+        });
+      }
+    }
+  }
+
+  console.log(
+    `CodeYam: analyzeAndGatherIndirectEntities: Analyzing ${indirectEntities.length} indirect imports of ${entities.length} entities`,
+  );
+
+  await analyzeEntities({
+    rootEntities: indirectEntities,
+    context,
+    project,
+    projectAnalyzer,
+    commit,
+    options: {
+      steps,
+      force: !!process.env.FORCE,
+      forceAll: !!process.env.FORCE_ALL,
+    },
+  });
+
+  for (const entity of entities) {
+    if (!['visual', 'library'].includes(entity.entityType)) continue;
+
+    if (!entity.metadata.importedBy) {
+      continue;
+    }
+
+    for (const path of Object.keys(entity.metadata.importedBy)) {
+      for (const entityName of Object.keys(entity.metadata.importedBy[path])) {
+        const analysis = context.getAnalysis(path, entityName);
+        if (!analysis) continue;
+
+        const newEntity = context.getMutableEntity(path, entityName);
+        if (!newEntity) continue;
+
+        if (entities.findIndex((e) => e.sha === newEntity.sha) === -1) {
+          entities.push(newEntity);
+        }
+      }
+    }
+  }
+
+  return {
+    entities,
+  };
+}

package/analyzer-template/project/analyzeBaselineCommit.ts

@@ -0,0 +1,186 @@
+import { AnalysisContext, ProjectAnalyzer } from '~codeyam/analyze';
+import { Analysis, Commit, Project } from '~codeyam/types';
+import analyzeBranchCommit from './analyzeBranchCommit';
+import {
+  loadAnalysisBranches,
+  upsertAnalysisBranches,
+  updateCommitMetadata,
+  updateProjectMetadata,
+} from '~codeyam/supabase';
+import { awsLog } from '~codeyam/utils';
+import { LazyFileStore } from './LazyFileStore';
+
+export interface AnalyzeBaselineCommitArgs {
+  project: Project;
+  projectAnalyzer: ProjectAnalyzer;
+  branchCommit: Commit;
+  analysisIds?: string[];
+  scenarioIds?: string[];
+  context: AnalysisContext;
+  steps?: string[];
+  desiredAnalyses?: Analysis[];
+  force?: boolean;
+  fileStore?: LazyFileStore;
+}
+
+export interface AnalyzeBaselineCommitResult {
+  projectAnalyzer: ProjectAnalyzer;
+  readyToBeCaptured: Analysis[];
+  context: AnalysisContext;
+}
+
+export default async function analyzeBaselineCommit({
+  project,
+  projectAnalyzer,
+  branchCommit,
+  analysisIds,
+  scenarioIds,
+  context,
+  steps,
+  desiredAnalyses = [],
+  force = false,
+  fileStore,
+}: AnalyzeBaselineCommitArgs): Promise<AnalyzeBaselineCommitResult> {
+  awsLog('CodeYam: Analyze - branchCommit (baseline) path', {
+    analysisIds,
+  });
+
+  if (project.metadata?.baselineCompleted && !analysisIds && !force) {
+    throw new Error(
+      'CodeYam Error: Cannot analyze branch commits after baseline is completed',
+    );
+  }
+
+  const primaryBranchId = project.branches.find((b) => b.primary)?.id;
+  if (!primaryBranchId) {
+    throw new Error(`Primary branch not found for project ${project.slug}`);
+  }
+
+  const { entities } = await analyzeBranchCommit({
+    project,
+    projectAnalyzer,
+    branchCommit,
+    analysisIds,
+    context,
+    steps,
+    fileStore,
+  });
+
+  let analysisBranches = Object.values(context.getAnalysisMap())
+    .flatMap((a) => Object.values(a))
+    .map((a) => ({
+      branchId: primaryBranchId,
+      entitySha: a.entitySha,
+      analysisId: a.id,
+      active: true,
+    }));
+
+  const existingAnalysisBranches = await loadAnalysisBranches({
+    projectId: project.id,
+    branchId: primaryBranchId,
+    analysisIds: analysisBranches.map((a) => a.analysisId).filter(Boolean),
+  });
+
+  analysisBranches = analysisBranches.filter(
+    (a) =>
+      a.analysisId &&
+      !existingAnalysisBranches.find((ab) => ab.analysisId === a.analysisId),
+  );
+
+  await upsertAnalysisBranches(analysisBranches);
+
+  const now = new Date().toISOString();
+  branchCommit.metadata = await updateCommitMetadata({
+    commitId: branchCommit.id,
+    runStatusUpdate: {
+      analysisCompletedAt: now,
+    },
+    metadataUpdate: {
+      entitiesAnalyzedAt: now,
+    },
+    archiveCurrentRun: true,
+  });
+
+  awsLog(`CodeYam: Marking baseline completed for project ${project.slug}`);
+
+  project.metadata = await updateProjectMetadata({
+    projectId: project.id,
+    metadataUpdate: {
+      baselineCompleted: true,
+    },
+  });
+
+  const readyToBeCaptured: Analysis[] = Object.values(context.getAnalysisMap())
+    .flatMap(Object.values)
+    .filter((analysis) => {
+      if (analysisIds) {
+        return analysisIds.includes(analysis.id);
+      }
+
+      if (scenarioIds) {
+        return analysis.scenarios?.some((s: { id?: string }) =>
+          scenarioIds.includes(s.id),
+        );
+      }
+
+      return analysis.status.readyToBeCaptured;
+    });
+
+  if (analysisIds || scenarioIds) {
+    if (readyToBeCaptured.length === 0) {
+      awsLog(
+        `CodeYam: branchCommit path complete with specified analysisIds but no analyses ready to be captured...`,
+        {
+          analysisIds,
+          scenarioIds,
+          desiredAnalyses: desiredAnalyses.map((a) => ({
+            id: a.id,
+            filePath: a.filePath,
+            entityName: a.entityName,
+          })),
+          allAnalyses: Object.keys(context.getAnalysisMap())
+            .filter(
+              (filePath) =>
+                desiredAnalyses.length === 0 ||
+                desiredAnalyses.some((a) => a.filePath === filePath),
+            )
+            .flatMap((filePath) =>
+              Object.keys(context.getAnalysisMap()[filePath] || {})
+                .filter(
+                  (entityName) =>
+                    desiredAnalyses.length === 0 ||
+                    desiredAnalyses.some(
+                      (a) =>
+                        a.filePath === filePath && a.entityName === entityName,
+                    ),
+                )
+                .flatMap((entityName) => {
+                  const analysis =
+                    context.getAnalysisMap()[filePath][entityName];
+                  return {
+                    id: analysis.id,
+                    filePath,
+                    entityName,
+                    readyToBeCaptured: analysis.status.readyToBeCaptured,
+                  };
+                }),
+            ),
+        },
+      );
+      await new Promise((resolve) => setTimeout(resolve, 1000));
+      throw new Error(
+        `CodeYam Error: No analyses ready to be captured for the specified analysisIds or scenarioIds: ${analysisIds} or ${scenarioIds}`,
+      );
+    }
+  } else {
+    awsLog(
+      `CodeYam: branchCommit path complete, ${readyToBeCaptured.length} analyses ready to be captured! Exiting...`,
+    );
+  }
+
+  return {
+    projectAnalyzer,
+    readyToBeCaptured,
+    context,
+  };
+}

package/analyzer-template/project/analyzeBranchCommit.ts

@@ -0,0 +1,256 @@
+import { Commit, Project } from '~codeyam/types';
+import createEntitiesAndSortFiles, {
+  EntityWithImportCount,
+} from './createEntitiesAndSortFiles';
+import {
+  AnalysisContext,
+  analyzeEntities,
+  ProjectAnalyzer,
+} from '~codeyam/analyze';
+import {
+  loadAnalyses,
+  loadFile,
+  updateCommitMetadata,
+  updateProjectMetadata,
+} from '~codeyam/supabase';
+import checkApprovedPath from './checkApprovedPath';
+import { LazyFileStore } from './LazyFileStore';
+
+interface AnalyzeBranchCommitArgs {
+  project: Project;
+  projectAnalyzer: ProjectAnalyzer;
+  branchCommit: Commit;
+  analysisIds?: string[];
+  context: AnalysisContext;
+  steps: string[];
+  fileStore?: LazyFileStore;
+}
+
+export default async function analyzeBranchCommit({
+  project,
+  projectAnalyzer,
+  branchCommit,
+  analysisIds,
+  context,
+  steps,
+  fileStore,
+}: AnalyzeBranchCommitArgs) {
+  console.log('CodeYam: Starting Branch Commit Analysis', branchCommit.sha, {
+    analysisIds,
+  });
+  const shouldUpdateBaselineStatus = !analysisIds || analysisIds.length === 0;
+
+  if (shouldUpdateBaselineStatus) {
+    branchCommit.metadata = await updateCommitMetadata({
+      commitId: branchCommit.id,
+      runStatusUpdate: {
+        createdAt: new Date().toISOString(),
+        fileCount: fileStore ? fileStore.size : project.files.length,
+        filesCompleted: 0,
+        entityCount: 0,
+        analysesCompleted: 0,
+      },
+    });
+
+    project.metadata = await updateProjectMetadata({
+      projectId: project.id,
+      metadataUpdate: {
+        baselineSha: branchCommit.sha,
+      },
+    });
+  }
+  // Step 1: Create the sorted entities and map
+  const sortedEntitiesAndMap = await createEntitiesAndSortFiles({
+    project,
+    branch: branchCommit.branch,
+    projectAnalyzer,
+    branchCommit,
+    fileStore,
+  });
+
+  // Step 2: Initialize context with all entities from the map
+  for (const fileEntities of Object.values(sortedEntitiesAndMap.entityMap)) {
+    for (const [entityName, entity] of Object.entries(fileEntities)) {
+      if (entity.name !== entityName) {
+        // for entities that were double-registered under an 'alias', preserve that alias
+        context.addEntity(entity, entityName);
+      } else {
+        context.addEntity(entity);
+      }
+    }
+  }
+
+  // Step 3: Determine which entities to analyze
+  let entitiesToAnalyze: EntityWithImportCount[] = [];
+
+  if (analysisIds) {
+    // If we have analysis IDs, load those specific entities
+    const analyses = await loadAnalyses({
+      projectId: project.id,
+      ids: analysisIds,
+    });
+
+    if (!analyses || analyses.length === 0) {
+      console.log(
+        'CodeYam Error: Analyses not found in analyzeBranchCommit for ids',
+        analysisIds,
+      );
+      throw new Error('Specified analyses not found');
+    }
+
+    for (const analysis of analyses) {
+      const entity = analysis.entity;
+      // Use fileStore for O(1) lookup when available
+      const file = fileStore
+        ? fileStore.getByPath(entity.filePath)
+        : project.files.find((f) => f.path === entity.filePath);
+      const fileAnalyzer = projectAnalyzer.getFileAnalyzer(file);
+      entity.code = fileAnalyzer.getEntityCode(entity.name);
+      entitiesToAnalyze.push(entity);
+    }
+
+    console.log(
+      'CodeYam Analyzing specified entities',
+      entitiesToAnalyze.map((e) => ({ path: e.filePath, name: e.name })),
+    );
+  } else {
+    // Otherwise, use the sorted entities
+    entitiesToAnalyze = sortedEntitiesAndMap.sortedEntities;
+
+    // Useful for running analysis for specific entities (if already analyzed use analysisIds)
+    // entitiesToAnalyze = Object.values(sortedEntitiesAndMap.sortedEntities)
+    //   .flat()
+    //   .filter(
+    //     (e) =>
+    //       e.sha ===
+    //       'aca34a70dfbc6ad07d9abdacb35404f4493bed44790aea6d533a56a5274f54b1',
+    //   );
+
+    console.log('Analyzing entities', entitiesToAnalyze.length);
+  }
+
+  // Step 4: Filter out entities from unapproved paths and already completed
+  const filteredEntities = [];
+  for (const entity of entitiesToAnalyze) {
+    // if (
+    //   entity.filePath !==
+    //     'dashboard/app/routes/dashboard.projects.$id.(home).tsx' ||
+    //     !['HomeContent', 'default'].includes(entity.name)
+
+    //   // entity.filePath !==
+    //   //   'dashboard/app/routes/dashboard.projects.$id.recent.($branchId).tsx' ||
+    //   //   !['RecentChangesPage', 'default'].includes(entity.name)
+    // )
+    //   continue;
+
+    // Skip unapproved paths
+    if (!checkApprovedPath({ project, path: entity.filePath })) {
+      console.log('CodeYam: Skipping unapproved path', entity.filePath);
+      continue;
+    }
+
+    // Skip already completed entities
+    if (entity.completed) {
+      console.log(
+        'Entity already completed',
+        entity.filePath,
+        entity.name,
+        entity.completed ? 'Completed' : 'Not Completed',
+      );
+      continue;
+    }
+
+    // Find the entity's file - use fileStore for O(1) lookup when available
+    let file = fileStore
+      ? fileStore.getById(entity.fileId)
+      : project.files.find((f) => f.id === entity.fileId);
+    if (!file) {
+      try {
+        file = await loadFile({ id: entity.fileId });
+        if (!file) {
+          console.log(
+            'CodeYam Error: Branch commit prep file not found for entity',
+            JSON.stringify(
+              {
+                entitySha: entity.sha,
+                entityFileId: entity.fileId,
+                entityFilePath: entity.filePath,
+                entityName: entity.name,
+              },
+              null,
+              2,
+            ),
+          );
+          continue; // Skip this entity rather than failing the whole batch
+        }
+      } catch (error) {
+        console.log(
+          'Error loading file for entity',
+          entity.filePath,
+          entity.name,
+          error,
+        );
+        continue;
+      }
+    }
+
+    // Associate the file with the entity for later use
+    entity.file = file;
+
+    // Mark entity as started
+    entity.started = new Date().toISOString();
+
+    filteredEntities.push(entity);
+  }
+
+  if (shouldUpdateBaselineStatus) {
+    // now we know the total number of entities, we can update the baseline status
+    branchCommit.metadata = await updateCommitMetadata({
+      commitId: branchCommit.id,
+      runStatusUpdate: {
+        entityCount: filteredEntities.length,
+      },
+    });
+  }
+
+  // Step 5: Analyze the filtered entities
+  if (filteredEntities.length > 0) {
+    try {
+      console.log(
+        `CodeYam: analyzeBranchCommit: Starting analysis of ${filteredEntities.length} root entities`,
+      );
+
+      await analyzeEntities({
+        rootEntities: filteredEntities,
+        context,
+        project,
+        projectAnalyzer,
+        commit: branchCommit,
+        options: {
+          branchCommitSha: branchCommit.sha,
+          steps,
+          force: !!process.env.FORCE,
+          forceAll: !!process.env.FORCE_ALL,
+        },
+      });
+    } catch (error) {
+      console.log(`CodeyYam Error: Baseline ${branchCommit.sha} FAILED`, error);
+      throw error;
+    }
+
+    // Mark all entities as completed
+    for (const entity of filteredEntities) {
+      entity.completed = new Date().toISOString();
+    }
+  }
+
+  // Final log for branch commit analysis
+  if (!analysisIds) {
+    console.log('Completed Branch Commit Analysis', branchCommit.sha);
+  }
+
+  // Return all entities that were processed
+  return {
+    entities: filteredEntities,
+  };
+}