@codeyam/codeyam-cli 0.0.0 → 0.1.0-staging.036391e

package/analyzer-template/project/LazyFileStore.ts

@@ -0,0 +1,537 @@
+import { Entity, File, Project } from '~codeyam/types';
+import { ProjectAnalyzer } from '~codeyam/analyze';
+import {
+  loadFiles,
+  loadEntities,
+  saveFiles,
+  generateSha,
+} from '~codeyam/database';
+
+export interface LazyFileStoreOptions {
+  project: Project;
+  projectAnalyzer: ProjectAnalyzer;
+  requestedEntityShas?: string[];
+  silent?: boolean;
+}
+
+interface SourceFileInfo {
+  path: string;
+  absolutePath: string;
+  content: string;
+}
+
+/**
+ * LazyFileStore provides O(1) file lookups with on-demand content/entity loading.
+ *
+ * Design Principles:
+ * 1. File metadata (id, path, name) is always loaded upfront - it's lightweight
+ * 2. File content is loaded on-demand from ProjectAnalyzer (AST) or DB
+ * 3. Entities are extracted on-demand from FileAnalyzer
+ * 4. All loaded data is cached for subsequent access
+ * 5. Provides both sync (cached) and async (load-if-needed) access patterns
+ */
+export class LazyFileStore {
+  private project: Project;
+  private projectAnalyzer: ProjectAnalyzer;
+  private silent: boolean;
+
+  // O(1) lookup indexes
+  private filesByPath: Map<string, File> = new Map();
+  private filesById: Map<string, File> = new Map();
+
+  // Track what's been loaded
+  private contentLoaded: Set<string> = new Set();
+  private entitiesLoaded: Set<string> = new Set();
+
+  // Prevent duplicate concurrent loads
+  private loadingContent: Map<string, Promise<File | undefined>> = new Map();
+  private loadingEntities: Map<string, Promise<File | undefined>> = new Map();
+
+  // Source file cache from ProjectAnalyzer (maps path to content)
+  private sourceFileMap: Map<string, SourceFileInfo> = new Map();
+
+  private constructor(options: LazyFileStoreOptions) {
+    this.project = options.project;
+    this.projectAnalyzer = options.projectAnalyzer;
+    this.silent = options.silent ?? false;
+  }
+
+  /**
+   * Factory method to create and initialize a LazyFileStore
+   */
+  static async create(options: LazyFileStoreOptions): Promise<LazyFileStore> {
+    const store = new LazyFileStore(options);
+    await store.initialize(options.requestedEntityShas);
+    return store;
+  }
+
+  /**
+   * Initialize the store with file metadata from the repository.
+   * This is lightweight - only metadata, no content or entities.
+   */
+  private async initialize(requestedEntityShas?: string[]): Promise<void> {
+    const { project, projectAnalyzer } = this;
+    const startTime = Date.now();
+
+    // Get all source files from TypeScript (enumeration only, content stored lazily)
+    const sourceFiles = projectAnalyzer.getAllSourceFiles();
+    if (!this.silent) {
+      console.log(
+        `CodeYam: LazyFileStore enumerating ${sourceFiles.length} source files for ${project.slug} (${Date.now() - startTime}ms)`,
+      );
+    }
+
+    // Cache source file content references
+    for (const sf of sourceFiles) {
+      this.sourceFileMap.set(sf.path, sf);
+    }
+
+    // Filter valid source files (same logic as getFilesWithEntitiesFromRepo)
+    const validSourceFilePaths = sourceFiles
+      .filter((sf) => {
+        if (sf.absolutePath.indexOf('node_modules') > -1) return false;
+        if (sf.content.length === 0) return false;
+        if (project.metadata?.approvedPaths) {
+          const approved = project.metadata.approvedPaths.find((path) =>
+            sf.absolutePath.includes(path),
+          );
+          if (!approved) return false;
+        }
+        return true;
+      })
+      .map((sf) => sf.path);
+
+    if (!this.silent) {
+      console.log(
+        `CodeYam: LazyFileStore filtered to ${validSourceFilePaths.length} valid files (${Date.now() - startTime}ms)`,
+      );
+    }
+
+    // Separate files that exist in project vs need to be loaded from DB
+    const existingFilesByPath = new Map<string, File>();
+    for (const file of project.files ?? []) {
+      existingFilesByPath.set(file.path, file);
+    }
+
+    const pathsNeedingDbLookup = validSourceFilePaths.filter(
+      (path) => !existingFilesByPath.has(path),
+    );
+
+    // Index existing files (without loading content)
+    for (const path of validSourceFilePaths) {
+      const existingFile = existingFilesByPath.get(path);
+      if (existingFile) {
+        // Don't load content yet - just index the metadata
+        this.filesByPath.set(existingFile.path, existingFile);
+        this.filesById.set(existingFile.id, existingFile);
+      }
+    }
+
+    if (!this.silent) {
+      console.log(
+        `CodeYam: LazyFileStore indexed ${this.filesByPath.size} existing files, ${pathsNeedingDbLookup.length} need DB lookup (${Date.now() - startTime}ms)`,
+      );
+    }
+
+    // Bulk load missing files from DB (metadata only)
+    if (pathsNeedingDbLookup.length > 0) {
+      const filesFromDb = await loadFiles({
+        projectId: project.id,
+        filePaths: pathsNeedingDbLookup,
+      });
+
+      if (filesFromDb) {
+        for (const file of filesFromDb) {
+          this.filesByPath.set(file.path, file);
+          this.filesById.set(file.id, file);
+        }
+      }
+
+      if (!this.silent) {
+        console.log(
+          `CodeYam: LazyFileStore loaded ${filesFromDb?.length ?? 0} files from DB (${Date.now() - startTime}ms)`,
+        );
+      }
+    }
+
+    // Identify files that still need to be created in DB
+    const filesToCreate: Array<{
+      name: string;
+      path: string;
+      project_id: string;
+    }> = [];
+
+    for (const path of pathsNeedingDbLookup) {
+      if (!this.filesByPath.has(path)) {
+        const name = path.split('/').pop()!;
+        filesToCreate.push({
+          name,
+          path,
+          project_id: project.id.toString(),
+        });
+      }
+    }
+
+    // Bulk save new files to DB
+    if (filesToCreate.length > 0) {
+      if (!this.silent) {
+        console.log(
+          `CodeYam: LazyFileStore creating ${filesToCreate.length} new files in DB (${Date.now() - startTime}ms)`,
+        );
+      }
+      const newFiles = await saveFiles(filesToCreate);
+      for (const newFile of newFiles) {
+        this.filesByPath.set(newFile.path, newFile);
+        this.filesById.set(newFile.id, newFile);
+      }
+    }
+
+    // If specific entity SHAs were requested, load them from the database
+    if (requestedEntityShas && requestedEntityShas.length > 0) {
+      if (!this.silent) {
+        console.log(
+          `CodeYam: LazyFileStore loading ${requestedEntityShas.length} requested entities from DB (${Date.now() - startTime}ms)`,
+        );
+      }
+      await this.loadEntitiesFromDb(requestedEntityShas);
+    }
+
+    if (!this.silent) {
+      console.log(
+        `CodeYam: LazyFileStore ready with ${this.filesByPath.size} files (${Date.now() - startTime}ms total)`,
+      );
+    }
+  }
+
+  // ============ O(1) LOOKUP METHODS ============
+
+  /**
+   * Get file by path - O(1) lookup
+   */
+  getByPath(path: string): File | undefined {
+    return this.filesByPath.get(path);
+  }
+
+  /**
+   * Get file by ID - O(1) lookup
+   */
+  getById(id: string): File | undefined {
+    return this.filesById.get(id);
+  }
+
+  /**
+   * Check if a file exists
+   */
+  has(path: string): boolean {
+    return this.filesByPath.has(path);
+  }
+
+  /**
+   * Get all file paths (for iteration)
+   */
+  paths(): IterableIterator<string> {
+    return this.filesByPath.keys();
+  }
+
+  /**
+   * Get all files as values (for iteration)
+   */
+  values(): IterableIterator<File> {
+    return this.filesByPath.values();
+  }
+
+  /**
+   * Get count of files
+   */
+  get size(): number {
+    return this.filesByPath.size;
+  }
+
+  // ============ LAZY LOADING METHODS ============
+
+  /**
+   * Check if content has been loaded for a file
+   */
+  isContentLoaded(path: string): boolean {
+    return this.contentLoaded.has(path);
+  }
+
+  /**
+   * Check if entities have been loaded for a file
+   */
+  isEntitiesLoaded(path: string): boolean {
+    return this.entitiesLoaded.has(path);
+  }
+
+  /**
+   * Ensure file content is loaded. Returns the file with content populated.
+   * Content comes from ProjectAnalyzer (AST) which is always authoritative.
+   */
+  async ensureContent(path: string): Promise<File | undefined> {
+    const file = this.filesByPath.get(path);
+    if (!file) return undefined;
+
+    // Already loaded
+    if (this.contentLoaded.has(path)) {
+      return file;
+    }
+
+    // Check if already loading (prevent duplicate concurrent loads)
+    const existingPromise = this.loadingContent.get(path);
+    if (existingPromise !== undefined) {
+      return existingPromise;
+    }
+
+    // Start loading
+    const loadPromise = this.doLoadContent(path, file);
+    this.loadingContent.set(path, loadPromise);
+
+    try {
+      return await loadPromise;
+    } finally {
+      this.loadingContent.delete(path);
+    }
+  }
+
+  private doLoadContent(path: string, file: File): Promise<File | undefined> {
+    const sourceFile = this.sourceFileMap.get(path);
+    if (sourceFile) {
+      file.content = sourceFile.content;
+      this.contentLoaded.add(path);
+    } else {
+      // Fallback: try to get from TypeScript program directly
+      const tsSourceFile = this.projectAnalyzer.getSourceFile(path);
+      if (tsSourceFile) {
+        file.content = tsSourceFile.getFullText();
+        this.contentLoaded.add(path);
+      }
+    }
+
+    return Promise.resolve(file);
+  }
+
+  /**
+   * Ensure entities are loaded for a file.
+   * Entities are extracted from the FileAnalyzer on demand.
+   */
+  async ensureEntities(path: string): Promise<File | undefined> {
+    const file = this.filesByPath.get(path);
+    if (!file) return undefined;
+
+    // Already loaded
+    if (this.entitiesLoaded.has(path)) {
+      return file;
+    }
+
+    // Check if already loading (prevent duplicate concurrent loads)
+    const existingPromise = this.loadingEntities.get(path);
+    if (existingPromise !== undefined) {
+      return existingPromise;
+    }
+
+    // Start loading
+    const loadPromise = this.doLoadEntities(path, file);
+    this.loadingEntities.set(path, loadPromise);
+
+    try {
+      return await loadPromise;
+    } finally {
+      this.loadingEntities.delete(path);
+    }
+  }
+
+  private async doLoadEntities(
+    path: string,
+    file: File,
+  ): Promise<File | undefined> {
+    // Ensure content is loaded first (needed for entity extraction)
+    await this.ensureContent(path);
+
+    const fileAnalyzer = this.projectAnalyzer.getFileAnalyzer(file);
+    if (!fileAnalyzer) {
+      file.entities = [];
+      this.entitiesLoaded.add(path);
+      return file;
+    }
+
+    const allEntityNodes = fileAnalyzer.getAllEntityNodesOnlyAliases();
+
+    file.entities = [];
+    for (const entityName in allEntityNodes) {
+      const entityType = fileAnalyzer.getEntityType(entityName);
+      const entityCode = fileAnalyzer.getEntityCode(entityName);
+      const sha = generateSha(file.path, entityName, entityCode);
+
+      file.entities.push({
+        name: entityName,
+        entityType,
+        code: entityCode,
+        sha,
+        fileId: file.id,
+        projectId: this.project.id,
+        filePath: file.path,
+        metadata: {
+          notExported: !fileAnalyzer.isExported(entityName),
+          namedExport: !fileAnalyzer.isDefaultExport(entityName),
+        },
+      });
+    }
+
+    this.entitiesLoaded.add(path);
+    return file;
+  }
+
+  /**
+   * Batch load content for multiple files (optimization for sequential processing)
+   */
+  async ensureContentBatch(paths: string[]): Promise<void> {
+    await Promise.all(paths.map((path) => this.ensureContent(path)));
+  }
+
+  /**
+   * Batch load entities for multiple files
+   */
+  async ensureEntitiesBatch(paths: string[]): Promise<void> {
+    await Promise.all(paths.map((path) => this.ensureEntities(path)));
+  }
+
+  // ============ ENTITY LOADING FROM DB ============
+
+  /**
+   * Load specific entities by SHA from the database.
+   * Used when file content has changed but we need old entity versions.
+   */
+  async loadEntitiesFromDb(shas: string[]): Promise<Entity[]> {
+    // First, check which SHAs we already have
+    const existingShas = new Set<string>();
+    for (const file of this.filesByPath.values()) {
+      if (file.entities) {
+        for (const entity of file.entities) {
+          existingShas.add(entity.sha);
+        }
+      }
+    }
+
+    const missingShas = shas.filter((sha) => !existingShas.has(sha));
+    if (missingShas.length === 0) {
+      return [];
+    }
+
+    if (!this.silent) {
+      console.log(
+        `CodeYam: Loading ${missingShas.length} requested entities from database that don't match current file content`,
+      );
+    }
+
+    const dbEntities = await loadEntities({
+      projectId: this.project.id,
+      shas: missingShas,
+    });
+
+    if (!dbEntities || dbEntities.length === 0) {
+      return [];
+    }
+
+    // Group entities by file path
+    const entitiesByFilePath = new Map<string, Entity[]>();
+    for (const entity of dbEntities) {
+      if (!entity.filePath) continue;
+      if (!entitiesByFilePath.has(entity.filePath)) {
+        entitiesByFilePath.set(entity.filePath, []);
+      }
+      entitiesByFilePath.get(entity.filePath)!.push(entity);
+    }
+
+    // Add entities to their files
+    for (const [filePath, entities] of entitiesByFilePath.entries()) {
+      let file = this.filesByPath.get(filePath);
+
+      // If file doesn't exist in store, we may need to load it
+      if (!file) {
+        const filesFromDb = await loadFiles({
+          projectId: this.project.id,
+          filePaths: [filePath],
+        });
+
+        if (filesFromDb && filesFromDb.length > 0) {
+          file = filesFromDb[0];
+          const sourceFile = this.sourceFileMap.get(filePath);
+          if (sourceFile) {
+            file.content = sourceFile.content;
+            this.contentLoaded.add(filePath);
+          }
+          file.entities = [];
+          this.filesByPath.set(file.path, file);
+          this.filesById.set(file.id, file);
+        }
+      }
+
+      if (file) {
+        // Add the DB entities to the file
+        file.entities = file.entities || [];
+        for (const entity of entities) {
+          // Only add if not already present
+          if (!file.entities.some((e) => e.sha === entity.sha)) {
+            file.entities.push(entity);
+          }
+        }
+      }
+    }
+
+    return dbEntities;
+  }
+
+  // ============ COMPATIBILITY LAYER ============
+
+  /**
+   * Get all files as an array (for backward compatibility)
+   * WARNING: This defeats lazy loading - use sparingly during migration
+   */
+  toArray(): File[] {
+    return Array.from(this.filesByPath.values());
+  }
+
+  /**
+   * Get all files with their entities loaded
+   * This is the memory-intensive operation - use only when truly needed
+   */
+  async toArrayWithEntities(): Promise<File[]> {
+    const paths = Array.from(this.filesByPath.keys());
+    await this.ensureEntitiesBatch(paths);
+    return this.toArray();
+  }
+
+  /**
+   * Iterate over files without loading content/entities
+   */
+  *[Symbol.iterator](): Iterator<File> {
+    for (const file of this.filesByPath.values()) {
+      yield file;
+    }
+  }
+
+  /**
+   * forEach with index (for migration from array-based iteration)
+   */
+  forEach(callback: (file: File, index: number) => void): void {
+    let index = 0;
+    for (const file of this.filesByPath.values()) {
+      callback(file, index++);
+    }
+  }
+
+  /**
+   * Get stats about loaded state
+   */
+  getStats(): {
+    totalFiles: number;
+    contentLoaded: number;
+    entitiesLoaded: number;
+  } {
+    return {
+      totalFiles: this.filesByPath.size,
+      contentLoaded: this.contentLoaded.size,
+      entitiesLoaded: this.entitiesLoaded.size,
+    };
+  }
+}

package/analyzer-template/project/ScenarioIterator.ts

@@ -0,0 +1,133 @@
+import { TaskCompleteType } from './controller/startController';
+
+export interface ScenarioWithId {
+  id: string;
+  name: string;
+}
+
+/**
+ * Manages scenario iteration state for multi-scenario capture.
+ *
+ * This class handles the coordination between:
+ * - Server errors (which advance to the next scenario)
+ * - taskComplete events from capture (which should only advance if they match the current scenario)
+ *
+ * The key invariant is that stale taskComplete events (from scenarios that already
+ * advanced due to errors) are ignored to prevent skipping scenarios.
+ */
+export class ScenarioIterator {
+  private scenarios: ScenarioWithId[];
+  private currentIndex = 0;
+  private currentResolve: (() => void) | null = null;
+  private _taskCompletion: TaskCompleteType | undefined;
+  private _failedScenarios: string[] = [];
+  private pollingInterval: NodeJS.Timeout | undefined;
+
+  constructor(scenarios: ScenarioWithId[]) {
+    this.scenarios = scenarios;
+  }
+
+  get currentScenario(): ScenarioWithId | undefined {
+    return this.scenarios[this.currentIndex];
+  }
+
+  get currentScenarioIndex(): number {
+    return this.currentIndex;
+  }
+
+  get taskCompletion(): TaskCompleteType | undefined {
+    return this._taskCompletion;
+  }
+
+  get failedScenarios(): string[] {
+    return this._failedScenarios;
+  }
+
+  get isLastScenario(): boolean {
+    return this.currentIndex === this.scenarios.length - 1;
+  }
+
+  get totalScenarios(): number {
+    return this.scenarios.length;
+  }
+
+  setPollingInterval(interval: NodeJS.Timeout): void {
+    this.pollingInterval = interval;
+  }
+
+  clearPollingInterval(): void {
+    if (this.pollingInterval) {
+      clearInterval(this.pollingInterval);
+      this.pollingInterval = undefined;
+    }
+  }
+
+  /**
+   * Handle taskComplete event from capture process.
+   * Returns true if the event was handled, false if it was ignored (stale event).
+   */
+  handleTaskComplete(
+    completeType: TaskCompleteType,
+    receivedScenarioId?: string,
+  ): boolean {
+    const current = this.currentScenario;
+
+    // VALIDATE: If scenarioId was provided, it must match the current scenario
+    // Ignore stale taskComplete events from scenarios that already advanced due to errors
+    if (receivedScenarioId && current && receivedScenarioId !== current.id) {
+      return false;
+    }
+
+    this.clearPollingInterval();
+    this._taskCompletion = completeType;
+
+    if (this.currentResolve) {
+      this.currentResolve();
+      this.currentResolve = null;
+    }
+    return true;
+  }
+
+  /**
+   * Handle server error for current scenario.
+   * Marks the scenario as failed and resolves the wait.
+   */
+  handleServerError(scenarioName: string): void {
+    this._failedScenarios.push(scenarioName);
+
+    if (this.currentResolve) {
+      this.currentResolve();
+      this.currentResolve = null;
+    }
+  }
+
+  /**
+   * Check if a scenario has failed.
+   */
+  hasScenarioFailed(scenarioName: string): boolean {
+    return this._failedScenarios.includes(scenarioName);
+  }
+
+  /**
+   * Wait for the current scenario to complete.
+   */
+  waitForCompletion(): Promise<void> {
+    return new Promise<void>((resolve) => {
+      this.currentResolve = resolve;
+    });
+  }
+
+  /**
+   * Advance to the next scenario. Call this after waitForCompletion resolves.
+   */
+  advanceToNext(): void {
+    this.currentIndex++;
+  }
+
+  /**
+   * Reset state for a new scenario iteration.
+   */
+  resetForScenario(): void {
+    this._taskCompletion = undefined;
+  }
+}