@codeyam/codeyam-cli 0.1.0-bleeding-edge.8afd3ee

package/analyzer-template/project/orchestrateCapture/SequentialCaptureTaskRunner.ts

@@ -0,0 +1,380 @@
+import type {
+  CaptureTaskRunner,
+  QueueStats,
+  TaskRunnerConfig,
+} from './taskRunner';
+import type { StartArgs } from '../start';
+import type { UniversalMock } from '~codeyam/utils';
+import type { EnvironmentVariable } from '~codeyam/types';
+import spawnTaskProcess from './spawnTaskProcess';
+import buildCliArgs from '../buildCliArgs';
+import * as path from 'path';
+import * as fs from 'fs';
+import {
+  rsyncCopy,
+  sanitizeEnvFiles,
+  applyUniversalMocks,
+} from '~codeyam/utils/server';
+
+/**
+ * Sequential capture task runner for local development and debugging.
+ *
+ * Processes batches sequentially with paired start+capture processes.
+ * Much simpler than AWS runner - easier to debug and understand.
+ *
+ * Architecture:
+ * - Duplicates both codeyam/ and project/ into capturer-1/ during creation (satisfies hardcoded '../project' refs)
+ * - start.ts polls controller.hasCompletedRepoCheckout() before creating runner to avoid race
+ * - Creates nested capturer directory (parent/capturer-1/{codeyam,project})
+ * - For EACH batch:
+ *   - Archives previous log.txt to log-{timestamp}.txt
+ *   - Spawns BOTH start.js and capture.js with cwd=capturer-1/codeyam
+ *   - start.js gets ANALYSIS_IDS env var and fast=true (skip checkout/build)
+ *   - capture.js polls log for controller port, processes all analyses in batch
+ *   - Both processes cleaned up when batch completes (or times out)
+ */
+
+export interface SequentialCaptureTaskRunnerContext {
+  absoluteProjectRootPath: string;
+  absoluteTempPath?: string;
+  startArgs: StartArgs;
+  /** Universal mocks to apply to the capturer's project directory */
+  universalMocks?: UniversalMock[];
+  /** Environment variables to pass to the spawned processes */
+  environmentVariables?: EnvironmentVariable[];
+}
+
+export default class SequentialCaptureTaskRunner implements CaptureTaskRunner {
+  private queue: string[][] = [];
+  private batchCompletedCallback?: (analysisIds: string[]) => void;
+  private isProcessing = false;
+  private isCleanedUp = false;
+  private processingLoop: Promise<void> | null = null;
+
+  private config: TaskRunnerConfig;
+  private context: SequentialCaptureTaskRunnerContext;
+  private capturerPath: string;
+
+  private constructor(
+    config: TaskRunnerConfig,
+    context: SequentialCaptureTaskRunnerContext,
+    capturerPath: string,
+  ) {
+    this.config = config;
+    this.context = context;
+    this.capturerPath = capturerPath;
+  }
+
+  /**
+   * Create and initialize a sequential task runner.
+   */
+  static async create(
+    config: TaskRunnerConfig,
+    context: SequentialCaptureTaskRunnerContext,
+  ): Promise<SequentialCaptureTaskRunner> {
+    console.log('CodeYam DEBUG: Creating SequentialCaptureTaskRunner...');
+
+    // Validate that cwd and project path share parent directory
+    // Use realpathSync to resolve symlinks (e.g., /tmp -> /private/tmp on macOS)
+    const parentOfCodeyam = fs.realpathSync(path.dirname(process.cwd()));
+    const parentOfProject = fs.realpathSync(
+      path.dirname(context.absoluteProjectRootPath),
+    );
+    if (parentOfCodeyam !== parentOfProject) {
+      throw new Error(
+        `CodeYam: CWD (${process.cwd()}) and project path (${context.absoluteProjectRootPath}) ` +
+          `must share parent directory for sequential runner. ` +
+          `Got parents: ${parentOfCodeyam} vs ${parentOfProject}`,
+      );
+    }
+
+    // Derive paths
+    const parentDir = parentOfCodeyam;
+    const capturerPath = path.normalize(path.join(parentDir, 'capturer-1'));
+
+    console.log(`CodeYam: Sequential runner using capturer at ${capturerPath}`);
+
+    // Create capturer base directory
+    if (!fs.existsSync(capturerPath)) {
+      console.log(
+        `CodeYam DEBUG: Creating capturer directory at ${capturerPath}`,
+      );
+      fs.mkdirSync(capturerPath, { recursive: true });
+    }
+
+    // Copy both codeyam/ and project/ into capturer-1/
+    // start.ts waits for repo checkout to complete before creating this runner
+    console.log('CodeYam: Copying codeyam/ and project/ to capturer-1/...');
+    await rsyncCopy({
+      sourcePath: parentDir,
+      destinationPath: capturerPath,
+      excludes: ['codeyam/log.txt', 'capturer-*'],
+      silent: false,
+    });
+    console.log('CodeYam DEBUG: Directory copy completed');
+
+    // Sanitize env files in capturer project (replaces real values with mock placeholders)
+    const capturerProjectPath = path.join(capturerPath, 'project');
+    console.log('CodeYam: Sanitizing env files in capturer project...');
+    console.log(`CodeYam DEBUG: capturerProjectPath = ${capturerProjectPath}`);
+    const sanitizeResult = sanitizeEnvFiles({
+      projectPath: capturerProjectPath,
+      log: (msg) => console.log(`CodeYam Sanitize: ${msg}`),
+    });
+    console.log(
+      `CodeYam DEBUG: Env files sanitized - created: ${sanitizeResult.createdFiles.length}, sanitized: ${sanitizeResult.sanitizedFiles.length}`,
+    );
+    if (sanitizeResult.sanitizedFiles.length > 0) {
+      console.log(
+        `CodeYam DEBUG: Sanitized files: ${sanitizeResult.sanitizedFiles.join(', ')}`,
+      );
+    }
+
+    // Apply universal mocks to capturer project
+    if (context.universalMocks && context.universalMocks.length > 0) {
+      console.log(
+        `CodeYam: Applying ${context.universalMocks.length} universal mock(s) to capturer project...`,
+      );
+      await applyUniversalMocks({
+        mocks: context.universalMocks,
+        projectPath: capturerProjectPath,
+      });
+      console.log('CodeYam DEBUG: Universal mocks applied');
+    }
+
+    const runner = new SequentialCaptureTaskRunner(
+      config,
+      context,
+      capturerPath,
+    );
+
+    // Start background processing loop
+    // Each batch will spawn its own paired start+capture processes
+    console.log('CodeYam DEBUG: Starting processing loop...');
+    runner.startProcessingLoop();
+
+    console.log(
+      'CodeYam DEBUG: SequentialCaptureTaskRunner created successfully',
+    );
+    return runner;
+  }
+
+  private archiveLog(logPath: string): void {
+    if (!fs.existsSync(logPath)) {
+      return;
+    }
+
+    try {
+      const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+      const archivePath = logPath.replace('.txt', `-${timestamp}.txt`);
+      fs.copyFileSync(logPath, archivePath);
+      console.log(`CodeYam: Archived log to ${path.basename(archivePath)}`);
+    } catch (error) {
+      console.warn(`CodeYam: Failed to archive log: ${error.message}`);
+    }
+  }
+
+  private startProcessingLoop(): void {
+    this.processingLoop = this.runProcessingLoop();
+  }
+
+  private async runProcessingLoop(): Promise<void> {
+    console.log('CodeYam DEBUG: Processing loop started');
+    while (!this.isCleanedUp) {
+      if (this.queue.length === 0) {
+        await new Promise((resolve) => setTimeout(resolve, 1000));
+        continue;
+      }
+
+      console.log(
+        `CodeYam DEBUG: Processing queue - ${this.queue.length} batches waiting`,
+      );
+      this.isProcessing = true;
+      const batch = this.queue.shift()!;
+
+      try {
+        await this.processBatch(batch);
+
+        console.log(`CodeYam DEBUG: Batch completed successfully`);
+        // Notify callback of completion
+        if (this.batchCompletedCallback) {
+          this.batchCompletedCallback(batch);
+        }
+      } catch (error) {
+        console.error('CodeYam: Error processing batch:', error);
+        // Still notify callback so orchestration can track failures
+        if (this.batchCompletedCallback) {
+          this.batchCompletedCallback(batch);
+        }
+      } finally {
+        this.isProcessing = false;
+      }
+    }
+    console.log('CodeYam DEBUG: Processing loop exited');
+  }
+
+  private async processBatch(analysisIds: string[]): Promise<void> {
+    console.log(
+      `CodeYam: Processing batch of ${analysisIds.length} analyses: ${analysisIds.join(', ')}`,
+    );
+
+    // Working directory is capturer's codeyam/ so '../project' resolves correctly
+    const capturerCodeyamPath = path.join(this.capturerPath, 'codeyam');
+    const capturerLogPath = path.join(capturerCodeyamPath, 'log.txt');
+    const capturerProjectPath = path.join(this.capturerPath, 'project');
+
+    // Archive previous log and truncate for clean slate
+    this.archiveLog(capturerLogPath);
+    if (fs.existsSync(capturerLogPath)) {
+      fs.unlinkSync(capturerLogPath);
+      console.log('CodeYam: Truncated log.txt for new batch');
+    }
+
+    // Use same args as parent, but override paths and add fast=true
+    const modifiedStartArgs: StartArgs = {
+      ...this.context.startArgs,
+      absoluteProjectRootPath: capturerProjectPath,
+      absoluteTempPath: this.context.absoluteTempPath,
+      noServer: false, // Must run server for capture to work
+      fast: true, // Skip slow checkout/build - we already copied everything
+    };
+
+    const cliArgs = buildCliArgs(modifiedStartArgs);
+
+    // Build environment variables from config
+    const configEnvVars: Record<string, string> = {};
+    for (const envVar of this.context.environmentVariables || []) {
+      if (envVar.key && envVar.value !== undefined) {
+        configEnvVars[envVar.key] = envVar.value;
+      }
+    }
+    if (Object.keys(configEnvVars).length > 0) {
+      console.log(
+        `CodeYam: Applying ${Object.keys(configEnvVars).length} environment variable(s) from config`,
+      );
+    }
+
+    // Spawn both start.js and capture.js for this batch
+    console.log('CodeYam: Spawning start.js and capture.js for batch...');
+    const startResult = spawnTaskProcess({
+      command: 'node',
+      args: ['--enable-source-maps', './dist/project/start.js', ...cliArgs],
+      env: {
+        ...configEnvVars,
+        ANALYSIS_IDS: analysisIds.join(','),
+        READY_TO_BE_CAPTURED: 'true',
+        PROJECT_SLUG: this.config.projectSlug,
+        CODEYAM_LOG_PATH: capturerLogPath,
+      },
+      workingDir: capturerCodeyamPath,
+      processName: 'project',
+      silent: false,
+    });
+
+    const captureResult = spawnTaskProcess({
+      command: 'node',
+      args: [
+        '--enable-source-maps',
+        './dist/playwright/capture.js',
+        '--slug',
+        this.config.projectSlug,
+      ],
+      env: {
+        CODEYAM_LOG_PATH: capturerLogPath,
+      },
+      workingDir: capturerCodeyamPath,
+      processName: 'playwright',
+      silent: false,
+    });
+
+    // Dynamic timeout based on batch size (5 + 5 * batchSize minutes)
+    const timeoutMinutes = 5 + 5 * analysisIds.length;
+    const timeoutMs = timeoutMinutes * 60 * 1000;
+    let timeoutHandle: NodeJS.Timeout | null = null;
+    const timeoutPromise = new Promise<never>((_, reject) => {
+      timeoutHandle = setTimeout(() => {
+        reject(new Error(`Batch timeout after ${timeoutMinutes} minutes`));
+        timeoutHandle = null;
+      }, timeoutMs);
+    });
+
+    try {
+      // Race: whichever finishes first (capture exits when done, or timeout)
+      await Promise.race([
+        captureResult.promise,
+        startResult.promise,
+        timeoutPromise,
+      ]);
+      console.log('CodeYam: Batch processing completed successfully');
+    } finally {
+      // Clean up timeout handle
+      if (timeoutHandle) {
+        clearTimeout(timeoutHandle);
+      }
+
+      // Kill both processes - whoever's still running needs cleanup
+      try {
+        startResult.process.kill('SIGTERM');
+      } catch (e) {
+        // Process may have already exited
+      }
+      try {
+        captureResult.process.kill('SIGTERM');
+      } catch (e) {
+        // Process may have already exited
+      }
+
+      // Give processes a moment to exit gracefully
+      await new Promise((resolve) => setTimeout(resolve, 2000));
+    }
+  }
+
+  enqueueWork(analysisIds: string[]): Promise<void> {
+    console.log(
+      `CodeYam DEBUG: Enqueueing batch of ${analysisIds.length} analyses`,
+    );
+    this.queue.push(analysisIds);
+    return Promise.resolve();
+  }
+
+  getQueueStats(): Promise<QueueStats> {
+    return Promise.resolve({
+      unreceived: this.queue.length,
+      inflight: this.isProcessing ? 1 : 0,
+    });
+  }
+
+  canContinue(): Promise<boolean> {
+    // Sequential runner spawns processes per batch, so always can continue
+    // unless explicitly cleaned up
+    return Promise.resolve(!this.isCleanedUp);
+  }
+
+  isRunning(): Promise<boolean> {
+    // Still running if we have work or are processing
+    return Promise.resolve(this.queue.length > 0 || this.isProcessing);
+  }
+
+  async cleanup(): Promise<void> {
+    if (this.isCleanedUp) return;
+    this.isCleanedUp = true;
+
+    console.log('CodeYam: Cleaning up sequential task runner...');
+
+    // Wait for current batch to complete if processing
+    // The processBatch method handles cleanup of its paired processes
+    if (this.processingLoop !== null) {
+      console.log('CodeYam: Waiting for processing loop to complete...');
+      await Promise.race([
+        this.processingLoop,
+        new Promise((resolve) => setTimeout(resolve, 10000)), // 10s timeout
+      ]);
+    }
+
+    console.log('CodeYam: Sequential task runner cleanup complete');
+  }
+
+  onBatchCompleted(callback: (analysisIds: string[]) => void): void {
+    this.batchCompletedCallback = callback;
+  }
+}

package/analyzer-template/project/orchestrateCapture/SupabaseAnalysisLoader.ts

@@ -0,0 +1,85 @@
+import type { Commit, Project } from '~codeyam/types';
+import { getSupabase } from '~codeyam/supabase';
+import type { AnalysisLoader, ReadyAnalysisData } from './taskRunner';
+
+interface SupabaseAnalysisLoaderConfig {
+  project: Project;
+  commit?: Commit | null;
+  analysisIds?: string[];
+  entityShas?: string[];
+}
+
+/**
+ * Supabase implementation of AnalysisLoader.
+ *
+ * Queries the database for analyses ready to be captured using the
+ * get_ready_to_be_captured_analyses RPC function.
+ */
+export default class SupabaseAnalysisLoader implements AnalysisLoader {
+  private config: SupabaseAnalysisLoaderConfig;
+
+  constructor(config: SupabaseAnalysisLoaderConfig) {
+    this.config = config;
+  }
+
+  async getReadyAnalyses(): Promise<ReadyAnalysisData[] | null> {
+    const args: { [key: string]: any } = {
+      projectid: this.config.project.id,
+    };
+
+    if (this.config.analysisIds) {
+      args['analysisids'] = this.config.analysisIds;
+    }
+
+    if (this.config.commit) {
+      if (this.config.commit.metadata?.baseline) {
+        args['branchcommitsha'] = this.config.commit.sha;
+      } else {
+        args['commitids'] = this.config.commit.id;
+      }
+    }
+
+    if (this.config.entityShas) {
+      args['entityshas'] = this.config.entityShas;
+    }
+
+    const pageSize = 1000;
+    const allData: ReadyAnalysisData[] = [];
+    let pageOffset = 0;
+
+    while (true) {
+      const query = getSupabase()
+        .rpc('get_ready_to_be_captured_analyses', args)
+        .select(
+          'id, commit_id, branch_commit_sha, status->processingForCaptureAt',
+        )
+        .range(pageOffset, pageOffset + pageSize - 1)
+        .returns<ReadyAnalysisData[]>();
+
+      const { data, error } = await query;
+
+      if (error) {
+        console.error(
+          'CodeYam Capture: Error: Failed to load readyToBeCaptured analyses',
+          args,
+          error,
+        );
+        return null;
+      }
+
+      if (!data || data.length === 0) {
+        break;
+      }
+
+      allData.push(...data);
+
+      if (data.length < pageSize) {
+        break;
+      }
+
+      pageOffset += pageSize;
+    }
+
+    return allData;
+  }
+}

package/analyzer-template/project/orchestrateCapture/spawnTaskProcess.ts

@@ -0,0 +1,112 @@
+import { ChildProcess, spawn } from 'child_process';
+import * as fs from 'fs';
+
+/**
+ * Minimal spawn wrapper for task runner processes.
+ *
+ * Unlike execAsync variants, this is:
+ * - Lightweight (no background job integration)
+ * - Designed for orchestration task runners
+ * - Returns both promise and process handle
+ * - Writes output to CODEYAM_LOG_PATH if set in env
+ */
+
+export interface SpawnTaskProcessArgs {
+  command: string;
+  args: string[];
+  env?: Record<string, string>;
+  workingDir?: string;
+  processName?: string;
+  silent?: boolean;
+}
+
+export interface SpawnTaskProcessResult {
+  promise: Promise<number>;
+  process: ChildProcess;
+}
+
+export default function spawnTaskProcess({
+  command,
+  args,
+  env = {},
+  workingDir,
+  processName = 'task',
+  silent = false,
+}: SpawnTaskProcessArgs): SpawnTaskProcessResult {
+  const spawnEnv = {
+    ...process.env,
+    ...env,
+  };
+
+  const childProcess = spawn(command, args, {
+    cwd: workingDir,
+    env: spawnEnv,
+    stdio: 'pipe', // Always pipe so we can capture and log
+  });
+
+  // Helper to add timestamps to multi-line output
+  const timestampLines = (data: string, prefix: string = ''): string => {
+    const timestamp = new Date().toLocaleString();
+    const lines = data.split('\n');
+
+    // Timestamp each non-empty line, preserve empty lines
+    return lines
+      .map((line) => {
+        if (line.trim()) {
+          return `[${timestamp}]${prefix} ${line}`;
+        }
+        return line; // Preserve empty lines without timestamp
+      })
+      .join('\n');
+  };
+
+  const writeToLog = (text: string) => {
+    const logPath = spawnEnv.CODEYAM_LOG_PATH;
+    if (!logPath) return;
+
+    fs.appendFile(logPath, text, (err) => {
+      if (err) {
+        console.log('CodeYam Error: Error writing to log file:', err);
+      }
+    });
+  };
+
+  // Capture stdout
+  childProcess.stdout?.on('data', (data) => {
+    const dataString = data?.toString() ?? '';
+    const timestampedOutput = timestampLines(dataString);
+
+    if (!silent) {
+      console.log(timestampedOutput);
+    }
+    writeToLog(timestampedOutput + '\n');
+  });
+
+  // Capture stderr
+  childProcess.stderr?.on('data', (data) => {
+    const dataString = data?.toString() ?? '';
+    const timestampedOutput = timestampLines(dataString, ' <STDERR>');
+
+    if (!silent) {
+      console.error(timestampedOutput);
+    }
+    writeToLog(timestampedOutput + '\n');
+  });
+
+  const promise = new Promise<number>((resolve, reject) => {
+    childProcess.on('exit', (code) => {
+      if (code === null) {
+        // Process was killed by signal
+        reject(new Error(`${processName} process was killed`));
+      } else {
+        resolve(code);
+      }
+    });
+
+    childProcess.on('error', (error) => {
+      reject(error);
+    });
+  });
+
+  return { promise, process: childProcess };
+}

package/analyzer-template/project/orchestrateCapture/taskRunner.ts

@@ -0,0 +1,121 @@
+/**
+ * Queue statistics for monitoring work distribution and progress.
+ */
+export interface QueueStats {
+  /** Work enqueued but not yet claimed by workers */
+  unreceived: number;
+  /** Work claimed and currently being processed */
+  inflight: number;
+}
+
+/**
+ * Analysis data for ready-to-be-captured analyses.
+ */
+export interface ReadyAnalysisData {
+  id: string;
+  commit_id: string | null;
+  branch_commit_sha: string | null;
+  processingForCaptureAt?: string;
+}
+
+/**
+ * Abstract analysis loader for fetching ready-to-be-captured analyses.
+ *
+ * Abstracts the database query layer to make orchestration testable.
+ * Implementations may use Supabase, mock data, or any other data source.
+ */
+export interface AnalysisLoader {
+  /**
+   * Fetch all analyses ready to be captured based on configured criteria.
+   *
+   * Returns null on error (orchestration will mark DB as stale and continue
+   * with existing data).
+   */
+  getReadyAnalyses(): Promise<ReadyAnalysisData[] | null>;
+}
+
+/**
+ * Configuration for initializing a task runner.
+ * Provides context needed for workers to execute capture batches.
+ */
+export interface TaskRunnerConfig {
+  /** Number of concurrent workers to run */
+  workerCount: number;
+  /** Project slug identifier */
+  projectSlug: string;
+  /** Web framework (e.g., 'next', 'remix') */
+  framework: string;
+  /** Package manager (e.g., 'npm', 'pnpm', 'yarn') */
+  packageManager: string;
+  /** Optional ECS cluster name for AWS implementation */
+  ecsCluster?: string;
+}
+
+/**
+ * Abstract task runner for distributed capture work.
+ *
+ * The runner manages worker lifecycle, work distribution, and completion tracking.
+ * Workers autonomously poll for work - the orchestrator enqueues batches but doesn't
+ * assign work to specific workers. This allows flexible implementations (cloud-based,
+ * local filesystem-based, etc.) while maintaining the same orchestration logic.
+ *
+ * Implementation contract:
+ * - Workers pull work autonomously (not pushed from orchestrator)
+ * - Workers load additional analysis details from database as needed
+ * - Batch completion is signaled via callback
+ * - All resources must be cleaned up in cleanup()
+ */
+export interface CaptureTaskRunner {
+  /**
+   * Enqueue a batch of analysis IDs for processing.
+   *
+   * Workers autonomously claim and process work. The orchestrator doesn't
+   * assign batches to specific workers.
+   */
+  enqueueWork(analysisIds: string[]): Promise<void>;
+
+  /**
+   * Get current work queue statistics.
+   *
+   * Used for progress logging and drain decisions.
+   */
+  getQueueStats(): Promise<QueueStats>;
+
+  /**
+   * Health check - returns false on catastrophic/unexpected failures.
+   *
+   * Should trigger immediate abort of orchestration. Implementations may
+   * log diagnostic details as needed.
+   *
+   * Examples of failures: workers crashed unexpectedly, AWS service errors,
+   * resource exhaustion, etc.
+   */
+  canContinue(): Promise<boolean>;
+
+  /**
+   * Returns true while workers are alive and capable of processing work.
+   *
+   * Returns false when all workers have naturally stopped (idle timeout,
+   * work completed, etc). Used as completion signal during drain phase.
+   *
+   * Unlike canContinue(), returning false is not necessarily an error - it
+   * indicates workers have gracefully shutdown.
+   */
+  isRunning(): Promise<boolean>;
+
+  /**
+   * Cleanup all resources (workers, queues, processes, etc.).
+   *
+   * Must be idempotent and safe to call multiple times.
+   * Should be awaited before orchestration exits.
+   */
+  cleanup(): Promise<void>;
+
+  /**
+   * Register callback for when workers complete batch processing.
+   *
+   * Critical for retry logic and progress tracking in orchestration.
+   * Callback receives the analysis IDs that were completed.
+   */
+  onBatchCompleted(callback: (analysisIds: string[]) => void): void;
+}