@codeyam/codeyam-cli 0.1.0-staging.ad31e3e → 0.1.0-staging.b8a55ba

package/analyzer-template/project/orchestrateCapture.ts

@@ -23,7 +23,7 @@
  */
 
 import type { Commit, Project } from '~codeyam/types';
-import { updateCommitMetadata } from '~codeyam/supabase';
+import { updateCommitMetadata } from '~codeyam/database';
 
 import type {
   AnalysisLoader,
@@ -331,7 +331,9 @@ export default async function orchestrateCapture({
       }
 
       // Early exit: if we've processed work and the runner has naturally stopped
-      if (knownAnalyses.size > 0 && enqueuedAnalyses.size > 0) {
+      // IMPORTANT: Only exit early if we're draining (analysis phase complete).
+      // Without isDraining check, we might exit before slow-running analyses finish.
+      if (isDraining && knownAnalyses.size > 0 && enqueuedAnalyses.size > 0) {
         const isStillRunning = await taskRunner.isRunning();
         if (
           !isStillRunning &&
@@ -347,8 +349,10 @@ export default async function orchestrateCapture({
       }
 
       if (isDraining) {
-        // Exit immediately if analysis complete but no entities ready for capture
-        if (knownAnalyses.size === 0) {
+        // Exit if analysis complete but no entities ready for capture
+        // Only check this after at least one poll (not firstLoop) to ensure we've
+        // had a chance to query the database for ready analyses
+        if (knownAnalyses.size === 0 && !firstLoop) {
           console.log(
             'CodeYam Capture: Analysis complete but no entities ready for capture, exiting',
           );

package/analyzer-template/project/prepareRepo.ts

@@ -39,7 +39,7 @@ export default async function prepareRepo(args: PrepareRepoArgs) {
       throw new Error(`Local project path does not exist: ${localProjectPath}`);
     }
 
-    awsLogDebugLevel(1, 'Rsyncing repo...');
+    awsLogDebugLevel(1, 'Syncing repo...');
 
     // Rsync project files respecting .gitignore to avoid copying secrets
     // (.env files, credentials, private keys, etc.) while preserving node_modules

package/analyzer-template/project/reconcileMockDataKeys.ts

@@ -5,28 +5,56 @@
  * (e.g., "eq('user_id', 'user-12345')") when the mock code expects variable
  * references (e.g., "eq('user_id', user.id)").
  *
- * This function remaps mismatched keys to their expected equivalents.
+ * This function also handles canonical key format (EntityName::hookName::index)
+ * and remaps old variable-based keys (variableName <- functionName) to canonical keys.
+ *
+ * @param mockData The mock data object to reconcile
+ * @param expectedKeys The expected keys from the structure
+ * @param entityName Optional entity name for canonical key remapping
  */
 export function reconcileMockDataKeys(
   mockData: Record<string, unknown>,
   expectedKeys: string[],
+  entityName?: string,
 ): Record<string, unknown> {
   const result: Record<string, unknown> = {};
+  // Track which canonical keys have been used for multiple calls to same hook
+  const usedCanonicalKeys = new Set<string>();
 
   for (const [key, value] of Object.entries(mockData)) {
     // Recursively reconcile nested objects
     const reconciledValue =
       value && typeof value === 'object' && !Array.isArray(value)
-        ? reconcileMockDataKeys(value as Record<string, unknown>, expectedKeys)
+        ? reconcileMockDataKeys(
+            value as Record<string, unknown>,
+            expectedKeys,
+            entityName,
+          )
         : value;
 
     // Check if key matches an expected key exactly
     if (expectedKeys.includes(key)) {
       result[key] = reconciledValue;
+      usedCanonicalKeys.add(key);
       continue;
     }
 
-    // Try to find closest match
+    // Try to remap variable-based key to canonical key
+    if (entityName) {
+      const canonicalKey = remapToCanonicalKey(
+        key,
+        expectedKeys,
+        entityName,
+        usedCanonicalKeys,
+      );
+      if (canonicalKey) {
+        result[canonicalKey] = reconciledValue;
+        usedCanonicalKeys.add(canonicalKey);
+        continue;
+      }
+    }
+
+    // Try to find closest match using existing logic
     const closestKey = findClosestKey(key, expectedKeys);
     if (closestKey) {
       result[closestKey] = reconciledValue;
@@ -39,6 +67,79 @@ export function reconcileMockDataKeys(
   return result;
 }
 
+/**
+ * Attempt to remap a variable-based key to a canonical key.
+ *
+ * Variable-based format: "variableName <- functionName"
+ * Canonical format: "EntityName::hookName::index"
+ *
+ * @example
+ * remapToCanonicalKey("currentCommit <- useLoaderData", ["DashboardPage::useLoaderData::0"], "DashboardPage", new Set())
+ * // returns "DashboardPage::useLoaderData::0"
+ */
+function remapToCanonicalKey(
+  key: string,
+  expectedCanonicalKeys: string[],
+  entityName: string,
+  usedKeys: Set<string>,
+): string | null {
+  // Check if key is in variable-based format: "varName <- functionName"
+  const arrowMatch = key.match(/<-\s*(\w+)/);
+  if (!arrowMatch) {
+    // Not a variable-based key, try other patterns
+    // Check if it's a simple function call like "useLoaderData()"
+    const funcCallMatch = key.match(/^(\w+)\(\)/);
+    if (funcCallMatch) {
+      const hookName = funcCallMatch[1];
+      return findMatchingCanonicalKey(
+        hookName,
+        entityName,
+        expectedCanonicalKeys,
+        usedKeys,
+      );
+    }
+    return null;
+  }
+
+  const hookName = arrowMatch[1];
+
+  return findMatchingCanonicalKey(
+    hookName,
+    entityName,
+    expectedCanonicalKeys,
+    usedKeys,
+  );
+}
+
+/**
+ * Find a matching canonical key for a hook name within an entity.
+ */
+function findMatchingCanonicalKey(
+  hookName: string,
+  entityName: string,
+  expectedCanonicalKeys: string[],
+  usedKeys: Set<string>,
+): string | null {
+  // Find all canonical keys for this entity and hook
+  const candidatePrefix = `${entityName}::${hookName}::`;
+  const candidates = expectedCanonicalKeys.filter(
+    (k) => k.startsWith(candidatePrefix) && !usedKeys.has(k),
+  );
+
+  if (candidates.length === 0) {
+    return null;
+  }
+
+  // Sort by index to prefer lower indices
+  candidates.sort((a, b) => {
+    const aIndex = parseInt(a.split('::')[2], 10);
+    const bIndex = parseInt(b.split('::')[2], 10);
+    return aIndex - bIndex;
+  });
+
+  return candidates[0];
+}
+
 /**
  * Find the closest matching key from expectedKeys for a given key.
  *

package/analyzer-template/project/runAnalysis.ts

@@ -16,7 +16,7 @@ import {
   loadEntity,
   loadScenario,
   updateCommitMetadata,
-} from '~codeyam/supabase';
+} from '~codeyam/database';
 import analyzeReadyToBeCaptured from './analyzeReadyToBeCaptured';
 import analyzeBaselineCommit from './analyzeBaselineCommit';
 import analyzeRegularCommit from './analyzeRegularCommit';

package/analyzer-template/project/runMultiScenarioServer.ts

@@ -11,7 +11,7 @@ import { startServer } from './startServer';
 import { awsLog, safeFileName } from '~codeyam/utils';
 import { recordErrorMessageForScenario } from './utils/errorHandling';
 import { Controller, TaskCompleteType } from './controller/startController';
-import { updateFreshAnalysisStatus, loadAnalysis } from '~codeyam/supabase';
+import { updateFreshAnalysisStatus, loadAnalysis } from '~codeyam/database';
 import getScenarioUrl from './getScenarioUrl';
 import { chromium } from 'playwright';
 import { ScenarioIterator } from './ScenarioIterator';
@@ -184,10 +184,6 @@ export async function runMultiScenarioServer({
     // Start the server (this runs in the background)
     const serverPromise = startServer({
       startCommand,
-      packageManager,
-      exitAfterSuccessfulAppStart,
-      framework,
-
       onStart: async (port: number) => {
         projectPort = port;
         serverStarted = true;
@@ -306,14 +302,14 @@ export async function runMultiScenarioServer({
             message: message.slice(0, 1000),
           });
 
-          // Save error to database
-          await recordErrorMessageForScenario(analysis, currentScenario, {
-            message: messageToSave,
-          } as Error);
-
-          // Only signal scenario failed for RELEVANT crash errors
-          // Non-crash errors (like SSR warnings) should be recorded but not stop capture
+          // Only save error and signal scenario failed for RELEVANT crash errors
+          // Non-crash errors (like SSR warnings) should be logged but not stop capture
+          // or mark the scenario as finished (which would prevent screenshot capture)
           if (relevantError) {
+            // Save crash error to database (sets finishedAt on scenario)
+            await recordErrorMessageForScenario(analysis, currentScenario, {
+              message: messageToSave,
+            } as Error);
             awsLog(
               'CodeYam SERVER DEBUG: Calling handleServerError for crash error',
               {
@@ -324,8 +320,9 @@ export async function runMultiScenarioServer({
             );
             iterator.handleServerError(currentScenario.name);
           } else {
+            // Non-crash errors are logged but don't affect scenario status
             awsLog(
-              'CodeYam SERVER DEBUG: Non-crash error recorded, NOT stopping scenario',
+              'CodeYam SERVER DEBUG: Non-crash error logged, NOT stopping scenario',
               {
                 analysisId: analysis.id,
                 scenarioName: currentScenario.name,
@@ -505,13 +502,13 @@ export async function runMultiScenarioServer({
 
       if (!iterator.hasScenarioFailed(scenario.name)) {
         completedScenarios.push(scenario.name);
-        awsLog('CodeYam SERVER DEBUG: Scenario marked as completed', {
+        awsLog('CodeYam: Scenario marked as completed', {
           analysisId: analysis.id,
           scenarioName: scenario.name,
           scenarioId: scenario.id,
         });
       } else {
-        awsLog('CodeYam SERVER DEBUG: Scenario marked as FAILED', {
+        awsLog('CodeYam: Scenario marked as FAILED', {
           analysisId: analysis.id,
           scenarioName: scenario.name,
           scenarioId: scenario.id,
@@ -519,7 +516,7 @@ export async function runMultiScenarioServer({
       }
 
       // Advance iterator to next scenario
-      awsLog('CodeYam SERVER DEBUG: Advancing to next scenario', {
+      awsLog('CodeYam: Advancing to next scenario', {
         analysisId: analysis.id,
         currentScenarioName: scenario.name,
         nextScenarioIndex: i + 1,

package/analyzer-template/project/runScenarioServer.ts

@@ -11,7 +11,7 @@ import { startServer } from './startServer';
 import { awsLog, safeFileName } from '~codeyam/utils';
 import { recordErrorMessageForScenario } from './utils/errorHandling';
 import { Controller, TaskCompleteType } from './controller/startController';
-import { updateFreshAnalysisStatus, loadAnalysis } from '~codeyam/supabase';
+import { updateFreshAnalysisStatus, loadAnalysis } from '~codeyam/database';
 import getScenarioUrl from './getScenarioUrl';
 
 const closedScenarios: string[] = [];
@@ -119,10 +119,6 @@ export async function runScenarioServer({
   try {
     const code = await startServer({
       startCommand,
-      packageManager,
-      exitAfterSuccessfulAppStart,
-      framework,
-
       onStart: async (port: number) => {
         if (taskCompletion) {
           console.log(

package/analyzer-template/project/serverOnlyModules.ts

@@ -0,0 +1,288 @@
+import ts from 'typescript';
+
+/**
+ * Configuration for modules that require client-side mocking.
+ *
+ * These are Node.js built-ins, server-only npm packages, or framework-specific
+ * server APIs that cannot run in the browser and need mock implementations.
+ */
+
+export interface ServerOnlyModuleMock {
+  /** The module specifier pattern (exact match or regex) */
+  module: string | RegExp;
+  /** Description of why this module needs mocking */
+  reason: string;
+  /** Category for grouping */
+  category: 'node-builtin' | 'npm-package' | 'framework';
+  /**
+   * Mock code generator function.
+   * @param importNames - The names being imported (for named imports)
+   * @param defaultImportName - The name used for default import (if any)
+   * @returns The mock code to replace the import with
+   */
+  getMockCode: (importNames: string[], defaultImportName?: string) => string;
+}
+
+/**
+ * Known server-only modules that need client-side mocking.
+ * This list can be extended by users via project configuration.
+ */
+export const SERVER_ONLY_MODULES: ServerOnlyModuleMock[] = [
+  // Node.js built-ins
+  {
+    module: 'node:crypto',
+    reason: 'Node.js crypto module is not available in browser',
+    category: 'node-builtin',
+    getMockCode: (importNames, defaultImportName) => {
+      if (defaultImportName) {
+        return `// Mock for node:crypto (Node.js built-in, not available in browser)
+const ${defaultImportName} = {
+  createHash: (algorithm: string) => ({
+    update: (data: string) => ({
+      digest: (encoding: string) => 'mock-hash-' + data.substring(0, 8),
+    }),
+  }),
+  randomBytes: (size: number) => new Uint8Array(size),
+  randomUUID: () => 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
+    const r = Math.random() * 16 | 0;
+    return (c === 'x' ? r : (r & 0x3 | 0x8)).toString(16);
+  }),
+};
+`;
+      }
+      // Named imports
+      let code =
+        '// Mock for node:crypto (Node.js built-in, not available in browser)\n';
+      for (const name of importNames) {
+        const [originalName, aliasName] = name
+          .split(/\s+as\s+/)
+          .map((s) => s.trim());
+        const finalName = aliasName || originalName;
+        if (originalName === 'createHash') {
+          code += `const ${finalName} = (algorithm: string) => ({
+  update: (data: string) => ({
+    digest: (encoding: string) => 'mock-hash-' + data.substring(0, 8),
+  }),
+});
+`;
+        } else if (originalName === 'randomBytes') {
+          code += `const ${finalName} = (size: number) => new Uint8Array(size);
+`;
+        } else if (originalName === 'randomUUID') {
+          code += `const ${finalName} = () => 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
+  const r = Math.random() * 16 | 0;
+  return (c === 'x' ? r : (r & 0x3 | 0x8)).toString(16);
+});
+`;
+        } else {
+          code += `const ${finalName} = () => { throw new Error('${originalName} is not available in browser'); };
+`;
+        }
+      }
+      return code;
+    },
+  },
+
+  // npm packages that require Node.js
+  {
+    module: 'https-proxy-agent',
+    reason: 'Requires Node.js net/tls modules',
+    category: 'npm-package',
+    getMockCode: (importNames) => {
+      return `// Mock for https-proxy-agent (requires Node.js net/tls modules)
+class HttpsProxyAgent {
+  constructor() {}
+}
+`;
+    },
+  },
+  {
+    module: 'node-fetch',
+    reason: 'Requires Node.js http/https modules',
+    category: 'npm-package',
+    getMockCode: (importNames, defaultImportName) => {
+      if (defaultImportName) {
+        return `// Mock for node-fetch (requires Node.js http/https modules)
+const ${defaultImportName} = async (url: string, options?: any) => ({
+  ok: true,
+  status: 200,
+  json: async () => ({}),
+  text: async () => '',
+  headers: new Headers(),
+});
+`;
+      }
+      // Named imports - browser has native Response, Headers, etc.
+      return '// node-fetch named imports removed - using browser natives\n';
+    },
+  },
+
+  // Framework-specific server APIs
+  {
+    module: 'next/headers',
+    reason: 'Next.js server-only API for reading request headers',
+    category: 'framework',
+    getMockCode: () => {
+      return `// Mock for next/headers (server-only in Next.js)
+const headers = async () => ({
+  get: (name: string) => null,
+  has: (name: string) => false,
+  entries: () => [][Symbol.iterator](),
+  keys: () => [][Symbol.iterator](),
+  values: () => [][Symbol.iterator](),
+  forEach: () => {},
+});
+const cookies = async () => ({
+  get: (name: string) => undefined,
+  getAll: () => [],
+  has: (name: string) => false,
+  set: () => {},
+  delete: () => {},
+});
+`;
+    },
+  },
+];
+
+/**
+ * Find a server-only module configuration by module specifier.
+ */
+export function findServerOnlyModule(
+  moduleSpecifier: string,
+): ServerOnlyModuleMock | undefined {
+  return SERVER_ONLY_MODULES.find((config) => {
+    if (typeof config.module === 'string') {
+      return config.module === moduleSpecifier;
+    }
+    return config.module.test(moduleSpecifier);
+  });
+}
+
+/**
+ * Check if a module specifier matches any known server-only module.
+ */
+export function isServerOnlyModule(moduleSpecifier: string): boolean {
+  return findServerOnlyModule(moduleSpecifier) !== undefined;
+}
+
+/**
+ * Parse an import statement to extract module specifier and import names.
+ */
+export function parseImportStatement(importStatement: string): {
+  moduleSpecifier: string;
+  importNames: string[];
+  defaultImportName?: string;
+} | null {
+  // Match: import { a, b } from "module"
+  const namedMatch = importStatement.match(
+    /import\s+\{([^}]+)\}\s+from\s+["']([^"']+)["']/,
+  );
+  if (namedMatch) {
+    const names = namedMatch[1]
+      .split(',')
+      .map((s) => s.trim())
+      .filter(Boolean);
+    return {
+      moduleSpecifier: namedMatch[2],
+      importNames: names,
+    };
+  }
+
+  // Match: import foo from "module" or import * as foo from "module"
+  const defaultMatch = importStatement.match(
+    /import\s+(?:\*\s+as\s+)?(\w+)\s+from\s+["']([^"']+)["']/,
+  );
+  if (defaultMatch) {
+    return {
+      moduleSpecifier: defaultMatch[2],
+      importNames: [],
+      defaultImportName: defaultMatch[1],
+    };
+  }
+
+  return null;
+}
+
+/**
+ * Apply server-only module mocks to file content.
+ * Replaces imports from known server-only modules with mock implementations.
+ */
+export function applyServerOnlyMocks(fileContent: string): string {
+  // Always use AST parsing - regex with nested quantifiers can cause catastrophic
+  // backtracking that hangs on a single .exec() call (before iteration limits kick in)
+  return applyServerOnlyMocksAst(fileContent);
+}
+
+/**
+ * AST-based implementation of applyServerOnlyMocks for large files.
+ * Uses TypeScript parser instead of regex to avoid catastrophic backtracking.
+ */
+function applyServerOnlyMocksAst(fileContent: string): string {
+  try {
+    const sourceFile = ts.createSourceFile(
+      'temp.ts',
+      fileContent,
+      ts.ScriptTarget.Latest,
+      true,
+    );
+
+    const replacements: { start: number; end: number; replacement: string }[] =
+      [];
+
+    for (const statement of sourceFile.statements) {
+      if (!ts.isImportDeclaration(statement)) continue;
+
+      const moduleSpecifier = statement.moduleSpecifier;
+      if (!ts.isStringLiteral(moduleSpecifier)) continue;
+
+      const modulePath = moduleSpecifier.text;
+      const config = findServerOnlyModule(modulePath);
+      if (!config) continue;
+
+      // Extract import names
+      const importNames: string[] = [];
+      let defaultImportName: string | undefined;
+
+      if (statement.importClause) {
+        // Default import
+        if (statement.importClause.name) {
+          defaultImportName = statement.importClause.name.text;
+        }
+
+        // Named imports
+        if (statement.importClause.namedBindings) {
+          if (ts.isNamedImports(statement.importClause.namedBindings)) {
+            for (const element of statement.importClause.namedBindings
+              .elements) {
+              importNames.push(element.name.text);
+            }
+          } else if (
+            ts.isNamespaceImport(statement.importClause.namedBindings)
+          ) {
+            // Namespace import like: import * as foo from 'module'
+            importNames.push(statement.importClause.namedBindings.name.text);
+          }
+        }
+      }
+
+      const mockCode = config.getMockCode(importNames, defaultImportName);
+      replacements.push({
+        start: statement.getStart(sourceFile),
+        end: statement.getEnd(),
+        replacement: mockCode,
+      });
+    }
+
+    // Apply replacements in reverse order to preserve positions
+    let result = fileContent;
+    replacements.sort((a, b) => b.start - a.start);
+    for (const { start, end, replacement } of replacements) {
+      result = result.slice(0, start) + replacement + result.slice(end);
+    }
+
+    return result;
+  } catch (error) {
+    console.warn('[applyServerOnlyMocksAst] Failed to parse file:', error);
+    return fileContent; // Return original content on error
+  }
+}

package/analyzer-template/project/start.ts

@@ -5,7 +5,8 @@ import {
   loadBackgroundJob,
   loadProject,
   updateBackgroundJobProgress,
-} from '~codeyam/supabase';
+  updateCommitMetadata,
+} from '~codeyam/database';
 import { Commit, ProjectFramework, ServerCommand } from '~codeyam/types';
 import runCaptureOrchestration from './orchestrateCapture';
 import AwsCaptureTaskRunner from './orchestrateCapture/AwsCaptureTaskRunner';
@@ -172,6 +173,24 @@ async function main({
         awsLog(
           'CodeYam Capture: analysis completed, signaling orchestration to begin draining...',
         );
+
+        // Update commit metadata to mark analysis as complete
+        // This allows the UI to show the "Capturing..." state
+        // With orchestration, the same process handles both phases,
+        // so we clear analyzerPid and set capturePid for a clean state transition
+        if (sharedContext.commit?.sha) {
+          await updateCommitMetadata({
+            commitSha: sharedContext.commit.sha,
+            runStatusUpdate: {
+              analysisCompletedAt: new Date().toISOString(),
+              analyzerPid: undefined,
+              capturePid: process.pid,
+            },
+          }).catch((error) => {
+            awsLog(`Warning: Failed to update commit metadata: ${error}`);
+          });
+        }
+
         if (sendControlMessage) {
           sendControlMessage('drain');
         } else {
@@ -213,12 +232,8 @@ async function main({
               absoluteTempPath,
               startArgs: {
                 packageManager,
-                absoluteProjectRootPath,
-                absoluteTempPath,
                 framework,
-                noServer: false,
                 steps,
-                fast,
               },
               // Pass universal mocks so they're applied to the capturer's project directory
               universalMocks: sharedContext.project.metadata?.universalMocks,
@@ -251,8 +266,18 @@ async function main({
             orchestrateCapture === 'local-sequential'
               ? null
               : (sharedContext.commit ?? sharedContext.branchCommit),
-          analysisIds: envConfig.analysisIds,
-          entityShas: envConfig.entityShas,
+          // In local-sequential mode, don't filter by analysisIds/entityShas since
+          // analysis may create new entities with different SHAs than the input filters.
+          // We want to capture all ready analyses for the project.
+          analysisIds:
+            orchestrateCapture === 'local-sequential'
+              ? undefined
+              : envConfig.analysisIds,
+          entityShas:
+            orchestrateCapture === 'local-sequential'
+              ? undefined
+              : envConfig.entityShas,
+          full: orchestrateCapture === 'local-sequential',
           taskRunner,
         });
         awsLog('CodeYam Capture: runCaptureOrchestration() completed');
@@ -378,17 +403,7 @@ async function main({
       });
       updateBackgroundJobProgress(backgroundJob, {
         initiated: backgroundJob.progress.initiated,
-        steps: [
-          {
-            name: 'build-project',
-            initiated:
-              backgroundJob.progress.steps.find(
-                (s) => s.name === 'build-project',
-              )?.initiated ?? new Date().toISOString(),
-            error: error.message,
-            completed: new Date().toISOString(),
-          },
-        ],
+        steps: [],
         completed: new Date().toISOString(),
       }).catch(() => {});
     }
@@ -425,6 +440,16 @@ async function main({
         `CodeYam: Exiting start.js process with ${hadError ? 'ERROR' : 'SUCCESS'}`,
       );
       process.exit(hadError ? 1 : 0);
+    } else if (process.env.CAPTURE_ONLY === 'true') {
+      awsLog(
+        'CodeYam: CAPTURE_ONLY mode - keeping files and exiting cleanly',
+        `Preserved ${generatedFiles.length} files`,
+      );
+      // Don't cleanup files in capture-only mode to preserve manual edits
+      awsLog(
+        `CodeYam: Exiting start.js process with ${hadError ? 'ERROR' : 'SUCCESS'}`,
+      );
+      process.exit(hadError ? 1 : 0);
     } else {
       awsLog(`CodeYam: Removing ${generatedFiles.length} generated files...`);
       removeScenario(generatedFiles);