@codeyam/codeyam-cli 0.1.0-staging.1 → 0.1.0-staging.596f0eb

package/analyzer-template/project/runScenarioServer.ts

@@ -11,7 +11,7 @@ import { startServer } from './startServer';
 import { awsLog, safeFileName } from '~codeyam/utils';
 import { recordErrorMessageForScenario } from './utils/errorHandling';
 import { Controller, TaskCompleteType } from './controller/startController';
-import { updateFreshAnalysisStatus, loadAnalysis } from '~codeyam/supabase';
+import { updateFreshAnalysisStatus, loadAnalysis } from '~codeyam/database';
 import getScenarioUrl from './getScenarioUrl';
 
 const closedScenarios: string[] = [];
@@ -119,10 +119,6 @@ export async function runScenarioServer({
   try {
     const code = await startServer({
       startCommand,
-      packageManager,
-      exitAfterSuccessfulAppStart,
-      framework,
-
       onStart: async (port: number) => {
         if (taskCompletion) {
           console.log(

package/analyzer-template/project/serverOnlyModules.ts

@@ -0,0 +1,240 @@
+/**
+ * Configuration for modules that require client-side mocking.
+ *
+ * These are Node.js built-ins, server-only npm packages, or framework-specific
+ * server APIs that cannot run in the browser and need mock implementations.
+ */
+
+export interface ServerOnlyModuleMock {
+  /** The module specifier pattern (exact match or regex) */
+  module: string | RegExp;
+  /** Description of why this module needs mocking */
+  reason: string;
+  /** Category for grouping */
+  category: 'node-builtin' | 'npm-package' | 'framework';
+  /**
+   * Mock code generator function.
+   * @param importNames - The names being imported (for named imports)
+   * @param defaultImportName - The name used for default import (if any)
+   * @returns The mock code to replace the import with
+   */
+  getMockCode: (importNames: string[], defaultImportName?: string) => string;
+}
+
+/**
+ * Known server-only modules that need client-side mocking.
+ * This list can be extended by users via project configuration.
+ */
+export const SERVER_ONLY_MODULES: ServerOnlyModuleMock[] = [
+  // Node.js built-ins
+  {
+    module: 'node:crypto',
+    reason: 'Node.js crypto module is not available in browser',
+    category: 'node-builtin',
+    getMockCode: (importNames, defaultImportName) => {
+      if (defaultImportName) {
+        return `// Mock for node:crypto (Node.js built-in, not available in browser)
+const ${defaultImportName} = {
+  createHash: (algorithm: string) => ({
+    update: (data: string) => ({
+      digest: (encoding: string) => 'mock-hash-' + data.substring(0, 8),
+    }),
+  }),
+  randomBytes: (size: number) => new Uint8Array(size),
+  randomUUID: () => 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
+    const r = Math.random() * 16 | 0;
+    return (c === 'x' ? r : (r & 0x3 | 0x8)).toString(16);
+  }),
+};
+`;
+      }
+      // Named imports
+      let code =
+        '// Mock for node:crypto (Node.js built-in, not available in browser)\n';
+      for (const name of importNames) {
+        const [originalName, aliasName] = name
+          .split(/\s+as\s+/)
+          .map((s) => s.trim());
+        const finalName = aliasName || originalName;
+        if (originalName === 'createHash') {
+          code += `const ${finalName} = (algorithm: string) => ({
+  update: (data: string) => ({
+    digest: (encoding: string) => 'mock-hash-' + data.substring(0, 8),
+  }),
+});
+`;
+        } else if (originalName === 'randomBytes') {
+          code += `const ${finalName} = (size: number) => new Uint8Array(size);
+`;
+        } else if (originalName === 'randomUUID') {
+          code += `const ${finalName} = () => 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
+  const r = Math.random() * 16 | 0;
+  return (c === 'x' ? r : (r & 0x3 | 0x8)).toString(16);
+});
+`;
+        } else {
+          code += `const ${finalName} = () => { throw new Error('${originalName} is not available in browser'); };
+`;
+        }
+      }
+      return code;
+    },
+  },
+
+  // npm packages that require Node.js
+  {
+    module: 'https-proxy-agent',
+    reason: 'Requires Node.js net/tls modules',
+    category: 'npm-package',
+    getMockCode: (importNames) => {
+      return `// Mock for https-proxy-agent (requires Node.js net/tls modules)
+class HttpsProxyAgent {
+  constructor() {}
+}
+`;
+    },
+  },
+  {
+    module: 'node-fetch',
+    reason: 'Requires Node.js http/https modules',
+    category: 'npm-package',
+    getMockCode: (importNames, defaultImportName) => {
+      if (defaultImportName) {
+        return `// Mock for node-fetch (requires Node.js http/https modules)
+const ${defaultImportName} = async (url: string, options?: any) => ({
+  ok: true,
+  status: 200,
+  json: async () => ({}),
+  text: async () => '',
+  headers: new Headers(),
+});
+`;
+      }
+      // Named imports - browser has native Response, Headers, etc.
+      return '// node-fetch named imports removed - using browser natives\n';
+    },
+  },
+
+  // Framework-specific server APIs
+  {
+    module: 'next/headers',
+    reason: 'Next.js server-only API for reading request headers',
+    category: 'framework',
+    getMockCode: () => {
+      return `// Mock for next/headers (server-only in Next.js)
+const headers = async () => ({
+  get: (name: string) => null,
+  has: (name: string) => false,
+  entries: () => [][Symbol.iterator](),
+  keys: () => [][Symbol.iterator](),
+  values: () => [][Symbol.iterator](),
+  forEach: () => {},
+});
+const cookies = async () => ({
+  get: (name: string) => undefined,
+  getAll: () => [],
+  has: (name: string) => false,
+  set: () => {},
+  delete: () => {},
+});
+`;
+    },
+  },
+];
+
+/**
+ * Find a server-only module configuration by module specifier.
+ */
+export function findServerOnlyModule(
+  moduleSpecifier: string,
+): ServerOnlyModuleMock | undefined {
+  return SERVER_ONLY_MODULES.find((config) => {
+    if (typeof config.module === 'string') {
+      return config.module === moduleSpecifier;
+    }
+    return config.module.test(moduleSpecifier);
+  });
+}
+
+/**
+ * Check if a module specifier matches any known server-only module.
+ */
+export function isServerOnlyModule(moduleSpecifier: string): boolean {
+  return findServerOnlyModule(moduleSpecifier) !== undefined;
+}
+
+/**
+ * Parse an import statement to extract module specifier and import names.
+ */
+export function parseImportStatement(importStatement: string): {
+  moduleSpecifier: string;
+  importNames: string[];
+  defaultImportName?: string;
+} | null {
+  // Match: import { a, b } from "module"
+  const namedMatch = importStatement.match(
+    /import\s+\{([^}]+)\}\s+from\s+["']([^"']+)["']/,
+  );
+  if (namedMatch) {
+    const names = namedMatch[1]
+      .split(',')
+      .map((s) => s.trim())
+      .filter(Boolean);
+    return {
+      moduleSpecifier: namedMatch[2],
+      importNames: names,
+    };
+  }
+
+  // Match: import foo from "module" or import * as foo from "module"
+  const defaultMatch = importStatement.match(
+    /import\s+(?:\*\s+as\s+)?(\w+)\s+from\s+["']([^"']+)["']/,
+  );
+  if (defaultMatch) {
+    return {
+      moduleSpecifier: defaultMatch[2],
+      importNames: [],
+      defaultImportName: defaultMatch[1],
+    };
+  }
+
+  return null;
+}
+
+/**
+ * Apply server-only module mocks to file content.
+ * Replaces imports from known server-only modules with mock implementations.
+ */
+export function applyServerOnlyMocks(fileContent: string): string {
+  let result = fileContent;
+
+  // Find all import statements
+  const importRegex =
+    /import\s+(?:(?:\{[^}]*\}|\*\s+as\s+\w+|\w+)\s*,?\s*)*\s*from\s*["']([^"']+)["'];?\s*\n?/g;
+
+  let match;
+  const replacements: { original: string; replacement: string }[] = [];
+
+  while ((match = importRegex.exec(fileContent)) !== null) {
+    const fullImport = match[0];
+    const parsed = parseImportStatement(fullImport);
+
+    if (!parsed) continue;
+
+    const config = findServerOnlyModule(parsed.moduleSpecifier);
+    if (!config) continue;
+
+    const mockCode = config.getMockCode(
+      parsed.importNames,
+      parsed.defaultImportName,
+    );
+    replacements.push({ original: fullImport, replacement: mockCode });
+  }
+
+  // Apply replacements (in reverse order to preserve positions)
+  for (const { original, replacement } of replacements) {
+    result = result.replace(original, replacement);
+  }
+
+  return result;
+}

package/analyzer-template/project/start.ts

@@ -5,7 +5,8 @@ import {
   loadBackgroundJob,
   loadProject,
   updateBackgroundJobProgress,
-} from '~codeyam/supabase';
+  updateCommitMetadata,
+} from '~codeyam/database';
 import { Commit, ProjectFramework, ServerCommand } from '~codeyam/types';
 import runCaptureOrchestration from './orchestrateCapture';
 import AwsCaptureTaskRunner from './orchestrateCapture/AwsCaptureTaskRunner';
@@ -172,6 +173,24 @@ async function main({
         awsLog(
           'CodeYam Capture: analysis completed, signaling orchestration to begin draining...',
         );
+
+        // Update commit metadata to mark analysis as complete
+        // This allows the UI to show the "Capturing..." state
+        // With orchestration, the same process handles both phases,
+        // so we clear analyzerPid and set capturePid for a clean state transition
+        if (sharedContext.commit?.sha) {
+          await updateCommitMetadata({
+            commitSha: sharedContext.commit.sha,
+            runStatusUpdate: {
+              analysisCompletedAt: new Date().toISOString(),
+              analyzerPid: undefined,
+              capturePid: process.pid,
+            },
+          }).catch((error) => {
+            awsLog(`Warning: Failed to update commit metadata: ${error}`);
+          });
+        }
+
         if (sendControlMessage) {
           sendControlMessage('drain');
         } else {
@@ -213,12 +232,8 @@ async function main({
               absoluteTempPath,
               startArgs: {
                 packageManager,
-                absoluteProjectRootPath,
-                absoluteTempPath,
                 framework,
-                noServer: false,
                 steps,
-                fast,
               },
               // Pass universal mocks so they're applied to the capturer's project directory
               universalMocks: sharedContext.project.metadata?.universalMocks,
@@ -251,8 +266,18 @@ async function main({
             orchestrateCapture === 'local-sequential'
               ? null
               : (sharedContext.commit ?? sharedContext.branchCommit),
-          analysisIds: envConfig.analysisIds,
-          entityShas: envConfig.entityShas,
+          // In local-sequential mode, don't filter by analysisIds/entityShas since
+          // analysis may create new entities with different SHAs than the input filters.
+          // We want to capture all ready analyses for the project.
+          analysisIds:
+            orchestrateCapture === 'local-sequential'
+              ? undefined
+              : envConfig.analysisIds,
+          entityShas:
+            orchestrateCapture === 'local-sequential'
+              ? undefined
+              : envConfig.entityShas,
+          full: orchestrateCapture === 'local-sequential',
           taskRunner,
         });
         awsLog('CodeYam Capture: runCaptureOrchestration() completed');
@@ -378,17 +403,7 @@ async function main({
       });
       updateBackgroundJobProgress(backgroundJob, {
         initiated: backgroundJob.progress.initiated,
-        steps: [
-          {
-            name: 'build-project',
-            initiated:
-              backgroundJob.progress.steps.find(
-                (s) => s.name === 'build-project',
-              )?.initiated ?? new Date().toISOString(),
-            error: error.message,
-            completed: new Date().toISOString(),
-          },
-        ],
+        steps: [],
         completed: new Date().toISOString(),
       }).catch(() => {});
     }

package/analyzer-template/project/startScenarioCapture.ts

@@ -13,7 +13,7 @@ import {
 import removeScenario from './removeScenario';
 import { recordErrorMessageForScenario } from './utils/errorHandling';
 import { Controller } from './controller/startController';
-import { loadFile, updateFreshAnalysisStatus } from '~codeyam/supabase';
+import { loadFile, updateFreshAnalysisStatus } from '~codeyam/database';
 import {
   awsLog,
   awsLogDebugLevel,
@@ -24,6 +24,11 @@ import {
 } from '~codeyam/utils';
 import getAppPath from './getAppPath';
 import { restoreAllBackups, restoreFromBackup } from './backupFiles';
+import {
+  readTrackingFile,
+  writeTrackingFile,
+  clearTrackingFile,
+} from './trackGeneratedFiles';
 import captureLibraryFunctionDirect from './captureLibraryFunctionDirect';
 import {
   getFrameworkForComponent,
@@ -79,6 +84,21 @@ export async function startScenarioCapture({
     restoreAllBackups(fullPath);
   }
 
+  // Clean up any leftover generated files from previous debug sessions
+  const previouslyTracked = readTrackingFile();
+  if (previouslyTracked) {
+    awsLog('CodeYam: Cleaning up files from previous session', {
+      generatedCount: previouslyTracked.generatedFiles.length,
+      backupCount: previouslyTracked.backupFiles.length,
+      timestamp: previouslyTracked.timestamp,
+    });
+    removeScenario(previouslyTracked.generatedFiles);
+    for (const backupPath of previouslyTracked.backupFiles) {
+      restoreFromBackup(backupPath);
+    }
+    clearTrackingFile();
+  }
+
   const sortedReadyToBeCaptured = readyToBeCaptured.sort((a, b) => {
     if (a.indirect !== b.indirect) {
       return a.indirect ? 1 : -1;
@@ -446,6 +466,9 @@ export async function startScenarioCapture({
         controller?.port,
       );
       allGeneratedFiles.push(codeyamPagePath);
+
+      // Persist file tracking for crash/debug recovery
+      writeTrackingFile(allGeneratedFiles, backupFiles);
     }
 
     // Skip if no scenarios to capture
@@ -565,6 +588,7 @@ export async function startScenarioCapture({
         for (const backupPath of backupsToRestore) {
           restoreFromBackup(backupPath);
         }
+        clearTrackingFile();
       }
     };
 

package/analyzer-template/project/startServer.ts

@@ -1,25 +1,32 @@
 import getPort from 'get-port';
 import execAsync from '../common/execAsync';
-import { ProjectFramework, ServerCommand } from '~codeyam/types';
+import { ServerCommand } from '~codeyam/types';
 import { Socket } from 'net';
 import { awsLogDebugLevel } from '~codeyam/utils';
 
 export interface StartServerArgs {
-  startCommand?: ServerCommand;
-  packageManager: string;
-  exitAfterSuccessfulAppStart?: boolean;
+  startCommand: ServerCommand;
   onStart: (port: number) => Promise<void>;
   onError: (data: string) => Promise<void>;
-  framework: ProjectFramework;
   projectPort?: number;
 }
 
+/**
+ * Start the project's dev server.
+ *
+ * Requires a complete startCommand with $PORT placeholder in args.
+ * The startCommand should be fully specified in the project's webapp configuration,
+ * including any package manager invocation and framework-specific flags.
+ *
+ * Examples of valid startCommand configurations:
+ * - npm: { command: 'npm', args: ['run', 'dev', '--', '--port', '$PORT'] }
+ * - pnpm: { command: 'pnpm', args: ['dev', '--port', '$PORT'] }
+ * - monorepo: { command: 'sh', args: ['-c', 'cd subdir && pnpm dev --port $PORT'] }
+ */
 export async function startServer({
   startCommand,
-  packageManager,
   onStart,
   onError,
-  framework,
   projectPort,
 }: StartServerArgs) {
   const acceptableErrorsRegexes = [
@@ -27,7 +34,7 @@ export async function startServer({
     /favicon\.ico/,
     /DeprecationWarning/,
     /warn deprecated/,
-    /One of your dependencies, babel-preset-react-app, is importing the/, // This error is thrown in CRA and is not a problem. TODO: How to not throw for warnings
+    /One of your dependencies, babel-preset-react-app, is importing the/,
     /No route matches URL/,
     /Browserslist: caniuse-lite is outdated/,
     /npx update-browserslist-db@latest/,
@@ -37,8 +44,27 @@ export async function startServer({
   ];
 
   try {
-    // both 'undefined' and 0 triggers port auto-selection, which we do here as late as
-    // possible to mimimise the risk of race conditions & port conflicts downstream
+    // Validate startCommand
+    if (!startCommand?.command) {
+      throw new Error(
+        'startCommand.command is required. ' +
+          'Please configure a complete startCommand in the webapp configuration.',
+      );
+    }
+    if (!startCommand.args?.length) {
+      throw new Error(
+        'startCommand.args is required and must be non-empty. ' +
+          'Please configure startCommand.args with $PORT placeholder.',
+      );
+    }
+    if (!startCommand.args.some((arg) => arg.includes('$PORT'))) {
+      throw new Error(
+        'startCommand.args must include $PORT placeholder. ' +
+          'Example: ["run", "dev", "--", "--port", "$PORT"]',
+      );
+    }
+
+    // Port auto-selection (as late as possible to minimize race conditions)
     if (!projectPort) {
       projectPort = await getPort();
       console.log(`CodeYam: Auto-selected project port: ${projectPort}`);
@@ -68,7 +94,6 @@ export async function startServer({
 
         socket.connect(port, 'localhost');
 
-        // Don't leave socket hanging
         timer.current = setTimeout(() => {
           console.log(`CodeYam: checkPort: Timeout`);
           socket.destroy();
@@ -89,66 +114,22 @@ export async function startServer({
       }
     };
 
-    // figure out the right CLI invocation for starting the server
-    const command = startCommand?.command ?? packageManager;
-    let args = startCommand?.args;
-
-    if (args) {
-      // Custom startCommand provided - clone and process it
-      args = [...args];
-
-      // Replace any $PORT placeholders with the actual port
-      let hasPort = false;
-      for (let i = 0; i < args.length; i++) {
-        if (args[i].includes('$PORT')) {
-          args[i] = args[i].replace('$PORT', projectPort.toString());
-          hasPort = true;
-        } else if (args[i] === '--port' || args[i] === '-p') {
-          hasPort = true;
-        }
-      }
+    // Build command from startCommand (substituting $PORT)
+    const command = startCommand.command;
+    const args = startCommand.args.map((arg) =>
+      arg.replace(/\$PORT/g, projectPort.toString()),
+    );
 
-      // If no port argument exists, add it.
-      // Use -- separator for compatibility with Turbo-based projects.
-      if (!hasPort) {
-        if (framework === ProjectFramework.Remix) {
-          args.push('--', '--port', projectPort.toString(), '--strictPort');
-        } else {
-          args.push('--', '--port', projectPort.toString());
-        }
-      }
-    } else {
-      if (framework === ProjectFramework.CodeYam) {
-        args = ['start', projectPort.toString()];
-      } else {
-        // there's no guarantee that the logic below will work for all frameworks, and
-        // certainly no guarantee that it'll work on any one individual project... but
-        // it's a reasonable default, and we have `startCommand` for anything bespoke
-        //
-        // All package managers support -- to pass arguments to scripts.
-        // npm requires it, and Turbo-based projects (monorepos) also require it.
-        // Using -- universally is safe and ensures compatibility.
-        args = ['run', 'dev', '--', '--port', projectPort.toString()];
-        if (framework === ProjectFramework.Remix) {
-          args.push('--strictPort');
-        }
-      }
-    }
-    let env = startCommand?.env;
-    if (!env) {
-      env = {
-        LOCAL: 'local',
-        PORT: projectPort.toString(),
-      };
-      if (framework === ProjectFramework.CRA) {
-        env.BROWSER = 'none';
-      }
-    } else {
-      // Replace $PORT placeholders in env values
-      env = { ...env };
+    // Handle env: $PORT substitution
+    let env: Record<string, string> = {
+      LOCAL: 'local',
+      PORT: projectPort.toString(),
+    };
+    if (startCommand.env) {
+      env = { ...env, ...startCommand.env };
       for (const key of Object.keys(env)) {
         if (typeof env[key] === 'string' && env[key].includes('$PORT')) {
-          env[key] = env[key].replace('$PORT', projectPort.toString());
+          env[key] = env[key].replace(/\$PORT/g, projectPort.toString());
         }
       }
     }
@@ -166,7 +147,6 @@ export async function startServer({
         env,
         workingDir: '../project',
         acceptableErrorsRegexes,
-        // logErrors: false, // we do more selective logging ourselves
         onError,
         processName: 'local-project-server',
       }),
@@ -183,6 +163,6 @@ export async function startServer({
     console.log(
       `CodeYam Error: Error executing start script: ${error.message}`,
     );
-    throw error; // Rethrow the error to be caught in the main function
+    throw error;
   }
 }

package/analyzer-template/project/trackGeneratedFiles.ts

@@ -0,0 +1,41 @@
+import { existsSync, readFileSync, writeFileSync, unlinkSync } from 'fs';
+
+const TRACKING_FILE_PATH = '../project/.codeyam-generated-files.json';
+
+export interface TrackedFiles {
+  generatedFiles: string[];
+  backupFiles: string[];
+  timestamp: string;
+}
+
+export function getTrackingFilePath(): string {
+  return TRACKING_FILE_PATH;
+}
+
+export function writeTrackingFile(
+  generatedFiles: string[],
+  backupFiles: string[],
+): void {
+  const data: TrackedFiles = {
+    generatedFiles,
+    backupFiles,
+    timestamp: new Date().toISOString(),
+  };
+  writeFileSync(TRACKING_FILE_PATH, JSON.stringify(data, null, 2), 'utf8');
+}
+
+export function readTrackingFile(): TrackedFiles | null {
+  if (!existsSync(TRACKING_FILE_PATH)) return null;
+  try {
+    const content = readFileSync(TRACKING_FILE_PATH, 'utf8');
+    return JSON.parse(content);
+  } catch {
+    return null;
+  }
+}
+
+export function clearTrackingFile(): void {
+  if (existsSync(TRACKING_FILE_PATH)) {
+    unlinkSync(TRACKING_FILE_PATH);
+  }
+}

package/analyzer-template/project/updateCommitBackgroundJob.ts

@@ -1,4 +1,4 @@
-import { updateBackgroundJobProgress } from '~codeyam/supabase';
+import { updateBackgroundJobProgress } from '~codeyam/database';
 import { BackgroundJob } from '~codeyam/types';
 export default async function updateCommitBackgroundJob(
   backgroundJob: BackgroundJob,

package/analyzer-template/project/utils/errorHandling.ts

@@ -1,6 +1,6 @@
 import { Analysis, Scenario } from '~codeyam/types';
 import emailErrorReport from '../emailErrorReport';
-import { updateFreshAnalysisStatus } from '~codeyam/supabase';
+import { updateFreshAnalysisStatus } from '~codeyam/database';
 
 export async function recordErrorMessageForScenario(
   analysis: Analysis,