@codeyam/codeyam-cli 0.1.0-staging.596f0eb → 0.1.0-staging.62d4615

package/analyzer-template/project/runMultiScenarioServer.ts

@@ -302,14 +302,14 @@ export async function runMultiScenarioServer({
             message: message.slice(0, 1000),
           });
 
-          // Save error to database
-          await recordErrorMessageForScenario(analysis, currentScenario, {
-            message: messageToSave,
-          } as Error);
-
-          // Only signal scenario failed for RELEVANT crash errors
-          // Non-crash errors (like SSR warnings) should be recorded but not stop capture
+          // Only save error and signal scenario failed for RELEVANT crash errors
+          // Non-crash errors (like SSR warnings) should be logged but not stop capture
+          // or mark the scenario as finished (which would prevent screenshot capture)
           if (relevantError) {
+            // Save crash error to database (sets finishedAt on scenario)
+            await recordErrorMessageForScenario(analysis, currentScenario, {
+              message: messageToSave,
+            } as Error);
             awsLog(
               'CodeYam SERVER DEBUG: Calling handleServerError for crash error',
               {
@@ -320,8 +320,9 @@ export async function runMultiScenarioServer({
             );
             iterator.handleServerError(currentScenario.name);
           } else {
+            // Non-crash errors are logged but don't affect scenario status
             awsLog(
-              'CodeYam SERVER DEBUG: Non-crash error recorded, NOT stopping scenario',
+              'CodeYam SERVER DEBUG: Non-crash error logged, NOT stopping scenario',
               {
                 analysisId: analysis.id,
                 scenarioName: currentScenario.name,
@@ -501,13 +502,13 @@ export async function runMultiScenarioServer({
 
       if (!iterator.hasScenarioFailed(scenario.name)) {
         completedScenarios.push(scenario.name);
-        awsLog('CodeYam SERVER DEBUG: Scenario marked as completed', {
+        awsLog('CodeYam: Scenario marked as completed', {
           analysisId: analysis.id,
           scenarioName: scenario.name,
           scenarioId: scenario.id,
         });
       } else {
-        awsLog('CodeYam SERVER DEBUG: Scenario marked as FAILED', {
+        awsLog('CodeYam: Scenario marked as FAILED', {
           analysisId: analysis.id,
           scenarioName: scenario.name,
           scenarioId: scenario.id,

package/analyzer-template/project/serverOnlyModules.ts

@@ -1,3 +1,5 @@
+import ts from 'typescript';
+
 /**
  * Configuration for modules that require client-side mocking.
  *
@@ -206,35 +208,206 @@ export function parseImportStatement(importStatement: string): {
  * Replaces imports from known server-only modules with mock implementations.
  */
 export function applyServerOnlyMocks(fileContent: string): string {
-  let result = fileContent;
+  // Always use AST parsing - regex with nested quantifiers can cause catastrophic
+  // backtracking that hangs on a single .exec() call (before iteration limits kick in)
+  return applyServerOnlyMocksAst(fileContent);
+}
+
+/**
+ * Check if a file exports an "env" variable that's derived from a node module import.
+ * This is a generic pattern for env configuration packages (like @t3-oss/env-*, envalid, etc.)
+ * that create server-only environment objects.
+ *
+ * Pattern detected:
+ *   import { someFunc } from "some-package";
+ *   export const env = someFunc({...});
+ *
+ * Returns the name of the imported function if detected, null otherwise.
+ */
+function detectEnvStoreFromNodeModule(sourceFile: ts.SourceFile): {
+  importedFunctionName: string;
+  modulePath: string;
+} | null {
+  // First, collect all imported function names from node modules
+  const nodeModuleImports = new Map<string, string>(); // functionName -> modulePath
 
-  // Find all import statements
-  const importRegex =
-    /import\s+(?:(?:\{[^}]*\}|\*\s+as\s+\w+|\w+)\s*,?\s*)*\s*from\s*["']([^"']+)["'];?\s*\n?/g;
+  for (const statement of sourceFile.statements) {
+    if (!ts.isImportDeclaration(statement)) continue;
 
-  let match;
-  const replacements: { original: string; replacement: string }[] = [];
+    const moduleSpecifier = statement.moduleSpecifier;
+    if (!ts.isStringLiteral(moduleSpecifier)) continue;
 
-  while ((match = importRegex.exec(fileContent)) !== null) {
-    const fullImport = match[0];
-    const parsed = parseImportStatement(fullImport);
+    const modulePath = moduleSpecifier.text;
+    // Skip relative imports - only look at node modules
+    if (modulePath.startsWith('.') || modulePath.startsWith('/')) continue;
 
-    if (!parsed) continue;
+    if (
+      statement.importClause?.namedBindings &&
+      ts.isNamedImports(statement.importClause.namedBindings)
+    ) {
+      for (const element of statement.importClause.namedBindings.elements) {
+        nodeModuleImports.set(element.name.text, modulePath);
+      }
+    }
+    if (statement.importClause?.name) {
+      nodeModuleImports.set(statement.importClause.name.text, modulePath);
+    }
+  }
 
-    const config = findServerOnlyModule(parsed.moduleSpecifier);
-    if (!config) continue;
+  // Now look for: export const env = importedFunc(...)
+  for (const statement of sourceFile.statements) {
+    if (!ts.isVariableStatement(statement)) continue;
 
-    const mockCode = config.getMockCode(
-      parsed.importNames,
-      parsed.defaultImportName,
+    // Check if it's exported
+    const isExported = statement.modifiers?.some(
+      (m) => m.kind === ts.SyntaxKind.ExportKeyword,
     );
-    replacements.push({ original: fullImport, replacement: mockCode });
-  }
+    if (!isExported) continue;
+
+    for (const decl of statement.declarationList.declarations) {
+      // Check if variable is named "env"
+      if (!ts.isIdentifier(decl.name) || decl.name.text !== 'env') continue;
 
-  // Apply replacements (in reverse order to preserve positions)
-  for (const { original, replacement } of replacements) {
-    result = result.replace(original, replacement);
+      // Check if initializer is a call expression
+      if (!decl.initializer || !ts.isCallExpression(decl.initializer)) continue;
+
+      // Get the function being called
+      const callExpr = decl.initializer.expression;
+      let funcName: string | null = null;
+
+      if (ts.isIdentifier(callExpr)) {
+        funcName = callExpr.text;
+      } else if (
+        ts.isPropertyAccessExpression(callExpr) &&
+        ts.isIdentifier(callExpr.name)
+      ) {
+        funcName = callExpr.name.text;
+      }
+
+      if (funcName && nodeModuleImports.has(funcName)) {
+        return {
+          importedFunctionName: funcName,
+          modulePath: nodeModuleImports.get(funcName)!,
+        };
+      }
+    }
   }
 
-  return result;
+  return null;
+}
+
+/**
+ * Generate mock code for env store patterns.
+ * Replaces the imported function with one that returns a proxy.
+ */
+function generateEnvStoreMock(functionName: string): string {
+  return `// Mock for env store function (server-only env vars)
+const ${functionName} = (config: any) => {
+  // Return a proxy that returns undefined for any property access
+  // This allows the code to run on the client without throwing
+  return new Proxy({} as any, {
+    get: (target, prop) => {
+      if (prop === 'then' || prop === Symbol.toStringTag) return undefined;
+      return undefined;
+    },
+  });
+};
+`;
+}
+
+/**
+ * AST-based implementation of applyServerOnlyMocks for large files.
+ * Uses TypeScript parser instead of regex to avoid catastrophic backtracking.
+ */
+function applyServerOnlyMocksAst(fileContent: string): string {
+  try {
+    const sourceFile = ts.createSourceFile(
+      'temp.ts',
+      fileContent,
+      ts.ScriptTarget.Latest,
+      true,
+    );
+
+    const replacements: { start: number; end: number; replacement: string }[] =
+      [];
+
+    // Check if this file exports an "env" variable from a node module function call
+    // This is a generic pattern for env configuration packages
+    const envStoreInfo = detectEnvStoreFromNodeModule(sourceFile);
+
+    for (const statement of sourceFile.statements) {
+      if (!ts.isImportDeclaration(statement)) continue;
+
+      const moduleSpecifier = statement.moduleSpecifier;
+      if (!ts.isStringLiteral(moduleSpecifier)) continue;
+
+      const modulePath = moduleSpecifier.text;
+
+      // Extract import names
+      const importNames: string[] = [];
+      let defaultImportName: string | undefined;
+
+      if (statement.importClause) {
+        // Default import
+        if (statement.importClause.name) {
+          defaultImportName = statement.importClause.name.text;
+        }
+
+        // Named imports
+        if (statement.importClause.namedBindings) {
+          if (ts.isNamedImports(statement.importClause.namedBindings)) {
+            for (const element of statement.importClause.namedBindings
+              .elements) {
+              importNames.push(element.name.text);
+            }
+          } else if (
+            ts.isNamespaceImport(statement.importClause.namedBindings)
+          ) {
+            // Namespace import like: import * as foo from 'module'
+            importNames.push(statement.importClause.namedBindings.name.text);
+          }
+        }
+      }
+
+      // Check if this import provides the function used to create the env store
+      if (
+        envStoreInfo &&
+        modulePath === envStoreInfo.modulePath &&
+        importNames.includes(envStoreInfo.importedFunctionName)
+      ) {
+        const mockCode = generateEnvStoreMock(
+          envStoreInfo.importedFunctionName,
+        );
+        replacements.push({
+          start: statement.getStart(sourceFile),
+          end: statement.getEnd(),
+          replacement: mockCode,
+        });
+        continue;
+      }
+
+      // Check for known server-only modules (from explicit list)
+      const config = findServerOnlyModule(modulePath);
+      if (!config) continue;
+
+      const mockCode = config.getMockCode(importNames, defaultImportName);
+      replacements.push({
+        start: statement.getStart(sourceFile),
+        end: statement.getEnd(),
+        replacement: mockCode,
+      });
+    }
+
+    // Apply replacements in reverse order to preserve positions
+    let result = fileContent;
+    replacements.sort((a, b) => b.start - a.start);
+    for (const { start, end, replacement } of replacements) {
+      result = result.slice(0, start) + replacement + result.slice(end);
+    }
+
+    return result;
+  } catch (error) {
+    console.warn('[applyServerOnlyMocksAst] Failed to parse file:', error);
+    return fileContent; // Return original content on error
+  }
 }

package/analyzer-template/project/start.ts

@@ -1,4 +1,5 @@
 import 'dotenv/config';
+import { once } from 'events';
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
 import {
@@ -16,14 +17,15 @@ import { startScenarioCapture } from './startScenarioCapture';
 import { createController } from './controller/startController';
 import { awsLog, awsLogDebugLevel } from '~codeyam/utils';
 import * as v8 from 'v8';
-import { initializeAIServiceMode } from '~codeyam/ai';
+import * as os from 'os';
+import { initializeAIServiceMode, destroyWorkerPool } from '~codeyam/ai';
 import runAnalysis, { EnvConfig, SharedContext } from './runAnalysis';
 import {
   loadOrCreateCommit,
   syncHeadBranches,
   syncPrimaryBranch,
 } from '~codeyam/github';
-import { installSignalHandlers } from '../process';
+import { installSignalHandlers } from '~codeyam/process';
 
 // Set process title for Activity Monitor visibility
 if (process.env.CODEYAM_PROCESS_NAME) {
@@ -118,28 +120,31 @@ async function main({
     // and attempted to access project/commit/branch data before it was loaded.
     const sharedContext = await loadSharedContext(envConfig);
 
-    // Heap monitoring: pulse every 1s, log every 30s when normal, warn every 1s when high
+    // Memory monitoring: tracks both V8 heap and RSS (which includes worker threads)
+    const containerMemoryMB = process.env.ECS_CONTAINER_MEMORY_MB
+      ? parseInt(process.env.ECS_CONTAINER_MEMORY_MB, 10)
+      : Math.round(os.totalmem() / 1024 / 1024);
     let lastLogTime = 0;
     setInterval(() => {
       const mem = process.memoryUsage();
       const heap = v8.getHeapStatistics();
-      const usagePercent = (mem.heapUsed / heap.heap_size_limit) * 100;
+      const heapMB = Math.round(mem.heapUsed / 1024 / 1024);
+      const rssMB = Math.round(mem.rss / 1024 / 1024);
+      const heapLimitMB = Math.round(heap.heap_size_limit / 1024 / 1024);
+      const rssPercent = (rssMB / containerMemoryMB) * 100;
       const now = Date.now();
 
-      if (usagePercent >= 80) {
-        // High usage: warn every second
+      if (rssPercent >= 80) {
         console.warn(
-          `CodeYam Heap Warning: ${Math.round(mem.heapUsed / 1024 / 1024)}MB used / ${Math.round(heap.heap_size_limit / 1024 / 1024)}MB limit (${Math.round(usagePercent)}%) ⚠️ HIGH USAGE`,
+          `CodeYam Memory Warning: RSS ${rssMB}MB/${containerMemoryMB}MB (${Math.round(rssPercent)}%), heap ${heapMB}MB/${heapLimitMB}MB ⚠️ HIGH`,
         );
-      } else if (usagePercent >= 50 && now - lastLogTime >= 5000) {
-        // Elevated usage: log every 5 seconds
+      } else if (rssPercent >= 50 && now - lastLogTime >= 5000) {
         console.log(
-          `CodeYam Heap Notice: ${Math.round(mem.heapUsed / 1024 / 1024)}MB used / ${Math.round(heap.heap_size_limit / 1024 / 1024)}MB limit (${Math.round(usagePercent)}%) ⚠️ Elevated`,
+          `CodeYam Memory Notice: RSS ${rssMB}MB/${containerMemoryMB}MB (${Math.round(rssPercent)}%), heap ${heapMB}MB/${heapLimitMB}MB ⚠️ Elevated`,
         );
       } else if (now - lastLogTime >= 30000) {
-        // Normal usage: log every 30 seconds
         console.log(
-          `CodeYam Heap Stats: ${Math.round(mem.heapUsed / 1024 / 1024)}MB used / ${Math.round(heap.heap_size_limit / 1024 / 1024)}MB limit (${Math.round(usagePercent)}%) 📊 OK`,
+          `CodeYam Memory Stats: RSS ${rssMB}MB/${containerMemoryMB}MB (${Math.round(rssPercent)}%), heap ${heapMB}MB/${heapLimitMB}MB 📊 OK`,
         );
       }
 
@@ -186,6 +191,11 @@ async function main({
               analyzerPid: undefined,
               capturePid: process.pid,
             },
+            updateCallback(metadata) {
+              if (metadata.currentRun?.captureCompletedAt) {
+                metadata.currentRun.completedAt = new Date().toISOString();
+              }
+            },
           }).catch((error) => {
             awsLog(`Warning: Failed to update commit metadata: ${error}`);
           });
@@ -207,7 +217,20 @@ async function main({
         while (!controller?.hasCompletedRepoCheckout()) {
           await new Promise((resolve) => setTimeout(resolve, 1000));
         }
-        awsLog('CodeYam: Repo checkout complete, starting orchestration...');
+        awsLog('CodeYam: Repo checkout complete.');
+
+        // Wait for analysis to enter producing phase (after prepareDataStructures)
+        // This prevents the 20-minute orchestration timeout from triggering before
+        // any analyses become ready to capture
+        awsLog('CodeYam: Waiting for analysis to start producing...');
+        if (!controller.hasStartedProducing()) {
+          await once(controller, 'analysisProducing');
+        }
+        awsLog('CodeYam: Analysis producing, starting orchestration...');
+
+        // Worker pool is no longer needed after data structure prep phase.
+        // Freeing it reclaims ~2GB before capture workers ramp up.
+        await destroyWorkerPool();
 
         // Create task runner based on orchestration mode
         let taskRunner;
@@ -219,6 +242,17 @@ async function main({
             framework: framework,
             packageManager: packageManager,
           });
+
+          const captureCommit =
+            sharedContext.commit ?? sharedContext.branchCommit;
+          if (captureCommit) {
+            await updateCommitMetadata({
+              commitId: captureCommit.id,
+              runStatusUpdate: {
+                captureTaskArns: taskRunner.getTaskArns(),
+              },
+            });
+          }
         } else if (orchestrateCapture === 'local-sequential') {
           taskRunner = await SequentialCaptureTaskRunner.create(
             {
@@ -260,8 +294,8 @@ async function main({
             }
           },
           project: sharedContext.project,
-          // In local-sequential mode, don't filter by commit since each analysis run
-          // creates a new fake commit. We want to capture all ready analyses for the project.
+          // In local-sequential mode, don't filter by commit (full=true handles that).
+          // For AWS mode, filter by the specific commit being analyzed.
           commit:
             orchestrateCapture === 'local-sequential'
               ? null
@@ -279,6 +313,8 @@ async function main({
               : envConfig.entityShas,
           full: orchestrateCapture === 'local-sequential',
           taskRunner,
+          // Always track capture progress on the commit, even when not filtering by it.
+          metadataCommit: sharedContext.commit ?? sharedContext.branchCommit,
         });
         awsLog('CodeYam Capture: runCaptureOrchestration() completed');
       };
@@ -440,6 +476,16 @@ async function main({
         `CodeYam: Exiting start.js process with ${hadError ? 'ERROR' : 'SUCCESS'}`,
       );
       process.exit(hadError ? 1 : 0);
+    } else if (process.env.CAPTURE_ONLY === 'true') {
+      awsLog(
+        'CodeYam: CAPTURE_ONLY mode - keeping files and exiting cleanly',
+        `Preserved ${generatedFiles.length} files`,
+      );
+      // Don't cleanup files in capture-only mode to preserve manual edits
+      awsLog(
+        `CodeYam: Exiting start.js process with ${hadError ? 'ERROR' : 'SUCCESS'}`,
+      );
+      process.exit(hadError ? 1 : 0);
     } else {
       awsLog(`CodeYam: Removing ${generatedFiles.length} generated files...`);
       removeScenario(generatedFiles);

package/analyzer-template/project/startScenarioCapture.ts

@@ -36,6 +36,7 @@ import {
 } from './utils/webappMapping';
 import mergeEnvVarsIntoStartCommand from './utils/mergeEnvVarsIntoStartCommand';
 import { LazyFileStore } from './LazyFileStore';
+import { DEFAULT_SCENARIO_NAME } from '../common/constants';
 
 export interface StartScenarioCaptureArgs {
   readyToBeCaptured: Analysis[];
@@ -85,8 +86,9 @@ export async function startScenarioCapture({
   }
 
   // Clean up any leftover generated files from previous debug sessions
+  // Skip in CAPTURE_ONLY mode - we want to preserve the files that were edited
   const previouslyTracked = readTrackingFile();
-  if (previouslyTracked) {
+  if (previouslyTracked && process.env.CAPTURE_ONLY !== 'true') {
     awsLog('CodeYam: Cleaning up files from previous session', {
       generatedCount: previouslyTracked.generatedFiles.length,
       backupCount: previouslyTracked.backupFiles.length,
@@ -97,6 +99,13 @@ export async function startScenarioCapture({
       restoreFromBackup(backupPath);
     }
     clearTrackingFile();
+  } else if (previouslyTracked && process.env.CAPTURE_ONLY === 'true') {
+    awsLog(
+      'CodeYam: CAPTURE_ONLY mode - preserving files from previous session',
+      {
+        generatedCount: previouslyTracked.generatedFiles.length,
+      },
+    );
   }
 
   const sortedReadyToBeCaptured = readyToBeCaptured.sort((a, b) => {
@@ -218,6 +227,11 @@ export async function startScenarioCapture({
     }
 
     const sortedScenarios = (analysis.scenarios ?? []).sort((a, b) => {
+      // Default Scenario always goes first
+      if (a.name === DEFAULT_SCENARIO_NAME) return -1;
+      if (b.name === DEFAULT_SCENARIO_NAME) return 1;
+
+      // Then prioritize new or revised scenarios
       const newOrRevisedA =
         !a.previousVersionId || a.metadata?.revisedFromPreviousVersion;
       const newOrRevisedB =
@@ -352,7 +366,12 @@ export async function startScenarioCapture({
     const mocksDir = `${appPath}/${CODEYAM_MOCKS_DIRNAME}/`;
 
     for (const scenario of sortedScenarios) {
-      if (scenarioIds && !scenarioIds.includes(scenario.id)) {
+      // Check both id and name since scenarioIds may contain either (recapture uses names)
+      if (
+        scenarioIds &&
+        !scenarioIds.includes(scenario.id) &&
+        !scenarioIds.includes(scenario.name)
+      ) {
         awsLog('CodeYam: Skipping scenario', {
           thisScenarioId: scenario.id,
           thisScenarioName: scenario.name,
@@ -379,6 +398,7 @@ export async function startScenarioCapture({
       if (
         !process.env['CODEYAM_LOCAL_PROJECT_PATH'] &&
         !scenarioIds?.includes(scenario.id) &&
+        !scenarioIds?.includes(scenario.name) &&
         !analysisIds?.includes(analysis.id) &&
         (scenarioStatus?.finishedAt ||
           scenarioStatus?.error ||
@@ -389,40 +409,50 @@ export async function startScenarioCapture({
 
       awsLogDebugLevel(1, `Preparing ${analysis.entityName}: ${scenario.name}`);
 
-      // Write scenario files upfront
-      const scenarioGeneratedFiles = await writeScenario({
-        analysis,
-        scenario,
-        context,
-        projectAnalyzer,
-        framework,
-        fileStore,
-      });
-
-      // For non-routes: write root.tsx now (all scenarios can share it)
-      // For routes: defer writeSimpleRoot to per-scenario execution (each needs different mocks)
-      // EXCEPTION: In PREP_ONLY mode (debug), routes also need root.tsx written here since we exit early
-      if (!isRoute || process.env.PREP_ONLY === 'true') {
-        const simpleRootResult = await writeSimpleRoot(
-          project,
-          framework,
+      // In CAPTURE_ONLY mode, skip file generation and use existing files
+      // This preserves manual fixes made during debugging
+      let scenarioGeneratedFiles: string[] = [];
+      if (process.env.CAPTURE_ONLY !== 'true') {
+        // Write scenario files upfront
+        scenarioGeneratedFiles = await writeScenario({
+          analysis,
           scenario,
-          analysis.filePath,
-          {
-            file: analysis.file,
-            entity: analysis.entity,
-            routeAnalysis: analysis,
-            mocksDir,
-            scenarioComponentPaths: scenarioGeneratedFiles,
-          },
-        );
-        if (simpleRootResult.filePath) {
-          if (simpleRootResult.backupPath) {
-            backupFiles.push(simpleRootResult.backupPath);
-          } else {
-            scenarioGeneratedFiles.push(simpleRootResult.filePath);
+          context,
+          projectAnalyzer,
+          framework,
+          fileStore,
+        });
+
+        // For non-routes: write root.tsx now (all scenarios can share it)
+        // For routes: defer writeSimpleRoot to per-scenario execution (each needs different mocks)
+        // EXCEPTION: In PREP_ONLY mode (debug), routes also need root.tsx written here since we exit early
+        if (!isRoute || process.env.PREP_ONLY === 'true') {
+          const simpleRootResult = await writeSimpleRoot(
+            project,
+            framework,
+            scenario,
+            analysis.filePath,
+            {
+              file: analysis.file,
+              entity: analysis.entity,
+              routeAnalysis: analysis,
+              mocksDir,
+              scenarioComponentPaths: scenarioGeneratedFiles,
+            },
+          );
+          if (simpleRootResult.filePath) {
+            if (simpleRootResult.backupPath) {
+              backupFiles.push(simpleRootResult.backupPath);
+            } else {
+              scenarioGeneratedFiles.push(simpleRootResult.filePath);
+            }
           }
         }
+      } else {
+        awsLogDebugLevel(
+          1,
+          `CAPTURE_ONLY mode: skipping file generation for ${scenario.name}`,
+        );
       }
 
       // Mark scenario as initializing
@@ -678,14 +708,22 @@ export async function startScenarioCapture({
         allGeneratedFiles,
       );
 
-      // Clean up all generated files
-      console.log(
-        'CodeYam: Removing all scenario files for analysis',
-        analysis.id,
-        analysis.entityName,
-      );
-      cleanupFiles(allGeneratedFiles, backupFiles);
-      backupFiles = [];
+      // Clean up all generated files (skip in CAPTURE_ONLY mode to preserve edits)
+      if (process.env.CAPTURE_ONLY !== 'true') {
+        console.log(
+          'CodeYam: Removing all scenario files for analysis',
+          analysis.id,
+          analysis.entityName,
+        );
+        cleanupFiles(allGeneratedFiles, backupFiles);
+        backupFiles = [];
+      } else {
+        console.log(
+          'CodeYam: CAPTURE_ONLY mode - preserving scenario files for analysis',
+          analysis.id,
+          analysis.entityName,
+        );
+      }
 
       forceStop = result.forceStop;
     }