@codeyam/codeyam-cli 0.1.0-staging.b8a55ba → 0.1.0-staging.bd4126b

package/analyzer-template/project/serverOnlyModules.ts

@@ -213,6 +213,108 @@ export function applyServerOnlyMocks(fileContent: string): string {
   return applyServerOnlyMocksAst(fileContent);
 }
 
+/**
+ * Check if a file exports an "env" variable that's derived from a node module import.
+ * This is a generic pattern for env configuration packages (like @t3-oss/env-*, envalid, etc.)
+ * that create server-only environment objects.
+ *
+ * Pattern detected:
+ *   import { someFunc } from "some-package";
+ *   export const env = someFunc({...});
+ *
+ * Returns the name of the imported function if detected, null otherwise.
+ */
+function detectEnvStoreFromNodeModule(sourceFile: ts.SourceFile): {
+  importedFunctionName: string;
+  modulePath: string;
+} | null {
+  // First, collect all imported function names from node modules
+  const nodeModuleImports = new Map<string, string>(); // functionName -> modulePath
+
+  for (const statement of sourceFile.statements) {
+    if (!ts.isImportDeclaration(statement)) continue;
+
+    const moduleSpecifier = statement.moduleSpecifier;
+    if (!ts.isStringLiteral(moduleSpecifier)) continue;
+
+    const modulePath = moduleSpecifier.text;
+    // Skip relative imports - only look at node modules
+    if (modulePath.startsWith('.') || modulePath.startsWith('/')) continue;
+
+    if (
+      statement.importClause?.namedBindings &&
+      ts.isNamedImports(statement.importClause.namedBindings)
+    ) {
+      for (const element of statement.importClause.namedBindings.elements) {
+        nodeModuleImports.set(element.name.text, modulePath);
+      }
+    }
+    if (statement.importClause?.name) {
+      nodeModuleImports.set(statement.importClause.name.text, modulePath);
+    }
+  }
+
+  // Now look for: export const env = importedFunc(...)
+  for (const statement of sourceFile.statements) {
+    if (!ts.isVariableStatement(statement)) continue;
+
+    // Check if it's exported
+    const isExported = statement.modifiers?.some(
+      (m) => m.kind === ts.SyntaxKind.ExportKeyword,
+    );
+    if (!isExported) continue;
+
+    for (const decl of statement.declarationList.declarations) {
+      // Check if variable is named "env"
+      if (!ts.isIdentifier(decl.name) || decl.name.text !== 'env') continue;
+
+      // Check if initializer is a call expression
+      if (!decl.initializer || !ts.isCallExpression(decl.initializer)) continue;
+
+      // Get the function being called
+      const callExpr = decl.initializer.expression;
+      let funcName: string | null = null;
+
+      if (ts.isIdentifier(callExpr)) {
+        funcName = callExpr.text;
+      } else if (
+        ts.isPropertyAccessExpression(callExpr) &&
+        ts.isIdentifier(callExpr.name)
+      ) {
+        funcName = callExpr.name.text;
+      }
+
+      if (funcName && nodeModuleImports.has(funcName)) {
+        return {
+          importedFunctionName: funcName,
+          modulePath: nodeModuleImports.get(funcName)!,
+        };
+      }
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Generate mock code for env store patterns.
+ * Replaces the imported function with one that returns a proxy.
+ */
+function generateEnvStoreMock(functionName: string): string {
+  return `// Mock for env store function (server-only env vars)
+const ${functionName} = (config: any) => {
+  // Return a proxy that returns undefined for any property access
+  // This allows the code to run on the client without throwing
+  return new Proxy({} as any, {
+    get: (target, prop) => {
+      if (prop === 'then' || prop === Symbol.toStringTag) return undefined;
+      return undefined;
+    },
+  });
+};
+`;
+}
+
 /**
  * AST-based implementation of applyServerOnlyMocks for large files.
  * Uses TypeScript parser instead of regex to avoid catastrophic backtracking.
@@ -229,6 +331,10 @@ function applyServerOnlyMocksAst(fileContent: string): string {
     const replacements: { start: number; end: number; replacement: string }[] =
       [];
 
+    // Check if this file exports an "env" variable from a node module function call
+    // This is a generic pattern for env configuration packages
+    const envStoreInfo = detectEnvStoreFromNodeModule(sourceFile);
+
     for (const statement of sourceFile.statements) {
       if (!ts.isImportDeclaration(statement)) continue;
 
@@ -236,8 +342,6 @@ function applyServerOnlyMocksAst(fileContent: string): string {
       if (!ts.isStringLiteral(moduleSpecifier)) continue;
 
       const modulePath = moduleSpecifier.text;
-      const config = findServerOnlyModule(modulePath);
-      if (!config) continue;
 
       // Extract import names
       const importNames: string[] = [];
@@ -265,6 +369,27 @@ function applyServerOnlyMocksAst(fileContent: string): string {
         }
       }
 
+      // Check if this import provides the function used to create the env store
+      if (
+        envStoreInfo &&
+        modulePath === envStoreInfo.modulePath &&
+        importNames.includes(envStoreInfo.importedFunctionName)
+      ) {
+        const mockCode = generateEnvStoreMock(
+          envStoreInfo.importedFunctionName,
+        );
+        replacements.push({
+          start: statement.getStart(sourceFile),
+          end: statement.getEnd(),
+          replacement: mockCode,
+        });
+        continue;
+      }
+
+      // Check for known server-only modules (from explicit list)
+      const config = findServerOnlyModule(modulePath);
+      if (!config) continue;
+
       const mockCode = config.getMockCode(importNames, defaultImportName);
       replacements.push({
         start: statement.getStart(sourceFile),

package/analyzer-template/project/start.ts

@@ -1,4 +1,5 @@
 import 'dotenv/config';
+import { once } from 'events';
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
 import {
@@ -16,14 +17,15 @@ import { startScenarioCapture } from './startScenarioCapture';
 import { createController } from './controller/startController';
 import { awsLog, awsLogDebugLevel } from '~codeyam/utils';
 import * as v8 from 'v8';
-import { initializeAIServiceMode } from '~codeyam/ai';
+import * as os from 'os';
+import { initializeAIServiceMode, destroyWorkerPool } from '~codeyam/ai';
 import runAnalysis, { EnvConfig, SharedContext } from './runAnalysis';
 import {
   loadOrCreateCommit,
   syncHeadBranches,
   syncPrimaryBranch,
 } from '~codeyam/github';
-import { installSignalHandlers } from '../process';
+import { installSignalHandlers } from '~codeyam/process';
 
 // Set process title for Activity Monitor visibility
 if (process.env.CODEYAM_PROCESS_NAME) {
@@ -118,28 +120,31 @@ async function main({
     // and attempted to access project/commit/branch data before it was loaded.
     const sharedContext = await loadSharedContext(envConfig);
 
-    // Heap monitoring: pulse every 1s, log every 30s when normal, warn every 1s when high
+    // Memory monitoring: tracks both V8 heap and RSS (which includes worker threads)
+    const containerMemoryMB = process.env.ECS_CONTAINER_MEMORY_MB
+      ? parseInt(process.env.ECS_CONTAINER_MEMORY_MB, 10)
+      : Math.round(os.totalmem() / 1024 / 1024);
     let lastLogTime = 0;
     setInterval(() => {
       const mem = process.memoryUsage();
       const heap = v8.getHeapStatistics();
-      const usagePercent = (mem.heapUsed / heap.heap_size_limit) * 100;
+      const heapMB = Math.round(mem.heapUsed / 1024 / 1024);
+      const rssMB = Math.round(mem.rss / 1024 / 1024);
+      const heapLimitMB = Math.round(heap.heap_size_limit / 1024 / 1024);
+      const rssPercent = (rssMB / containerMemoryMB) * 100;
       const now = Date.now();
 
-      if (usagePercent >= 80) {
-        // High usage: warn every second
+      if (rssPercent >= 80) {
         console.warn(
-          `CodeYam Heap Warning: ${Math.round(mem.heapUsed / 1024 / 1024)}MB used / ${Math.round(heap.heap_size_limit / 1024 / 1024)}MB limit (${Math.round(usagePercent)}%) ⚠️ HIGH USAGE`,
+          `CodeYam Memory Warning: RSS ${rssMB}MB/${containerMemoryMB}MB (${Math.round(rssPercent)}%), heap ${heapMB}MB/${heapLimitMB}MB ⚠️ HIGH`,
         );
-      } else if (usagePercent >= 50 && now - lastLogTime >= 5000) {
-        // Elevated usage: log every 5 seconds
+      } else if (rssPercent >= 50 && now - lastLogTime >= 5000) {
         console.log(
-          `CodeYam Heap Notice: ${Math.round(mem.heapUsed / 1024 / 1024)}MB used / ${Math.round(heap.heap_size_limit / 1024 / 1024)}MB limit (${Math.round(usagePercent)}%) ⚠️ Elevated`,
+          `CodeYam Memory Notice: RSS ${rssMB}MB/${containerMemoryMB}MB (${Math.round(rssPercent)}%), heap ${heapMB}MB/${heapLimitMB}MB ⚠️ Elevated`,
         );
       } else if (now - lastLogTime >= 30000) {
-        // Normal usage: log every 30 seconds
         console.log(
-          `CodeYam Heap Stats: ${Math.round(mem.heapUsed / 1024 / 1024)}MB used / ${Math.round(heap.heap_size_limit / 1024 / 1024)}MB limit (${Math.round(usagePercent)}%) 📊 OK`,
+          `CodeYam Memory Stats: RSS ${rssMB}MB/${containerMemoryMB}MB (${Math.round(rssPercent)}%), heap ${heapMB}MB/${heapLimitMB}MB 📊 OK`,
         );
       }
 
@@ -186,6 +191,11 @@ async function main({
               analyzerPid: undefined,
               capturePid: process.pid,
             },
+            updateCallback(metadata) {
+              if (metadata.currentRun?.captureCompletedAt) {
+                metadata.currentRun.completedAt = new Date().toISOString();
+              }
+            },
           }).catch((error) => {
             awsLog(`Warning: Failed to update commit metadata: ${error}`);
           });
@@ -207,7 +217,20 @@ async function main({
         while (!controller?.hasCompletedRepoCheckout()) {
           await new Promise((resolve) => setTimeout(resolve, 1000));
         }
-        awsLog('CodeYam: Repo checkout complete, starting orchestration...');
+        awsLog('CodeYam: Repo checkout complete.');
+
+        // Wait for analysis to enter producing phase (after prepareDataStructures)
+        // This prevents the 20-minute orchestration timeout from triggering before
+        // any analyses become ready to capture
+        awsLog('CodeYam: Waiting for analysis to start producing...');
+        if (!controller.hasStartedProducing()) {
+          await once(controller, 'analysisProducing');
+        }
+        awsLog('CodeYam: Analysis producing, starting orchestration...');
+
+        // Worker pool is no longer needed after data structure prep phase.
+        // Freeing it reclaims ~2GB before capture workers ramp up.
+        await destroyWorkerPool();
 
         // Create task runner based on orchestration mode
         let taskRunner;
@@ -219,6 +242,17 @@ async function main({
             framework: framework,
             packageManager: packageManager,
           });
+
+          const captureCommit =
+            sharedContext.commit ?? sharedContext.branchCommit;
+          if (captureCommit) {
+            await updateCommitMetadata({
+              commitId: captureCommit.id,
+              runStatusUpdate: {
+                captureTaskArns: taskRunner.getTaskArns(),
+              },
+            });
+          }
         } else if (orchestrateCapture === 'local-sequential') {
           taskRunner = await SequentialCaptureTaskRunner.create(
             {
@@ -260,8 +294,8 @@ async function main({
             }
           },
           project: sharedContext.project,
-          // In local-sequential mode, don't filter by commit since each analysis run
-          // creates a new fake commit. We want to capture all ready analyses for the project.
+          // In local-sequential mode, don't filter by commit (full=true handles that).
+          // For AWS mode, filter by the specific commit being analyzed.
           commit:
             orchestrateCapture === 'local-sequential'
               ? null
@@ -279,6 +313,8 @@ async function main({
               : envConfig.entityShas,
           full: orchestrateCapture === 'local-sequential',
           taskRunner,
+          // Always track capture progress on the commit, even when not filtering by it.
+          metadataCommit: sharedContext.commit ?? sharedContext.branchCommit,
         });
         awsLog('CodeYam Capture: runCaptureOrchestration() completed');
       };

package/analyzer-template/project/startScenarioCapture.ts

@@ -36,6 +36,7 @@ import {
 } from './utils/webappMapping';
 import mergeEnvVarsIntoStartCommand from './utils/mergeEnvVarsIntoStartCommand';
 import { LazyFileStore } from './LazyFileStore';
+import { DEFAULT_SCENARIO_NAME } from '../common/constants';
 
 export interface StartScenarioCaptureArgs {
   readyToBeCaptured: Analysis[];
@@ -226,6 +227,11 @@ export async function startScenarioCapture({
     }
 
     const sortedScenarios = (analysis.scenarios ?? []).sort((a, b) => {
+      // Default Scenario always goes first
+      if (a.name === DEFAULT_SCENARIO_NAME) return -1;
+      if (b.name === DEFAULT_SCENARIO_NAME) return 1;
+
+      // Then prioritize new or revised scenarios
       const newOrRevisedA =
         !a.previousVersionId || a.metadata?.revisedFromPreviousVersion;
       const newOrRevisedB =