@codeyam/codeyam-cli 0.1.0-staging.b8a55ba → 0.1.0-staging.b8f4f94

package/analyzer-template/project/serverOnlyModules.ts

@@ -213,6 +213,108 @@ export function applyServerOnlyMocks(fileContent: string): string {
   return applyServerOnlyMocksAst(fileContent);
 }
 
+/**
+ * Check if a file exports an "env" variable that's derived from a node module import.
+ * This is a generic pattern for env configuration packages (like @t3-oss/env-*, envalid, etc.)
+ * that create server-only environment objects.
+ *
+ * Pattern detected:
+ *   import { someFunc } from "some-package";
+ *   export const env = someFunc({...});
+ *
+ * Returns the name of the imported function if detected, null otherwise.
+ */
+function detectEnvStoreFromNodeModule(sourceFile: ts.SourceFile): {
+  importedFunctionName: string;
+  modulePath: string;
+} | null {
+  // First, collect all imported function names from node modules
+  const nodeModuleImports = new Map<string, string>(); // functionName -> modulePath
+
+  for (const statement of sourceFile.statements) {
+    if (!ts.isImportDeclaration(statement)) continue;
+
+    const moduleSpecifier = statement.moduleSpecifier;
+    if (!ts.isStringLiteral(moduleSpecifier)) continue;
+
+    const modulePath = moduleSpecifier.text;
+    // Skip relative imports - only look at node modules
+    if (modulePath.startsWith('.') || modulePath.startsWith('/')) continue;
+
+    if (
+      statement.importClause?.namedBindings &&
+      ts.isNamedImports(statement.importClause.namedBindings)
+    ) {
+      for (const element of statement.importClause.namedBindings.elements) {
+        nodeModuleImports.set(element.name.text, modulePath);
+      }
+    }
+    if (statement.importClause?.name) {
+      nodeModuleImports.set(statement.importClause.name.text, modulePath);
+    }
+  }
+
+  // Now look for: export const env = importedFunc(...)
+  for (const statement of sourceFile.statements) {
+    if (!ts.isVariableStatement(statement)) continue;
+
+    // Check if it's exported
+    const isExported = statement.modifiers?.some(
+      (m) => m.kind === ts.SyntaxKind.ExportKeyword,
+    );
+    if (!isExported) continue;
+
+    for (const decl of statement.declarationList.declarations) {
+      // Check if variable is named "env"
+      if (!ts.isIdentifier(decl.name) || decl.name.text !== 'env') continue;
+
+      // Check if initializer is a call expression
+      if (!decl.initializer || !ts.isCallExpression(decl.initializer)) continue;
+
+      // Get the function being called
+      const callExpr = decl.initializer.expression;
+      let funcName: string | null = null;
+
+      if (ts.isIdentifier(callExpr)) {
+        funcName = callExpr.text;
+      } else if (
+        ts.isPropertyAccessExpression(callExpr) &&
+        ts.isIdentifier(callExpr.name)
+      ) {
+        funcName = callExpr.name.text;
+      }
+
+      if (funcName && nodeModuleImports.has(funcName)) {
+        return {
+          importedFunctionName: funcName,
+          modulePath: nodeModuleImports.get(funcName)!,
+        };
+      }
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Generate mock code for env store patterns.
+ * Replaces the imported function with one that returns a proxy.
+ */
+function generateEnvStoreMock(functionName: string): string {
+  return `// Mock for env store function (server-only env vars)
+const ${functionName} = (config: any) => {
+  // Return a proxy that returns undefined for any property access
+  // This allows the code to run on the client without throwing
+  return new Proxy({} as any, {
+    get: (target, prop) => {
+      if (prop === 'then' || prop === Symbol.toStringTag) return undefined;
+      return undefined;
+    },
+  });
+};
+`;
+}
+
 /**
  * AST-based implementation of applyServerOnlyMocks for large files.
  * Uses TypeScript parser instead of regex to avoid catastrophic backtracking.
@@ -229,6 +331,10 @@ function applyServerOnlyMocksAst(fileContent: string): string {
     const replacements: { start: number; end: number; replacement: string }[] =
       [];
 
+    // Check if this file exports an "env" variable from a node module function call
+    // This is a generic pattern for env configuration packages
+    const envStoreInfo = detectEnvStoreFromNodeModule(sourceFile);
+
     for (const statement of sourceFile.statements) {
       if (!ts.isImportDeclaration(statement)) continue;
 
@@ -236,8 +342,6 @@ function applyServerOnlyMocksAst(fileContent: string): string {
       if (!ts.isStringLiteral(moduleSpecifier)) continue;
 
       const modulePath = moduleSpecifier.text;
-      const config = findServerOnlyModule(modulePath);
-      if (!config) continue;
 
       // Extract import names
       const importNames: string[] = [];
@@ -265,6 +369,27 @@ function applyServerOnlyMocksAst(fileContent: string): string {
         }
       }
 
+      // Check if this import provides the function used to create the env store
+      if (
+        envStoreInfo &&
+        modulePath === envStoreInfo.modulePath &&
+        importNames.includes(envStoreInfo.importedFunctionName)
+      ) {
+        const mockCode = generateEnvStoreMock(
+          envStoreInfo.importedFunctionName,
+        );
+        replacements.push({
+          start: statement.getStart(sourceFile),
+          end: statement.getEnd(),
+          replacement: mockCode,
+        });
+        continue;
+      }
+
+      // Check for known server-only modules (from explicit list)
+      const config = findServerOnlyModule(modulePath);
+      if (!config) continue;
+
       const mockCode = config.getMockCode(importNames, defaultImportName);
       replacements.push({
         start: statement.getStart(sourceFile),

package/analyzer-template/project/start.ts

@@ -1,4 +1,5 @@
 import 'dotenv/config';
+import { once } from 'events';
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
 import {
@@ -16,14 +17,15 @@ import { startScenarioCapture } from './startScenarioCapture';
 import { createController } from './controller/startController';
 import { awsLog, awsLogDebugLevel } from '~codeyam/utils';
 import * as v8 from 'v8';
-import { initializeAIServiceMode } from '~codeyam/ai';
+import * as os from 'os';
+import { initializeAIServiceMode, destroyWorkerPool } from '~codeyam/ai';
 import runAnalysis, { EnvConfig, SharedContext } from './runAnalysis';
 import {
   loadOrCreateCommit,
   syncHeadBranches,
   syncPrimaryBranch,
 } from '~codeyam/github';
-import { installSignalHandlers } from '../process';
+import { installSignalHandlers } from '~codeyam/process';
 
 // Set process title for Activity Monitor visibility
 if (process.env.CODEYAM_PROCESS_NAME) {
@@ -118,28 +120,31 @@ async function main({
     // and attempted to access project/commit/branch data before it was loaded.
     const sharedContext = await loadSharedContext(envConfig);
 
-    // Heap monitoring: pulse every 1s, log every 30s when normal, warn every 1s when high
+    // Memory monitoring: tracks both V8 heap and RSS (which includes worker threads)
+    const containerMemoryMB = process.env.ECS_CONTAINER_MEMORY_MB
+      ? parseInt(process.env.ECS_CONTAINER_MEMORY_MB, 10)
+      : Math.round(os.totalmem() / 1024 / 1024);
     let lastLogTime = 0;
     setInterval(() => {
       const mem = process.memoryUsage();
       const heap = v8.getHeapStatistics();
-      const usagePercent = (mem.heapUsed / heap.heap_size_limit) * 100;
+      const heapMB = Math.round(mem.heapUsed / 1024 / 1024);
+      const rssMB = Math.round(mem.rss / 1024 / 1024);
+      const heapLimitMB = Math.round(heap.heap_size_limit / 1024 / 1024);
+      const rssPercent = (rssMB / containerMemoryMB) * 100;
       const now = Date.now();
 
-      if (usagePercent >= 80) {
-        // High usage: warn every second
+      if (rssPercent >= 80) {
         console.warn(
-          `CodeYam Heap Warning: ${Math.round(mem.heapUsed / 1024 / 1024)}MB used / ${Math.round(heap.heap_size_limit / 1024 / 1024)}MB limit (${Math.round(usagePercent)}%) ⚠️ HIGH USAGE`,
+          `CodeYam Memory Warning: RSS ${rssMB}MB/${containerMemoryMB}MB (${Math.round(rssPercent)}%), heap ${heapMB}MB/${heapLimitMB}MB ⚠️ HIGH`,
         );
-      } else if (usagePercent >= 50 && now - lastLogTime >= 5000) {
-        // Elevated usage: log every 5 seconds
+      } else if (rssPercent >= 50 && now - lastLogTime >= 5000) {
         console.log(
-          `CodeYam Heap Notice: ${Math.round(mem.heapUsed / 1024 / 1024)}MB used / ${Math.round(heap.heap_size_limit / 1024 / 1024)}MB limit (${Math.round(usagePercent)}%) ⚠️ Elevated`,
+          `CodeYam Memory Notice: RSS ${rssMB}MB/${containerMemoryMB}MB (${Math.round(rssPercent)}%), heap ${heapMB}MB/${heapLimitMB}MB ⚠️ Elevated`,
         );
       } else if (now - lastLogTime >= 30000) {
-        // Normal usage: log every 30 seconds
         console.log(
-          `CodeYam Heap Stats: ${Math.round(mem.heapUsed / 1024 / 1024)}MB used / ${Math.round(heap.heap_size_limit / 1024 / 1024)}MB limit (${Math.round(usagePercent)}%) 📊 OK`,
+          `CodeYam Memory Stats: RSS ${rssMB}MB/${containerMemoryMB}MB (${Math.round(rssPercent)}%), heap ${heapMB}MB/${heapLimitMB}MB 📊 OK`,
         );
       }
 
@@ -186,6 +191,11 @@ async function main({
               analyzerPid: undefined,
               capturePid: process.pid,
             },
+            updateCallback(metadata) {
+              if (metadata.currentRun?.captureCompletedAt) {
+                metadata.currentRun.completedAt = new Date().toISOString();
+              }
+            },
           }).catch((error) => {
             awsLog(`Warning: Failed to update commit metadata: ${error}`);
           });
@@ -207,7 +217,20 @@ async function main({
         while (!controller?.hasCompletedRepoCheckout()) {
           await new Promise((resolve) => setTimeout(resolve, 1000));
         }
-        awsLog('CodeYam: Repo checkout complete, starting orchestration...');
+        awsLog('CodeYam: Repo checkout complete.');
+
+        // Wait for analysis to enter producing phase (after prepareDataStructures)
+        // This prevents the 20-minute orchestration timeout from triggering before
+        // any analyses become ready to capture
+        awsLog('CodeYam: Waiting for analysis to start producing...');
+        if (!controller.hasStartedProducing()) {
+          await once(controller, 'analysisProducing');
+        }
+        awsLog('CodeYam: Analysis producing, starting orchestration...');
+
+        // Worker pool is no longer needed after data structure prep phase.
+        // Freeing it reclaims ~2GB before capture workers ramp up.
+        await destroyWorkerPool();
 
         // Create task runner based on orchestration mode
         let taskRunner;
@@ -219,6 +242,17 @@ async function main({
             framework: framework,
             packageManager: packageManager,
           });
+
+          const captureCommit =
+            sharedContext.commit ?? sharedContext.branchCommit;
+          if (captureCommit) {
+            await updateCommitMetadata({
+              commitId: captureCommit.id,
+              runStatusUpdate: {
+                captureTaskArns: taskRunner.getTaskArns(),
+              },
+            });
+          }
         } else if (orchestrateCapture === 'local-sequential') {
           taskRunner = await SequentialCaptureTaskRunner.create(
             {
@@ -260,8 +294,8 @@ async function main({
             }
           },
           project: sharedContext.project,
-          // In local-sequential mode, don't filter by commit since each analysis run
-          // creates a new fake commit. We want to capture all ready analyses for the project.
+          // In local-sequential mode, don't filter by commit (full=true handles that).
+          // For AWS mode, filter by the specific commit being analyzed.
           commit:
             orchestrateCapture === 'local-sequential'
               ? null
@@ -279,6 +313,11 @@ async function main({
               : envConfig.entityShas,
           full: orchestrateCapture === 'local-sequential',
           taskRunner,
+          // Poll faster in local mode - no DB load concerns like in cloud
+          pollingInterval:
+            orchestrateCapture === 'local-sequential' ? 5 * 1000 : undefined,
+          // Always track capture progress on the commit, even when not filtering by it.
+          metadataCommit: sharedContext.commit ?? sharedContext.branchCommit,
         });
         awsLog('CodeYam Capture: runCaptureOrchestration() completed');
       };

package/analyzer-template/project/startScenarioCapture.ts

@@ -6,6 +6,7 @@ import writeUniversalMocks from './writeUniversalMocks';
 import writeScenario from './writeScenario';
 import writeSimpleRoot from './writeSimpleRoot';
 import writeCodeYamPage from './writeCodeYamPage';
+import writeClientLogRoute from './writeClientLogRoute';
 import {
   runMultiScenarioServer,
   ScenarioToCapture,
@@ -36,6 +37,7 @@ import {
 } from './utils/webappMapping';
 import mergeEnvVarsIntoStartCommand from './utils/mergeEnvVarsIntoStartCommand';
 import { LazyFileStore } from './LazyFileStore';
+import { DEFAULT_SCENARIO_NAME } from '../common/constants';
 
 export interface StartScenarioCaptureArgs {
   readyToBeCaptured: Analysis[];
@@ -226,6 +228,11 @@ export async function startScenarioCapture({
     }
 
     const sortedScenarios = (analysis.scenarios ?? []).sort((a, b) => {
+      // Default Scenario always goes first
+      if (a.name === DEFAULT_SCENARIO_NAME) return -1;
+      if (b.name === DEFAULT_SCENARIO_NAME) return 1;
+
+      // Then prioritize new or revised scenarios
       const newOrRevisedA =
         !a.previousVersionId || a.metadata?.revisedFromPreviousVersion;
       const newOrRevisedB =
@@ -491,6 +498,14 @@ export async function startScenarioCapture({
       );
       allGeneratedFiles.push(codeyamPagePath);
 
+      // Write client-log API route so client-side errors appear in the server log
+      const clientLogPath = await writeClientLogRoute(
+        project,
+        framework,
+        analysis.filePath,
+      );
+      if (clientLogPath) allGeneratedFiles.push(clientLogPath);
+
       // Persist file tracking for crash/debug recovery
       writeTrackingFile(allGeneratedFiles, backupFiles);
     }

package/analyzer-template/project/writeClientLogRoute.ts

@@ -0,0 +1,125 @@
+import { Project, ProjectFramework } from '~codeyam/types';
+import { findWebAppForFile, PROJECT_RELATIVE_PATH } from '~codeyam/utils';
+import writeFile from '../common/writeFile';
+import * as path from 'path';
+
+/**
+ * Writes a client-log API route to the project so that client-side errors
+ * (window.onerror, unhandledrejection, console.error) are reported back to
+ * the server log where they can be read from the terminal.
+ *
+ * The route receives JSON POST requests and console.log's them, which sends
+ * the output to the dev server's stdout (captured in the server log file).
+ */
+export default async function writeClientLogRoute(
+  project: Project,
+  framework: ProjectFramework,
+  analysisFilePath?: string,
+) {
+  const relativePath = PROJECT_RELATIVE_PATH;
+  const fullAppPath = getFullAppPath(project, analysisFilePath);
+
+  let routePath: string;
+  let content: string;
+
+  switch (framework) {
+    case ProjectFramework.Remix:
+    case ProjectFramework.CRA:
+    case ProjectFramework.CodeYam:
+      routePath = `${relativePath}/${fullAppPath}/routes/api.client-log.ts`;
+      content = getRemixRoute();
+      break;
+    case ProjectFramework.Next:
+      routePath = `${relativePath}/${fullAppPath}/api/client-log/route.ts`;
+      content = getNextAppRoute();
+      break;
+    case ProjectFramework.NextPages:
+      routePath = `${relativePath}/pages/api/client-log.ts`;
+      content = getNextPagesRoute();
+      break;
+    default:
+      return;
+  }
+
+  await writeFile(routePath, content);
+  return routePath;
+}
+
+function getFullAppPath(project: Project, analysisFilePath?: string): string {
+  if (analysisFilePath) {
+    const webapp = findWebAppForFile(
+      analysisFilePath,
+      project.metadata?.webapps ?? [],
+    );
+    if (webapp) {
+      const webappPath = webapp.path === '.' ? '' : webapp.path;
+      const appDirectory = webapp.appDirectory ?? 'app';
+      return webappPath ? path.join(webappPath, appDirectory) : appDirectory;
+    }
+  }
+  const firstWebapp = project.metadata?.webapps?.[0];
+  if (firstWebapp) {
+    const webappPath = firstWebapp.path === '.' ? '' : firstWebapp.path;
+    const appDirectory = firstWebapp.appDirectory ?? 'app';
+    return webappPath ? path.join(webappPath, appDirectory) : appDirectory;
+  }
+  return 'app';
+}
+
+function formatLogEntry(): string {
+  return `
+function formatEntry(body) {
+  const ts = new Date().toLocaleString();
+  const scenario = body.scenario ? ' [' + body.scenario + ']' : '';
+  const type = (body.type || 'error').toUpperCase();
+  const lines = ['[' + ts + '] [CLIENT ' + type + ']' + scenario + ' ' + (body.message || 'Unknown error')];
+  if (body.stack) lines.push(body.stack);
+  if (body.url) lines.push('  at ' + body.url);
+  return lines.join('\\n');
+}`;
+}
+
+function getRemixRoute(): string {
+  return `// Auto-generated by CodeYam — client error reporting endpoint
+${formatLogEntry()}
+
+export async function action({ request }) {
+  try {
+    const body = await request.json();
+    console.log(formatEntry(body));
+  } catch (e) {}
+  return new Response('ok', { status: 200 });
+}
+`;
+}
+
+function getNextAppRoute(): string {
+  return `// Auto-generated by CodeYam — client error reporting endpoint
+import { NextResponse } from 'next/server';
+${formatLogEntry()}
+
+export async function POST(request: Request) {
+  try {
+    const body = await request.json();
+    console.log(formatEntry(body));
+  } catch (e) {}
+  return NextResponse.json({ ok: true });
+}
+`;
+}
+
+function getNextPagesRoute(): string {
+  return `// Auto-generated by CodeYam — client error reporting endpoint
+import type { NextApiRequest, NextApiResponse } from 'next';
+${formatLogEntry()}
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  if (req.method !== 'POST') return res.status(405).end();
+  try {
+    const body = typeof req.body === 'string' ? JSON.parse(req.body) : req.body;
+    console.log(formatEntry(body));
+  } catch (e) {}
+  res.status(200).json({ ok: true });
+}
+`;
+}