@codeyam/codeyam-cli 0.1.0-staging.1669d45 → 0.1.0-staging.2a88920

package/analyzer-template/project/start.ts

@@ -17,7 +17,8 @@ import { startScenarioCapture } from './startScenarioCapture';
 import { createController } from './controller/startController';
 import { awsLog, awsLogDebugLevel } from '~codeyam/utils';
 import * as v8 from 'v8';
-import { initializeAIServiceMode } from '~codeyam/ai';
+import * as os from 'os';
+import { initializeAIServiceMode, destroyWorkerPool } from '~codeyam/ai';
 import runAnalysis, { EnvConfig, SharedContext } from './runAnalysis';
 import {
   loadOrCreateCommit,
@@ -119,28 +120,31 @@ async function main({
     // and attempted to access project/commit/branch data before it was loaded.
     const sharedContext = await loadSharedContext(envConfig);
 
-    // Heap monitoring: pulse every 1s, log every 30s when normal, warn every 1s when high
+    // Memory monitoring: tracks both V8 heap and RSS (which includes worker threads)
+    const containerMemoryMB = process.env.ECS_CONTAINER_MEMORY_MB
+      ? parseInt(process.env.ECS_CONTAINER_MEMORY_MB, 10)
+      : Math.round(os.totalmem() / 1024 / 1024);
     let lastLogTime = 0;
     setInterval(() => {
       const mem = process.memoryUsage();
       const heap = v8.getHeapStatistics();
-      const usagePercent = (mem.heapUsed / heap.heap_size_limit) * 100;
+      const heapMB = Math.round(mem.heapUsed / 1024 / 1024);
+      const rssMB = Math.round(mem.rss / 1024 / 1024);
+      const heapLimitMB = Math.round(heap.heap_size_limit / 1024 / 1024);
+      const rssPercent = (rssMB / containerMemoryMB) * 100;
       const now = Date.now();
 
-      if (usagePercent >= 80) {
-        // High usage: warn every second
+      if (rssPercent >= 80) {
         console.warn(
-          `CodeYam Heap Warning: ${Math.round(mem.heapUsed / 1024 / 1024)}MB used / ${Math.round(heap.heap_size_limit / 1024 / 1024)}MB limit (${Math.round(usagePercent)}%) ⚠️ HIGH USAGE`,
+          `CodeYam Memory Warning: RSS ${rssMB}MB/${containerMemoryMB}MB (${Math.round(rssPercent)}%), heap ${heapMB}MB/${heapLimitMB}MB ⚠️ HIGH`,
         );
-      } else if (usagePercent >= 50 && now - lastLogTime >= 5000) {
-        // Elevated usage: log every 5 seconds
+      } else if (rssPercent >= 50 && now - lastLogTime >= 5000) {
         console.log(
-          `CodeYam Heap Notice: ${Math.round(mem.heapUsed / 1024 / 1024)}MB used / ${Math.round(heap.heap_size_limit / 1024 / 1024)}MB limit (${Math.round(usagePercent)}%) ⚠️ Elevated`,
+          `CodeYam Memory Notice: RSS ${rssMB}MB/${containerMemoryMB}MB (${Math.round(rssPercent)}%), heap ${heapMB}MB/${heapLimitMB}MB ⚠️ Elevated`,
         );
       } else if (now - lastLogTime >= 30000) {
-        // Normal usage: log every 30 seconds
         console.log(
-          `CodeYam Heap Stats: ${Math.round(mem.heapUsed / 1024 / 1024)}MB used / ${Math.round(heap.heap_size_limit / 1024 / 1024)}MB limit (${Math.round(usagePercent)}%) 📊 OK`,
+          `CodeYam Memory Stats: RSS ${rssMB}MB/${containerMemoryMB}MB (${Math.round(rssPercent)}%), heap ${heapMB}MB/${heapLimitMB}MB 📊 OK`,
         );
       }
 
@@ -187,6 +191,11 @@ async function main({
               analyzerPid: undefined,
               capturePid: process.pid,
             },
+            updateCallback(metadata) {
+              if (metadata.currentRun?.captureCompletedAt) {
+                metadata.currentRun.completedAt = new Date().toISOString();
+              }
+            },
           }).catch((error) => {
             awsLog(`Warning: Failed to update commit metadata: ${error}`);
           });
@@ -219,6 +228,10 @@ async function main({
         }
         awsLog('CodeYam: Analysis producing, starting orchestration...');
 
+        // Worker pool is no longer needed after data structure prep phase.
+        // Freeing it reclaims ~2GB before capture workers ramp up.
+        await destroyWorkerPool();
+
         // Create task runner based on orchestration mode
         let taskRunner;
 
@@ -229,6 +242,17 @@ async function main({
             framework: framework,
             packageManager: packageManager,
           });
+
+          const captureCommit =
+            sharedContext.commit ?? sharedContext.branchCommit;
+          if (captureCommit) {
+            await updateCommitMetadata({
+              commitId: captureCommit.id,
+              runStatusUpdate: {
+                captureTaskArns: taskRunner.getTaskArns(),
+              },
+            });
+          }
         } else if (orchestrateCapture === 'local-sequential') {
           taskRunner = await SequentialCaptureTaskRunner.create(
             {

package/analyzer-template/project/writeMockDataTsx.ts

@@ -43,6 +43,108 @@ function normalizeParamsMockData(
   return result;
 }
 
+/**
+ * Convert comma-separated string values to arrays when they look like array data.
+ * This handles cases where the LLM generates strings like "color,size" instead
+ * of arrays like ["color", "size"] due to schema type misdetection.
+ */
+function convertCommaSeparatedStringsToArrays(
+  mockData: Record<string, unknown>,
+): Record<string, unknown> {
+  const result: Record<string, unknown> = {};
+
+  for (const [key, value] of Object.entries(mockData)) {
+    if (typeof value === 'string') {
+      const asArray = parseCommaSeparatedStringAsArray(value, key);
+      if (asArray !== null) {
+        result[key] = asArray;
+        console.log(
+          `CodeYam: Converted comma-separated string to array for key "${key}": "${value}" -> [${asArray.map((s) => `"${s}"`).join(', ')}]`,
+        );
+      } else {
+        result[key] = value;
+      }
+    } else if (
+      value !== null &&
+      typeof value === 'object' &&
+      !Array.isArray(value)
+    ) {
+      // Recursively process nested objects
+      result[key] = convertCommaSeparatedStringsToArrays(
+        value as Record<string, unknown>,
+      );
+    } else if (Array.isArray(value)) {
+      // Recursively process arrays (each element could be an object)
+      result[key] = value.map((item) => {
+        if (item !== null && typeof item === 'object' && !Array.isArray(item)) {
+          return convertCommaSeparatedStringsToArrays(
+            item as Record<string, unknown>,
+          );
+        }
+        return item;
+      });
+    } else {
+      result[key] = value;
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Detect if a string looks like a comma-separated list that should be an array.
+ */
+function parseCommaSeparatedStringAsArray(
+  value: string,
+  key: string,
+): string[] | null {
+  if (!value.includes(',')) {
+    return null;
+  }
+
+  // Heuristic: if the key name suggests it's an array field, convert it
+  // Check this FIRST before sentence detection, since "color, size" looks like
+  // a sentence but should be converted to ["color", "size"] for _attributes fields
+  const arrayFieldPatterns = [
+    /_attributes$/i,
+    /_ids$/i,
+    /_items$/i,
+    /_tags$/i,
+    /_values$/i,
+    /_types$/i,
+    /_names$/i,
+    /_keys$/i,
+    /^attributes$/i,
+    /^items$/i,
+    /^tags$/i,
+    /^values$/i,
+  ];
+
+  const looksLikeArrayField = arrayFieldPatterns.some((pattern) =>
+    pattern.test(key),
+  );
+
+  if (looksLikeArrayField) {
+    // For array-like fields, only reject if it contains newlines (likely prose)
+    if (value.includes('\n')) {
+      return null;
+    }
+    const parts = value.split(',').map((s) => s.trim());
+    return parts.filter((s) => s.length > 0);
+  }
+
+  // For non-array-named fields, apply stricter sentence detection
+  // Looks like a sentence: "Hello, world"
+  if (/,\s+[a-z]/.test(value)) {
+    return null;
+  }
+  if (value.includes('\n')) {
+    return null;
+  }
+
+  return null;
+}
+
 /**
  * Return true when `markup` contains 0 or >1 meaningful
  * top-level nodes (element *or* non-blank text), so React
@@ -632,13 +734,19 @@ const jsonToCode = (
               stringValue.startsWith('Promise.') ||
               stringValue.startsWith('new Promise');
 
+            // Check if the string contains newlines (potential issue for string literals)
+            const hasNewlines = stringValue.includes('\n');
+
             // Handle special type indicators that need transformation
             let outputValue: string;
             if (stringValue === 'promise' || looksLikePromise) {
               // The key is a function call (e.g., "revalidate()"), so output a function
               // that returns a Promise, not the Promise itself
+              // Important: Escape newlines inside Promise expressions to avoid
+              // "unterminated string literal" errors when the Promise contains
+              // multi-line string arguments like Promise.resolve('line1\nline2')
               const promiseExpr = looksLikePromise
-                ? stringValue
+                ? stringValue.replace(/\n/g, '\\n')
                 : 'Promise.resolve()';
               outputValue = `() => ${promiseExpr}`;
               console.log(
@@ -650,7 +758,12 @@ const jsonToCode = (
                 `CodeYam: Nested func transform (string->function): "${key}" -> "${funcName}": () => {}`,
               );
             } else if (looksLikeFunction) {
-              outputValue = stringValue;
+              // If the function expression contains newlines inside string literals,
+              // we need to escape them to avoid "unterminated string literal" errors
+              // This handles cases like: () => Promise.resolve('line1\nline2')
+              outputValue = hasNewlines
+                ? stringValue.replace(/\n/g, '\\n')
+                : stringValue;
               console.log(
                 `CodeYam: Nested func transform (string): "${key}" -> "${funcName}": ${stringValue.substring(0, 30)}...`,
               );
@@ -749,11 +862,16 @@ export default async function writeMockDataTsx(
 
   // Reconcile mock data keys to match expected keys from dataForMocks
   // This handles cases where LLM generated literal values instead of variable references
-  const defaultMockData = reconcileMockDataKeys(
+  const reconciledMockData = reconcileMockDataKeys(
     rawDefaultMockData,
     expectedKeys,
   );
 
+  // Convert comma-separated strings to arrays for fields that should be arrays
+  // This handles cases where LLM generates "color,size" instead of ["color", "size"]
+  const defaultMockData =
+    convertCommaSeparatedStringsToArrays(reconciledMockData);
+
   const mockDataComment = `// This file is auto-generated by CodeYam. Do not edit this file manually.
 // This file contains mock data required for:
 // Scenario: ${mockScenario.id} - ${mockScenario.name}
@@ -791,11 +909,16 @@ import React from 'react';
     };
 
     // Reconcile scenario-specific mock data keys as well
-    const scenarioMockData = reconcileMockDataKeys(
+    const reconciledScenarioMockData = reconcileMockDataKeys(
       rawScenarioMockData,
       expectedKeys,
     );
 
+    // Convert comma-separated strings to arrays for scenario data
+    const scenarioMockData = convertCommaSeparatedStringsToArrays(
+      reconciledScenarioMockData,
+    );
+
     const mergedArguments = defaultArguments.map((defaultArg, index) => {
       return deepMerge(defaultArg, scenarioArguments[index]);
     });

package/analyzer-template/project/writeScenarioComponents.ts

@@ -1205,6 +1205,21 @@ function addMockToContent(
         fileContent = fileContent.replace(importRegExp, '$1$2');
       }
 
+      // Also handle namespace imports (import * as foo from '...')
+      // These need to be renamed to foo__cyOriginal when the mock code spreads from the original.
+      // Note: We match any path (not just escapedImportPath) because the import path may have
+      // been rewritten by transitive import handling before this code runs.
+      if (shouldRenameToOriginal) {
+        const namespaceImportRegExp = new RegExp(
+          `(import\\s+\\*\\s+as\\s+)${escapeRegExp(firstPart)}(\\s+from\\s+['"][^'"]*['"])`,
+          'm',
+        );
+        fileContent = fileContent.replace(
+          namespaceImportRegExp,
+          `$1${firstPart}__cyOriginal$2`,
+        );
+      }
+
       // Remove empty imports entirely to avoid partial commenting issues with multiline imports
       // This handles both single-line and multiline empty imports
       fileContent = fileContent.replace(
@@ -1698,9 +1713,16 @@ export default async function writeScenarioComponents({
   });
 
   let importedExportIndex = 0;
+  const loopStartTime = Date.now();
+  console.log(
+    `[WriteScenario] Starting import loop for ${entity.name}: ${sortedImportedExports.length} imports`,
+  );
   for (const importedExport of sortedImportedExports) {
     importedExportIndex++;
     if (importedExportIndex % 5 === 0 || importedExportIndex === 1) {
+      console.log(
+        `[WriteScenario] ${entity.name} import ${importedExportIndex}/${sortedImportedExports.length}: ${importedExport.name} elapsed=${Date.now() - loopStartTime}ms`,
+      );
       debugLog(
         `Processing importedExport ${importedExportIndex}/${sortedImportedExports.length}`,
         {
@@ -1888,6 +1910,10 @@ export default async function writeScenarioComponents({
               importedExport.resolvedIsDefault === true &&
               importedExport.isDefault === false;
 
+            console.log(
+              `[WriteScenario] RECURSE START: ${entity.name} -> ${importedExportEntity.name}`,
+            );
+            const recurseStartTime = Date.now();
             debugLog(
               `Recursing into writeScenarioComponents for ${importedExportEntity.name}`,
               {
@@ -1919,7 +1945,10 @@ export default async function writeScenarioComponents({
                   ? importedExport.name
                   : undefined,
               }),
-              60000, // 1 minute timeout for recursive calls
+              180000, // 3 minute timeout for recursive calls (complex components need more time)
+            );
+            console.log(
+              `[WriteScenario] RECURSE END: ${entity.name} -> ${importedExportEntity.name} took ${Date.now() - recurseStartTime}ms`,
             );
             debugLog(
               `Completed recursive writeScenarioComponents for ${importedExportEntity.name}`,
@@ -2261,6 +2290,10 @@ ${exportKeyword}const ${functionName} = new Proxy(() => scenarios().data()?.["${
     }
   }
 
+  // Track post-import-loop timing
+  const postLoopStartTime = Date.now();
+  console.log(`[WriteScenario] POST-LOOP START: ${entity.name}`);
+
   // Collect universal mocks BEFORE processing nodeModuleImports
   // This is needed to check if a node module import is handled by a universal mock
   const universalMocks = project.metadata?.universalMocks ?? [];
@@ -2320,6 +2353,10 @@ ${exportKeyword}const ${functionName} = new Proxy(() => scenarios().data()?.["${
     );
   }
 
+  console.log(
+    `[WriteScenario] POST-LOOP ${entity.name}: node+universal mocks took ${Date.now() - postLoopStartTime}ms`,
+  );
+
   if (
     rootAnalysis.entitySha === entity.sha &&
     entity.metadata?.notExported &&
@@ -2398,6 +2435,10 @@ ${exportKeyword}const ${functionName} = new Proxy(() => scenarios().data()?.["${
   );
   debugLog('Completed rewriteRelativeModuleImports');
 
+  console.log(
+    `[WriteScenario] POST-LOOP ${entity.name}: transformations took ${Date.now() - postLoopStartTime}ms`,
+  );
+
   /**
    * Recursively process a file's imports to create transitive copies with server-only stripped.
    * This handles chains of arbitrary depth: A -> B -> C -> D where each needs server-only removed.
@@ -2418,7 +2459,7 @@ ${exportKeyword}const ${functionName} = new Proxy(() => scenarios().data()?.["${
     startTime: number = Date.now(),
   ): Promise<string> {
     // Global timeout for entire transitive processing
-    const GLOBAL_TIMEOUT_MS = 60000; // 1 minute max for all transitive processing
+    const GLOBAL_TIMEOUT_MS = 180000; // 3 minutes max for all transitive processing (complex components need more time)
     const elapsed = Date.now() - startTime;
     if (elapsed > GLOBAL_TIMEOUT_MS) {
       throw new Error(
@@ -2427,6 +2468,10 @@ ${exportKeyword}const ${functionName} = new Proxy(() => scenarios().data()?.["${
     }
 
     const importPaths = extractInternalImportPaths(content);
+    // Always log to help debug timeout issues
+    console.log(
+      `[TransitiveImports] depth=${depth} file=${path.basename(sourceFilePath)} imports=${importPaths.length} visited=${visitedPaths.size} elapsed=${Date.now() - startTime}ms`,
+    );
     debugLog(`processTransitiveImportsRecursively depth=${depth}`, {
       sourceFilePath,
       importCount: importPaths.length,
@@ -2481,8 +2526,10 @@ ${exportKeyword}const ${functionName} = new Proxy(() => scenarios().data()?.["${
       );
       const extension = importFile.name.split('.').pop();
       const isIndex = isIndexPath(importFile.path);
-      const pathHash = safeFileName(importFile.path);
-      const transitiveFilePath = `${PROJECT_RELATIVE_PATH}/${basePath}/${pathHash}_${isIndex ? 'index_' : ''}transitive_${safeFileName(scenario.name)}.${extension}`;
+      // Limit pathHash length to prevent ENAMETOOLONG errors on macOS (255 char limit)
+      const pathHash = safeFileName(importFile.path, { maxLength: 80 });
+      const scenarioSlug = safeFileName(scenario.name, { maxLength: 60 });
+      const transitiveFilePath = `${PROJECT_RELATIVE_PATH}/${basePath}/${pathHash}_${isIndex ? 'index_' : ''}transitive_${scenarioSlug}.${extension}`;
 
       // Check if this is a circular import (we're already processing this file)
       const isCircularImport = visitedPaths.has(resolvedPath);
@@ -2606,6 +2653,10 @@ ${exportKeyword}const ${functionName} = new Proxy(() => scenarios().data()?.["${
     return modifiedContent;
   }
 
+  console.log(
+    `[WriteScenario] POST-LOOP ${entity.name}: before remaining imports ${Date.now() - postLoopStartTime}ms`,
+  );
+
   // Process remaining internal imports that weren't in importedExports
   // This handles transitive dependencies: when the file content includes code (e.g., from
   // other functions in the same file) that imports from files with "server-only"
@@ -2681,8 +2732,10 @@ ${exportKeyword}const ${functionName} = new Proxy(() => scenarios().data()?.["${
     );
     const targetFileExtension = targetFile.name.split('.').pop();
     const targetFileIsIndex = isIndexPath(targetFile.path);
-    const filePathHash = safeFileName(targetFile.path);
-    const transformedFilePath = `${PROJECT_RELATIVE_PATH}/${targetFileBasePath}/${filePathHash}_${targetFileIsIndex ? 'index_' : ''}transitive_${safeFileName(scenario.name)}.${targetFileExtension}`;
+    // Limit path hash length to prevent ENAMETOOLONG errors
+    const filePathHash = safeFileName(targetFile.path, { maxLength: 80 });
+    const targetScenarioSlug = safeFileName(scenario.name, { maxLength: 60 });
+    const transformedFilePath = `${PROJECT_RELATIVE_PATH}/${targetFileBasePath}/${filePathHash}_${targetFileIsIndex ? 'index_' : ''}transitive_${targetScenarioSlug}.${targetFileExtension}`;
 
     // Check if we've already processed this file as a transitive copy
     // Note: __data_file_written__ is for entity-specific scenario files with different naming,
@@ -2737,8 +2790,12 @@ ${exportKeyword}const ${functionName} = new Proxy(() => scenarios().data()?.["${
         );
         const nestedExtension = nestedFile.name.split('.').pop();
         const nestedIsIndex = isIndexPath(nestedFile.path);
-        const nestedPathHash = safeFileName(nestedFile.path);
-        const nestedTransformedPath = `${PROJECT_RELATIVE_PATH}/${nestedBasePath}/${nestedPathHash}_${nestedIsIndex ? 'index_' : ''}transitive_${safeFileName(scenario.name)}.${nestedExtension}`;
+        // Limit path hash length to prevent ENAMETOOLONG errors
+        const nestedPathHash = safeFileName(nestedFile.path, { maxLength: 80 });
+        const nestedScenarioSlug = safeFileName(scenario.name, {
+          maxLength: 60,
+        });
+        const nestedTransformedPath = `${PROJECT_RELATIVE_PATH}/${nestedBasePath}/${nestedPathHash}_${nestedIsIndex ? 'index_' : ''}transitive_${nestedScenarioSlug}.${nestedExtension}`;
 
         // Check if already processed as a transitive file (we can rewrite to point to it)
         // Note: __data_file_written__ is for entity-specific scenario files with different naming,
@@ -2859,6 +2916,10 @@ ${exportKeyword}const ${functionName} = new Proxy(() => scenarios().data()?.["${
     fileContent = fileContent.replace(importRegex, `$1${safeRelativePath}$2`);
   }
 
+  console.log(
+    `[WriteScenario] POST-LOOP ${entity.name}: remaining imports loop took ${Date.now() - postLoopStartTime}ms`,
+  );
+
   const scenarioComponentComment = `// This file is auto-generated by CodeYam. Do not edit this file manually.
 // This file contains content for a scenario component:
 // Analyses being written: ${JSON.stringify(fileAnalyses?.map((a) => ({ id: a.id, entityName: a.entityName })))}
@@ -2870,6 +2931,34 @@ ${exportKeyword}const ${functionName} = new Proxy(() => scenarios().data()?.["${
 // Scenario: ${scenario.id} - ${scenario.name}
 `;
 
+  // Final pass: Rename any namespace imports (import * as X from '...') that have
+  // corresponding mock code using ...X__cyOriginal spread pattern.
+  // This handles cases where:
+  // 1. The import path was rewritten by transitive import handling
+  // 2. The import wasn't caught by the earlier renaming logic
+  // We scan for all __cyOriginal references in the mock code and ensure the imports are renamed.
+  const cyOriginalReferences = fileContent.match(/\.\.\.(\w+)__cyOriginal/g);
+  if (cyOriginalReferences) {
+    const uniqueNames = [
+      ...new Set(
+        cyOriginalReferences.map((ref) =>
+          ref.replace('...', '').replace('__cyOriginal', ''),
+        ),
+      ),
+    ];
+    for (const name of uniqueNames) {
+      // Match namespace imports for this name that haven't been renamed yet
+      const namespaceImportRegex = new RegExp(
+        `(import\\s+\\*\\s+as\\s+)${escapeRegExp(name)}(\\s+from\\s+['"][^'"]*['"])`,
+        'g',
+      );
+      fileContent = fileContent.replace(
+        namespaceImportRegex,
+        `$1${name}__cyOriginal$2`,
+      );
+    }
+  }
+
   // Use the directive that was extracted at the beginning of processing
   // This ensures it stays at the very top even after imports are prepended
   // NOTE: We only preserve "use client" directives, NOT "use server" directives.
@@ -2891,5 +2980,9 @@ ${exportKeyword}const ${functionName} = new Proxy(() => scenarios().data()?.["${
   debugLog('Successfully wrote scenario file');
   scenarioComponentPaths.push(scenarioComponentPath);
 
+  console.log(
+    `[WriteScenario] POST-LOOP ${entity.name}: COMPLETE total=${Date.now() - postLoopStartTime}ms`,
+  );
+
   return { scenarioComponentPaths, writtenScenarioComponents };
 }