@codeyam/codeyam-cli 0.1.0-staging.6e699e5 → 0.1.0-staging.7481014

package/analyzer-template/project/start.ts

@@ -17,7 +17,8 @@ import { startScenarioCapture } from './startScenarioCapture';
 import { createController } from './controller/startController';
 import { awsLog, awsLogDebugLevel } from '~codeyam/utils';
 import * as v8 from 'v8';
-import { initializeAIServiceMode } from '~codeyam/ai';
+import * as os from 'os';
+import { initializeAIServiceMode, destroyWorkerPool } from '~codeyam/ai';
 import runAnalysis, { EnvConfig, SharedContext } from './runAnalysis';
 import {
   loadOrCreateCommit,
@@ -119,28 +120,31 @@ async function main({
     // and attempted to access project/commit/branch data before it was loaded.
     const sharedContext = await loadSharedContext(envConfig);
 
-    // Heap monitoring: pulse every 1s, log every 30s when normal, warn every 1s when high
+    // Memory monitoring: tracks both V8 heap and RSS (which includes worker threads)
+    const containerMemoryMB = process.env.ECS_CONTAINER_MEMORY_MB
+      ? parseInt(process.env.ECS_CONTAINER_MEMORY_MB, 10)
+      : Math.round(os.totalmem() / 1024 / 1024);
     let lastLogTime = 0;
     setInterval(() => {
       const mem = process.memoryUsage();
       const heap = v8.getHeapStatistics();
-      const usagePercent = (mem.heapUsed / heap.heap_size_limit) * 100;
+      const heapMB = Math.round(mem.heapUsed / 1024 / 1024);
+      const rssMB = Math.round(mem.rss / 1024 / 1024);
+      const heapLimitMB = Math.round(heap.heap_size_limit / 1024 / 1024);
+      const rssPercent = (rssMB / containerMemoryMB) * 100;
       const now = Date.now();
 
-      if (usagePercent >= 80) {
-        // High usage: warn every second
+      if (rssPercent >= 80) {
         console.warn(
-          `CodeYam Heap Warning: ${Math.round(mem.heapUsed / 1024 / 1024)}MB used / ${Math.round(heap.heap_size_limit / 1024 / 1024)}MB limit (${Math.round(usagePercent)}%) ⚠️ HIGH USAGE`,
+          `CodeYam Memory Warning: RSS ${rssMB}MB/${containerMemoryMB}MB (${Math.round(rssPercent)}%), heap ${heapMB}MB/${heapLimitMB}MB ⚠️ HIGH`,
         );
-      } else if (usagePercent >= 50 && now - lastLogTime >= 5000) {
-        // Elevated usage: log every 5 seconds
+      } else if (rssPercent >= 50 && now - lastLogTime >= 5000) {
         console.log(
-          `CodeYam Heap Notice: ${Math.round(mem.heapUsed / 1024 / 1024)}MB used / ${Math.round(heap.heap_size_limit / 1024 / 1024)}MB limit (${Math.round(usagePercent)}%) ⚠️ Elevated`,
+          `CodeYam Memory Notice: RSS ${rssMB}MB/${containerMemoryMB}MB (${Math.round(rssPercent)}%), heap ${heapMB}MB/${heapLimitMB}MB ⚠️ Elevated`,
         );
       } else if (now - lastLogTime >= 30000) {
-        // Normal usage: log every 30 seconds
         console.log(
-          `CodeYam Heap Stats: ${Math.round(mem.heapUsed / 1024 / 1024)}MB used / ${Math.round(heap.heap_size_limit / 1024 / 1024)}MB limit (${Math.round(usagePercent)}%) 📊 OK`,
+          `CodeYam Memory Stats: RSS ${rssMB}MB/${containerMemoryMB}MB (${Math.round(rssPercent)}%), heap ${heapMB}MB/${heapLimitMB}MB 📊 OK`,
         );
       }
 
@@ -187,6 +191,11 @@ async function main({
               analyzerPid: undefined,
               capturePid: process.pid,
             },
+            updateCallback(metadata) {
+              if (metadata.currentRun?.captureCompletedAt) {
+                metadata.currentRun.completedAt = new Date().toISOString();
+              }
+            },
           }).catch((error) => {
             awsLog(`Warning: Failed to update commit metadata: ${error}`);
           });
@@ -219,6 +228,10 @@ async function main({
         }
         awsLog('CodeYam: Analysis producing, starting orchestration...');
 
+        // Worker pool is no longer needed after data structure prep phase.
+        // Freeing it reclaims ~2GB before capture workers ramp up.
+        await destroyWorkerPool();
+
         // Create task runner based on orchestration mode
         let taskRunner;
 
@@ -229,6 +242,17 @@ async function main({
             framework: framework,
             packageManager: packageManager,
           });
+
+          const captureCommit =
+            sharedContext.commit ?? sharedContext.branchCommit;
+          if (captureCommit) {
+            await updateCommitMetadata({
+              commitId: captureCommit.id,
+              runStatusUpdate: {
+                captureTaskArns: taskRunner.getTaskArns(),
+              },
+            });
+          }
         } else if (orchestrateCapture === 'local-sequential') {
           taskRunner = await SequentialCaptureTaskRunner.create(
             {
@@ -289,6 +313,9 @@ async function main({
               : envConfig.entityShas,
           full: orchestrateCapture === 'local-sequential',
           taskRunner,
+          // Poll faster in local mode - no DB load concerns like in cloud
+          pollingInterval:
+            orchestrateCapture === 'local-sequential' ? 5 * 1000 : undefined,
           // Always track capture progress on the commit, even when not filtering by it.
           metadataCommit: sharedContext.commit ?? sharedContext.branchCommit,
         });

package/analyzer-template/project/startScenarioCapture.ts

@@ -6,6 +6,7 @@ import writeUniversalMocks from './writeUniversalMocks';
 import writeScenario from './writeScenario';
 import writeSimpleRoot from './writeSimpleRoot';
 import writeCodeYamPage from './writeCodeYamPage';
+import writeClientLogRoute from './writeClientLogRoute';
 import {
   runMultiScenarioServer,
   ScenarioToCapture,
@@ -497,6 +498,14 @@ export async function startScenarioCapture({
       );
       allGeneratedFiles.push(codeyamPagePath);
 
+      // Write client-log API route so client-side errors appear in the server log
+      const clientLogPath = await writeClientLogRoute(
+        project,
+        framework,
+        analysis.filePath,
+      );
+      if (clientLogPath) allGeneratedFiles.push(clientLogPath);
+
       // Persist file tracking for crash/debug recovery
       writeTrackingFile(allGeneratedFiles, backupFiles);
     }

package/analyzer-template/project/writeClientLogRoute.ts

@@ -0,0 +1,125 @@
+import { Project, ProjectFramework } from '~codeyam/types';
+import { findWebAppForFile, PROJECT_RELATIVE_PATH } from '~codeyam/utils';
+import writeFile from '../common/writeFile';
+import * as path from 'path';
+
+/**
+ * Writes a client-log API route to the project so that client-side errors
+ * (window.onerror, unhandledrejection, console.error) are reported back to
+ * the server log where they can be read from the terminal.
+ *
+ * The route receives JSON POST requests and console.log's them, which sends
+ * the output to the dev server's stdout (captured in the server log file).
+ */
+export default async function writeClientLogRoute(
+  project: Project,
+  framework: ProjectFramework,
+  analysisFilePath?: string,
+) {
+  const relativePath = PROJECT_RELATIVE_PATH;
+  const fullAppPath = getFullAppPath(project, analysisFilePath);
+
+  let routePath: string;
+  let content: string;
+
+  switch (framework) {
+    case ProjectFramework.Remix:
+    case ProjectFramework.CRA:
+    case ProjectFramework.CodeYam:
+      routePath = `${relativePath}/${fullAppPath}/routes/api.client-log.ts`;
+      content = getRemixRoute();
+      break;
+    case ProjectFramework.Next:
+      routePath = `${relativePath}/${fullAppPath}/api/client-log/route.ts`;
+      content = getNextAppRoute();
+      break;
+    case ProjectFramework.NextPages:
+      routePath = `${relativePath}/pages/api/client-log.ts`;
+      content = getNextPagesRoute();
+      break;
+    default:
+      return;
+  }
+
+  await writeFile(routePath, content);
+  return routePath;
+}
+
+function getFullAppPath(project: Project, analysisFilePath?: string): string {
+  if (analysisFilePath) {
+    const webapp = findWebAppForFile(
+      analysisFilePath,
+      project.metadata?.webapps ?? [],
+    );
+    if (webapp) {
+      const webappPath = webapp.path === '.' ? '' : webapp.path;
+      const appDirectory = webapp.appDirectory ?? 'app';
+      return webappPath ? path.join(webappPath, appDirectory) : appDirectory;
+    }
+  }
+  const firstWebapp = project.metadata?.webapps?.[0];
+  if (firstWebapp) {
+    const webappPath = firstWebapp.path === '.' ? '' : firstWebapp.path;
+    const appDirectory = firstWebapp.appDirectory ?? 'app';
+    return webappPath ? path.join(webappPath, appDirectory) : appDirectory;
+  }
+  return 'app';
+}
+
+function formatLogEntry(): string {
+  return `
+function formatEntry(body) {
+  const ts = new Date().toLocaleString();
+  const scenario = body.scenario ? ' [' + body.scenario + ']' : '';
+  const type = (body.type || 'error').toUpperCase();
+  const lines = ['[' + ts + '] [CLIENT ' + type + ']' + scenario + ' ' + (body.message || 'Unknown error')];
+  if (body.stack) lines.push(body.stack);
+  if (body.url) lines.push('  at ' + body.url);
+  return lines.join('\\n');
+}`;
+}
+
+function getRemixRoute(): string {
+  return `// Auto-generated by CodeYam — client error reporting endpoint
+${formatLogEntry()}
+
+export async function action({ request }) {
+  try {
+    const body = await request.json();
+    console.log(formatEntry(body));
+  } catch (e) {}
+  return new Response('ok', { status: 200 });
+}
+`;
+}
+
+function getNextAppRoute(): string {
+  return `// Auto-generated by CodeYam — client error reporting endpoint
+import { NextResponse } from 'next/server';
+${formatLogEntry()}
+
+export async function POST(request: Request) {
+  try {
+    const body = await request.json();
+    console.log(formatEntry(body));
+  } catch (e) {}
+  return NextResponse.json({ ok: true });
+}
+`;
+}
+
+function getNextPagesRoute(): string {
+  return `// Auto-generated by CodeYam — client error reporting endpoint
+import type { NextApiRequest, NextApiResponse } from 'next';
+${formatLogEntry()}
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  if (req.method !== 'POST') return res.status(405).end();
+  try {
+    const body = typeof req.body === 'string' ? JSON.parse(req.body) : req.body;
+    console.log(formatEntry(body));
+  } catch (e) {}
+  res.status(200).json({ ok: true });
+}
+`;
+}

package/analyzer-template/project/writeMockDataTsx.ts

@@ -8,6 +8,7 @@ import type { AnyNode } from 'domhandler';
 import { safeFileName } from '~codeyam/utils';
 import { cleanOutBoundary } from '~codeyam/ai';
 import { reconcileMockDataKeys } from './reconcileMockDataKeys';
+import coerceObjectsToPrimitivesBySchema from '~codeyam/ai/src/lib/dataStructure/helpers/coerceObjectsToPrimitivesBySchema';
 
 /**
  * Normalize useParams mock data by fixing the React Router splat param (*).
@@ -192,6 +193,36 @@ const safeString = (s: string, level: number) => {
   return s?.replace ? s.replace(/"/g, "'").replace(/\n/g, '\\n') : s;
 };
 
+/**
+ * Clean up malformed LLM-generated function values.
+ * The LLM sometimes generates function values with an invalid "(function)" prefix like:
+ * - "(function)() => {}"
+ * - "(function)(event) => {}"
+ * - "(function)(id) => {}"
+ *
+ * This strips the prefix and adds TypeScript `any` type annotations to parameters.
+ * Returns null if the value doesn't match the malformed pattern.
+ */
+const cleanMalformedFunctionValue = (value: string): string | null => {
+  // Match "(function)" followed by optional params in parens, then "=>"
+  const match = value.match(/^\(function\)(\([^)]*\))\s*=>\s*(.*)$/);
+  if (!match) return null;
+
+  const [, paramsWithParens, body] = match;
+
+  // Extract parameter names and add `: any` type annotation
+  // e.g., "(event)" -> "(event: any)", "()" -> "()"
+  const params = paramsWithParens.slice(1, -1).trim(); // Remove outer parens
+  const typedParams = params
+    ? params
+        .split(',')
+        .map((p) => `${p.trim()}: any`)
+        .join(', ')
+    : '';
+
+  return `(${typedParams}) => ${body}`;
+};
+
 /**
  * Check if a mock key's schema indicates it should be instantiated as a Map.
  * A Map is indicated by:
@@ -310,6 +341,38 @@ function extractAllKeys(obj: JsonTypeDefinition | undefined): string[] {
  */
 const BARE_TYPE_MARKERS = ['function', 'async-function'];
 
+/**
+ * Check if a string is a TypeScript arrow function type annotation rather than
+ * a valid JavaScript arrow function expression.
+ *
+ * TypeScript arrow types look like function expressions (they contain '=>') but
+ * their "body" is a type name, not a JS expression:
+ * - "() => void"
+ * - "(event: MouseEvent) => void"
+ * - "() => string"
+ * - "() => Promise<void>"
+ * - "(id: string, name: string) => boolean"
+ *
+ * Valid JS arrow functions have actual expression bodies:
+ * - "() => {}"
+ * - "() => 'hello'"
+ * - "(x) => x + 1"
+ * - "() => { console.log('hi'); }"
+ */
+const ARROW_TYPE_RETURN_TYPES =
+  /^(void|string|number|boolean|any|unknown|never|null|undefined|object|Date|Promise<\w+>)$/i;
+
+const isArrowFunctionTypeAnnotation = (value: string): boolean => {
+  const arrowIndex = value.indexOf('=>');
+  if (arrowIndex === -1) return false;
+
+  const body = value.slice(arrowIndex + 2).trim();
+
+  // A real JS function would have {}, an expression, a string literal, etc.
+  // A type annotation has a bare type keyword as the "body".
+  return ARROW_TYPE_RETURN_TYPES.test(body);
+};
+
 /**
  * Check if a value is a bare function type marker that should be filtered out.
  * These markers indicate the LLM detected a function but couldn't generate meaningful mock data.
@@ -336,6 +399,59 @@ const isBareTypeMarker = (value: unknown): boolean => {
   return false;
 };
 
+/**
+ * Escape newlines that appear inside string literals within a code string,
+ * while preserving structural newlines (those between statements/lines of code).
+ *
+ * This handles cases where LLM generates function bodies like:
+ * - "() => { console.log('line1\nline2'); }" - escape the \n inside the string
+ * - "() => {\n  console.log('x');\n}" - preserve structural \n between statements
+ * - "() => Promise.resolve('a\nb')" - escape the \n inside the string argument
+ *
+ * The algorithm tracks whether we're inside a string literal (single, double, or
+ * backtick quotes) and only escapes newlines that appear within quotes.
+ */
+const escapeNewlinesInStringLiterals = (code: string): string => {
+  let result = '';
+  let inString = false;
+  let stringChar = ''; // The quote character that started the current string
+
+  for (let i = 0; i < code.length; i++) {
+    const char = code[i];
+    const prevChar = i > 0 ? code[i - 1] : '';
+
+    // Check for string literal boundaries (single, double, or backtick quotes)
+    // Skip escaped quotes (preceded by backslash)
+    if ((char === '"' || char === "'" || char === '`') && prevChar !== '\\') {
+      if (!inString) {
+        // Starting a string literal
+        inString = true;
+        stringChar = char;
+      } else if (char === stringChar) {
+        // Ending the current string literal
+        inString = false;
+        stringChar = '';
+      }
+      // If char is a different quote type while in a string, it's just a character
+    }
+
+    // Escape newlines only when inside a string literal
+    if (char === '\n') {
+      if (inString) {
+        // Inside a string literal - escape the newline
+        result += '\\n';
+      } else {
+        // Outside string - keep the structural newline
+        result += char;
+      }
+    } else {
+      result += char;
+    }
+  }
+
+  return result;
+};
+
 /**
  * Normalize mock data by merging entries that have the same normalized function name.
  * e.g., "useLastLogLine(projectSlug, !!simulatingEntitySha)" and
@@ -734,13 +850,19 @@ const jsonToCode = (
               stringValue.startsWith('Promise.') ||
               stringValue.startsWith('new Promise');
 
+            // Check if the string contains newlines (potential issue for string literals)
+            const hasNewlines = stringValue.includes('\n');
+
             // Handle special type indicators that need transformation
             let outputValue: string;
             if (stringValue === 'promise' || looksLikePromise) {
               // The key is a function call (e.g., "revalidate()"), so output a function
               // that returns a Promise, not the Promise itself
+              // Important: Escape newlines inside Promise expressions to avoid
+              // "unterminated string literal" errors when the Promise contains
+              // multi-line string arguments like Promise.resolve('line1\nline2')
               const promiseExpr = looksLikePromise
-                ? stringValue
+                ? stringValue.replace(/\n/g, '\\n')
                 : 'Promise.resolve()';
               outputValue = `() => ${promiseExpr}`;
               console.log(
@@ -752,10 +874,49 @@ const jsonToCode = (
                 `CodeYam: Nested func transform (string->function): "${key}" -> "${funcName}": () => {}`,
               );
             } else if (looksLikeFunction) {
-              outputValue = stringValue;
-              console.log(
-                `CodeYam: Nested func transform (string): "${key}" -> "${funcName}": ${stringValue.substring(0, 30)}...`,
-              );
+              // Check for "[native code]" in function body — this is what
+              // Function.prototype.toString() returns for built-in functions.
+              // The LLM sometimes generates "function () { [native code] }" which
+              // is not valid JavaScript (the "[" is parsed as array destructuring).
+              if (stringValue.includes('[native code]')) {
+                outputValue = '() => {}';
+                console.log(
+                  `CodeYam: Nested func transform (native-code->noop): "${key}" -> "${funcName}": () => {}`,
+                );
+              } else if (cleanMalformedFunctionValue(stringValue)) {
+                // Check for malformed LLM-generated function values like "(function)() => {}"
+                // These have an invalid "(function)" prefix that must be stripped
+                outputValue = cleanMalformedFunctionValue(stringValue)!;
+                console.log(
+                  `CodeYam: Nested func transform (malformed->clean): "${key}" -> "${funcName}": ${outputValue}`,
+                );
+              } else if (isArrowFunctionTypeAnnotation(stringValue)) {
+                // TypeScript arrow function type annotations like "() => void",
+                // "(event: MouseEvent) => void", "() => Promise<void>" contain '=>'
+                // so they pass the looksLikeFunction check, but they are TYPE syntax,
+                // not valid JavaScript expressions. Convert to a no-op function.
+                outputValue = '() => {}';
+                console.log(
+                  `CodeYam: Nested func transform (type-annotation->function): "${key}" -> "${funcName}": () => {}`,
+                );
+              } else {
+                // Handle newlines in function expressions:
+                // - Newlines inside string literals must be escaped to avoid
+                //   "unterminated string literal" errors
+                // - Structural newlines (between statements) should be kept as-is
+                //   to produce valid multi-line JavaScript
+                //
+                // We escape ONLY newlines that appear inside string literals (single,
+                // double, or template quotes). Structural newlines are preserved.
+                if (hasNewlines) {
+                  outputValue = escapeNewlinesInStringLiterals(stringValue);
+                } else {
+                  outputValue = stringValue;
+                }
+                console.log(
+                  `CodeYam: Nested func transform (string): "${key}" -> "${funcName}": ${stringValue.substring(0, 30)}...`,
+                );
+              }
             } else {
               // Plain string value - must be quoted
               outputValue = `"${safeString(stringValue, level)}"`;
@@ -783,9 +944,22 @@ const jsonToCode = (
             if (isBareTypeMarker(stringValue)) {
               continue;
             }
-            keyStrings.push(
-              `${levelSpacing}"${safeString(key, level)}": "${safeString(stringValue, level)}"`,
-            );
+            // Check for malformed LLM-generated function values like "(function)(id) => {}"
+            // These can appear on plain keys (e.g., "closeToast") and need to be output as
+            // actual functions, not quoted strings
+            const cleanedPlainKeyFn = cleanMalformedFunctionValue(stringValue);
+            if (cleanedPlainKeyFn) {
+              keyStrings.push(
+                `${levelSpacing}"${safeString(key, level)}": ${cleanedPlainKeyFn}`,
+              );
+              console.log(
+                `CodeYam: Plain key func transform (malformed->clean): "${key}": ${cleanedPlainKeyFn}`,
+              );
+            } else {
+              keyStrings.push(
+                `${levelSpacing}"${safeString(key, level)}": "${safeString(stringValue, level)}"`,
+              );
+            }
           }
         } else {
           // Skip plain keys that have function-call versions (at the same nesting level where
@@ -856,6 +1030,14 @@ export default async function writeMockDataTsx(
     expectedKeys,
   );
 
+  // Coerce objects/arrays to primitives when the schema expects a primitive type.
+  // Defense-in-depth: the generation pipeline already runs this coercion, but
+  // data in the DB may have wrong types from older code versions or edge cases.
+  // This prevents runtime errors like "TypeError: body.match is not a function".
+  if (dataForMocks) {
+    coerceObjectsToPrimitivesBySchema(reconciledMockData, dataForMocks);
+  }
+
   // Convert comma-separated strings to arrays for fields that should be arrays
   // This handles cases where LLM generates "color,size" instead of ["color", "size"]
   const defaultMockData =
@@ -903,6 +1085,14 @@ import React from 'react';
       expectedKeys,
     );
 
+    // Coerce objects/arrays to primitives for non-default scenario data too
+    if (dataForMocks) {
+      coerceObjectsToPrimitivesBySchema(
+        reconciledScenarioMockData,
+        dataForMocks,
+      );
+    }
+
     // Convert comma-separated strings to arrays for scenario data
     const scenarioMockData = convertCommaSeparatedStringsToArrays(
       reconciledScenarioMockData,