@codeyam/codeyam-cli 0.1.0-staging.15d0f46 → 0.1.0-staging.1669d45

package/analyzer-template/project/start.ts

@@ -1,4 +1,5 @@
 import 'dotenv/config';
+import { once } from 'events';
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
 import {
@@ -23,7 +24,7 @@ import {
   syncHeadBranches,
   syncPrimaryBranch,
 } from '~codeyam/github';
-import { installSignalHandlers } from '../process';
+import { installSignalHandlers } from '~codeyam/process';
 
 // Set process title for Activity Monitor visibility
 if (process.env.CODEYAM_PROCESS_NAME) {
@@ -207,7 +208,16 @@ async function main({
         while (!controller?.hasCompletedRepoCheckout()) {
           await new Promise((resolve) => setTimeout(resolve, 1000));
         }
-        awsLog('CodeYam: Repo checkout complete, starting orchestration...');
+        awsLog('CodeYam: Repo checkout complete.');
+
+        // Wait for analysis to enter producing phase (after prepareDataStructures)
+        // This prevents the 20-minute orchestration timeout from triggering before
+        // any analyses become ready to capture
+        awsLog('CodeYam: Waiting for analysis to start producing...');
+        if (!controller.hasStartedProducing()) {
+          await once(controller, 'analysisProducing');
+        }
+        awsLog('CodeYam: Analysis producing, starting orchestration...');
 
         // Create task runner based on orchestration mode
         let taskRunner;
@@ -260,8 +270,8 @@ async function main({
             }
           },
           project: sharedContext.project,
-          // In local-sequential mode, don't filter by commit since each analysis run
-          // creates a new fake commit. We want to capture all ready analyses for the project.
+          // In local-sequential mode, don't filter by commit (full=true handles that).
+          // For AWS mode, filter by the specific commit being analyzed.
           commit:
             orchestrateCapture === 'local-sequential'
               ? null
@@ -279,6 +289,8 @@ async function main({
               : envConfig.entityShas,
           full: orchestrateCapture === 'local-sequential',
           taskRunner,
+          // Always track capture progress on the commit, even when not filtering by it.
+          metadataCommit: sharedContext.commit ?? sharedContext.branchCommit,
         });
         awsLog('CodeYam Capture: runCaptureOrchestration() completed');
       };
@@ -440,6 +452,16 @@ async function main({
         `CodeYam: Exiting start.js process with ${hadError ? 'ERROR' : 'SUCCESS'}`,
       );
       process.exit(hadError ? 1 : 0);
+    } else if (process.env.CAPTURE_ONLY === 'true') {
+      awsLog(
+        'CodeYam: CAPTURE_ONLY mode - keeping files and exiting cleanly',
+        `Preserved ${generatedFiles.length} files`,
+      );
+      // Don't cleanup files in capture-only mode to preserve manual edits
+      awsLog(
+        `CodeYam: Exiting start.js process with ${hadError ? 'ERROR' : 'SUCCESS'}`,
+      );
+      process.exit(hadError ? 1 : 0);
     } else {
       awsLog(`CodeYam: Removing ${generatedFiles.length} generated files...`);
       removeScenario(generatedFiles);

package/analyzer-template/project/startScenarioCapture.ts

@@ -36,6 +36,7 @@ import {
 } from './utils/webappMapping';
 import mergeEnvVarsIntoStartCommand from './utils/mergeEnvVarsIntoStartCommand';
 import { LazyFileStore } from './LazyFileStore';
+import { DEFAULT_SCENARIO_NAME } from '../common/constants';
 
 export interface StartScenarioCaptureArgs {
   readyToBeCaptured: Analysis[];
@@ -85,8 +86,9 @@ export async function startScenarioCapture({
   }
 
   // Clean up any leftover generated files from previous debug sessions
+  // Skip in CAPTURE_ONLY mode - we want to preserve the files that were edited
   const previouslyTracked = readTrackingFile();
-  if (previouslyTracked) {
+  if (previouslyTracked && process.env.CAPTURE_ONLY !== 'true') {
     awsLog('CodeYam: Cleaning up files from previous session', {
       generatedCount: previouslyTracked.generatedFiles.length,
       backupCount: previouslyTracked.backupFiles.length,
@@ -97,6 +99,13 @@ export async function startScenarioCapture({
       restoreFromBackup(backupPath);
     }
     clearTrackingFile();
+  } else if (previouslyTracked && process.env.CAPTURE_ONLY === 'true') {
+    awsLog(
+      'CodeYam: CAPTURE_ONLY mode - preserving files from previous session',
+      {
+        generatedCount: previouslyTracked.generatedFiles.length,
+      },
+    );
   }
 
   const sortedReadyToBeCaptured = readyToBeCaptured.sort((a, b) => {
@@ -218,6 +227,11 @@ export async function startScenarioCapture({
     }
 
     const sortedScenarios = (analysis.scenarios ?? []).sort((a, b) => {
+      // Default Scenario always goes first
+      if (a.name === DEFAULT_SCENARIO_NAME) return -1;
+      if (b.name === DEFAULT_SCENARIO_NAME) return 1;
+
+      // Then prioritize new or revised scenarios
       const newOrRevisedA =
         !a.previousVersionId || a.metadata?.revisedFromPreviousVersion;
       const newOrRevisedB =
@@ -395,40 +409,50 @@ export async function startScenarioCapture({
 
       awsLogDebugLevel(1, `Preparing ${analysis.entityName}: ${scenario.name}`);
 
-      // Write scenario files upfront
-      const scenarioGeneratedFiles = await writeScenario({
-        analysis,
-        scenario,
-        context,
-        projectAnalyzer,
-        framework,
-        fileStore,
-      });
-
-      // For non-routes: write root.tsx now (all scenarios can share it)
-      // For routes: defer writeSimpleRoot to per-scenario execution (each needs different mocks)
-      // EXCEPTION: In PREP_ONLY mode (debug), routes also need root.tsx written here since we exit early
-      if (!isRoute || process.env.PREP_ONLY === 'true') {
-        const simpleRootResult = await writeSimpleRoot(
-          project,
-          framework,
+      // In CAPTURE_ONLY mode, skip file generation and use existing files
+      // This preserves manual fixes made during debugging
+      let scenarioGeneratedFiles: string[] = [];
+      if (process.env.CAPTURE_ONLY !== 'true') {
+        // Write scenario files upfront
+        scenarioGeneratedFiles = await writeScenario({
+          analysis,
           scenario,
-          analysis.filePath,
-          {
-            file: analysis.file,
-            entity: analysis.entity,
-            routeAnalysis: analysis,
-            mocksDir,
-            scenarioComponentPaths: scenarioGeneratedFiles,
-          },
-        );
-        if (simpleRootResult.filePath) {
-          if (simpleRootResult.backupPath) {
-            backupFiles.push(simpleRootResult.backupPath);
-          } else {
-            scenarioGeneratedFiles.push(simpleRootResult.filePath);
+          context,
+          projectAnalyzer,
+          framework,
+          fileStore,
+        });
+
+        // For non-routes: write root.tsx now (all scenarios can share it)
+        // For routes: defer writeSimpleRoot to per-scenario execution (each needs different mocks)
+        // EXCEPTION: In PREP_ONLY mode (debug), routes also need root.tsx written here since we exit early
+        if (!isRoute || process.env.PREP_ONLY === 'true') {
+          const simpleRootResult = await writeSimpleRoot(
+            project,
+            framework,
+            scenario,
+            analysis.filePath,
+            {
+              file: analysis.file,
+              entity: analysis.entity,
+              routeAnalysis: analysis,
+              mocksDir,
+              scenarioComponentPaths: scenarioGeneratedFiles,
+            },
+          );
+          if (simpleRootResult.filePath) {
+            if (simpleRootResult.backupPath) {
+              backupFiles.push(simpleRootResult.backupPath);
+            } else {
+              scenarioGeneratedFiles.push(simpleRootResult.filePath);
+            }
           }
         }
+      } else {
+        awsLogDebugLevel(
+          1,
+          `CAPTURE_ONLY mode: skipping file generation for ${scenario.name}`,
+        );
       }
 
       // Mark scenario as initializing
@@ -684,14 +708,22 @@ export async function startScenarioCapture({
         allGeneratedFiles,
       );
 
-      // Clean up all generated files
-      console.log(
-        'CodeYam: Removing all scenario files for analysis',
-        analysis.id,
-        analysis.entityName,
-      );
-      cleanupFiles(allGeneratedFiles, backupFiles);
-      backupFiles = [];
+      // Clean up all generated files (skip in CAPTURE_ONLY mode to preserve edits)
+      if (process.env.CAPTURE_ONLY !== 'true') {
+        console.log(
+          'CodeYam: Removing all scenario files for analysis',
+          analysis.id,
+          analysis.entityName,
+        );
+        cleanupFiles(allGeneratedFiles, backupFiles);
+        backupFiles = [];
+      } else {
+        console.log(
+          'CodeYam: CAPTURE_ONLY mode - preserving scenario files for analysis',
+          analysis.id,
+          analysis.entityName,
+        );
+      }
 
       forceStop = result.forceStop;
     }

package/analyzer-template/project/writeMockDataTsx.ts

@@ -9,6 +9,40 @@ import { safeFileName } from '~codeyam/utils';
 import { cleanOutBoundary } from '~codeyam/ai';
 import { reconcileMockDataKeys } from './reconcileMockDataKeys';
 
+/**
+ * Normalize useParams mock data by fixing the React Router splat param (*).
+ * The LLM often generates random string values for `*` (the splat param),
+ * but this param represents the URL path suffix after the matched route
+ * and should typically be empty string for default scenarios.
+ *
+ * Example: For route `/entity/:sha/*`, the `*` param contains everything
+ * after `/entity/:sha/`. An empty string means the default tab/view.
+ */
+function normalizeParamsMockData(
+  mockData: Record<string, unknown>,
+): Record<string, unknown> {
+  const result: Record<string, unknown> = {};
+
+  for (const [key, value] of Object.entries(mockData)) {
+    // Check if this is a useParams mock data entry
+    if (key.includes('useParams') && value && typeof value === 'object') {
+      const paramsObj = value as Record<string, unknown>;
+      // If there's a `*` key (splat param), default it to empty string
+      // The splat param represents URL path suffix and is typically empty for default views
+      if ('*' in paramsObj) {
+        result[key] = {
+          ...paramsObj,
+          '*': '', // Default splat to empty string for default tab/view
+        };
+        continue;
+      }
+    }
+    result[key] = value;
+  }
+
+  return result;
+}
+
 /**
  * Return true when `markup` contains 0 or >1 meaningful
  * top-level nodes (element *or* non-blank text), so React
@@ -47,12 +81,12 @@ const cleanOutTypes = (str: string) => {
 };
 
 const safeString = (s: string, level: number) => {
-  if (s?.endsWith && s.endsWith(')') && (level === 1 || level === 2)) {
-    // remove arguments from top level functions
-    s = s.split('(')[0] + '()';
-    // strip type parameters like <typeof loader>
-    s = cleanOutTypes(s);
-  }
+  // NOTE: We no longer strip arguments from top-level function keys (level 2)
+  // because normalizeMockData now handles merging duplicate keys while preserving
+  // original keys when there's only one call site. This ensures constructMockCode
+  // can look up data with the full key (e.g., "useCustomSizes(projectSlug)").
+  //
+  // Only escape quotes and newlines for safe string output.
   return s?.replace ? s.replace(/"/g, "'").replace(/\n/g, '\\n') : s;
 };
 
@@ -101,12 +135,14 @@ const shouldBeMap = (mockKey: string, mergedDataStructure: any): boolean => {
 };
 
 /**
- * Normalize a function call key by stripping arguments and type parameters,
- * but preserving variable qualifiers (::variableName suffix).
+ * Normalize a function call key by stripping arguments but PRESERVING type parameters.
+ * Type parameters like <typeof loader> are important for disambiguation.
+ * Variable qualifiers (::variableName suffix) are also preserved.
+ *
  * e.g., "useLastLogLine(projectSlug, !!simulatingEntitySha)" -> "useLastLogLine()"
- * e.g., "useLoaderData<typeof loader>()" -> "useLoaderData()"
+ * e.g., "useLoaderData<typeof loader>()" -> "useLoaderData<typeof loader>()" (PRESERVED)
  * e.g., "useFetcher()::entityDiffFetcher" -> "useFetcher()::entityDiffFetcher" (preserved)
- * e.g., "useFetcher<T>()::entityDiffFetcher" -> "useFetcher()::entityDiffFetcher"
+ * e.g., "useFetcher<T>()::entityDiffFetcher" -> "useFetcher<T>()::entityDiffFetcher" (PRESERVED)
  */
 const normalizeKey = (key: string): string => {
   // Check for variable qualifier (::variableName)
@@ -116,41 +152,36 @@ const normalizeKey = (key: string): string => {
 
   // Check if key ends with ) - indicating a function call (after removing qualifier)
   if (keyWithoutQualifier.endsWith(')') && keyWithoutQualifier.includes('(')) {
-    // Strip arguments: "foo(a, b)" -> "foo()"
-    const normalized = keyWithoutQualifier.split('(')[0] + '()';
-    // Strip type parameters: "foo<T>()" -> "foo()"
-    return cleanOutTypes(normalized) + qualifier;
-  }
-  return key;
-};
+    // Find the opening paren that matches the closing paren (accounting for nested parens and generics)
+    // We want to strip arguments but preserve type parameters
+    // e.g., "useLoaderData<typeof loader>(arg1, arg2)" -> "useLoaderData<typeof loader>()"
+    // e.g., "foo(a, b)" -> "foo()"
+
+    // Find where type parameters end (if any) by finding matching < >
+    let typeParamEnd = 0;
+    if (keyWithoutQualifier.includes('<')) {
+      let depth = 0;
+      for (let i = 0; i < keyWithoutQualifier.length; i++) {
+        if (keyWithoutQualifier[i] === '<') depth++;
+        if (keyWithoutQualifier[i] === '>') {
+          depth--;
+          if (depth === 0) {
+            typeParamEnd = i + 1;
+            break;
+          }
+        }
+      }
+    }
 
-/**
- * Deep merge two objects, with source values taking precedence.
- * Arrays are replaced, not merged.
- */
-const deepMergeObjects = (
-  target: Record<string, unknown>,
-  source: Record<string, unknown>,
-): Record<string, unknown> => {
-  const result = { ...target };
-  for (const key of Object.keys(source)) {
-    if (
-      source[key] &&
-      typeof source[key] === 'object' &&
-      !Array.isArray(source[key]) &&
-      target[key] &&
-      typeof target[key] === 'object' &&
-      !Array.isArray(target[key])
-    ) {
-      result[key] = deepMergeObjects(
-        target[key] as Record<string, unknown>,
-        source[key] as Record<string, unknown>,
-      );
-    } else {
-      result[key] = source[key];
+    // Find the opening paren for function arguments (after type params)
+    const argsStart = keyWithoutQualifier.indexOf('(', typeParamEnd);
+    if (argsStart !== -1) {
+      // Preserve everything up to and including the opening paren, then close it
+      const normalized = keyWithoutQualifier.slice(0, argsStart) + '()';
+      return normalized + qualifier;
     }
   }
-  return result;
+  return key;
 };
 
 /**
@@ -240,7 +271,11 @@ const normalizeMockData = (
   // Second pass: merge values for duplicate normalized keys
   for (const [normKey, originalKeys] of keyMapping) {
     if (originalKeys.length === 1) {
-      // No duplicates, use the original key (may be normalized or not based on safeString)
+      // No duplicates - PRESERVE the original key with arguments
+      // This is critical because constructMockCode generates code that accesses
+      // data with the FULL key (e.g., scenarios().data()?.["useCustomSizes(projectSlug)"])
+      // If we normalize to "useCustomSizes()" but code looks for "useCustomSizes(projectSlug)",
+      // the lookup fails and returns undefined, causing runtime errors.
       const value = mockData[originalKeys[0]];
       // Skip bare type markers like "function" or "async-function"
       if (isBareTypeMarker(value)) {
@@ -251,7 +286,7 @@ const normalizeMockData = (
       if (isEmptyObject(value)) {
         continue;
       }
-      normalized[normKey] = value;
+      normalized[originalKeys[0]] = value;
     } else {
       // Multiple keys normalize to the same value - merge them
       let mergedValue: unknown = {};
@@ -269,7 +304,7 @@ const normalizeMockData = (
           typeof mergedValue === 'object' &&
           !Array.isArray(mergedValue)
         ) {
-          mergedValue = deepMergeObjects(
+          mergedValue = deepMerge(
             mergedValue as Record<string, unknown>,
             value as Record<string, unknown>,
           );
@@ -279,10 +314,12 @@ const normalizeMockData = (
         }
       }
       // Only add if we have actual data (not just filtered type markers)
+      // Note: typeof null === 'object' in JS, so we must check for null explicitly
       if (
         mergedValue !== undefined &&
         !isBareTypeMarker(mergedValue) &&
         !(
+          mergedValue !== null &&
           typeof mergedValue === 'object' &&
           Object.keys(mergedValue as object).length === 0
         )
@@ -310,8 +347,11 @@ const jsonToCode = (
     if (Array.isArray(json)) {
       const arrayStrings: string[] = [];
       for (const item of json) {
-        if (!item) continue;
-        if (typeof item === 'object') {
+        // Skip undefined but preserve null (important for ref props)
+        if (item === undefined) continue;
+        if (item === null) {
+          arrayStrings.push('null');
+        } else if (typeof item === 'object') {
           arrayStrings.push(jsonToCode(item, level + 1, mergedDataStructure));
         } else {
           arrayStrings.push(`"${safeString(item, level)}"`);
@@ -407,7 +447,7 @@ const jsonToCode = (
               `${levelSpacing}"${safeString(key, level)}": ${arrayString}`,
             );
           }
-        } else if (typeof jsonObj[key] === 'object') {
+        } else if (jsonObj[key] !== null && typeof jsonObj[key] === 'object') {
           const objKeys = Object.keys(jsonObj[key] as object);
 
           // Check if this is a nested function signature (e.g., "postMessage(message, targetOrigin)": { signature: [...] })
@@ -465,25 +505,39 @@ const jsonToCode = (
             objKeys[0].endsWith(')')
           ) {
             const signatureKey = objKeys[0];
-            // Extract arguments without parentheses for the combined key
+            // Extract arguments without parentheses for the function name
             const args = signatureKey.slice(1, -1);
-            const flattenedKey = `${key}(${args})`;
             const unwrappedValue = (jsonObj[key] as Record<string, unknown>)[
               signatureKey
             ];
-            if (typeof unwrappedValue === 'string') {
+
+            // Handle null values - generate arrow function stubs for function signatures with null
+            if (unwrappedValue === null) {
+              const funcName = key;
+              const argsString = args
+                .split(',')
+                .map((arg, i) => `${arg.trim()}: any`)
+                .join(', ');
+              keyStrings.push(
+                `${levelSpacing}"${safeString(funcName, level)}": (${argsString}) => {}`,
+              );
+            } else if (typeof unwrappedValue === 'string') {
+              const flattenedKey = `${key}(${args})`;
               keyStrings.push(
                 `${levelSpacing}"${safeString(flattenedKey, level)}": "${safeString(unwrappedValue, level)}"`,
               );
             } else if (typeof unwrappedValue === 'object') {
+              const flattenedKey = `${key}(${args})`;
               keyStrings.push(
                 `${levelSpacing}"${safeString(flattenedKey, level)}": ${jsonToCode(unwrappedValue, level + 1, mergedDataStructure)}`,
               );
             } else if (typeof unwrappedValue === 'boolean') {
+              const flattenedKey = `${key}(${args})`;
               keyStrings.push(
                 `${levelSpacing}"${safeString(flattenedKey, level)}": ${unwrappedValue}`,
               );
             } else {
+              const flattenedKey = `${key}(${args})`;
               keyStrings.push(
                 `${levelSpacing}"${safeString(flattenedKey, level)}": ${safeString(String(unwrappedValue), level)}`,
               );
@@ -564,10 +618,13 @@ const jsonToCode = (
             seenNormalizedFuncNames.add(funcName);
 
             // Check if the string value looks like a function expression
+            // Be careful not to match descriptive strings like "function listening" -
+            // a valid function expression must have parentheses like function(), function name(), or function (
             const looksLikeFunction =
               stringValue.includes('=>') ||
-              stringValue.startsWith('function') ||
-              stringValue.startsWith('async');
+              /^function\s*[\w]*\s*\(/.test(stringValue) ||
+              /^async\s+function\s*[\w]*\s*\(/.test(stringValue) ||
+              /^async\s*\(/.test(stringValue);
 
             // Check if the string value looks like a Promise expression
             // e.g., "Promise.resolve()", "new Promise(...)", etc.
@@ -714,10 +771,13 @@ import React from 'react';
   if (mockScenario.name === DEFAULT_SCENARIO_NAME) {
     // Normalize mock data to merge entries with same normalized function name
     const normalizedMockData = normalizeMockData(defaultMockData);
+    // Fix useParams splat param (*) to default to empty string
+    const paramsNormalizedMockData =
+      normalizeParamsMockData(normalizedMockData);
     dataCode = jsonToCode(
       {
         arguments: defaultArguments,
-        ...normalizedMockData,
+        ...paramsNormalizedMockData,
       },
       2,
       mergedDataStructure,
@@ -746,10 +806,13 @@ import React from 'react';
     const normalizedMockData = normalizeMockData(
       mergedMockData as Record<string, unknown>,
     );
+    // Fix useParams splat param (*) to default to empty string
+    const paramsNormalizedMockData =
+      normalizeParamsMockData(normalizedMockData);
 
     const mergedData = {
       arguments: mergedArguments,
-      ...normalizedMockData,
+      ...paramsNormalizedMockData,
     };
     // Use level=2 (same as default scenario) for consistent key formatting.
     // This ensures nested function arguments are kept (e.g., "from('workouts')")

package/analyzer-template/project/writeScenarioClientWrapper.ts

@@ -0,0 +1,21 @@
+import { writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import type { Scenario } from '~codeyam/types';
+import { generateScenarioClientWrapper } from '~codeyam/generate';
+
+/**
+ * Writes the ScenarioClientWrapper component to the mocks directory.
+ * This client component is used by Server Component pages to handle
+ * interactive features (iframe messaging, console interception, etc.)
+ */
+export default async function writeScenarioClientWrapper(
+  scenario: Scenario,
+  mocksDir: string,
+): Promise<string> {
+  const content = generateScenarioClientWrapper(scenario);
+  const filePath = join(mocksDir, 'ScenarioClientWrapper.tsx');
+
+  await writeFile(filePath, content);
+
+  return filePath;
+}