@codeyam/codeyam-cli 0.1.0-staging.1669d45 → 0.1.0-staging.323686

package/analyzer-template/packages/ai/src/lib/promptGenerators/simplifyKeysForLLM.ts

@@ -0,0 +1,391 @@
+import { JsonTypeDefinition } from '~codeyam/types';
+
+/**
+ * Simplify verbose function call keys for LLM consumption.
+ *
+ * The static analysis generates keys that include entire function call signatures
+ * with all arguments, including multi-line callback bodies. For example:
+ *
+ * ```
+ * "useQuery({ id: Number(id), pagination: debouncedPagination, onSuccess: async (quote) => { ...30 lines... } })"
+ * ```
+ *
+ * These verbose keys confuse the LLM and cause it to miss generating data for some keys.
+ * This function simplifies keys to just the essential parts:
+ *
+ * ```
+ * "useQuery({ id, pagination, onSuccess: () => <truncated> })"
+ * ```
+ *
+ * The reconcileMockDataKeys function already does fuzzy matching, so the LLM's
+ * simplified response keys will still match back to the original verbose keys.
+ */
+export default function simplifyKeysForLLM(
+  data: JsonTypeDefinition,
+): JsonTypeDefinition {
+  if (typeof data !== 'object' || data === null) {
+    return data;
+  }
+
+  const result: JsonTypeDefinition = {};
+
+  for (const key of Object.keys(data)) {
+    const simplifiedKey = simplifyFunctionCallKey(key);
+    const value = data[key];
+
+    // Recursively simplify nested objects
+    if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
+      result[simplifiedKey] = simplifyKeysForLLM(value as JsonTypeDefinition);
+    } else if (Array.isArray(value)) {
+      // For arrays, recursively simplify each object element
+      result[simplifiedKey] = value.map((item) =>
+        typeof item === 'object' && item !== null && !Array.isArray(item)
+          ? simplifyKeysForLLM(item as JsonTypeDefinition)
+          : item,
+      ) as JsonTypeDefinition[];
+    } else {
+      result[simplifiedKey] = value;
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Simplify a single function call key.
+ *
+ * Rules:
+ * 1. If key doesn't look like a function call, return as-is
+ * 2. For function calls with object arguments:
+ *    - Keep property names
+ *    - Replace complex values with simplified versions
+ *    - Replace callbacks with () => ...
+ * 3. Cap total length at MAX_KEY_LENGTH
+ */
+function simplifyFunctionCallKey(key: string): string {
+  // If it doesn't look like a function call, return as-is
+  if (!key.includes('(')) {
+    return key;
+  }
+
+  // Extract function name and arguments
+  const funcNameMatch = key.match(/^([a-zA-Z_][a-zA-Z0-9_]*)\s*\(/);
+  if (!funcNameMatch) {
+    return key;
+  }
+
+  const funcName = funcNameMatch[1];
+  const argsStart = key.indexOf('(');
+  const argsContent = key.slice(argsStart + 1, -1); // Remove outer parens
+
+  // If args are empty or very short, return as-is
+  if (argsContent.trim().length < 50) {
+    return key;
+  }
+
+  // Parse and simplify the arguments
+  const simplifiedArgs = simplifyArguments(argsContent);
+  const simplified = `${funcName}(${simplifiedArgs})`;
+
+  // Cap length - use <truncated> instead of ... to prevent LLMs from
+  // interpreting it as a fill-in placeholder (Issue #14)
+  const MAX_KEY_LENGTH = 150;
+  if (simplified.length > MAX_KEY_LENGTH) {
+    return simplified.slice(0, MAX_KEY_LENGTH - 11) + '<truncated>';
+  }
+
+  return simplified;
+}
+
+/**
+ * Simplify function call arguments.
+ *
+ * Handles multiple arguments separated by commas, where each argument
+ * could be an object literal, a simple value, or a complex expression.
+ */
+function simplifyArguments(argsContent: string): string {
+  const trimmed = argsContent.trim();
+
+  // Split arguments at top-level commas (not inside nested structures)
+  const args = splitTopLevelArguments(trimmed);
+
+  const simplifiedArgs = args.map((arg) => {
+    const argTrimmed = arg.trim();
+
+    // Check if it's an object literal (starts with {)
+    if (argTrimmed.startsWith('{')) {
+      return simplifyObjectLiteral(argTrimmed);
+    }
+
+    // For non-object arguments, just truncate if needed
+    // Use <truncated> instead of ... to prevent LLMs from interpreting as fill-in
+    if (argTrimmed.length > 30) {
+      return argTrimmed.slice(0, 19) + '<truncated>';
+    }
+
+    return argTrimmed;
+  });
+
+  return simplifiedArgs.join(', ');
+}
+
+/**
+ * Split arguments at top-level commas, respecting nested braces/parens.
+ */
+function splitTopLevelArguments(argsContent: string): string[] {
+  const args: string[] = [];
+  let current = '';
+  let depth = 0;
+  let inString = false;
+  let stringChar = '';
+
+  for (let i = 0; i < argsContent.length; i++) {
+    const char = argsContent[i];
+    const prevChar = i > 0 ? argsContent[i - 1] : '';
+
+    // Handle string boundaries
+    if ((char === '"' || char === "'" || char === '`') && prevChar !== '\\') {
+      if (!inString) {
+        inString = true;
+        stringChar = char;
+      } else if (char === stringChar) {
+        inString = false;
+      }
+    }
+
+    if (!inString) {
+      if (char === '{' || char === '(' || char === '[') {
+        depth++;
+      } else if (char === '}' || char === ')' || char === ']') {
+        depth--;
+      } else if (char === ',' && depth === 0) {
+        // End of argument
+        if (current.trim()) {
+          args.push(current.trim());
+        }
+        current = '';
+        continue;
+      }
+    }
+
+    current += char;
+  }
+
+  // Don't forget the last argument
+  if (current.trim()) {
+    args.push(current.trim());
+  }
+
+  return args;
+}
+
+/**
+ * Simplify an object literal, keeping property names but simplifying values.
+ */
+function simplifyObjectLiteral(objStr: string): string {
+  // Extract property names and their values
+  const properties = extractObjectProperties(objStr);
+
+  if (properties.length === 0) {
+    return '{ }';
+  }
+
+  const simplifiedProps = properties.map(({ name, value }) => {
+    const simplifiedValue = simplifyPropertyValue(value);
+    // If value is same as name (shorthand), just use name
+    if (simplifiedValue === name) {
+      return name;
+    }
+    return `${name}: ${simplifiedValue}`;
+  });
+
+  return `{ ${simplifiedProps.join(', ')} }`;
+}
+
+/**
+ * Extract property names and values from an object literal string.
+ */
+function extractObjectProperties(
+  objStr: string,
+): Array<{ name: string; value: string }> {
+  const properties: Array<{ name: string; value: string }> = [];
+
+  // Remove outer braces and trim
+  let content = objStr.trim();
+  if (content.startsWith('{')) content = content.slice(1);
+  if (content.endsWith('}')) content = content.slice(0, -1);
+  content = content.trim();
+
+  // Parse properties, handling nested structures
+  let current = '';
+  let depth = 0;
+  let inString = false;
+  let stringChar = '';
+
+  for (let i = 0; i < content.length; i++) {
+    const char = content[i];
+    const prevChar = i > 0 ? content[i - 1] : '';
+
+    // Handle string boundaries
+    if ((char === '"' || char === "'" || char === '`') && prevChar !== '\\') {
+      if (!inString) {
+        inString = true;
+        stringChar = char;
+      } else if (char === stringChar) {
+        inString = false;
+      }
+    }
+
+    if (!inString) {
+      if (char === '{' || char === '(' || char === '[') {
+        depth++;
+      } else if (char === '}' || char === ')' || char === ']') {
+        depth--;
+      } else if (char === ',' && depth === 0) {
+        // End of property
+        const prop = parseProperty(current.trim());
+        if (prop) properties.push(prop);
+        current = '';
+        continue;
+      }
+    }
+
+    current += char;
+  }
+
+  // Don't forget the last property
+  if (current.trim()) {
+    const prop = parseProperty(current.trim());
+    if (prop) properties.push(prop);
+  }
+
+  return properties;
+}
+
+/**
+ * Parse a single property string into name and value.
+ */
+function parseProperty(
+  propStr: string,
+): { name: string; value: string } | null {
+  // Handle shorthand properties (just a name)
+  const shorthandMatch = propStr.match(/^([a-zA-Z_][a-zA-Z0-9_]*)$/);
+  if (shorthandMatch) {
+    return { name: shorthandMatch[1], value: shorthandMatch[1] };
+  }
+
+  // Handle name: value
+  const colonIndex = findPropertyColon(propStr);
+  if (colonIndex === -1) {
+    return null;
+  }
+
+  const name = propStr.slice(0, colonIndex).trim();
+  const value = propStr.slice(colonIndex + 1).trim();
+
+  // Clean up property name (remove quotes if present)
+  const cleanName = name.replace(/^['"]|['"]$/g, '');
+
+  return { name: cleanName, value };
+}
+
+/**
+ * Find the colon that separates property name from value.
+ * Must handle cases like { 'key': value } and { key: value }
+ */
+function findPropertyColon(propStr: string): number {
+  let depth = 0;
+  let inString = false;
+  let stringChar = '';
+
+  for (let i = 0; i < propStr.length; i++) {
+    const char = propStr[i];
+    const prevChar = i > 0 ? propStr[i - 1] : '';
+
+    if ((char === '"' || char === "'" || char === '`') && prevChar !== '\\') {
+      if (!inString) {
+        inString = true;
+        stringChar = char;
+      } else if (char === stringChar) {
+        inString = false;
+      }
+    }
+
+    if (!inString) {
+      if (char === '{' || char === '(' || char === '[') {
+        depth++;
+      } else if (char === '}' || char === ')' || char === ']') {
+        depth--;
+      } else if (char === ':' && depth === 0) {
+        return i;
+      }
+    }
+  }
+
+  return -1;
+}
+
+/**
+ * Simplify a property value.
+ */
+function simplifyPropertyValue(value: string): string {
+  const trimmed = value.trim();
+
+  // If it's a simple identifier, return as-is
+  if (/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(trimmed)) {
+    return trimmed;
+  }
+
+  // If it's a simple literal (number, boolean, null, undefined), return as-is
+  if (/^(true|false|null|undefined|\d+(\.\d+)?)$/.test(trimmed)) {
+    return trimmed;
+  }
+
+  // If it's a short string literal, return as-is
+  if (
+    (trimmed.startsWith('"') || trimmed.startsWith("'")) &&
+    trimmed.length < 30
+  ) {
+    return trimmed;
+  }
+
+  // If it looks like a function/arrow function, simplify
+  // Use <truncated> instead of ... to prevent LLMs from interpreting as fill-in
+  if (
+    trimmed.startsWith('function') ||
+    trimmed.startsWith('async function') ||
+    trimmed.includes('=>') ||
+    (trimmed.startsWith('(') && trimmed.includes('=>'))
+  ) {
+    return '() => <truncated>';
+  }
+
+  // If it's an object literal, recursively simplify
+  if (trimmed.startsWith('{')) {
+    const inner = simplifyObjectLiteral(trimmed);
+    if (inner.length > 50) {
+      return '{ <truncated> }';
+    }
+    return inner;
+  }
+
+  // If it's an array, simplify
+  // Use <truncated> instead of ... to prevent LLMs from interpreting as fill-in
+  if (trimmed.startsWith('[')) {
+    return '[<truncated>]';
+  }
+
+  // For complex expressions, extract the first identifier or truncate
+  // Use <truncated> instead of ... to prevent LLMs from interpreting as fill-in
+  const identMatch = trimmed.match(/^([a-zA-Z_][a-zA-Z0-9_]*)/);
+  if (identMatch && trimmed.length > 30) {
+    return identMatch[1] + '<truncated>';
+  }
+
+  // Truncate long values
+  if (trimmed.length > 30) {
+    return trimmed.slice(0, 19) + '<truncated>';
+  }
+
+  return trimmed;
+}

package/analyzer-template/packages/ai/src/lib/resolvePathToControllable.ts

@@ -336,6 +336,7 @@ function findControllablePrefix(
     // Check direct match in attributesMap
     if (prefix in attributesMap) {
       const type = attributesMap[prefix];
+
       // For objects, arrays, or unknown types, nested property access is controllable
       if (
         isArrayType(type) ||
@@ -375,16 +376,6 @@ function findControllableBaseForDerivedPath(
   path: string,
   attributesMap: Record<string, string>,
 ): string | null {
-  // Use cleanPathOfNonTransformingFunctions to strip non-transforming methods
-  // like .filter(), .sort(), .slice() etc. This handles all method calls generically
-  // without hardcoding specific method names.
-  const cleanedPath = cleanPathOfNonTransformingFunctions(path);
-
-  // If the path wasn't changed, there are no non-transforming functions to strip
-  if (cleanedPath === path) {
-    return null;
-  }
-
   // Helper to check if a path is in attributesMap
   const checkInAttributesMap = (pathToCheck: string): string | null => {
     // Direct match
@@ -401,21 +392,121 @@ function findControllableBaseForDerivedPath(
     return null;
   };
 
-  // Check the cleaned path directly
+  // Use cleanPathOfNonTransformingFunctions to strip non-transforming methods
+  // like .filter(), .sort(), .slice() etc. This handles all method calls generically
+  // without hardcoding specific method names.
+  const cleanedPath = cleanPathOfNonTransformingFunctions(path);
+
+  // Check the cleaned path directly (even if unchanged, later steps may find something)
   const directResult = checkInAttributesMap(cleanedPath);
   if (directResult) {
     return directResult;
   }
 
-  // Extract the short path (after functionCallReturnValue) for attributesMap lookup
-  const functionReturnMatch = cleanedPath.match(
+  // Fix 38: For controllability purposes, strip ALL derivation operations
+  // including transforming methods like .split(...) and array indices.
+  // The key insight is: if we control the base, we control all derived values.
+  //
+  // Pattern: base.split('...').filter(...)[N] → find base
+  // We progressively strip: [N] → filter(...) → split(...) → until we find something controllable
+
+  // Methods that derive new values (should be stripped for controllability)
+  const derivingMethods = [
+    'split',
+    'match',
+    'filter',
+    'map',
+    'find',
+    'findIndex',
+    'slice',
+    'substring',
+    'substr',
+    'trim',
+    'trimStart',
+    'trimEnd',
+    'toLowerCase',
+    'toUpperCase',
+    'replace',
+    'replaceAll',
+    'sort',
+    'reverse',
+    'flat',
+    'flatMap',
+  ];
+
+  // Build regex to match method calls: .methodName(...)
+  // Use a simpler approach: progressively strip trailing derivations
+  let workingPath = cleanedPath;
+
+  // Keep stripping derivations until we find something controllable or can't strip anymore
+  // Max 20 iterations to prevent infinite loops
+  for (let iteration = 0; iteration < 20; iteration++) {
+    // Try current path
+    const result = checkInAttributesMap(workingPath);
+    if (result) {
+      return result;
+    }
+
+    // Also check via functionCallReturnValue extraction
+    // Note: We return workingPath (the FULL path) if the short path is found,
+    // because the full path is more meaningful for execution flows
+    const functionReturnMatch = workingPath.match(
+      /\.functionCallReturnValue\.(.+)$/,
+    );
+    if (functionReturnMatch) {
+      const shortPath = functionReturnMatch[1];
+      const shortResult = checkInAttributesMap(shortPath);
+      if (shortResult) {
+        // Return the full path, not just the short result
+        // This ensures execution flows have meaningful paths like
+        // "useParams().functionCallReturnValue.*" instead of just "*"
+        return workingPath;
+      }
+    }
+
+    // Try to strip trailing array index: path[0] → path
+    const arrayIndexMatch = workingPath.match(/^(.+)\[\d+\]$/);
+    if (arrayIndexMatch) {
+      workingPath = arrayIndexMatch[1];
+      continue;
+    }
+
+    // Try to strip trailing method call: path.method(...) → path
+    let strippedMethod = false;
+    for (const method of derivingMethods) {
+      // Match .method(...) at the end, handling nested parentheses
+      const methodRegex = new RegExp(
+        `\\.${method}\\([^()]*(?:\\([^()]*\\)[^()]*)*\\)$`,
+      );
+      if (methodRegex.test(workingPath)) {
+        workingPath = workingPath.replace(methodRegex, '');
+        strippedMethod = true;
+        break;
+      }
+    }
+
+    if (!strippedMethod) {
+      // Can't strip anymore
+      break;
+    }
+  }
+
+  // Final check after stripping
+  const finalResult = checkInAttributesMap(workingPath);
+  if (finalResult) {
+    return finalResult;
+  }
+
+  // Also check via functionCallReturnValue extraction for final path
+  const finalFunctionReturnMatch = workingPath.match(
     /\.functionCallReturnValue\.(.+)$/,
   );
-  if (functionReturnMatch) {
-    const shortPath = functionReturnMatch[1];
+  if (finalFunctionReturnMatch) {
+    const shortPath = finalFunctionReturnMatch[1];
     const shortResult = checkInAttributesMap(shortPath);
     if (shortResult) {
-      return shortResult;
+      // Return the full path, not just the short result
+      return workingPath;
     }
   }
 
@@ -593,10 +684,11 @@ export default function resolvePathToControllable(
       };
     }
 
-    // Fix 34: Handle derived paths from controllable bases
+    // Fix 34/38: Handle derived paths from controllable bases
     // Paths like "useParams().functionCallReturnValue.*.split('/').filter(Boolean)[0]"
     // are derived from "useParams().functionCallReturnValue.*" via string methods.
-    // If the base path is controllable, the derived path should also be controllable.
+    // If the base path is controllable, return the BASE path (not the derived path)
+    // because that's what we need to mock in the data structure.
     const derivedBaseResult = findControllableBaseForDerivedPath(
       fullResolvedPath,
       attributesMap,
@@ -604,7 +696,9 @@ export default function resolvePathToControllable(
     if (derivedBaseResult) {
       chain.push(`derived from controllable base: ${derivedBaseResult}`);
       return {
-        resolvedPath: fullResolvedPath,
+        // Return the controllable BASE path, not the derived path
+        // This ensures execution flows contain paths that exist in attributesMap
+        resolvedPath: derivedBaseResult,
         resolutionChain: chain,
         isControllable: true,
       };
@@ -632,21 +726,48 @@ export default function resolvePathToControllable(
   // 5. Fix 35: Check for controllable prefix on the local path directly
   // This handles cases where the localPath itself (before any resolution)
   // has a controllable prefix in attributesMap
-  const localPrefixResult = findControllablePrefix(
-    localPath,
-    attributesMap,
-    fullToShortPathMap,
+  //
+  // IMPORTANT: Only do prefix matching if:
+  // a) The root variable is known in equivalentSignatureVariables, OR
+  // b) The path looks like an already-translated path (contains .functionCallReturnValue.)
+  //
+  // If the root variable is NOT known AND it's not an already-translated path, it's likely
+  // from an internal scope (callback, async function) and shouldn't be matched
+  // against similarly-named variables from different scopes.
+  // For example, "data.status" from inside `await response.json()` should NOT
+  // match the "data" from a useFetcher hook.
+  const localVarNameForPrefix = extractLocalVariableName(localPath);
+  const isKnownVariable = localVarNameForPrefix in equivalentSignatureVariables;
+
+  // Fix 35b: Also allow prefix matching for already-translated paths from child components
+  // These paths start with hook patterns like useLoaderData<...>().functionCallReturnValue
+  // and should be checked for controllable prefixes even if the hook itself isn't
+  // in equivalentSignatureVariables (because it's a fully resolved path, not a local variable)
+  const isAlreadyTranslatedPath = localPath.includes(
+    '.functionCallReturnValue.',
   );
-  if (localPrefixResult) {
-    chain.push(`nested property of controllable prefix (direct)`);
-    return {
-      resolvedPath: localPath,
-      resolutionChain: chain,
-      isControllable: true,
-    };
+
+  if (isKnownVariable || isAlreadyTranslatedPath) {
+    const localPrefixResult = findControllablePrefix(
+      localPath,
+      attributesMap,
+      fullToShortPathMap,
+    );
+    if (localPrefixResult) {
+      chain.push(`nested property of controllable prefix (direct)`);
+      return {
+        resolvedPath: localPath,
+        resolutionChain: chain,
+        isControllable: true,
+      };
+    }
+  } else {
+    chain.push(
+      `root variable '${localVarNameForPrefix}' not in equivalentSignatureVariables - skipping prefix match`,
+    );
   }
 
-  // 5b. Fix 36: Check for derived paths on localPath directly
+  // 5b. Fix 36/38: Check for derived paths on localPath directly
   // When localPath contains derivation methods like .filter(cyScope()), check if
   // the base path (before .filter()) is controllable
   const localDerivedResult = findControllableBaseForDerivedPath(
@@ -658,7 +779,8 @@ export default function resolvePathToControllable(
       `derived from controllable base (direct): ${localDerivedResult}`,
     );
     return {
-      resolvedPath: localPath,
+      // Return the controllable BASE path, not the derived path
+      resolvedPath: localDerivedResult,
       resolutionChain: chain,
       isControllable: true,
     };

package/analyzer-template/packages/ai/src/lib/worker/SerializableDataStructure.ts

@@ -13,6 +13,7 @@ import {
 import type {
   ConditionalEffect,
   CompoundConditional,
+  JsxRenderingUsage,
 } from '../astScopes/types';
 
 export interface SerializableScopeVariable {
@@ -101,7 +102,8 @@ export interface EnrichedConditionalUsage {
       | 'and'
       | 'arrayIncludes'
       | 'arraySome'
-      | 'arrayEvery';
+      | 'arrayEvery'
+      | 'arrayLength';
     /** For comparison operations, the value being compared to */
     comparedValue?: string;
   };
@@ -154,6 +156,15 @@ export interface SerializableDataStructure {
    * This allows us to merge child component execution flows with parent gating conditions.
    */
   childBoundaryGatingConditions?: Record<string, EnrichedConditionalUsage[]>;
+
+  /**
+   * JSX rendering usages - tracks arrays rendered via .map() and strings interpolated in JSX.
+   * Used to generate execution flows for array sizes and text lengths without explicit conditionals.
+   *
+   * Example:
+   * {items.map(i => <Item {...i} />)} produces an array-map usage for 'items'
+   */
+  jsxRenderingUsages?: JsxRenderingUsage[];
 }
 
 /**
@@ -329,3 +340,13 @@ export function getChildBoundaryGatingConditions(
 ): Record<string, EnrichedConditionalUsage[]> {
   return dataStructure.childBoundaryGatingConditions ?? {};
 }
+
+/**
+ * Get JSX rendering usages (arrays rendered via .map() and strings interpolated in JSX).
+ * Returns the array of JSX rendering usages, or empty array if not available.
+ */
+export function getJsxRenderingUsages(
+  dataStructure: SerializableDataStructure,
+): JsxRenderingUsage[] {
+  return dataStructure.jsxRenderingUsages ?? [];
+}