@codeyam/codeyam-cli 0.1.0-staging.1669d45 → 0.1.0-staging.25a2014

package/analyzer-template/packages/ai/src/lib/promptGenerators/gatherAttributesMap.ts

@@ -139,16 +139,19 @@ export function gatherAttributesMap(
       // Use merged type if available, otherwise fall back to isolated type
       const typeValue =
         mergedSignatureSchema[key] ?? isolatedSignatureSchema[key];
-      const equivalentSignatureVariable =
-        equivalentSignatureVariables[keyParts[0]];
-      const equivalentSignatureVariableParts = splitOutsideParenthesesAndArrays(
-        equivalentSignatureVariable,
-      );
+      // Handle array case (OR expressions) - use first element if array
+      const rawEquivalent = equivalentSignatureVariables[keyParts[0]];
+      const equivalentSignatureVariable = Array.isArray(rawEquivalent)
+        ? rawEquivalent[0]
+        : rawEquivalent;
+      const equivalentSignatureVariableParts = equivalentSignatureVariable
+        ? splitOutsideParenthesesAndArrays(equivalentSignatureVariable)
+        : [];
       if (
         equivalentSignatureVariable &&
-        !equivalentSignatureVariableParts[0].includes('(')
+        !equivalentSignatureVariableParts[0]?.includes('(')
       ) {
-        const equivalentKey = equivalentSignatureVariables[keyParts[0]];
+        const equivalentKey = equivalentSignatureVariable;
         const equivalentPath = joinParenthesesAndArrays([
           equivalentKey,
           ...keyParts.slice(1),

package/analyzer-template/packages/ai/src/lib/promptGenerators/generateChunkPrompt.ts

@@ -0,0 +1,82 @@
+import {
+  ExecutionFlow,
+  Scenario,
+  ScenariosDataStructure,
+} from '~codeyam/types';
+import simplifyKeysForLLM from './simplifyKeysForLLM';
+import noErrorAttributes from './noErrorAttributes';
+import { JsonTypeDefinition } from '~codeyam/types';
+
+type RequiredValue = NonNullable<ExecutionFlow['requiredValues']>[number];
+
+interface GenerateChunkPromptArgs {
+  scenario: Scenario;
+  chunk: ScenariosDataStructure['dataForMocks'];
+  chunkIndex: number;
+  totalChunks: number;
+  relevantRequiredValues: RequiredValue[];
+}
+
+/**
+ * Generate a prompt for a focused LLM call to generate mock data for a single chunk.
+ *
+ * Large data structures overwhelm LLMs, causing them to make mistakes on some keys.
+ * By processing smaller chunks, each key gets more focused attention.
+ */
+export default function generateChunkPrompt({
+  scenario,
+  chunk,
+  chunkIndex,
+  totalChunks,
+  relevantRequiredValues,
+}: GenerateChunkPromptArgs): string {
+  // Clean up the schema for LLM consumption
+  const cleanedSchema = simplifyKeysForLLM(
+    noErrorAttributes(chunk) as JsonTypeDefinition,
+  );
+
+  const keysInChunk = Object.keys(chunk || {});
+
+  // Build requirements section if there are relevant requiredValues
+  const requirementsSection =
+    relevantRequiredValues.length > 0
+      ? `## Execution Flow Requirements for These Keys
+These specific values MUST be set to produce the correct scenario behavior:
+\`\`\`json
+${JSON.stringify(relevantRequiredValues, null, 2)}
+\`\`\`
+
+`
+      : '';
+
+  return `Generate mock data for chunk ${chunkIndex + 1} of ${totalChunks}.
+
+## Scenario
+\`\`\`json
+${JSON.stringify({ name: scenario.name, description: scenario.description }, null, 2)}
+\`\`\`
+
+${requirementsSection}## Keys in This Chunk (${keysInChunk.length} keys)
+Generate data for ONLY these keys:
+${keysInChunk.map((k) => `- \`${k}\``).join('\n')}
+
+## Schema for This Chunk
+\`\`\`json
+${JSON.stringify(cleanedSchema, null, 2)}
+\`\`\`
+
+## Instructions
+1. Match the EXACT structure shown in the schema
+2. Keys with special characters like \`*\` are LITERAL keys, not patterns
+3. If requiredValues are shown above, set values that satisfy them
+4. For paths like \`functionCall().functionCallReturnValue.property\`, the mock data key is just \`functionCall()\`
+
+Return ONLY a JSON object with this structure:
+\`\`\`json
+{
+  "mockData": {
+    // fill in ONLY the keys from this chunk
+  }
+}
+\`\`\``;
+}

package/analyzer-template/packages/ai/src/lib/promptGenerators/generateCriticalKeysPrompt.ts

@@ -0,0 +1,103 @@
+import {
+  ExecutionFlow,
+  Scenario,
+  ScenariosDataStructure,
+} from '~codeyam/types';
+import simplifyKeysForLLM from './simplifyKeysForLLM';
+import noErrorAttributes from './noErrorAttributes';
+import { JsonTypeDefinition } from '~codeyam/types';
+
+interface GenerateCriticalKeysPromptArgs {
+  scenario: Scenario;
+  executionFlows: ExecutionFlow[] | undefined;
+  criticalKeys: string[];
+  fullDataForMocks: ScenariosDataStructure['dataForMocks'];
+}
+
+/**
+ * Generate a prompt for a focused LLM call to generate only the critical mock data keys.
+ *
+ * When a scenario has covered flows with requiredValues, only a subset of the full
+ * dataForMocks schema is actually needed to satisfy those requirements. This prompt
+ * focuses the LLM on JUST those critical keys, avoiding confusion from large schemas.
+ *
+ * This is called BEFORE the main data generation to ensure critical keys are correctly
+ * generated with full attention from the LLM.
+ */
+export default function generateCriticalKeysPrompt({
+  scenario,
+  executionFlows,
+  criticalKeys,
+  fullDataForMocks,
+}: GenerateCriticalKeysPromptArgs): string {
+  // Build the subset schema with only critical keys
+  const criticalSchema: Record<string, any> = {};
+  if (fullDataForMocks && typeof fullDataForMocks === 'object') {
+    for (const key of criticalKeys) {
+      if (key in fullDataForMocks) {
+        criticalSchema[key] = (fullDataForMocks as Record<string, any>)[key];
+      }
+    }
+  }
+
+  // Clean up the schema for LLM consumption
+  const cleanedSchema = simplifyKeysForLLM(
+    noErrorAttributes(criticalSchema) as JsonTypeDefinition,
+  );
+
+  // Get the covered flows and their requirements
+  const coveredFlowIds = scenario.metadata?.coveredFlows || [];
+  const coveredFlowRequirements = coveredFlowIds
+    .map((flowId) => {
+      const flow = executionFlows?.find((f) => f.id === flowId);
+      if (!flow) return null;
+      return {
+        flowId: flow.id,
+        flowName: flow.name,
+        flowDescription: flow.description,
+        requiredValues: flow.requiredValues,
+      };
+    })
+    .filter(Boolean);
+
+  return `Generate mock data for these CRITICAL keys. Pay close attention to the requiredValues.
+
+## IMPORTANT: These keys are critical for satisfying the execution flow requirements
+The keys below directly control the scenario's behavior. Generate values that EXACTLY satisfy the requiredValues.
+
+## Scenario
+\`\`\`json
+${JSON.stringify({ name: scenario.name, description: scenario.description }, null, 2)}
+\`\`\`
+
+## Execution Flow Requirements
+These are the specific values that MUST be set:
+\`\`\`json
+${JSON.stringify(coveredFlowRequirements, null, 2)}
+\`\`\`
+
+## Critical Keys Schema (generate data for ONLY these keys)
+\`\`\`json
+${JSON.stringify(cleanedSchema, null, 2)}
+\`\`\`
+
+## Instructions
+1. Match the EXACT structure shown in the schema
+2. Keys with special characters like \`*\` are LITERAL keys, not patterns
+3. Set values that satisfy ALL the requiredValues above
+4. For paths like \`functionCall().functionCallReturnValue.property\`, the mock data key is just \`functionCall()\`
+
+Return ONLY a JSON object with this structure:
+\`\`\`json
+{
+  "scenarioData": {
+    "scenarioName": "${scenario.name}",
+    "data": {
+      "mockData": {
+        // fill in ONLY the critical keys above
+      }
+    }
+  }
+}
+\`\`\``;
+}

package/analyzer-template/packages/ai/src/lib/promptGenerators/generateEntityScenarioDataGenerator.ts

@@ -2,9 +2,12 @@ import {
   ScenariosDataStructure,
   Scenario,
   ScenarioData,
+  JsonTypeDefinition,
   ExecutionFlow,
 } from '~codeyam/types';
 import noErrorAttributes from './noErrorAttributes';
+import simplifyKeysForLLM from './simplifyKeysForLLM';
+import collapseNullableObjects from './collapseNullableObjects';
 
 /**
  * Sort object keys by the size of their JSON-stringified values (smallest first).
@@ -21,12 +24,18 @@ function sortKeysBySize(obj: Record<string, any>): Record<string, any> {
   return Object.fromEntries(entries);
 }
 
+interface GeneratePromptOptions {
+  /** When true, mockData has already been generated via chunks - skip it in main call */
+  mockDataAlreadyGenerated?: boolean;
+}
+
 export default function generateEntityScenarioDataGenerator(
   structure: ScenariosDataStructure,
   scenario: Scenario,
   executionFlows: ExecutionFlow[] | undefined,
   defaultScenarioData?: ScenarioData,
   incompleteResponse?: string,
+  options?: GeneratePromptOptions,
 ) {
   if (incompleteResponse) {
     return incompleteResponse;
@@ -85,11 +94,23 @@ ${JSON.stringify(defaultScenarioData.data, null, 2)}
 `
     : '';
 
-  // Sort keys by size (smallest first) so small keys appear before large keys,
-  // increasing the chance the LLM generates all keys for large schemas
-  const cleanedDataForMocks = structure.dataForMocks
-    ? noErrorAttributes(structure.dataForMocks)
+  // Simplify verbose function call keys for LLM consumption, then sort by size.
+  // The static analysis generates keys like "useQuery({ id: Number(id), onSuccess: async (quote) => { ...30 lines... } })"
+  // which confuse the LLM. simplifyKeysForLLM reduces them to "useQuery({ id, onSuccess: () => <truncated> })"
+  // while the reconcileMockDataKeys function will still match the simplified response back to original keys.
+  // Then sort keys by size (smallest first) so small keys appear before large keys,
+  // increasing the chance the LLM generates all keys for large schemas.
+
+  const afterNoError = structure.dataForMocks
+    ? (noErrorAttributes(structure.dataForMocks) as JsonTypeDefinition)
     : null;
+
+  const afterSimplify = afterNoError ? simplifyKeysForLLM(afterNoError) : null;
+
+  const cleanedDataForMocks = afterSimplify
+    ? collapseNullableObjects(afterSimplify)
+    : null;
+
   const sortedDataForMocks =
     cleanedDataForMocks &&
     typeof cleanedDataForMocks === 'object' &&
@@ -97,12 +118,28 @@ ${JSON.stringify(defaultScenarioData.data, null, 2)}
       ? sortKeysBySize(cleanedDataForMocks as Record<string, any>)
       : cleanedDataForMocks;
 
-  const mockDataSection = sortedDataForMocks
-    ? `## mockData Structure
+  // Log nullable-related content in final prompt JSON
+  if (sortedDataForMocks) {
+    const finalJson = JSON.stringify(sortedDataForMocks, null, 2);
+    const hasNullableMarker = finalJson.includes('_nullable');
+    const nullCollapsed = (finalJson.match(/"object \| null"/g) || []).length;
+    const arrayCollapsed = (finalJson.match(/"array \| null"/g) || []).length;
+    if (hasNullableMarker || nullCollapsed > 0 || arrayCollapsed > 0) {
+      console.log(
+        `[generateEntityScenarioDataGenerator] final prompt: ${nullCollapsed} "object | null", ${arrayCollapsed} "array | null"${hasNullableMarker ? ', WARNING: _nullable still present!' : ''}`,
+      );
+    }
+  }
+
+  // When mockData has already been generated via chunks, skip it in the main call
+  const mockDataSection = options?.mockDataAlreadyGenerated
+    ? '## mockData Structure\nmockData has been pre-generated—return `{}` for mockData.'
+    : sortedDataForMocks
+      ? `## mockData Structure
 \`\`\`json
 ${JSON.stringify(sortedDataForMocks, null, 2)}
 \`\`\``
-    : '## mockData Structure\nNone—return `{}` for mockData.';
+      : '## mockData Structure\nNone—return `{}` for mockData.';
 
   const argumentsSection = structure.arguments
     ? `## argumentsData Structure

package/analyzer-template/packages/ai/src/lib/promptGenerators/simplifyKeysForLLM.ts

@@ -0,0 +1,391 @@
+import { JsonTypeDefinition } from '~codeyam/types';
+
+/**
+ * Simplify verbose function call keys for LLM consumption.
+ *
+ * The static analysis generates keys that include entire function call signatures
+ * with all arguments, including multi-line callback bodies. For example:
+ *
+ * ```
+ * "useQuery({ id: Number(id), pagination: debouncedPagination, onSuccess: async (quote) => { ...30 lines... } })"
+ * ```
+ *
+ * These verbose keys confuse the LLM and cause it to miss generating data for some keys.
+ * This function simplifies keys to just the essential parts:
+ *
+ * ```
+ * "useQuery({ id, pagination, onSuccess: () => <truncated> })"
+ * ```
+ *
+ * The reconcileMockDataKeys function already does fuzzy matching, so the LLM's
+ * simplified response keys will still match back to the original verbose keys.
+ */
+export default function simplifyKeysForLLM(
+  data: JsonTypeDefinition,
+): JsonTypeDefinition {
+  if (typeof data !== 'object' || data === null) {
+    return data;
+  }
+
+  const result: JsonTypeDefinition = {};
+
+  for (const key of Object.keys(data)) {
+    const simplifiedKey = simplifyFunctionCallKey(key);
+    const value = data[key];
+
+    // Recursively simplify nested objects
+    if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
+      result[simplifiedKey] = simplifyKeysForLLM(value as JsonTypeDefinition);
+    } else if (Array.isArray(value)) {
+      // For arrays, recursively simplify each object element
+      result[simplifiedKey] = value.map((item) =>
+        typeof item === 'object' && item !== null && !Array.isArray(item)
+          ? simplifyKeysForLLM(item as JsonTypeDefinition)
+          : item,
+      ) as JsonTypeDefinition[];
+    } else {
+      result[simplifiedKey] = value;
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Simplify a single function call key.
+ *
+ * Rules:
+ * 1. If key doesn't look like a function call, return as-is
+ * 2. For function calls with object arguments:
+ *    - Keep property names
+ *    - Replace complex values with simplified versions
+ *    - Replace callbacks with () => ...
+ * 3. Cap total length at MAX_KEY_LENGTH
+ */
+function simplifyFunctionCallKey(key: string): string {
+  // If it doesn't look like a function call, return as-is
+  if (!key.includes('(')) {
+    return key;
+  }
+
+  // Extract function name and arguments
+  const funcNameMatch = key.match(/^([a-zA-Z_][a-zA-Z0-9_]*)\s*\(/);
+  if (!funcNameMatch) {
+    return key;
+  }
+
+  const funcName = funcNameMatch[1];
+  const argsStart = key.indexOf('(');
+  const argsContent = key.slice(argsStart + 1, -1); // Remove outer parens
+
+  // If args are empty or very short, return as-is
+  if (argsContent.trim().length < 50) {
+    return key;
+  }
+
+  // Parse and simplify the arguments
+  const simplifiedArgs = simplifyArguments(argsContent);
+  const simplified = `${funcName}(${simplifiedArgs})`;
+
+  // Cap length - use <truncated> instead of ... to prevent LLMs from
+  // interpreting it as a fill-in placeholder (Issue #14)
+  const MAX_KEY_LENGTH = 150;
+  if (simplified.length > MAX_KEY_LENGTH) {
+    return simplified.slice(0, MAX_KEY_LENGTH - 11) + '<truncated>';
+  }
+
+  return simplified;
+}
+
+/**
+ * Simplify function call arguments.
+ *
+ * Handles multiple arguments separated by commas, where each argument
+ * could be an object literal, a simple value, or a complex expression.
+ */
+function simplifyArguments(argsContent: string): string {
+  const trimmed = argsContent.trim();
+
+  // Split arguments at top-level commas (not inside nested structures)
+  const args = splitTopLevelArguments(trimmed);
+
+  const simplifiedArgs = args.map((arg) => {
+    const argTrimmed = arg.trim();
+
+    // Check if it's an object literal (starts with {)
+    if (argTrimmed.startsWith('{')) {
+      return simplifyObjectLiteral(argTrimmed);
+    }
+
+    // For non-object arguments, just truncate if needed
+    // Use <truncated> instead of ... to prevent LLMs from interpreting as fill-in
+    if (argTrimmed.length > 30) {
+      return argTrimmed.slice(0, 19) + '<truncated>';
+    }
+
+    return argTrimmed;
+  });
+
+  return simplifiedArgs.join(', ');
+}
+
+/**
+ * Split arguments at top-level commas, respecting nested braces/parens.
+ */
+function splitTopLevelArguments(argsContent: string): string[] {
+  const args: string[] = [];
+  let current = '';
+  let depth = 0;
+  let inString = false;
+  let stringChar = '';
+
+  for (let i = 0; i < argsContent.length; i++) {
+    const char = argsContent[i];
+    const prevChar = i > 0 ? argsContent[i - 1] : '';
+
+    // Handle string boundaries
+    if ((char === '"' || char === "'" || char === '`') && prevChar !== '\\') {
+      if (!inString) {
+        inString = true;
+        stringChar = char;
+      } else if (char === stringChar) {
+        inString = false;
+      }
+    }
+
+    if (!inString) {
+      if (char === '{' || char === '(' || char === '[') {
+        depth++;
+      } else if (char === '}' || char === ')' || char === ']') {
+        depth--;
+      } else if (char === ',' && depth === 0) {
+        // End of argument
+        if (current.trim()) {
+          args.push(current.trim());
+        }
+        current = '';
+        continue;
+      }
+    }
+
+    current += char;
+  }
+
+  // Don't forget the last argument
+  if (current.trim()) {
+    args.push(current.trim());
+  }
+
+  return args;
+}
+
+/**
+ * Simplify an object literal, keeping property names but simplifying values.
+ */
+function simplifyObjectLiteral(objStr: string): string {
+  // Extract property names and their values
+  const properties = extractObjectProperties(objStr);
+
+  if (properties.length === 0) {
+    return '{ }';
+  }
+
+  const simplifiedProps = properties.map(({ name, value }) => {
+    const simplifiedValue = simplifyPropertyValue(value);
+    // If value is same as name (shorthand), just use name
+    if (simplifiedValue === name) {
+      return name;
+    }
+    return `${name}: ${simplifiedValue}`;
+  });
+
+  return `{ ${simplifiedProps.join(', ')} }`;
+}
+
+/**
+ * Extract property names and values from an object literal string.
+ */
+function extractObjectProperties(
+  objStr: string,
+): Array<{ name: string; value: string }> {
+  const properties: Array<{ name: string; value: string }> = [];
+
+  // Remove outer braces and trim
+  let content = objStr.trim();
+  if (content.startsWith('{')) content = content.slice(1);
+  if (content.endsWith('}')) content = content.slice(0, -1);
+  content = content.trim();
+
+  // Parse properties, handling nested structures
+  let current = '';
+  let depth = 0;
+  let inString = false;
+  let stringChar = '';
+
+  for (let i = 0; i < content.length; i++) {
+    const char = content[i];
+    const prevChar = i > 0 ? content[i - 1] : '';
+
+    // Handle string boundaries
+    if ((char === '"' || char === "'" || char === '`') && prevChar !== '\\') {
+      if (!inString) {
+        inString = true;
+        stringChar = char;
+      } else if (char === stringChar) {
+        inString = false;
+      }
+    }
+
+    if (!inString) {
+      if (char === '{' || char === '(' || char === '[') {
+        depth++;
+      } else if (char === '}' || char === ')' || char === ']') {
+        depth--;
+      } else if (char === ',' && depth === 0) {
+        // End of property
+        const prop = parseProperty(current.trim());
+        if (prop) properties.push(prop);
+        current = '';
+        continue;
+      }
+    }
+
+    current += char;
+  }
+
+  // Don't forget the last property
+  if (current.trim()) {
+    const prop = parseProperty(current.trim());
+    if (prop) properties.push(prop);
+  }
+
+  return properties;
+}
+
+/**
+ * Parse a single property string into name and value.
+ */
+function parseProperty(
+  propStr: string,
+): { name: string; value: string } | null {
+  // Handle shorthand properties (just a name)
+  const shorthandMatch = propStr.match(/^([a-zA-Z_][a-zA-Z0-9_]*)$/);
+  if (shorthandMatch) {
+    return { name: shorthandMatch[1], value: shorthandMatch[1] };
+  }
+
+  // Handle name: value
+  const colonIndex = findPropertyColon(propStr);
+  if (colonIndex === -1) {
+    return null;
+  }
+
+  const name = propStr.slice(0, colonIndex).trim();
+  const value = propStr.slice(colonIndex + 1).trim();
+
+  // Clean up property name (remove quotes if present)
+  const cleanName = name.replace(/^['"]|['"]$/g, '');
+
+  return { name: cleanName, value };
+}
+
+/**
+ * Find the colon that separates property name from value.
+ * Must handle cases like { 'key': value } and { key: value }
+ */
+function findPropertyColon(propStr: string): number {
+  let depth = 0;
+  let inString = false;
+  let stringChar = '';
+
+  for (let i = 0; i < propStr.length; i++) {
+    const char = propStr[i];
+    const prevChar = i > 0 ? propStr[i - 1] : '';
+
+    if ((char === '"' || char === "'" || char === '`') && prevChar !== '\\') {
+      if (!inString) {
+        inString = true;
+        stringChar = char;
+      } else if (char === stringChar) {
+        inString = false;
+      }
+    }
+
+    if (!inString) {
+      if (char === '{' || char === '(' || char === '[') {
+        depth++;
+      } else if (char === '}' || char === ')' || char === ']') {
+        depth--;
+      } else if (char === ':' && depth === 0) {
+        return i;
+      }
+    }
+  }
+
+  return -1;
+}
+
+/**
+ * Simplify a property value.
+ */
+function simplifyPropertyValue(value: string): string {
+  const trimmed = value.trim();
+
+  // If it's a simple identifier, return as-is
+  if (/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(trimmed)) {
+    return trimmed;
+  }
+
+  // If it's a simple literal (number, boolean, null, undefined), return as-is
+  if (/^(true|false|null|undefined|\d+(\.\d+)?)$/.test(trimmed)) {
+    return trimmed;
+  }
+
+  // If it's a short string literal, return as-is
+  if (
+    (trimmed.startsWith('"') || trimmed.startsWith("'")) &&
+    trimmed.length < 30
+  ) {
+    return trimmed;
+  }
+
+  // If it looks like a function/arrow function, simplify
+  // Use <truncated> instead of ... to prevent LLMs from interpreting as fill-in
+  if (
+    trimmed.startsWith('function') ||
+    trimmed.startsWith('async function') ||
+    trimmed.includes('=>') ||
+    (trimmed.startsWith('(') && trimmed.includes('=>'))
+  ) {
+    return '() => <truncated>';
+  }
+
+  // If it's an object literal, recursively simplify
+  if (trimmed.startsWith('{')) {
+    const inner = simplifyObjectLiteral(trimmed);
+    if (inner.length > 50) {
+      return '{ <truncated> }';
+    }
+    return inner;
+  }
+
+  // If it's an array, simplify
+  // Use <truncated> instead of ... to prevent LLMs from interpreting as fill-in
+  if (trimmed.startsWith('[')) {
+    return '[<truncated>]';
+  }
+
+  // For complex expressions, extract the first identifier or truncate
+  // Use <truncated> instead of ... to prevent LLMs from interpreting as fill-in
+  const identMatch = trimmed.match(/^([a-zA-Z_][a-zA-Z0-9_]*)/);
+  if (identMatch && trimmed.length > 30) {
+    return identMatch[1] + '<truncated>';
+  }
+
+  // Truncate long values
+  if (trimmed.length > 30) {
+    return trimmed.slice(0, 19) + '<truncated>';
+  }
+
+  return trimmed;
+}