@codeyam/codeyam-cli 0.1.0-staging.e38f7bd → 0.1.0

package/analyzer-template/packages/ai/src/lib/extractCriticalDataKeys.ts

@@ -0,0 +1,120 @@
+import type { ExecutionFlow, ScenariosDataStructure } from '~codeyam/types';
+
+/**
+ * Extract the data structure keys that are critical for satisfying execution flow requirements.
+ *
+ * When generating scenario mock data, only a subset of the full dataForMocks structure
+ * is actually needed to satisfy the scenario's covered flows' requiredValues.
+ * This function identifies those critical keys so they can be processed with more focus
+ * (e.g., in a separate LLM call or prioritized in the prompt).
+ *
+ * @param dataForMocks - The full data structure schema
+ * @param executionFlows - All execution flows for the entity
+ * @param coveredFlowIds - The flow IDs covered by the current scenario
+ * @returns Array of top-level keys from dataForMocks that are referenced by the covered flows
+ */
+export function extractCriticalDataKeys(
+  dataForMocks: ScenariosDataStructure['dataForMocks'],
+  executionFlows: ExecutionFlow[],
+  coveredFlowIds: string[],
+): string[] {
+  if (!dataForMocks || typeof dataForMocks !== 'object') {
+    return [];
+  }
+
+  const dataKeys = Object.keys(dataForMocks);
+  const criticalKeys = new Set<string>();
+
+  // Get the covered flows
+  const coveredFlows = executionFlows.filter((flow) =>
+    coveredFlowIds.includes(flow.id),
+  );
+
+  // For each covered flow, check if any requiredValue references a dataForMocks key
+  for (const flow of coveredFlows) {
+    if (!flow.requiredValues) continue;
+
+    for (const rv of flow.requiredValues) {
+      const path = rv.attributePath;
+      if (!path) continue;
+
+      // Try to match the attributePath to a dataForMocks key
+      // The path format is: "functionCall().functionCallReturnValue.property"
+      // or: "functionCall<Type>().functionCallReturnValue.property"
+      // We need to extract the function call part and match it to a key
+
+      for (const key of dataKeys) {
+        if (pathMatchesKey(path, key)) {
+          criticalKeys.add(key);
+          break; // One match per requiredValue is enough
+        }
+      }
+    }
+  }
+
+  return Array.from(criticalKeys);
+}
+
+/**
+ * Check if an attributePath references a given dataForMocks key.
+ *
+ * Handles various path formats:
+ * - "useParams().functionCallReturnValue.*" matches "useParams()"
+ * - "useLoaderData<LoaderData>().functionCallReturnValue.data" matches "useLoaderData<LoaderData>()"
+ * - "useFetcher().data.success" matches "useFetcher()"
+ */
+function pathMatchesKey(attributePath: string, dataKey: string): boolean {
+  // Normalize both for comparison
+  // Remove .functionCallReturnValue from the path
+  const normalizedPath = attributePath.replace('.functionCallReturnValue', '');
+
+  // Check if the path starts with the key (exact match for function call keys)
+  if (normalizedPath.startsWith(dataKey)) {
+    // Ensure it's not a partial match (e.g., "use" shouldn't match "useState")
+    const afterKey = normalizedPath.slice(dataKey.length);
+    // Valid continuations: end of string, '.', or '['
+    if (
+      afterKey === '' ||
+      afterKey.startsWith('.') ||
+      afterKey.startsWith('[')
+    ) {
+      return true;
+    }
+  }
+
+  // Try matching without type parameters if needed
+  // e.g., "useLoaderData<LoaderData>()" should match attributePaths with same base
+  const keyWithoutTypes = removeTypeParameters(dataKey);
+  const pathWithoutTypes = removeTypeParameters(normalizedPath);
+
+  if (pathWithoutTypes.startsWith(keyWithoutTypes)) {
+    const afterKey = pathWithoutTypes.slice(keyWithoutTypes.length);
+    if (
+      afterKey === '' ||
+      afterKey.startsWith('.') ||
+      afterKey.startsWith('[')
+    ) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+/**
+ * Remove type parameters from a function call string.
+ * e.g., "useLoaderData<LoaderData>()" -> "useLoaderData()"
+ */
+function removeTypeParameters(str: string): string {
+  // Match and remove <...> including nested angle brackets
+  let result = str;
+  let prevLength = 0;
+
+  // Keep removing until no more changes (handles nested generics)
+  while (result.length !== prevLength) {
+    prevLength = result.length;
+    result = result.replace(/<[^<>]*>/g, '');
+  }
+
+  return result;
+}

package/analyzer-template/packages/ai/src/lib/generateChangesEntityScenarioData.ts

@@ -16,6 +16,8 @@ import type {
 import { LlmCall } from '~codeyam/types';
 import validateJson from './validateJson';
 import checkAllAttributes from './checkAllAttributes';
+import convertNullToUndefinedBySchema from './dataStructure/helpers/convertNullToUndefinedBySchema';
+import deepEqual from './deepEqual';
 import { awsLog } from '~codeyam/utils';
 import { AI, parseJsonSafe } from '~codeyam/ai';
 
@@ -59,12 +61,14 @@ async function generateChangesDataForScenario({
   llmCalls: LlmCall[];
 }> {
   if (
-    structure === existingStructure &&
-    scenario.metadata.keyAttributeInstructions ===
-      existingScenario.metadata.keyAttributeInstructions
+    deepEqual(structure, existingStructure) &&
+    deepEqual(
+      scenario.metadata?.coveredFlows,
+      existingScenario?.metadata?.coveredFlows,
+    )
   ) {
     console.log(
-      "CodeYam: No changes in scenario's structure or keyAttributeInstructions - returning existing scenario data",
+      "CodeYam: No changes in scenario's structure or coveredFlows - returning existing scenario data",
       {
         analysisId: scenario.analysisId,
         scenarioId: scenario.id,
@@ -95,6 +99,7 @@ async function generateChangesDataForScenario({
       prompt = generateEntityScenarioDataGenerator(
         structure,
         scenario,
+        undefined, // executionFlows - not available in changes context
         defaultScenarioData,
         incompleteResponse,
       );
@@ -219,8 +224,8 @@ async function generateChangesDataForScenario({
     if (!Array.isArray(argumentsData)) {
       argumentsData = [];
     }
-    fullScenarioData.data.argumentsData = argumentsData.map((arg) =>
-      checkAllAttributes(arg),
+    fullScenarioData.data.argumentsData = argumentsData.map(
+      (arg) => checkAllAttributes(arg) as { [key: string]: unknown },
     );
 
     if (structure.dataForMocks && !fullScenarioData.data.mockData) {
@@ -232,7 +237,28 @@ async function generateChangesDataForScenario({
     }
     fullScenarioData.data.mockData = checkAllAttributes(
       fullScenarioData.data.mockData,
-    );
+    ) as { [key: string]: unknown };
+
+    // Convert null values to undefined based on schema type constraints.
+    // LLM uses null for "no value" (JSON doesn't support undefined), but TypeScript
+    // types like "string | undefined" don't accept null. This converts null→undefined
+    // for fields typed as "T | undefined" (but preserves null for "T | null").
+    if (structure.dataForMocks && fullScenarioData.data.mockData) {
+      convertNullToUndefinedBySchema(
+        fullScenarioData.data.mockData,
+        structure.dataForMocks,
+      );
+    }
+    if (structure.arguments && fullScenarioData.data.argumentsData) {
+      for (let i = 0; i < fullScenarioData.data.argumentsData.length; i++) {
+        if (structure.arguments[i]) {
+          convertNullToUndefinedBySchema(
+            fullScenarioData.data.argumentsData[i],
+            structure.arguments[i],
+          );
+        }
+      }
+    }
 
     return {
       llmCalls: [{ name: scenario.name, id: llmCall.id }],
@@ -346,6 +372,47 @@ NEVER include "error" fields in responses. Skip them entirely.
 - If structure has \`{ data: {...}, error: {...} }\`, only fill \`data\`
 - Leave out any attribute named "error"—do not set to null, omit entirely
 
+## CRITICAL: Preserve Exact Structure
+Your response MUST mirror the EXACT nested structure provided in mockData Structure.
+- Do NOT reorganize, split, or create duplicate keys
+- The hierarchy of nested objects must match exactly what was provided
+- Only change the leaf VALUES (replacing type descriptions like "string" with actual data like "hello")
+- Arrays should have 3-5 items to provide realistic test data variety
+
+**Example** - if given this structure:
+\`\`\`json
+{
+  "createClient()": {
+    "from(\\"users\\")": {
+      "select(\\"*\\")": { "data": [{ "id": "string", "name": "string" }] },
+      "delete()": { "eq(\\"id\\", id)": {} }
+    }
+  }
+}
+\`\`\`
+
+Return ONE \`from("users")\` key containing BOTH \`select("*")\` and \`delete()\`:
+\`\`\`json
+{
+  "createClient()": {
+    "from(\\"users\\")": {
+      "select(\\"*\\")": { "data": [{ "id": "user-1", "name": "Alice" }, { "id": "user-2", "name": "Bob" }] },
+      "delete()": { "eq(\\"id\\", \\"user-1\\")": {} }
+    }
+  }
+}
+\`\`\`
+
+**WRONG** - do NOT create duplicate keys like this:
+\`\`\`json
+{
+  "createClient()": {
+    "from(\\"users\\")": { "select(\\"*\\")": { ... } },
+    "from(\\"users\\")": { "delete()": { ... } }
+  }
+}
+\`\`\`
+
 ## Special Markers
 
 ### Dynamic Dates (\`~~codeyam-code~~\`)

package/analyzer-template/packages/ai/src/lib/generateChangesEntityScenarios.ts

@@ -5,21 +5,19 @@ import generateChangesEntityScenariosGenerator from './promptGenerators/generate
 import {
   Analysis,
   Entity,
+  ExecutionFlow,
   ReadonlyAnalysisMap,
   Scenario,
 } from '~codeyam/types';
 import { ScenarioResult } from './generateEntityScenarios';
-import findMatchingAttribute from './findMatchingAttribute';
 import { awsLog } from '~codeyam/utils';
 import { sanitizePlaywrightInstructions } from './validatePlaywrightInstructions';
-import gatherRelevantDependentKeyAttributes from './gatherRelevantDependentKeyAttributes';
 import { AI, DEFAULT_LARGER_MODEL, parseJsonSafe } from '~codeyam/ai';
 
 interface GenerateChangesEntityScenariosArgs {
   entity: Entity;
-  keyAttributes: Analysis['metadata']['keyAttributes'];
-  addedKeyAttributes: Analysis['metadata']['keyAttributes'];
-  removedKeyAttributes: Analysis['metadata']['keyAttributes'];
+  executionFlows?: ExecutionFlow[];
+  previousExecutionFlows?: ExecutionFlow[];
   dependentAnalyses?: ReadonlyAnalysisMap;
   existingScenarios: Scenario[];
   commitDiff: string;
@@ -33,10 +31,8 @@ const DEFAULT_SCENARIO_NAME = 'Default Scenario';
 
 export default async function generateChangesEntityScenarios({
   entity,
-  keyAttributes,
-  addedKeyAttributes,
-  removedKeyAttributes,
-  dependentAnalyses,
+  executionFlows,
+  previousExecutionFlows,
   existingScenarios,
   commitDiff,
   analysis,
@@ -44,16 +40,22 @@ export default async function generateChangesEntityScenarios({
   model,
   changedDataStructureFields = [],
 }: GenerateChangesEntityScenariosArgs) {
-  const relevantDependentKeyAttributes = gatherRelevantDependentKeyAttributes(
-    entity,
-    analysis.metadata.mergedDataStructure,
-    dependentAnalyses,
+  // Determine added/removed flows
+  const currentFlowIds = new Set((executionFlows ?? []).map((f) => f.id));
+  const previousFlowIds = new Set(
+    (previousExecutionFlows ?? []).map((f) => f.id),
   );
+
+  const addedFlows =
+    executionFlows?.filter((f) => !previousFlowIds.has(f.id)) ?? [];
+  const removedFlows =
+    previousExecutionFlows?.filter((f) => !currentFlowIds.has(f.id)) ?? [];
+
+  // If no changes, return existing scenarios
   if (
-    (!addedKeyAttributes || addedKeyAttributes.length === 0) &&
-    (!removedKeyAttributes || removedKeyAttributes.length === 0) &&
-    !existingScenarios.some((s) => s.metadata?.playwrightInstructions) &&
-    Object.keys(relevantDependentKeyAttributes).length === 0
+    addedFlows.length === 0 &&
+    removedFlows.length === 0 &&
+    !existingScenarios.some((s) => s.metadata?.playwrightInstructions)
   ) {
     return { scenarios: existingScenarios };
   }
@@ -62,21 +64,20 @@ export default async function generateChangesEntityScenarios({
     `CodeYam: Generating ${error ? 'error ' : ''}changes entity scenarios`,
     {
       filePath: entity.filePath,
-      entityNam: entity.name,
-      analysis: analysis.id,
-      keyAttributes: keyAttributes.map((a) => a.externalPath),
-      addedKeyAttributes: addedKeyAttributes?.map((a) => a.externalPath),
-      removedKeyAttributes: removedKeyAttributes?.map((a) => a.externalPath),
+      entityName: entity.name,
+      analysisId: analysis.id,
+      currentFlowCount: executionFlows?.length ?? 0,
+      addedFlows: addedFlows.map((f) => f.id),
+      removedFlows: removedFlows.map((f) => f.id),
     },
   );
 
   const prompt = generateChangesEntityScenariosGenerator({
     entity,
     mergedDataStructure: analysis.metadata.mergedDataStructure,
-    keyAttributes,
-    addedKeyAttributes,
-    removedKeyAttributes,
-    dependentAnalyses,
+    executionFlows,
+    addedFlows,
+    removedFlows,
     existingScenarios,
     commitDiff,
     error,
@@ -93,7 +94,7 @@ export default async function generateChangesEntityScenarios({
     type: `generate${error ? 'Error' : ''}ChangesEntityScenarios`,
     systemMessage: error
       ? ERROR_SYSTEM_MESSAGE
-      : generateChangesSystemMessage(context, scenarioCount, false),
+      : generateChangesSystemMessage(context, scenarioCount),
     prompt,
     model: model ?? DEFAULT_LARGER_MODEL,
   });
@@ -111,32 +112,17 @@ export default async function generateChangesEntityScenarios({
           path: entity.file.path,
         },
       },
-      keyAttributes,
-      addedKeyAttributes,
-      removedKeyAttributes,
-      dependentAnalyses: Object.fromEntries(
-        Object.entries(dependentAnalyses ?? {}).map(([path, analyses]) => [
-          path,
-          Object.fromEntries(
-            Object.entries(analyses).map(([name, analysis]) => [
-              name,
-              {
-                metadata: {
-                  keyAttributes: analysis.metadata?.keyAttributes,
-                },
-              },
-            ]),
-          ),
-        ]),
-      ),
+      executionFlows,
+      addedFlows,
+      removedFlows,
       existingScenarios: existingScenarios.map((s) => ({
         id: s.id,
         name: s.name,
         description: s.description,
         metadata: {
-          error: s.metadata.error,
-          keyAttributeInstructions: s.metadata.keyAttributeInstructions,
-          playwrightInstructions: s.metadata.playwrightInstructions,
+          error: s.metadata?.error,
+          coveredFlows: s.metadata?.coveredFlows,
+          playwrightInstructions: s.metadata?.playwrightInstructions,
         },
       })),
       commitDiff,
@@ -146,7 +132,7 @@ export default async function generateChangesEntityScenarios({
           isolatedDataStructure: {
             equivalentSignatureVariables:
               entity.metadata.isolatedDataStructure
-                .equivalentSignatureVariables,
+                ?.equivalentSignatureVariables,
           },
         },
       },
@@ -191,7 +177,7 @@ export default async function generateChangesEntityScenarios({
     );
 
     const revisedFromPreviousVersion = previousVersion
-      ? !!scenarioInfo.keyAttributeInstructions
+      ? !!scenarioInfo.coveredFlows
       : undefined;
     const useScenarioData = !previousVersion || revisedFromPreviousVersion;
 
@@ -204,58 +190,42 @@ export default async function generateChangesEntityScenarios({
         : previousVersion.description,
       metadata: {
         testName:
-          useScenarioData || !previousVersion.metadata.testName
+          useScenarioData || !previousVersion?.metadata?.testName
             ? scenarioInfo.testName
             : previousVersion.metadata.testName,
         error,
         revisedFromPreviousVersion,
         codeSnippet: useScenarioData
           ? scenarioInfo.codeSnippet
-          : previousVersion.metadata.codeSnippet,
-        fix: useScenarioData ? scenarioInfo.fix : previousVersion.metadata.fix,
+          : previousVersion?.metadata?.codeSnippet,
+        fix: useScenarioData
+          ? scenarioInfo.fix
+          : previousVersion?.metadata?.fix,
         fixSnippet: useScenarioData
           ? scenarioInfo.fixSnippet
-          : previousVersion.metadata.fixSnippet,
+          : previousVersion?.metadata?.fixSnippet,
         likelihood: useScenarioData
           ? scenarioInfo.likelihood
-          : previousVersion.metadata.likelihood,
+          : previousVersion?.metadata?.likelihood,
         likelihoodDescription: useScenarioData
           ? scenarioInfo.likelihoodDescription
-          : previousVersion.metadata.likelihoodDescription,
+          : previousVersion?.metadata?.likelihoodDescription,
         severity: useScenarioData
           ? scenarioInfo.severity
-          : previousVersion.metadata.severity,
+          : previousVersion?.metadata?.severity,
         severityDescription: useScenarioData
           ? scenarioInfo.severityDescription
-          : previousVersion.metadata.severityDescription,
-        dataMapping: useScenarioData
-          ? scenarioInfo.dataMapping
-          : previousVersion.metadata.dataMapping,
-        keyAttributeInstructions: useScenarioData
-          ? Object.keys(scenarioInfo.keyAttributeInstructions).reduce(
-              (acc: any, key) => {
-                const attributesList = keyAttributes.map(
-                  (attr) => attr.internalPath,
-                );
-                const matchingKey = findMatchingAttribute(
-                  key,
-                  attributesList,
-                  entity.metadata.isolatedDataStructure
-                    .equivalentSignatureVariables,
-                );
-                acc[matchingKey] = scenarioInfo.keyAttributeInstructions[key];
-                return acc;
-              },
-              {},
-            )
-          : previousVersion.metadata.keyAttributeInstructions,
+          : previousVersion?.metadata?.severityDescription,
+        coveredFlows: useScenarioData
+          ? scenarioInfo.coveredFlows || []
+          : previousVersion?.metadata?.coveredFlows || [],
         playwrightInstructions: useScenarioData
           ? sanitizePlaywrightInstructions(
               scenarioInfo.playwrightInstructions || [],
               entity.name,
               scenarioInfo.name,
             )
-          : previousVersion.metadata.playwrightInstructions,
+          : previousVersion?.metadata?.playwrightInstructions,
       },
       previousVersionId: previousVersion?.id,
     });
@@ -271,7 +241,6 @@ export default async function generateChangesEntityScenarios({
 function generateChangesSystemMessage(
   context?: string,
   scenarioCount?: number,
-  _isError?: boolean,
 ): string {
   const contextSection = context
     ? `## Business Context
@@ -288,27 +257,35 @@ Focus scenario updates on demonstrating this change.${scenarioCount ? ` Generate
 
   return `You are updating data scenarios after a code change. Determine which existing scenarios need updating, which should be removed, and if new ones are needed.
 
-${contextSection}## Update Rules
+${contextSection}## Your Goal: Maintain Full Execution Flow Coverage
+
+You are given:
+- \`executionFlows\`: All current distinct outcomes/behaviors this component can produce
+- \`addedFlows\`: New flows that didn't exist before
+- \`removedFlows\`: Flows that no longer exist
+- \`existingScenarios\`: Current scenarios with their coveredFlows
+
+**Your job**: Update scenarios to maintain coverage of ALL current execution flows.
 
-### Removed Key Attributes
-- Update scenarios using removed attributes—replace with new attributes or remove if no longer relevant
-- CRITICAL: "${DEFAULT_SCENARIO_NAME}" must NEVER contain removed attributes—update or replace them
-- Try to maintain 3+ scenarios; create replacements if removing too many
+## Update Rules
 
-### Added Key Attributes
-- Create or update scenarios to demonstrate new behavior
+### Removed Flows
+- If an existing scenario only covers removed flows, remove the scenario
+- If a scenario covers both removed and current flows, update its coveredFlows to remove the deleted ones
 
-### Playwright Instructions
-- Validate existing instructions still work after code changes
-- Update any that reference changed UI elements
+### Added Flows
+- Create new scenarios or update existing ones to cover new flows
+- Follow the same rules as initial scenario generation:
+  - Blocking flows (blocksOtherFlows: true) get their own dedicated scenarios
+  - Non-blocking flows can be combined with compatible existing scenarios
 
 ### Unchanged Scenarios
-- Return name only (no other fields needed)
+- Return name only (no other fields needed) if the scenario's covered flows are all still valid
 
 ### Default Scenario
 - First scenario must be "${DEFAULT_SCENARIO_NAME}" (exact name)
 - NEVER include playwrightInstructions—captures initial state only
-- Must include all current key attributes with valid values
+- Should cover as many non-blocking, non-error flows as possible
 
 ## Response Format
 
@@ -326,27 +303,31 @@ Otherwise:
   "overview": "Added empty state scenario for new display behavior.",
   "dataScenarios": [
     { "name": "${DEFAULT_SCENARIO_NAME}" },
-    { "name": "Many notifications" },
+    { "name": "Settings Modal" },
     {
-      "name": "No notifications",
-      "testName": "it(\\"shows empty message when no notifications\\")",
-      "description": "User has no notifications.",
-      "keyAttributeInstructions": {
-        "notifications[]": { "localVariable": "notifications[]", "instruction": "Empty array" }
-      }
+      "name": "Empty State",
+      "testName": "it(\\"shows empty message when no items\\")",
+      "description": "User has no items to display.",
+      "coveredFlows": ["empty-list"]
     }
   ]
 }
 \`\`\`
 
+## Field Descriptions
+
+- **name**: Human-readable scenario name (letters, numbers, spaces only)
+- **testName**: Jest-style test description (only for new/updated scenarios)
+- **description**: What the user sees (only for new/updated scenarios)
+- **coveredFlows**: Array of execution flow IDs this scenario covers (only for new/updated scenarios)
+- **playwrightInstructions**: User interactions needed (only for new/updated scenarios that require interaction)
+
 ## Rules
 - Scenario names: letters, numbers, spaces only
 - testName: Jest style with specific expectations
-- Use values from \`validValueOptions\` in key attributes list
-- Include dependency key attributes if dependencies involved
-- React elements: simple only (\`<div>\`, \`<span>\`)
-- No external library references—plain JS/TS values
-- Relative dates OK: "today", "tomorrow", "last month"
+- All values must be valid JSON
+- EVERY execution flow must be covered by at least one scenario
+- Blocking flows get dedicated scenarios
 `;
 }
 
@@ -381,27 +362,23 @@ Be conservative with high scores.
 - No external library references—mock with plain JS
 - Each scenario causes a DISTINCT error type
 
-Note: In JS/TS, any non-zero value satisfies a boolean check—this won't error.
-
 ## Response Format
 \`\`\`json
 {
   "errorDataScenarios": [
     { "name": "Existing valid error" },
     {
-      "dataMapping": { "notifications array": "props.notifications" },
       "name": "Non-utf8 encoding error",
-      "codeSnippet": "const encoded = encodeURIComponent(message);",
+      "testName": "it(\\"throws on non-utf8 characters\\")",
       "description": "encodeURIComponent throws on non-utf8 characters",
+      "codeSnippet": "const encoded = encodeURIComponent(message);",
       "fix": "Wrap in try/catch",
       "fixSnippet": "try { encoded = encodeURIComponent(message); } catch (e) { ... }",
       "likelihood": 7,
       "likelihoodDescription": "International users may input non-utf8 characters",
       "severity": 2,
       "severityDescription": "Function throws uncaught error",
-      "keyAttributeInstructions": {
-        "notifications[]": "Array with message containing non-utf8 characters"
-      }
+      "coveredFlows": []
     }
   ]
 }