@codeyam/codeyam-cli 0.1.0-staging.d0ad4ae → 0.1.0-staging.d52b24f

package/analyzer-template/packages/ai/src/lib/generateEntityScenarios.ts

@@ -1,55 +1,45 @@
 import completionCall from './completionCall';
 import { saveLlmCall } from '~codeyam/aws/dynamodb';
 import generateEntityScenariosGenerator from './promptGenerators/generateEntityScenariosGenerator';
+import { getFlowsForScenarioGeneration } from './generateExecutionFlows';
 
 import {
   Analysis,
   Entity,
+  ExecutionFlow,
   PlaywrightInstruction,
-  ReadonlyAnalysisMap,
   Scenario,
-  ValueOptionRef,
 } from '~codeyam/types';
-// import findMatchingAttribute from './findMatchingAttribute';
 import { awsLog } from '~codeyam/utils';
 import { sanitizePlaywrightInstructions } from './validatePlaywrightInstructions';
 import { AI, DEFAULT_LARGER_MODEL, parseJsonSafe } from '~codeyam/ai';
 
+/**
+ * Maximum number of execution flows to send to the LLM for scenario generation.
+ * When there are more flows than this threshold, we filter to the top N
+ * highest priority flows to avoid overwhelming the LLM and wasting tokens.
+ */
+const MAX_FLOWS_FOR_SCENARIO_GENERATION = 30;
+
 export interface ScenarioResult {
   name: string;
   testName: string;
   description: string;
-  /**
-   * Key attribute instructions with structured value references.
-   * Format: { dataStructurePath: { localVariable, instruction, valueOptionRef? } }
-   */
-  keyAttributeInstructions: {
-    [key: string]:
-      | string
-      | {
-          localVariable: string;
-          instruction: string;
-          /** Reference to validValueOption index for tracking coverage */
-          valueOptionRef?: ValueOptionRef;
-          /** Reference to errorValueOption index for error scenarios */
-          errorOptionRef?: ValueOptionRef;
-        };
-  };
-  codeSnippet: string;
-  fix: string;
-  fixSnippet: string;
-  likelihood: number;
-  likelihoodDescription: string;
-  severity: number;
-  severityDescription: string;
-  dataMapping: { [key: string]: string };
+  /** IDs of execution flows this scenario covers */
+  coveredFlows: string[];
+  codeSnippet?: string;
+  fix?: string;
+  fixSnippet?: string;
+  likelihood?: number;
+  likelihoodDescription?: string;
+  severity?: number;
+  severityDescription?: string;
   playwrightInstructions?: PlaywrightInstruction[];
 }
 
 interface GenerateEntityScenariosArgs {
   entity: Pick<Entity, 'projectId' | 'filePath' | 'name' | 'code' | 'metadata'>;
-  keyAttributes: Analysis['metadata']['keyAttributes'];
-  dependentAnalyses?: ReadonlyAnalysisMap;
+  executionFlows?: ExecutionFlow[];
   analysis: Analysis;
   error?: boolean;
   model?: AI.Model;
@@ -59,7 +49,7 @@ const DEFAULT_SCENARIO_NAME = 'Default Scenario';
 
 export default async function generateEntityScenarios({
   entity,
-  keyAttributes,
+  executionFlows,
   analysis,
   error,
   model,
@@ -69,9 +59,50 @@ export default async function generateEntityScenarios({
     entityName: entity.name,
   });
 
+  // If no execution flows are available (and not generating error scenarios),
+  // return only the Default Scenario without calling LLM
+  if (!error && (!executionFlows || executionFlows.length === 0)) {
+    console.log(
+      `CodeYam: No execution flows available for ${entity.filePath} ${entity.name}, creating Default Scenario only`,
+    );
+    const defaultScenario: Scenario = {
+      projectId: entity.projectId,
+      analysisId: analysis.id,
+      name: DEFAULT_SCENARIO_NAME,
+      description: 'Default component rendering with standard data',
+      metadata: {
+        testName: 'it("renders the component with default data")',
+        error: false,
+        coveredFlows: [],
+        playwrightInstructions: [],
+      },
+    };
+    return {
+      scenarios: [defaultScenario],
+      llmCall: null,
+    };
+  }
+
+  // Filter execution flows to top N highest priority when there are many flows
+  // This prevents overwhelming the LLM and wasting tokens on low-priority flows
+  let filteredFlows = executionFlows;
+  if (
+    executionFlows &&
+    executionFlows.length > MAX_FLOWS_FOR_SCENARIO_GENERATION
+  ) {
+    filteredFlows = getFlowsForScenarioGeneration(executionFlows, {
+      maxFlows: MAX_FLOWS_FOR_SCENARIO_GENERATION,
+      ensureGroupDiversity: true,
+    }) as ExecutionFlow[];
+
+    console.log(
+      `CodeYam: Filtered ${executionFlows.length} flows to ${filteredFlows.length} for scenario generation`,
+    );
+  }
+
   const prompt = generateEntityScenariosGenerator({
     entity,
-    keyAttributes,
+    executionFlows: filteredFlows,
     error,
   });
 
@@ -100,14 +131,14 @@ export default async function generateEntityScenarios({
         filePath: entity.filePath,
         code: entity.code,
       },
-      keyAttributes,
+      executionFlows,
       analysis: {
         id: analysis.id,
         metadata: {
           isolatedDataStructure: {
             equivalentSignatureVariables:
               entity.metadata.isolatedDataStructure
-                .equivalentSignatureVariables,
+                ?.equivalentSignatureVariables,
           },
         },
       },
@@ -145,11 +176,15 @@ export default async function generateEntityScenarios({
     return { scenarios: [], llmCall };
   }
 
-  const scenarios = (
-    result[
-      `${error ? 'errorD' : 'd'}ataScenarios`
-    ] as unknown as ScenarioResult[]
-  ).map(
+  const dataScenarios = result[`${error ? 'errorD' : 'd'}ataScenarios`];
+
+  // Gracefully handle missing dataScenarios (e.g., when LLM returns unexpected format
+  // or when test fixtures don't include the expected key)
+  if (!Array.isArray(dataScenarios)) {
+    return { scenarios: [], llmCall };
+  }
+
+  const scenarios = (dataScenarios as unknown as ScenarioResult[]).map(
     (scenarioInfo) =>
       ({
         projectId: entity.projectId,
@@ -166,23 +201,7 @@ export default async function generateEntityScenarios({
           likelihoodDescription: scenarioInfo.likelihoodDescription,
           severity: scenarioInfo.severity,
           severityDescription: scenarioInfo.severityDescription,
-          dataMapping: scenarioInfo.dataMapping,
-          keyAttributeInstructions: scenarioInfo.keyAttributeInstructions,
-          // Object.keys(
-          //   scenarioInfo.keyAttributeInstructions,
-          // ).reduce((acc, key) => {
-          //   const attributesList = keyAttributes.map(
-          //     (attr) => attr.internalPath,
-          //   );
-          //   const matchingKey = findMatchingAttribute(
-          //     key,
-          //     attributesList,
-          //     entity.metadata.isolatedDataStructure
-          //       .equivalentSignatureVariables,
-          //   );
-          //   acc[matchingKey] = scenarioInfo.keyAttributeInstructions[key];
-          //   return acc;
-          // }, {}),
+          coveredFlows: scenarioInfo.coveredFlows || [],
           playwrightInstructions: sanitizePlaywrightInstructions(
             scenarioInfo.playwrightInstructions || [],
             entity.name,
@@ -204,7 +223,7 @@ function generateSystemMessage(
 ): string {
   const scenarioCountText = scenarioCount
     ? `Generate exactly ${scenarioCount} scenario${scenarioCount > 1 ? 's' : ''}.`
-    : 'Generate 1-4 scenarios.';
+    : 'Generate 3-6 scenarios MAXIMUM. Do NOT exceed 6 scenarios.';
 
   const contextSection = context
     ? `## Business Context
@@ -215,92 +234,136 @@ Focus scenarios on demonstrating this specific behavior change.
 `
     : '';
 
-  return `You are a test scenario designer. Generate data scenarios that demonstrate a function's behavior.
+  return `You are a test scenario designer. Generate data scenarios that demonstrate a function's behavior by covering its execution flows.
 
 ${contextSection}${scenarioCountText}
 
-## Coverage Goals (IMPORTANT)
-Your primary goal is to MAXIMIZE coverage of key attributes across all scenarios:
+## Your Primary Goal: Cover ALL Execution Flows EFFICIENTLY
 
-1. **Use ALL validValueOptions**: Each key attribute has indexed \`validValueOptions\`. Across your scenarios, try to use EVERY option at least once. If an attribute has 3 options (indices 0, 1, 2), create scenarios that collectively cover all 3.
+You are given \`executionFlows\` - all the distinct outcomes/behaviors this component can produce. Each flow has:
+- \`id\`: Unique identifier
+- \`name\`: Human-readable name (e.g., "Modal visible", "Loading state")
+- \`description\`: What the user sees or what behavior occurs
+- \`requiredValues\`: Attribute values needed to produce this outcome
+- \`impact\`: Visual/behavioral importance ("high", "medium", "low")
+- \`blocksOtherFlows\`: If true, this outcome hides other content (modals, overlays)
+- \`isError\`: If true, this represents an error/exception scenario
+- \`exclusiveGroup\`: If set, flows with the same group are MUTUALLY EXCLUSIVE (only one can be active)
 
-2. **Prioritize attributes with dependencies**: Key attributes with a \`dependencies\` array are HIGH PRIORITY - they control conditionally-rendered content. Create scenarios that satisfy these dependencies to reveal gated UI/behavior.
+**Your job**: Create the MINIMUM number of scenarios that cover ALL execution flows.
 
-3. **Cover conditional branches**: Use \`codeUsages\` to identify all conditional paths. If code shows \`status === 'active'\` and \`status === 'pending'\`, ensure scenarios cover BOTH values.
+## CRITICAL: Understanding Exclusive Groups
 
-4. **Vary values strategically**: Don't repeat the same validValueOption across scenarios unless necessary. Use different indices to maximize coverage.
+Flows with the same \`exclusiveGroup\` are MUTUALLY EXCLUSIVE - only ONE can be active at a time.
 
-## Default Scenario Rules
-The FIRST scenario MUST be named exactly "${DEFAULT_SCENARIO_NAME}" (case-sensitive).
-- Fill ALL key attributes using values from their \`validValueOptions\` list
-- If "children" is an argument, populate it (otherwise output may be blank)
-- Ignore error-related attributes (set to null)
-- NEVER include \`playwrightInstructions\` - we need a screenshot BEFORE any user interaction
-- Do NOT add playwrightInstructions even if the component has forms/buttons - default shows initial state only
-- **For attributes WITH dependencies**: The Default scenario should satisfy ALL dependencies so the attribute's effect is visible
+**Example exclusive groups:**
+- \`"status-badge"\`: loading, success, error, pending - only ONE badge shows
+- \`"active-tab"\`: users, settings, profile - only ONE tab is visible
+- \`"modal"\`: settings-modal, confirm-modal - only ONE modal at a time
 
-## Other Scenarios
-- Each produces DISTINCT output (visual: different screenshot; backend: different return/side-effect)
-- **Use DIFFERENT validValueOptions** than the Default scenario - this is how we achieve coverage
-- Only specify attributes that DIFFER from default
-- Target uncovered validValueOptions: check which indices you haven't used yet
-- May include \`playwrightInstructions\` for user interactions (clicks, form fills)
+5. **Maximize visual diversity**: For attributes that control what content is displayed (rather than just styling), vary values across scenarios so screenshots show meaningfully different views:
+   - Don't use the same display-controlling value in every scenario
+   - Ensure at least one scenario shows the primary content view
+   - Other scenarios can demonstrate alternative display states
 
-## Using codeUsages for Branch Coverage
-Key attributes may include a \`codeUsages\` array showing exactly where they're used in conditionals:
-\`\`\`json
-{
-  "codeUsages": [
-    { "line": 45, "code": "if (status === 'active')", "context": "Used in if statement (comparison check)" }
-  ]
-}
-\`\`\`
-Use this to ensure scenarios cover all conditional branches. If you see \`status === 'active'\`, create scenarios for both active AND non-active states.
+**Key insight**: You don't need a separate scenario for each flow in an exclusive group!
+- Pick the PRIMARY/DEFAULT option for the Default Scenario
+- Only create additional scenarios for IMPORTANT alternative states (e.g., error states, loading states)
+- Skip scenarios for minor variations within a group
+
+## Scenario Strategy
+
+### 1. Default Scenario (REQUIRED - must be first, named exactly "${DEFAULT_SCENARIO_NAME}")
+- For each exclusive group, pick the PRIMARY/HAPPY-PATH option:
+  - Status: "success" or "ready" (not loading, not error)
+  - Tabs: The main/first tab
+  - View modes: The default view
+- Combine as many NON-BLOCKING, NON-ERROR, NON-EXCLUSIVE-CONFLICT flows as possible
+- AVOID flows with \`blocksOtherFlows: true\`
+- NEVER include \`playwrightInstructions\`
+
+### 2. Blocking Flow Scenarios (ONE per blocking flow)
+Each flow with \`blocksOtherFlows: true\` gets its OWN scenario.
+- Modals, overlays, drawers hide other content
+- Don't combine blocking flows
+
+### 3. Important Alternative States (SELECTIVE)
+Only create scenarios for IMPORTANT alternatives within exclusive groups:
+- ✅ Loading state (if exists) - important UX state
+- ✅ Error state (if exists) - important UX state
+- ✅ Empty state (if exists) - important UX state
+- ❌ Each individual tab - NOT needed (tabs are just navigation)
+- ❌ Each status badge variant - just pick the important ones
 
-## Handling Dependencies (Compound Conditionals) - HIGH PRIORITY
-Key attributes with \`dependencies\` are the MOST IMPORTANT to cover because they reveal conditionally-gated content that only appears when multiple conditions are satisfied together.
+### 4. CRITICAL: Skip Low-Value Scenarios
+Do NOT create separate scenarios for:
+- **Different tabs** - Tabs are just navigation. The Default Scenario covers the main tab. SKIP others.
+- **Status badge variants** - Only need: Default (success/ready) + Loading + Error. SKIP queued, pending, etc.
+- **Banner variants** - Pick ONE important banner if any. SKIP creating a scenario for each banner type.
+- **Minor variations** - If two flows show essentially the same UI with small differences, combine them or skip one.
 
-**Why this matters**: If you see \`items.length > 0 && !isLoading && !hasError\`, the content only renders when ALL THREE conditions are true. Missing even one dependency means the UI won't show.
+**Remember**: Including a flow in the Default Scenario COUNTS as covering it. You don't need a dedicated scenario for each flow.
 
-**Strategy for dependent attributes**:
-1. **Default scenario**: MUST satisfy ALL dependencies of the highest-impact key attributes
-2. **Other scenarios**: Can vary the dependent values to show alternative states (e.g., what happens when isLoading=true)
+## Combining Flows
+
+Flows can combine if:
+1. They have DIFFERENT \`exclusiveGroup\` values (or no exclusiveGroup)
+2. Their \`requiredValues\` don't conflict on the same attribute
+
+Flows CANNOT combine if:
+1. They have the SAME \`exclusiveGroup\` - they're mutually exclusive
+2. Their \`requiredValues\` conflict (same attribute, different required value)
+
+When combining flows, the scenario's mock data must satisfy ALL the requiredValues from ALL covered flows.
+
+## Output Format
 
-When a key attribute has dependencies like:
 \`\`\`json
 {
-  "dependencies": [
-    { "path": "signature[0].isLoading", "requiredValue": false },
-    { "path": "signature[0].hasError", "requiredValue": false }
+  "dataScenarios": [
+    {
+      "name": "${DEFAULT_SCENARIO_NAME}",
+      "testName": "it(\\"displays primary content with all features visible\\")",
+      "description": "Main content view: success status, main tab selected, user logged in",
+      "coveredFlows": ["status-success", "tab-main", "user-authenticated", "content-loaded"]
+    },
+    {
+      "name": "Loading State",
+      "testName": "it(\\"shows loading spinner while fetching data\\")",
+      "description": "Data is being loaded, loading badge visible",
+      "coveredFlows": ["status-loading", "tab-main"]
+    },
+    {
+      "name": "Empty State",
+      "testName": "it(\\"shows empty message when no items\\")",
+      "description": "No items to display, empty state UI shown",
+      "coveredFlows": ["status-success", "tab-main", "empty-list"]
+    },
+    {
+      "name": "Settings Modal",
+      "testName": "it(\\"displays settings modal overlay\\")",
+      "description": "Settings modal is open and visible",
+      "coveredFlows": ["modal-settings"]
+    }
   ]
 }
 \`\`\`
-The Default scenario MUST set BOTH dependent attributes to their required values so the key attribute's effect is visible.
 
-**Creating scenarios for dependent pathways**:
-- Create at least one scenario where ALL dependencies are satisfied (usually Default)
-- Consider creating additional scenarios that intentionally break dependencies to show the "hidden" state
+**Note:** This example has ~20 flows but only 4 scenarios because:
+- Exclusive groups (status, tabs) only need ONE representative in Default Scenario
+- We don't create scenarios for every tab - they're just navigation
+- Only important states (loading, empty, modal) get dedicated scenarios
 
-## keyAttributeInstructions Format
-Use \`dataStructurePath\` as key. Include variable name, instruction, AND a valueOptionRef that references which validValueOption you're using:
-\`\`\`json
-{
-  "useDiffModal().diffView": {
-    "localVariable": "diffView",
-    "instruction": "Set to 'split' mode",
-    "valueOptionRef": { "index": 0, "refType": "exact" }
-  }
-}
-\`\`\`
+## Field Descriptions
 
-### valueOptionRef Types
-- \`"exact"\`: Uses the validValueOption at this index exactly
-- \`"derived"\`: Based on this validValueOption but modified (include \`derivation\` field explaining how)
-- \`"custom"\`: Value not from validValueOptions list (use sparingly)
+- **name**: Human-readable scenario name (letters, numbers, spaces only - no special characters)
+- **testName**: Jest-style test description, e.g. \`it("displays error when email invalid")\`
+- **description**: What the user sees or what behavior occurs in this scenario
+- **coveredFlows**: Array of execution flow IDs this scenario covers (from the executionFlows list)
+- **playwrightInstructions**: (optional) User interactions needed to reach this state
 
-The \`index\` refers to the validValueOptions array position (0 = first option).
+## playwrightInstructions Format (when needed)
 
-## playwrightInstructions (optional, non-default only)
 \`\`\`json
 {
   "playwrightInstructions": [
@@ -311,84 +374,27 @@ The \`index\` refers to the validValueOptions array position (0 = first option).
 \`\`\`
 Note: selectorArgs values must be JSON strings, not JavaScript regex literals.
 
-Use when: data only appears after user action (button click, form submit), or key attribute isn't in data structure.
-
 ## Rules
-- Relative dates OK in instructions: "today", "tomorrow", "last month"
-- React elements: use simple elements (\`<div>\`, \`<span>\`) - no custom JSX
-- No external library references - plain JS/TS values only
-- Scenario names: letters, numbers, spaces only (no special characters)
-- testName: Jest style with specific expectations, e.g. \`it("displays error when email invalid")\`
-- All values must be valid JSON (strings, numbers, booleans, arrays, objects) - no JavaScript literals like regex
 
-## Response Format
-\`\`\`json
-{
-  "dataScenarios": [
-    {
-      "name": "${DEFAULT_SCENARIO_NAME}",
-      "testName": "it(\"displays items list with data\")",
-      "description": "Shows item list when data is loaded and no errors",
-      "keyAttributeInstructions": {
-        "useData().items[]": {
-          "localVariable": "items[]",
-          "instruction": "Array with 3 items",
-          "valueOptionRef": { "index": 0, "refType": "exact" }
-        },
-        "useData().isLoading": {
-          "localVariable": "isLoading",
-          "instruction": "false - satisfies dependency for items to show",
-          "valueOptionRef": { "index": 1, "refType": "exact" }
-        },
-        "useData().status": {
-          "localVariable": "status",
-          "instruction": "Set to 'success'",
-          "valueOptionRef": { "index": 0, "refType": "exact" }
-        }
-      }
-    },
-    {
-      "name": "Loading state",
-      "testName": "it(\"shows spinner when loading\")",
-      "description": "Data is loading - items hidden, spinner shown",
-      "keyAttributeInstructions": {
-        "useData().isLoading": {
-          "localVariable": "isLoading",
-          "instruction": "true - breaks dependency, items won't show",
-          "valueOptionRef": { "index": 0, "refType": "exact" }
-        },
-        "useData().status": {
-          "localVariable": "status",
-          "instruction": "Set to 'loading' for coverage of loading state",
-          "valueOptionRef": { "index": 1, "refType": "exact" }
-        }
-      }
-    },
-    {
-      "name": "Empty state",
-      "testName": "it(\"shows empty message when no items\")",
-      "description": "No items to display",
-      "keyAttributeInstructions": {
-        "useData().items[]": {
-          "localVariable": "items[]",
-          "instruction": "Empty array [] to show empty state",
-          "valueOptionRef": { "index": 1, "refType": "exact" }
-        }
-      }
-    }
-  ]
-}
-\`\`\`
-Note how the example:
-- Default satisfies ALL dependencies (isLoading=false) so items content shows
-- "Loading state" uses a different valueOptionRef for status to cover that code path
-- "Empty state" covers the empty array case to maximize coverage
-- Instructions describe the VALUE, not the index (e.g., "Empty array []" not "uses index 1")
+1. **HARD LIMIT: 3-6 scenarios only** - If you generate more than 6 scenarios, you're doing it wrong. Aim for the minimum that covers important states.
+2. **${DEFAULT_SCENARIO_NAME}** must be first and named exactly that (case-sensitive)
+3. **Exclusive groups**: Pick ONE flow per group for Default Scenario. DON'T create scenarios for each option.
+4. **Blocking flows** (blocksOtherFlows: true) get dedicated scenarios
+5. **Only 2-3 additional scenarios**: Loading State, Empty State, maybe one Error or Modal
+6. **DO NOT cover every flow** - Many flows are fine being represented only in the Default Scenario
+7. Scenario names: letters, numbers, spaces only (no special characters)
+8. All values must be valid JSON
+
+**Example for a component with 27 flows:**
+- Default Scenario (covers 10+ flows including: success status, main tab, content loaded, etc.)
+- Loading State (covers loading flow)
+- Empty State (covers empty flow)
+- Modal Open (if there's an important modal)
+- Error State (if there's a critical error state)
+= 4-5 scenarios total, NOT 27
 `;
 }
 
-const SYSTEM_MESSAGE = generateSystemMessage();
-
 const ERROR_SYSTEM_MESSAGE = `You are a security/reliability analyst. Identify data inputs that will cause UNCAUGHT ERRORS in this function.
 
 ## What Qualifies as an Error Scenario
@@ -423,19 +429,17 @@ Note: In JS/TS, any non-zero value satisfies a boolean check—this won't error.
 {
   "errorDataScenarios": [
     {
-      "dataMapping": { "notifications array": "arguments - notifications" },
       "name": "Non-utf8 message encoding",
-      "codeSnippet": "const encoded = encodeURIComponent(message);",
+      "testName": "it(\\"throws on non-utf8 characters\\")",
       "description": "encodeURIComponent throws on non-utf8 characters",
+      "codeSnippet": "const encoded = encodeURIComponent(message);",
       "fix": "Wrap in try/catch",
       "fixSnippet": "try {\\n  encoded = encodeURIComponent(message);\\n} catch (e) {\\n  // handle\\n}",
       "likelihood": 7,
       "likelihoodDescription": "International users may input non-utf8 characters",
       "severity": 2,
       "severityDescription": "Function throws uncaught error",
-      "keyAttributeInstructions": {
-        "notifications[]": "Array with message containing non-utf8 characters"
-      }
+      "coveredFlows": []
     }
   ]
 }