@codeyam/codeyam-cli 0.1.0-staging.28f73cf → 0.1.0-staging.323686

package/analyzer-template/packages/ai/src/lib/generateEntityScenarios.ts

@@ -1,48 +1,45 @@
 import completionCall from './completionCall';
 import { saveLlmCall } from '~codeyam/aws/dynamodb';
 import generateEntityScenariosGenerator from './promptGenerators/generateEntityScenariosGenerator';
+import { getFlowsForScenarioGeneration } from './generateExecutionFlows';
 
 import {
   Analysis,
   Entity,
+  ExecutionFlow,
   PlaywrightInstruction,
-  ReadonlyAnalysisMap,
   Scenario,
 } from '~codeyam/types';
-// import findMatchingAttribute from './findMatchingAttribute';
 import { awsLog } from '~codeyam/utils';
 import { sanitizePlaywrightInstructions } from './validatePlaywrightInstructions';
 import { AI, DEFAULT_LARGER_MODEL, parseJsonSafe } from '~codeyam/ai';
 
+/**
+ * Maximum number of execution flows to send to the LLM for scenario generation.
+ * When there are more flows than this threshold, we filter to the top N
+ * highest priority flows to avoid overwhelming the LLM and wasting tokens.
+ */
+const MAX_FLOWS_FOR_SCENARIO_GENERATION = 30;
+
 export interface ScenarioResult {
   name: string;
   testName: string;
   description: string;
-  // New format: { dataStructurePath: { localVariable, instruction } }
-  // Old format (deprecated): { path: instruction_string }
-  keyAttributeInstructions: {
-    [key: string]:
-      | string
-      | {
-          localVariable: string;
-          instruction: string;
-        };
-  };
-  codeSnippet: string;
-  fix: string;
-  fixSnippet: string;
-  likelihood: number;
-  likelihoodDescription: string;
-  severity: number;
-  severityDescription: string;
-  dataMapping: { [key: string]: string };
+  /** IDs of execution flows this scenario covers */
+  coveredFlows: string[];
+  codeSnippet?: string;
+  fix?: string;
+  fixSnippet?: string;
+  likelihood?: number;
+  likelihoodDescription?: string;
+  severity?: number;
+  severityDescription?: string;
   playwrightInstructions?: PlaywrightInstruction[];
 }
 
 interface GenerateEntityScenariosArgs {
   entity: Pick<Entity, 'projectId' | 'filePath' | 'name' | 'code' | 'metadata'>;
-  keyAttributes: Analysis['metadata']['keyAttributes'];
-  dependentAnalyses?: ReadonlyAnalysisMap;
+  executionFlows?: ExecutionFlow[];
   analysis: Analysis;
   error?: boolean;
   model?: AI.Model;
@@ -52,7 +49,7 @@ const DEFAULT_SCENARIO_NAME = 'Default Scenario';
 
 export default async function generateEntityScenarios({
   entity,
-  keyAttributes,
+  executionFlows,
   analysis,
   error,
   model,
@@ -62,9 +59,50 @@ export default async function generateEntityScenarios({
     entityName: entity.name,
   });
 
+  // If no execution flows are available (and not generating error scenarios),
+  // return only the Default Scenario without calling LLM
+  if (!error && (!executionFlows || executionFlows.length === 0)) {
+    console.log(
+      `CodeYam: No execution flows available for ${entity.filePath} ${entity.name}, creating Default Scenario only`,
+    );
+    const defaultScenario: Scenario = {
+      projectId: entity.projectId,
+      analysisId: analysis.id,
+      name: DEFAULT_SCENARIO_NAME,
+      description: 'Default component rendering with standard data',
+      metadata: {
+        testName: 'it("renders the component with default data")',
+        error: false,
+        coveredFlows: [],
+        playwrightInstructions: [],
+      },
+    };
+    return {
+      scenarios: [defaultScenario],
+      llmCall: null,
+    };
+  }
+
+  // Filter execution flows to top N highest priority when there are many flows
+  // This prevents overwhelming the LLM and wasting tokens on low-priority flows
+  let filteredFlows = executionFlows;
+  if (
+    executionFlows &&
+    executionFlows.length > MAX_FLOWS_FOR_SCENARIO_GENERATION
+  ) {
+    filteredFlows = getFlowsForScenarioGeneration(executionFlows, {
+      maxFlows: MAX_FLOWS_FOR_SCENARIO_GENERATION,
+      ensureGroupDiversity: true,
+    }) as ExecutionFlow[];
+
+    console.log(
+      `CodeYam: Filtered ${executionFlows.length} flows to ${filteredFlows.length} for scenario generation`,
+    );
+  }
+
   const prompt = generateEntityScenariosGenerator({
     entity,
-    keyAttributes,
+    executionFlows: filteredFlows,
     error,
   });
 
@@ -93,14 +131,14 @@ export default async function generateEntityScenarios({
         filePath: entity.filePath,
         code: entity.code,
       },
-      keyAttributes,
+      executionFlows,
       analysis: {
         id: analysis.id,
         metadata: {
           isolatedDataStructure: {
             equivalentSignatureVariables:
               entity.metadata.isolatedDataStructure
-                .equivalentSignatureVariables,
+                ?.equivalentSignatureVariables,
           },
         },
       },
@@ -159,23 +197,7 @@ export default async function generateEntityScenarios({
           likelihoodDescription: scenarioInfo.likelihoodDescription,
           severity: scenarioInfo.severity,
           severityDescription: scenarioInfo.severityDescription,
-          dataMapping: scenarioInfo.dataMapping,
-          keyAttributeInstructions: scenarioInfo.keyAttributeInstructions,
-          // Object.keys(
-          //   scenarioInfo.keyAttributeInstructions,
-          // ).reduce((acc, key) => {
-          //   const attributesList = keyAttributes.map(
-          //     (attr) => attr.internalPath,
-          //   );
-          //   const matchingKey = findMatchingAttribute(
-          //     key,
-          //     attributesList,
-          //     entity.metadata.isolatedDataStructure
-          //       .equivalentSignatureVariables,
-          //   );
-          //   acc[matchingKey] = scenarioInfo.keyAttributeInstructions[key];
-          //   return acc;
-          // }, {}),
+          coveredFlows: scenarioInfo.coveredFlows || [],
           playwrightInstructions: sanitizePlaywrightInstructions(
             scenarioInfo.playwrightInstructions || [],
             entity.name,
@@ -197,7 +219,7 @@ function generateSystemMessage(
 ): string {
   const scenarioCountText = scenarioCount
     ? `Generate exactly ${scenarioCount} scenario${scenarioCount > 1 ? 's' : ''}.`
-    : 'Generate 1-4 scenarios.';
+    : 'Generate 3-6 scenarios MAXIMUM. Do NOT exceed 6 scenarios.';
 
   const contextSection = context
     ? `## Business Context
@@ -208,35 +230,136 @@ Focus scenarios on demonstrating this specific behavior change.
 `
     : '';
 
-  return `You are a test scenario designer. Generate data scenarios that demonstrate a function's behavior.
+  return `You are a test scenario designer. Generate data scenarios that demonstrate a function's behavior by covering its execution flows.
 
 ${contextSection}${scenarioCountText}
 
-## Default Scenario Rules
-The FIRST scenario MUST be named exactly "${DEFAULT_SCENARIO_NAME}" (case-sensitive).
-- Fill ALL key attributes using values from their \`validValueOptions\` list
-- If "children" is an argument, populate it (otherwise output may be blank)
-- Ignore error-related attributes (set to null)
-- NEVER include \`playwrightInstructions\` - we need a screenshot BEFORE any user interaction
-- Do NOT add playwrightInstructions even if the component has forms/buttons - default shows initial state only
-
-## Other Scenarios
-- Each produces DISTINCT output (visual: different screenshot; backend: different return/side-effect)
-- Only specify attributes that DIFFER from default
-- May include \`playwrightInstructions\` for user interactions (clicks, form fills)
-
-## keyAttributeInstructions Format
-Use \`dataStructurePath\` as key, include both variable name and instruction:
+## Your Primary Goal: Cover ALL Execution Flows EFFICIENTLY
+
+You are given \`executionFlows\` - all the distinct outcomes/behaviors this component can produce. Each flow has:
+- \`id\`: Unique identifier
+- \`name\`: Human-readable name (e.g., "Modal visible", "Loading state")
+- \`description\`: What the user sees or what behavior occurs
+- \`requiredValues\`: Attribute values needed to produce this outcome
+- \`impact\`: Visual/behavioral importance ("high", "medium", "low")
+- \`blocksOtherFlows\`: If true, this outcome hides other content (modals, overlays)
+- \`isError\`: If true, this represents an error/exception scenario
+- \`exclusiveGroup\`: If set, flows with the same group are MUTUALLY EXCLUSIVE (only one can be active)
+
+**Your job**: Create the MINIMUM number of scenarios that cover ALL execution flows.
+
+## CRITICAL: Understanding Exclusive Groups
+
+Flows with the same \`exclusiveGroup\` are MUTUALLY EXCLUSIVE - only ONE can be active at a time.
+
+**Example exclusive groups:**
+- \`"status-badge"\`: loading, success, error, pending - only ONE badge shows
+- \`"active-tab"\`: users, settings, profile - only ONE tab is visible
+- \`"modal"\`: settings-modal, confirm-modal - only ONE modal at a time
+
+5. **Maximize visual diversity**: For attributes that control what content is displayed (rather than just styling), vary values across scenarios so screenshots show meaningfully different views:
+   - Don't use the same display-controlling value in every scenario
+   - Ensure at least one scenario shows the primary content view
+   - Other scenarios can demonstrate alternative display states
+
+**Key insight**: You don't need a separate scenario for each flow in an exclusive group!
+- Pick the PRIMARY/DEFAULT option for the Default Scenario
+- Only create additional scenarios for IMPORTANT alternative states (e.g., error states, loading states)
+- Skip scenarios for minor variations within a group
+
+## Scenario Strategy
+
+### 1. Default Scenario (REQUIRED - must be first, named exactly "${DEFAULT_SCENARIO_NAME}")
+- For each exclusive group, pick the PRIMARY/HAPPY-PATH option:
+  - Status: "success" or "ready" (not loading, not error)
+  - Tabs: The main/first tab
+  - View modes: The default view
+- Combine as many NON-BLOCKING, NON-ERROR, NON-EXCLUSIVE-CONFLICT flows as possible
+- AVOID flows with \`blocksOtherFlows: true\`
+- NEVER include \`playwrightInstructions\`
+
+### 2. Blocking Flow Scenarios (ONE per blocking flow)
+Each flow with \`blocksOtherFlows: true\` gets its OWN scenario.
+- Modals, overlays, drawers hide other content
+- Don't combine blocking flows
+
+### 3. Important Alternative States (SELECTIVE)
+Only create scenarios for IMPORTANT alternatives within exclusive groups:
+- ✅ Loading state (if exists) - important UX state
+- ✅ Error state (if exists) - important UX state
+- ✅ Empty state (if exists) - important UX state
+- ❌ Each individual tab - NOT needed (tabs are just navigation)
+- ❌ Each status badge variant - just pick the important ones
+
+### 4. CRITICAL: Skip Low-Value Scenarios
+Do NOT create separate scenarios for:
+- **Different tabs** - Tabs are just navigation. The Default Scenario covers the main tab. SKIP others.
+- **Status badge variants** - Only need: Default (success/ready) + Loading + Error. SKIP queued, pending, etc.
+- **Banner variants** - Pick ONE important banner if any. SKIP creating a scenario for each banner type.
+- **Minor variations** - If two flows show essentially the same UI with small differences, combine them or skip one.
+
+**Remember**: Including a flow in the Default Scenario COUNTS as covering it. You don't need a dedicated scenario for each flow.
+
+## Combining Flows
+
+Flows can combine if:
+1. They have DIFFERENT \`exclusiveGroup\` values (or no exclusiveGroup)
+2. Their \`requiredValues\` don't conflict on the same attribute
+
+Flows CANNOT combine if:
+1. They have the SAME \`exclusiveGroup\` - they're mutually exclusive
+2. Their \`requiredValues\` conflict (same attribute, different required value)
+
+When combining flows, the scenario's mock data must satisfy ALL the requiredValues from ALL covered flows.
+
+## Output Format
+
 \`\`\`json
 {
-  "useDiffModal().diffView": {
-    "localVariable": "diffView",
-    "instruction": "Set to 'split' mode"
-  }
+  "dataScenarios": [
+    {
+      "name": "${DEFAULT_SCENARIO_NAME}",
+      "testName": "it(\\"displays primary content with all features visible\\")",
+      "description": "Main content view: success status, main tab selected, user logged in",
+      "coveredFlows": ["status-success", "tab-main", "user-authenticated", "content-loaded"]
+    },
+    {
+      "name": "Loading State",
+      "testName": "it(\\"shows loading spinner while fetching data\\")",
+      "description": "Data is being loaded, loading badge visible",
+      "coveredFlows": ["status-loading", "tab-main"]
+    },
+    {
+      "name": "Empty State",
+      "testName": "it(\\"shows empty message when no items\\")",
+      "description": "No items to display, empty state UI shown",
+      "coveredFlows": ["status-success", "tab-main", "empty-list"]
+    },
+    {
+      "name": "Settings Modal",
+      "testName": "it(\\"displays settings modal overlay\\")",
+      "description": "Settings modal is open and visible",
+      "coveredFlows": ["modal-settings"]
+    }
+  ]
 }
 \`\`\`
 
-## playwrightInstructions (optional, non-default only)
+**Note:** This example has ~20 flows but only 4 scenarios because:
+- Exclusive groups (status, tabs) only need ONE representative in Default Scenario
+- We don't create scenarios for every tab - they're just navigation
+- Only important states (loading, empty, modal) get dedicated scenarios
+
+## Field Descriptions
+
+- **name**: Human-readable scenario name (letters, numbers, spaces only - no special characters)
+- **testName**: Jest-style test description, e.g. \`it("displays error when email invalid")\`
+- **description**: What the user sees or what behavior occurs in this scenario
+- **coveredFlows**: Array of execution flow IDs this scenario covers (from the executionFlows list)
+- **playwrightInstructions**: (optional) User interactions needed to reach this state
+
+## playwrightInstructions Format (when needed)
+
 \`\`\`json
 {
   "playwrightInstructions": [
@@ -247,45 +370,27 @@ Use \`dataStructurePath\` as key, include both variable name and instruction:
 \`\`\`
 Note: selectorArgs values must be JSON strings, not JavaScript regex literals.
 
-Use when: data only appears after user action (button click, form submit), or key attribute isn't in data structure.
-
 ## Rules
-- Relative dates OK in instructions: "today", "tomorrow", "last month"
-- React elements: use simple elements (\`<div>\`, \`<span>\`) - no custom JSX
-- No external library references - plain JS/TS values only
-- Scenario names: letters, numbers, spaces only (no special characters)
-- testName: Jest style with specific expectations, e.g. \`it("displays error when email invalid")\`
-- All values must be valid JSON (strings, numbers, booleans, arrays, objects) - no JavaScript literals like regex
 
-## Response Format
-\`\`\`json
-{
-  "dataScenarios": [
-    {
-      "name": "${DEFAULT_SCENARIO_NAME}",
-      "testName": "it(\"displays notifications with unread indicator\")",
-      "description": "Default state with 3 unread and 2 read notifications",
-      "keyAttributeInstructions": {
-        "useNotifications().notifications[]": { "localVariable": "notifications[]", "instruction": "Create 5 notifications" },
-        "useNotifications().notifications[].read": { "localVariable": "notifications[].read", "instruction": "false for 3, true for 2" }
-      }
-    },
-    {
-      "name": "Empty state",
-      "testName": "it(\"shows placeholder when no notifications\")",
-      "description": "User has no notifications",
-      "keyAttributeInstructions": {
-        "useNotifications().notifications[]": { "localVariable": "notifications[]", "instruction": "Empty array" }
-      }
-    }
-  ]
-}
-\`\`\`
+1. **HARD LIMIT: 3-6 scenarios only** - If you generate more than 6 scenarios, you're doing it wrong. Aim for the minimum that covers important states.
+2. **${DEFAULT_SCENARIO_NAME}** must be first and named exactly that (case-sensitive)
+3. **Exclusive groups**: Pick ONE flow per group for Default Scenario. DON'T create scenarios for each option.
+4. **Blocking flows** (blocksOtherFlows: true) get dedicated scenarios
+5. **Only 2-3 additional scenarios**: Loading State, Empty State, maybe one Error or Modal
+6. **DO NOT cover every flow** - Many flows are fine being represented only in the Default Scenario
+7. Scenario names: letters, numbers, spaces only (no special characters)
+8. All values must be valid JSON
+
+**Example for a component with 27 flows:**
+- Default Scenario (covers 10+ flows including: success status, main tab, content loaded, etc.)
+- Loading State (covers loading flow)
+- Empty State (covers empty flow)
+- Modal Open (if there's an important modal)
+- Error State (if there's a critical error state)
+= 4-5 scenarios total, NOT 27
 `;
 }
 
-const SYSTEM_MESSAGE = generateSystemMessage();
-
 const ERROR_SYSTEM_MESSAGE = `You are a security/reliability analyst. Identify data inputs that will cause UNCAUGHT ERRORS in this function.
 
 ## What Qualifies as an Error Scenario
@@ -320,19 +425,17 @@ Note: In JS/TS, any non-zero value satisfies a boolean check—this won't error.
 {
   "errorDataScenarios": [
     {
-      "dataMapping": { "notifications array": "arguments - notifications" },
       "name": "Non-utf8 message encoding",
-      "codeSnippet": "const encoded = encodeURIComponent(message);",
+      "testName": "it(\\"throws on non-utf8 characters\\")",
       "description": "encodeURIComponent throws on non-utf8 characters",
+      "codeSnippet": "const encoded = encodeURIComponent(message);",
       "fix": "Wrap in try/catch",
       "fixSnippet": "try {\\n  encoded = encodeURIComponent(message);\\n} catch (e) {\\n  // handle\\n}",
       "likelihood": 7,
       "likelihoodDescription": "International users may input non-utf8 characters",
       "severity": 2,
       "severityDescription": "Function throws uncaught error",
-      "keyAttributeInstructions": {
-        "notifications[]": "Array with message containing non-utf8 characters"
-      }
+      "coveredFlows": []
     }
   ]
 }