@codeyam/codeyam-cli 0.1.0-staging.b8a55ba → 0.1.0-staging.c90f8c9

package/analyzer-template/packages/ai/src/lib/astScopes/types.ts

@@ -142,9 +142,147 @@ export interface VariablePath {
   segments: PathSegment[];
 }
 
+/**
+ * Represents the effect of a setter call within a conditional block.
+ * This tracks what state changes occur when specific conditions are met.
+ *
+ * Example: if (data.success) { setShowModal(true); }
+ * - condition: data.success = truthy
+ * - effect: setShowModal called with true
+ */
+export interface ConditionalEffect {
+  /**
+   * Single condition that controls this effect.
+   * Use this for simple if statements with a single condition.
+   */
+  condition?: {
+    /** The path being checked (e.g., "data.success") */
+    path: string;
+    /** Type of condition check */
+    conditionType: 'truthiness' | 'comparison' | 'switch';
+    /** Values being compared against (for comparison/switch) */
+    comparedValues?: string[];
+    /** The value required for this effect to occur */
+    requiredValue: string | boolean;
+  };
+
+  /**
+   * Multiple conditions that all must be true for this effect.
+   * Use this for compound conditions (a && b && c).
+   */
+  conditions?: Array<{
+    /** The path being checked */
+    path: string;
+    /** Type of condition check */
+    conditionType: 'truthiness' | 'comparison' | 'switch';
+    /** Values being compared against */
+    comparedValues?: string[];
+    /** The value required for this condition to be true */
+    requiredValue: string | boolean;
+  }>;
+
+  /**
+   * The effect that occurs when the condition(s) are met.
+   */
+  effect: {
+    /** Type of effect - currently only 'setter' for useState setters */
+    type: 'setter';
+    /** The state variable name (e.g., "showModal" from [showModal, setShowModal]) */
+    stateVariable: string;
+    /** The setter function name (e.g., "setShowModal") */
+    setterName: string;
+    /** The value being set (literal or expression text) */
+    value: string;
+    /** The type of value being set */
+    valueType: 'string' | 'number' | 'boolean' | 'null' | 'dynamic';
+  };
+
+  /**
+   * Source location for this conditional effect
+   */
+  sourceLocation?: {
+    lineNumber: number;
+    column: number;
+    codeSnippet: string;
+  };
+}
+
 /**
  * Results of AST-based scope analysis
  */
+
+/**
+ * Operation types for derived variables.
+ * - notNull/isNull: null checks (x !== null, x === null)
+ * - equals/notEquals: value comparisons (x === 'value', x !== 'value')
+ * - or/and: logical combinations (a || b, a && b)
+ * - arrayIncludes/arraySome/arrayEvery: array predicate methods
+ * - arrayLength: array.length property access
+ */
+export type DerivedVariableOperation =
+  | 'notNull'
+  | 'isNull'
+  | 'equals'
+  | 'notEquals'
+  | 'or'
+  | 'and'
+  | 'arrayIncludes'
+  | 'arraySome'
+  | 'arrayEvery'
+  | 'arrayLength';
+
+/**
+ * Information about how a variable was derived from other variables.
+ * Used to trace derived boolean variables back to their data sources.
+ */
+export interface DerivedVariableInfo {
+  /** For single-source derivations (notNull, equals, array predicates, etc.) */
+  sourcePath?: string;
+  /** For multi-source derivations (or, and) */
+  sourcePaths?: string[];
+  /** The operation that derived this variable */
+  operation: DerivedVariableOperation;
+  /** For comparison operations, the value being compared to */
+  comparedValue?: string;
+}
+
+/**
+ * Represents a JSX rendering usage where arrays or strings are rendered directly.
+ * Used to generate execution flows for array sizes and text lengths
+ * even without explicit conditionals.
+ *
+ * Examples:
+ * - `{items.map(i => <Item {...i} />)}` → array-map usage for 'items'
+ * - `{user.name}` → text-interpolation usage for 'user.name'
+ */
+export interface JsxRenderingUsage {
+  /**
+   * The path to the rendered value (e.g., "items", "data.users", "user.name")
+   */
+  path: string;
+
+  /**
+   * Type of rendering:
+   * - 'array-map': Array rendered via .map() method
+   * - 'text-interpolation': String rendered directly in JSX
+   */
+  renderingType: 'array-map' | 'text-interpolation';
+
+  /**
+   * The detected type of the value being rendered
+   */
+  valueType?: 'array' | 'string' | 'unknown';
+
+  /**
+   * Source location information
+   */
+  sourceLocation?: {
+    lineNumber: number;
+    column: number;
+    codeSnippet: string;
+  };
+}
+
 export interface AstScopeAnalysisResult {
   // Variable structure map (variable path -> type)
   structure: Record<string, string>;
@@ -162,6 +300,24 @@ export interface AstScopeAnalysisResult {
   // Compound conditionals grouped by chain (all conditions must be true together)
   compoundConditionals?: CompoundConditional[];
 
+  // Conditional effects - tracks what setter calls happen inside conditionals
+  // This allows us to determine what data conditions must be true for different state changes
+  conditionalEffects: ConditionalEffect[];
+
+  // Derived variables - variables computed from boolean expressions
+  // Maps variable name to derivation info (e.g., hasAnalysis -> { sourcePath: 'analysis', operation: 'notNull' })
+  derivedVariables?: Record<string, DerivedVariableInfo>;
+
+  // Child boundary gating conditions - tracks which conditions must be true for a child component to render
+  // Maps child component name to array of conditional usages that gate its rendering
+  // Example: {hasAnalysis && <ScenarioViewer />} → { ScenarioViewer: [{ path: 'hasAnalysis', ... }] }
+  childBoundaryGatingConditions?: Record<string, ConditionalUsage[]>;
+
+  // JSX rendering usages - tracks arrays rendered via .map() and strings interpolated in JSX
+  // Used to generate execution flows for array sizes and text lengths without explicit conditionals
+  // Example: {items.map(i => <Item {...i} />)} → array-map usage for 'items'
+  jsxRenderingUsages?: JsxRenderingUsage[];
+
   // Whether the analysis is complete enough to skip LLM calls
   isComplete: boolean;
 
@@ -217,6 +373,45 @@ export interface ConditionalUsage {
   chainExpression?: string;
   /** Whether this condition is negated (e.g., !foo) */
   isNegated?: boolean;
+  /**
+   * Whether this conditional controls JSX rendering.
+   * True when the conditional appears in a JSX expression like {cond && <Component />}
+   * or {cond ? <A /> : <B />}. Used to identify high-impact execution flows.
+   */
+  controlsJsxRendering?: boolean;
+
+  /**
+   * Information about how this variable was derived from other variables.
+   * Used to trace derived boolean variables back to their data sources.
+   *
+   * Examples:
+   * - `const hasAnalysis = analysis !== null` → { sourcePath: 'analysis', operation: 'notNull' }
+   * - `const isLoading = status === 'loading'` → { sourcePath: 'status', operation: 'equals', comparedValue: 'loading' }
+   * - `const isBusy = isRunning || isQueued` → { operation: 'or', sourcePaths: ['isRunning', 'isQueued'] }
+   */
+  derivedFrom?: {
+    /** For single-source derivations (notNull, equals, array predicates, etc.) */
+    sourcePath?: string;
+    /** For multi-source derivations (or, and) */
+    sourcePaths?: string[];
+    /** The operation that derived this variable */
+    operation: DerivedVariableOperation;
+    /** For comparison operations, the value being compared to */
+    comparedValue?: string;
+  };
+
+  /**
+   * The full original expression as a semantic constraint for LLM reasoning.
+   * Used when the path contains complex operations (filter, map, etc.) that
+   * can't be simply resolved to a mock value.
+   *
+   * Examples:
+   * - `scenarios.filter(s => s.active).length > 1`
+   * - `users.some(u => u.role === 'admin' && u.active)`
+   *
+   * The LLM can interpret this constraint to generate appropriate mock data.
+   */
+  constraintExpression?: string;
 }
 
 /**
@@ -236,6 +431,14 @@ export interface CompoundConditional {
     isNegated: boolean;
     /** Required value for this condition to be true */
     requiredValue?: string | boolean;
+    /**
+     * When conditions are part of an OR expression within an && chain,
+     * they share the same orGroupId. Conditions with the same orGroupId
+     * are alternatives - only ONE needs to be satisfied.
+     * Example: `A && (B || C)` creates conditions where B and C share an orGroupId.
+     * This allows flow generation to expand into separate flows: A&&B and A&&C.
+     */
+    orGroupId?: string;
   }[];
   /** Where this compound conditional occurs */
   location: 'if' | 'ternary' | 'logical-and' | 'switch';
@@ -245,6 +448,12 @@ export interface CompoundConditional {
     column: number;
     codeSnippet: string;
   };
+  /**
+   * Whether this compound conditional controls JSX rendering.
+   * True when the conditional appears in a JSX expression like {a && b && <Component />}
+   * Used to identify high-impact execution flows.
+   */
+  controlsJsxRendering?: boolean;
 }
 
 /**
@@ -372,6 +581,28 @@ export interface AnalysisContext {
   // Get all conditional usages
   getConditionalUsages(): Record<string, ConditionalUsage[]>;
 
+  // Add a gating condition for a child component
+  // Called when we see patterns like: {hasAnalysis && <ChildComponent />}
+  addChildBoundaryGatingCondition(
+    childComponentName: string,
+    gatingCondition: ConditionalUsage,
+  ): void;
+
+  // Get all child boundary gating conditions
+  getChildBoundaryGatingConditions(): Record<string, ConditionalUsage[]>;
+
+  // Record a derived variable and its source expression
+  // Called when we see patterns like: const hasAnalysis = analysis !== null
+  addDerivedVariable(
+    variableName: string,
+    derivationInfo: {
+      sourcePath?: string;
+      sourcePaths?: string[];
+      operation: DerivedVariableOperation;
+      comparedValue?: string;
+    },
+  ): void;
+
   // Add a compound conditional for key attribute dependency detection
   addCompoundConditional(compound: CompoundConditional): void;
 
@@ -381,4 +612,17 @@ export interface AnalysisContext {
   // Update a schema type, typically used when switch statements reveal
   // the valid values for a parameter (creating a union type)
   updateSchemaType(path: string | StructuredPath, newType: string): void;
+
+  // Add a conditional effect (setter call inside a conditional)
+  addConditionalEffect(effect: ConditionalEffect): void;
+
+  // Get all conditional effects
+  getConditionalEffects(): ConditionalEffect[];
+
+  // Add a JSX rendering usage (array.map() or text interpolation)
+  // Called when we see patterns like: {items.map(i => <Item />)} or {user.name}
+  addJsxRenderingUsage(usage: JsxRenderingUsage): void;
+
+  // Get all JSX rendering usages
+  getJsxRenderingUsages(): JsxRenderingUsage[];
 }

package/analyzer-template/packages/ai/src/lib/checkAllAttributes.ts

@@ -1,25 +1,44 @@
-export default function checkAllAttributes(data: { [key: string]: unknown }) {
-  for (const key of Object.keys(data || {})) {
+export default function checkAllAttributes(
+  data: { [key: string]: unknown } | unknown,
+): unknown {
+  // Handle non-object values (strings, numbers, etc.) - return them as-is
+  if (data === null || data === undefined) {
+    return data;
+  }
+  if (typeof data !== 'object') {
+    return data;
+  }
+  // Handle arrays - recurse into each element
+  if (Array.isArray(data)) {
+    return data.map((item) => checkAllAttributes(item));
+  }
+
+  const obj = data as { [key: string]: unknown };
+  for (const key of Object.keys(obj)) {
     try {
-      if (data[key] === undefined) {
-        delete data[key];
+      if (obj[key] === undefined) {
+        delete obj[key];
         continue;
       }
 
-      if (typeof data[key] === 'object') {
-        checkAllAttributes(data[key] as { [key: string]: unknown });
-      } else if (typeof data[key] === 'string') {
-        data[key] = (data[key] as string).replace(/\n/g, '\\n');
+      if (Array.isArray(obj[key])) {
+        obj[key] = (obj[key] as unknown[]).map((item) =>
+          checkAllAttributes(item),
+        );
+      } else if (typeof obj[key] === 'object' && obj[key] !== null) {
+        checkAllAttributes(obj[key]);
+      } else if (typeof obj[key] === 'string') {
+        obj[key] = (obj[key] as string).replace(/\n/g, '\\n');
       }
     } catch (e) {
       console.error('Error checking all attributes', {
         key,
-        data,
+        data: obj,
         error: e,
       });
       throw e;
     }
   }
 
-  return data;
+  return obj;
 }

package/analyzer-template/packages/ai/src/lib/completionCall.ts

@@ -1,5 +1,6 @@
 import OpenAI from 'openai';
 import * as lib from '../lib';
+import { trackDataSnapshot } from './e2eDataTracking';
 
 import PQueue from 'p-queue';
 import { default as PRetry, Options } from 'p-retry';
@@ -9,6 +10,7 @@ import { getModelInfo } from './modelInfo';
 import { AI } from './types';
 import { callClaudeCli } from './services/claudeCliAIService';
 import { isClaudeCliMode } from './services/aiServiceMode';
+import { awsLogDebugLevel } from '~codeyam/utils';
 
 // set to 'true' only locally for e.g. benchmarking unassisted error rates
 const RETRIES_DISABLED = false;
@@ -66,7 +68,11 @@ export default async function completionCall({
   // When CODEYAM_LLM_FIXTURES_DIR is set, load responses from captured JSON files
   // instead of making real API calls
   if (process.env.CODEYAM_LLM_FIXTURES_DIR) {
-    return await replayLlmCall(type, process.env.CODEYAM_LLM_FIXTURES_DIR);
+    return await replayLlmCall(
+      type,
+      process.env.CODEYAM_LLM_FIXTURES_DIR,
+      systemMessage,
+    );
   }
 
   console.log(
@@ -133,10 +139,35 @@ export default async function completionCall({
     () => {
       queueEndTime = Date.now();
       return PRetry(
-        () => {
-          return openai.chat.completions.create(params, {
-            timeout: 5 * 60 * 1000, // 5 minute timeout
-          });
+        async () => {
+          const callStartTime = Date.now();
+          const waitingMessages = [
+            'Waiting for LLM response',
+            'Still waiting for LLM response',
+            'LLM call in progress',
+            'Processing LLM request',
+            'Awaiting LLM completion',
+          ];
+
+          const logInterval = setInterval(() => {
+            const elapsedSeconds = Math.floor(
+              (Date.now() - callStartTime) / 1000,
+            );
+            const messageIndex =
+              Math.floor(elapsedSeconds / 10) % waitingMessages.length;
+            awsLogDebugLevel(
+              1,
+              `${waitingMessages[messageIndex]} [type=${type}, model=${model}, elapsed=${elapsedSeconds}s]`,
+            );
+          }, 10000);
+
+          try {
+            return await openai.chat.completions.create(params, {
+              timeout: 5 * 60 * 1000, // 5 minute timeout
+            });
+          } finally {
+            clearInterval(logInterval);
+          }
         },
         {
           ...defaultRetryOptions,
@@ -282,21 +313,46 @@ Please provide a corrected version with valid JSON only. Do not include any expl
         const correctionChatCompletion = await queue.add(
           () => {
             return PRetry(
-              () => {
-                return openai.chat.completions.create(
-                  {
-                    ...params,
-                    messages: [
-                      { role: 'system', content: systemMessage },
-                      { role: 'user', content: prompt },
-                      { role: 'assistant', content: completion },
-                      { role: 'user', content: correctionPrompt },
-                    ],
-                  },
-                  {
-                    timeout: 5 * 60 * 1000,
-                  },
-                );
+              async () => {
+                const correctionStartTime = Date.now();
+                const waitingMessages = [
+                  'Waiting for LLM correction response',
+                  'Still waiting for LLM correction',
+                  'LLM correction in progress',
+                  'Processing LLM correction request',
+                  'Awaiting LLM correction completion',
+                ];
+
+                const logInterval = setInterval(() => {
+                  const elapsedSeconds = Math.floor(
+                    (Date.now() - correctionStartTime) / 1000,
+                  );
+                  const messageIndex =
+                    Math.floor(elapsedSeconds / 10) % waitingMessages.length;
+                  awsLogDebugLevel(
+                    1,
+                    `${waitingMessages[messageIndex]} [type=${type}, model=${model}, elapsed=${elapsedSeconds}s]`,
+                  );
+                }, 10000);
+
+                try {
+                  return await openai.chat.completions.create(
+                    {
+                      ...params,
+                      messages: [
+                        { role: 'system', content: systemMessage },
+                        { role: 'user', content: prompt },
+                        { role: 'assistant', content: completion },
+                        { role: 'user', content: correctionPrompt },
+                      ],
+                    },
+                    {
+                      timeout: 5 * 60 * 1000,
+                    },
+                  );
+                } finally {
+                  clearInterval(logInterval);
+                }
               },
               {
                 ...defaultRetryOptions,
@@ -372,6 +428,12 @@ Please provide a corrected version with valid JSON only. Do not include any expl
     }
   }
 
+  // Track the completion for E2E debugging
+  trackDataSnapshot(`completionCall_${type}`, {
+    completion,
+    finishReason: chatCompletion.choices[0].finish_reason,
+  });
+
   return {
     finishReason: chatCompletion.choices[0].finish_reason,
     completion,
@@ -388,11 +450,13 @@ export const QueueSize = () => queue.size;
  *
  * @param type - The prompt type to look up (e.g., 'generateEntityScenarios')
  * @param fixturesDir - Directory containing captured LLM call JSON files
+ * @param systemMessage - The system message for the call (used for content-based matching)
  * @returns Mocked completion response matching the structure of a real LLM call
  */
 async function replayLlmCall(
   type: string,
   fixturesDir: string,
+  systemMessage?: string,
 ): Promise<{
   finishReason: string;
   completion: string;
@@ -443,43 +507,159 @@ async function replayLlmCall(
       }
     }
 
+    // Sort fixtures by created_at timestamp to ensure deterministic ordering
+    // This is critical because filesystem order is not guaranteed
+    for (const promptType of Object.keys(callsByType)) {
+      callsByType[promptType].sort((a, b) => {
+        const timeA = a.created_at ?? 0;
+        const timeB = b.created_at ?? 0;
+        return timeA - timeB;
+      });
+    }
+
     const matchingCalls = callsByType[type];
 
     if (!matchingCalls || matchingCalls.length === 0) {
       const availableTypes = Object.keys(callsByType).join(', ');
-      throw new Error(
-        `No captured LLM call found for type '${type}'. Available types: ${availableTypes}`,
+      console.warn(
+        `CodeYam Test: No captured LLM call found for type '${type}'. Available types: ${availableTypes}`,
       );
+      // Return empty response - callers should handle gracefully
+      // This allows new LLM call types to be added without breaking existing fixtures
+      return {
+        finishReason: 'stop',
+        completion: '{}',
+        stats: {
+          model: 'fixture-fallback',
+          prompt_type: type,
+          system_message: '',
+          prompt_text: '',
+          response: '{}',
+          input_tokens: 0,
+          output_tokens: 0,
+          cost: 0,
+        },
+      };
     }
 
-    // Track call count per type and cycle through multiple responses
-    const callKey = `${fixturesDir}::${type}`;
-    if (!callIndexByType[callKey]) {
-      callIndexByType[callKey] = 0;
+    // For scenario data generation calls, match by scenario name instead of cycling by index.
+    // This is critical because scenarios may be processed in parallel with Promise.all,
+    // causing non-deterministic call order. Matching by scenario name ensures each call
+    // gets the correct fixture regardless of execution order.
+    //
+    // We group fixtures by scenario name and track indices per scenario to handle
+    // multiple calls for the same scenario (e.g., main call + continuation call).
+    let call: any;
+    const scenarioBasedTypes = [
+      'generateEntityScenarioData',
+      'generateChunkMockData',
+      'generateMissingMockData',
+    ];
+
+    if (scenarioBasedTypes.includes(type) && systemMessage) {
+      // Extract scenario name from system message: 'Scenario name must match exactly: "NAME"'
+      // or from chunk system message which includes scenario name
+      const scenarioNameMatch = systemMessage.match(
+        /Scenario name must match exactly: "([^"]+)"/,
+      );
+      const requestedScenarioName = scenarioNameMatch?.[1];
+
+      if (requestedScenarioName) {
+        // Group fixtures by scenario name
+        const fixturesByScenario: Record<string, any[]> = {};
+        for (const fixture of matchingCalls) {
+          try {
+            const props = JSON.parse(fixture.props || '{}');
+            const scenarioName = props.scenario?.name || '__NO_SCENARIO__';
+            if (!fixturesByScenario[scenarioName]) {
+              fixturesByScenario[scenarioName] = [];
+            }
+            fixturesByScenario[scenarioName].push(fixture);
+          } catch {
+            // Skip fixtures that can't be parsed
+          }
+        }
+
+        const scenarioFixtures = fixturesByScenario[requestedScenarioName];
+        if (scenarioFixtures && scenarioFixtures.length > 0) {
+          // Track call index per (type, scenario) combination
+          const scenarioCallKey = `${fixturesDir}::${type}::${requestedScenarioName}`;
+          if (callIndexByType[scenarioCallKey] === undefined) {
+            callIndexByType[scenarioCallKey] = 0;
+          }
+          const scenarioCallIndex = callIndexByType[scenarioCallKey];
+          callIndexByType[scenarioCallKey] =
+            (scenarioCallIndex + 1) % scenarioFixtures.length;
+
+          call = scenarioFixtures[scenarioCallIndex];
+          console.log(
+            `CodeYam Test: ✅ Matched fixture for scenario '${requestedScenarioName}' [${scenarioCallIndex + 1}/${scenarioFixtures.length}]`,
+          );
+        } else {
+          const availableScenarios = Object.keys(fixturesByScenario).join(', ');
+          console.warn(
+            `CodeYam Test: ⚠️ No fixture found for scenario '${requestedScenarioName}'. Available: [${availableScenarios}]`,
+          );
+        }
+      } else {
+        console.warn(
+          `CodeYam Test: ⚠️ Could not extract scenario name from system message for type '${type}'`,
+        );
+      }
     }
-    const callIndex = callIndexByType[callKey];
-    callIndexByType[callKey] = (callIndex + 1) % matchingCalls.length;
 
-    const call = matchingCalls[callIndex];
+    // Fall back to index-based cycling if no scenario match was found
+    if (!call) {
+      const callKey = `${fixturesDir}::${type}`;
+      if (callIndexByType[callKey] === undefined) {
+        callIndexByType[callKey] = 0;
+      }
+      const callIndex = callIndexByType[callKey];
+      callIndexByType[callKey] = (callIndex + 1) % matchingCalls.length;
+      call = matchingCalls[callIndex];
 
-    console.log(
-      `CodeYam Test: Replaying LLM response for '${type}' [${callIndex + 1}/${matchingCalls.length}]`,
-    );
+      console.log(
+        `CodeYam Test: Replaying LLM response for '${type}' [${callIndex + 1}/${matchingCalls.length}]`,
+      );
+    }
 
     // Extract the actual completion content from the OpenAI response
     // The response field contains the full OpenAI API response as a JSON string
-    let completion: string;
+    let rawCompletion: string;
     try {
       const responseObj = JSON.parse(call.response);
-      completion = responseObj.choices?.[0]?.message?.content || call.response;
+      rawCompletion =
+        responseObj.choices?.[0]?.message?.content || call.response;
     } catch {
       // If parsing fails, use the response as-is (for backwards compatibility)
-      completion = call.response;
+      rawCompletion = call.response;
     }
 
+    // Apply the same post-processing as real completionCall() does
+    // This ensures replayed responses are processed identically to real API calls
+    // Step 1: Remove <think>...</think> sections that some models include
+    let processedCompletion = rawCompletion;
+    if (rawCompletion) {
+      processedCompletion = rawCompletion
+        .replace(/<think>[\s\S]*?<\/think>/g, '')
+        .trim();
+    }
+
+    // Step 2: Extract JSON from text that might have extra content around it
+    // (Only for JSON responses - we assume most LLM calls expect JSON)
+    const completion = processedCompletion
+      ? (processedCompletion.match(/\{[\s\S]*\}/)?.[0] ?? processedCompletion)
+      : processedCompletion;
+
+    // Track the replayed completion for E2E debugging
+    trackDataSnapshot(`completionCall_${type}`, {
+      completion: completion || '',
+      finishReason: 'stop',
+    });
+
     return {
       finishReason: 'stop',
-      completion,
+      completion: completion || '',
       stats: {
         model: call.model ?? 'fixture',
         prompt_type: type,