@codeyam/codeyam-cli 0.1.0-staging.1669d45 → 0.1.0-staging.323686

package/analyzer-template/packages/ai/src/lib/completionCall.ts

@@ -1,5 +1,6 @@
 import OpenAI from 'openai';
 import * as lib from '../lib';
+import { trackDataSnapshot } from './e2eDataTracking';
 
 import PQueue from 'p-queue';
 import { default as PRetry, Options } from 'p-retry';
@@ -9,6 +10,7 @@ import { getModelInfo } from './modelInfo';
 import { AI } from './types';
 import { callClaudeCli } from './services/claudeCliAIService';
 import { isClaudeCliMode } from './services/aiServiceMode';
+import { awsLogDebugLevel } from '~codeyam/utils';
 
 // set to 'true' only locally for e.g. benchmarking unassisted error rates
 const RETRIES_DISABLED = false;
@@ -66,7 +68,11 @@ export default async function completionCall({
   // When CODEYAM_LLM_FIXTURES_DIR is set, load responses from captured JSON files
   // instead of making real API calls
   if (process.env.CODEYAM_LLM_FIXTURES_DIR) {
-    return await replayLlmCall(type, process.env.CODEYAM_LLM_FIXTURES_DIR);
+    return await replayLlmCall(
+      type,
+      process.env.CODEYAM_LLM_FIXTURES_DIR,
+      systemMessage,
+    );
   }
 
   console.log(
@@ -133,10 +139,35 @@ export default async function completionCall({
     () => {
       queueEndTime = Date.now();
       return PRetry(
-        () => {
-          return openai.chat.completions.create(params, {
-            timeout: 5 * 60 * 1000, // 5 minute timeout
-          });
+        async () => {
+          const callStartTime = Date.now();
+          const waitingMessages = [
+            'Waiting for LLM response',
+            'Still waiting for LLM response',
+            'LLM call in progress',
+            'Processing LLM request',
+            'Awaiting LLM completion',
+          ];
+
+          const logInterval = setInterval(() => {
+            const elapsedSeconds = Math.floor(
+              (Date.now() - callStartTime) / 1000,
+            );
+            const messageIndex =
+              Math.floor(elapsedSeconds / 10) % waitingMessages.length;
+            awsLogDebugLevel(
+              1,
+              `${waitingMessages[messageIndex]} [type=${type}, model=${model}, elapsed=${elapsedSeconds}s]`,
+            );
+          }, 10000);
+
+          try {
+            return await openai.chat.completions.create(params, {
+              timeout: 5 * 60 * 1000, // 5 minute timeout
+            });
+          } finally {
+            clearInterval(logInterval);
+          }
         },
         {
           ...defaultRetryOptions,
@@ -282,21 +313,46 @@ Please provide a corrected version with valid JSON only. Do not include any expl
         const correctionChatCompletion = await queue.add(
           () => {
             return PRetry(
-              () => {
-                return openai.chat.completions.create(
-                  {
-                    ...params,
-                    messages: [
-                      { role: 'system', content: systemMessage },
-                      { role: 'user', content: prompt },
-                      { role: 'assistant', content: completion },
-                      { role: 'user', content: correctionPrompt },
-                    ],
-                  },
-                  {
-                    timeout: 5 * 60 * 1000,
-                  },
-                );
+              async () => {
+                const correctionStartTime = Date.now();
+                const waitingMessages = [
+                  'Waiting for LLM correction response',
+                  'Still waiting for LLM correction',
+                  'LLM correction in progress',
+                  'Processing LLM correction request',
+                  'Awaiting LLM correction completion',
+                ];
+
+                const logInterval = setInterval(() => {
+                  const elapsedSeconds = Math.floor(
+                    (Date.now() - correctionStartTime) / 1000,
+                  );
+                  const messageIndex =
+                    Math.floor(elapsedSeconds / 10) % waitingMessages.length;
+                  awsLogDebugLevel(
+                    1,
+                    `${waitingMessages[messageIndex]} [type=${type}, model=${model}, elapsed=${elapsedSeconds}s]`,
+                  );
+                }, 10000);
+
+                try {
+                  return await openai.chat.completions.create(
+                    {
+                      ...params,
+                      messages: [
+                        { role: 'system', content: systemMessage },
+                        { role: 'user', content: prompt },
+                        { role: 'assistant', content: completion },
+                        { role: 'user', content: correctionPrompt },
+                      ],
+                    },
+                    {
+                      timeout: 5 * 60 * 1000,
+                    },
+                  );
+                } finally {
+                  clearInterval(logInterval);
+                }
               },
               {
                 ...defaultRetryOptions,
@@ -372,6 +428,12 @@ Please provide a corrected version with valid JSON only. Do not include any expl
     }
   }
 
+  // Track the completion for E2E debugging
+  trackDataSnapshot(`completionCall_${type}`, {
+    completion,
+    finishReason: chatCompletion.choices[0].finish_reason,
+  });
+
   return {
     finishReason: chatCompletion.choices[0].finish_reason,
     completion,
@@ -388,11 +450,13 @@ export const QueueSize = () => queue.size;
  *
  * @param type - The prompt type to look up (e.g., 'generateEntityScenarios')
  * @param fixturesDir - Directory containing captured LLM call JSON files
+ * @param systemMessage - The system message for the call (used for content-based matching)
  * @returns Mocked completion response matching the structure of a real LLM call
  */
 async function replayLlmCall(
   type: string,
   fixturesDir: string,
+  systemMessage?: string,
 ): Promise<{
   finishReason: string;
   completion: string;
@@ -443,6 +507,16 @@ async function replayLlmCall(
       }
     }
 
+    // Sort fixtures by created_at timestamp to ensure deterministic ordering
+    // This is critical because filesystem order is not guaranteed
+    for (const promptType of Object.keys(callsByType)) {
+      callsByType[promptType].sort((a, b) => {
+        const timeA = a.created_at ?? 0;
+        const timeB = b.created_at ?? 0;
+        return timeA - timeB;
+      });
+    }
+
     const matchingCalls = callsByType[type];
 
     if (!matchingCalls || matchingCalls.length === 0) {
@@ -468,34 +542,124 @@ async function replayLlmCall(
       };
     }
 
-    // Track call count per type and cycle through multiple responses
-    const callKey = `${fixturesDir}::${type}`;
-    if (!callIndexByType[callKey]) {
-      callIndexByType[callKey] = 0;
+    // For scenario data generation calls, match by scenario name instead of cycling by index.
+    // This is critical because scenarios may be processed in parallel with Promise.all,
+    // causing non-deterministic call order. Matching by scenario name ensures each call
+    // gets the correct fixture regardless of execution order.
+    //
+    // We group fixtures by scenario name and track indices per scenario to handle
+    // multiple calls for the same scenario (e.g., main call + continuation call).
+    let call: any;
+    const scenarioBasedTypes = [
+      'generateEntityScenarioData',
+      'generateChunkMockData',
+      'generateMissingMockData',
+    ];
+
+    if (scenarioBasedTypes.includes(type) && systemMessage) {
+      // Extract scenario name from system message: 'Scenario name must match exactly: "NAME"'
+      // or from chunk system message which includes scenario name
+      const scenarioNameMatch = systemMessage.match(
+        /Scenario name must match exactly: "([^"]+)"/,
+      );
+      const requestedScenarioName = scenarioNameMatch?.[1];
+
+      if (requestedScenarioName) {
+        // Group fixtures by scenario name
+        const fixturesByScenario: Record<string, any[]> = {};
+        for (const fixture of matchingCalls) {
+          try {
+            const props = JSON.parse(fixture.props || '{}');
+            const scenarioName = props.scenario?.name || '__NO_SCENARIO__';
+            if (!fixturesByScenario[scenarioName]) {
+              fixturesByScenario[scenarioName] = [];
+            }
+            fixturesByScenario[scenarioName].push(fixture);
+          } catch {
+            // Skip fixtures that can't be parsed
+          }
+        }
+
+        const scenarioFixtures = fixturesByScenario[requestedScenarioName];
+        if (scenarioFixtures && scenarioFixtures.length > 0) {
+          // Track call index per (type, scenario) combination
+          const scenarioCallKey = `${fixturesDir}::${type}::${requestedScenarioName}`;
+          if (callIndexByType[scenarioCallKey] === undefined) {
+            callIndexByType[scenarioCallKey] = 0;
+          }
+          const scenarioCallIndex = callIndexByType[scenarioCallKey];
+          callIndexByType[scenarioCallKey] =
+            (scenarioCallIndex + 1) % scenarioFixtures.length;
+
+          call = scenarioFixtures[scenarioCallIndex];
+          console.log(
+            `CodeYam Test: ✅ Matched fixture for scenario '${requestedScenarioName}' [${scenarioCallIndex + 1}/${scenarioFixtures.length}]`,
+          );
+        } else {
+          const availableScenarios = Object.keys(fixturesByScenario).join(', ');
+          console.warn(
+            `CodeYam Test: ⚠️ No fixture found for scenario '${requestedScenarioName}'. Available: [${availableScenarios}]`,
+          );
+        }
+      } else {
+        console.warn(
+          `CodeYam Test: ⚠️ Could not extract scenario name from system message for type '${type}'`,
+        );
+      }
     }
-    const callIndex = callIndexByType[callKey];
-    callIndexByType[callKey] = (callIndex + 1) % matchingCalls.length;
 
-    const call = matchingCalls[callIndex];
+    // Fall back to index-based cycling if no scenario match was found
+    if (!call) {
+      const callKey = `${fixturesDir}::${type}`;
+      if (callIndexByType[callKey] === undefined) {
+        callIndexByType[callKey] = 0;
+      }
+      const callIndex = callIndexByType[callKey];
+      callIndexByType[callKey] = (callIndex + 1) % matchingCalls.length;
+      call = matchingCalls[callIndex];
 
-    console.log(
-      `CodeYam Test: Replaying LLM response for '${type}' [${callIndex + 1}/${matchingCalls.length}]`,
-    );
+      console.log(
+        `CodeYam Test: Replaying LLM response for '${type}' [${callIndex + 1}/${matchingCalls.length}]`,
+      );
+    }
 
     // Extract the actual completion content from the OpenAI response
     // The response field contains the full OpenAI API response as a JSON string
-    let completion: string;
+    let rawCompletion: string;
     try {
       const responseObj = JSON.parse(call.response);
-      completion = responseObj.choices?.[0]?.message?.content || call.response;
+      rawCompletion =
+        responseObj.choices?.[0]?.message?.content || call.response;
     } catch {
       // If parsing fails, use the response as-is (for backwards compatibility)
-      completion = call.response;
+      rawCompletion = call.response;
     }
 
+    // Apply the same post-processing as real completionCall() does
+    // This ensures replayed responses are processed identically to real API calls
+    // Step 1: Remove <think>...</think> sections that some models include
+    let processedCompletion = rawCompletion;
+    if (rawCompletion) {
+      processedCompletion = rawCompletion
+        .replace(/<think>[\s\S]*?<\/think>/g, '')
+        .trim();
+    }
+
+    // Step 2: Extract JSON from text that might have extra content around it
+    // (Only for JSON responses - we assume most LLM calls expect JSON)
+    const completion = processedCompletion
+      ? (processedCompletion.match(/\{[\s\S]*\}/)?.[0] ?? processedCompletion)
+      : processedCompletion;
+
+    // Track the replayed completion for E2E debugging
+    trackDataSnapshot(`completionCall_${type}`, {
+      completion: completion || '',
+      finishReason: 'stop',
+    });
+
     return {
       finishReason: 'stop',
-      completion,
+      completion: completion || '',
       stats: {
         model: call.model ?? 'fixture',
         prompt_type: type,

package/analyzer-template/packages/ai/src/lib/dataStructure/ScopeDataStructure.ts

@@ -82,6 +82,8 @@
 import { ScopeAnalysis } from '~codeyam/types';
 import { EquivalencyManager } from './equivalencyManagers/EquivalencyManager';
 import fillInSchemaGapsAndUnknowns from './helpers/fillInSchemaGapsAndUnknowns';
+import { clearCleanKnownObjectFunctionsCache } from './helpers/cleanKnownObjectFunctions';
+import { clearCleanNonObjectFunctionsCache } from './helpers/cleanNonObjectFunctions';
 
 /**
  * Patterns that indicate recursive type structures in schema paths.
@@ -332,6 +334,19 @@ export function resetScopeDataStructureMetrics() {
   followEquivalenciesEarlyExitPhase1Count = 0;
   followEquivalenciesWithWorkCount = 0;
   addEquivalencyCallCount = 0;
+
+  // Clear module-level caches to prevent unbounded memory growth across entities
+  const knownObjectCache = clearCleanKnownObjectFunctionsCache();
+  const nonObjectCache = clearCleanNonObjectFunctionsCache();
+  if (knownObjectCache.count > 0 || nonObjectCache.count > 0) {
+    const totalBytes =
+      knownObjectCache.estimatedBytes + nonObjectCache.estimatedBytes;
+    console.log('CodeYam: Cleared analysis caches', {
+      knownObjectCache: `${knownObjectCache.count} entries, ${(knownObjectCache.estimatedBytes / 1024).toFixed(1)}KB`,
+      nonObjectCache: `${nonObjectCache.count} entries, ${(nonObjectCache.estimatedBytes / 1024).toFixed(1)}KB`,
+      totalKB: `${(totalBytes / 1024).toFixed(1)}KB`,
+    });
+  }
 }
 
 // Performance: Pre-computed Sets for equivalency reason filtering (O(1) vs O(n))
@@ -436,6 +451,13 @@ export class ScopeDataStructure {
     import('../astScopes/types').ConditionalUsage[]
   > = {};
 
+  /**
+   * JSX rendering usages collected during AST analysis.
+   * Tracks arrays rendered via .map() and strings interpolated in JSX.
+   */
+  private rawJsxRenderingUsages: import('../astScopes/types').JsxRenderingUsage[] =
+    [];
+
   private lastAddToSchemaId = 0;
   private lastEquivalencyId = 0;
   private lastEquivalencyDatabaseId = 0;
@@ -3876,13 +3898,29 @@ export class ScopeDataStructure {
         // "useFetcher<...>().state" for execution flow validation
         if (equivalentValue.schemaPath.endsWith('.functionCallReturnValue')) {
           // Extract the hook call path (everything before .functionCallReturnValue)
-          const hookCallPath = equivalentValue.schemaPath.slice(
+          let hookCallPath = equivalentValue.schemaPath.slice(
             0,
             -'.functionCallReturnValue'.length,
           );
           // Only include if it looks like a hook call (contains parentheses)
           // and the variable name (path) is a simple identifier (no dots)
           if (hookCallPath.includes('(') && !path.includes('.')) {
+            // Special case: If hookCallPath is a callback scope (cyScope pattern),
+            // trace through it to find what the callback actually returns.
+            // This handles useState(() => { return prop; }) patterns.
+            const cyScopeMatch = hookCallPath.match(/^(cyScope\d+)\(\)$/);
+            if (cyScopeMatch) {
+              // Use the equivalency database to trace the callback's return value
+              // to its actual source (e.g., viewModeFromUrl -> segments -> params -> useParams)
+              const dbEntry = this.getEquivalenciesDatabaseEntry(
+                scopeNode.name, // Component scope
+                path, // variable name (e.g., viewMode)
+              );
+              if (dbEntry?.sourceCandidates?.length > 0) {
+                // Use the traced source instead of the callback scope
+                hookCallPath = dbEntry.sourceCandidates[0].schemaPath;
+              }
+            }
             equivalentSignatureVariables[path] = hookCallPath;
           }
         }
@@ -4000,6 +4038,7 @@ export class ScopeDataStructure {
     // E.g., analysis → currentEntityAnalysis → useLoaderData().functionCallReturnValue.currentEntityAnalysis
     // We need multiple passes because resolutions can depend on each other
     const maxIterations = 5; // Prevent infinite loops
+
     for (let iteration = 0; iteration < maxIterations; iteration++) {
       let changed = false;
 
@@ -4078,6 +4117,40 @@ export class ScopeDataStructure {
             }
           }
 
+          // Fix 38: Handle cyScope lazy initializer return values
+          // When we have viewMode -> cyScope20(), trace through to find what cyScope20 returns.
+          // The lazy initializer's return value should be the controllable data source.
+          // Pattern: cyScopeN() where N is a number
+          const cyScopeMatch = sourcePath.match(/^(cyScope\d+)\(\)$/);
+          if (cyScopeMatch) {
+            const cyScopeName = cyScopeMatch[1];
+            const cyScopeNode = this.scopeNodes[cyScopeName];
+
+            if (cyScopeNode?.equivalencies) {
+              // Look for returnValue equivalency in the cyScope
+              const returnValueEquivs =
+                cyScopeNode.equivalencies['returnValue'];
+              if (returnValueEquivs && returnValueEquivs.length > 0) {
+                // Get the first return value source
+                const returnSource = returnValueEquivs[0].schemaPath;
+
+                // If the return source is a simple variable (not a complex path),
+                // resolve varName directly to that variable
+                if (
+                  returnSource &&
+                  !returnSource.includes('(') &&
+                  !returnSource.includes('[')
+                ) {
+                  // Update varName to point to the return source
+                  if (equivalentSignatureVariables[varName] !== returnSource) {
+                    equivalentSignatureVariables[varName] = returnSource;
+                    changed = true;
+                  }
+                }
+              }
+            }
+          }
+
           continue;
         }
 
@@ -4503,6 +4576,33 @@ export class ScopeDataStructure {
     return enriched;
   }
 
+  /**
+   * Add JSX rendering usages from AST analysis.
+   * These track arrays rendered via .map() and strings interpolated in JSX.
+   */
+  addJsxRenderingUsages(
+    usages: import('../astScopes/types').JsxRenderingUsage[],
+  ): void {
+    // Add usages, avoiding duplicates based on path and renderingType
+    for (const usage of usages) {
+      const exists = this.rawJsxRenderingUsages.some(
+        (existing) =>
+          existing.path === usage.path &&
+          existing.renderingType === usage.renderingType,
+      );
+      if (!exists) {
+        this.rawJsxRenderingUsages.push(usage);
+      }
+    }
+  }
+
+  /**
+   * Get JSX rendering usages collected during analysis.
+   */
+  getJsxRenderingUsages(): import('../astScopes/types').JsxRenderingUsage[] {
+    return this.rawJsxRenderingUsages;
+  }
+
   toSerializable(): SerializableDataStructure {
     // Helper to clean cyScope and cyDuplicateKey from a string for output
     const cleanCyScope = (str: string): string =>
@@ -5003,6 +5103,12 @@ export class ScopeDataStructure {
         ? enrichedGatingConditions
         : undefined;
 
+    // Get JSX rendering usages (arrays via .map(), strings via interpolation)
+    const jsxRenderingUsages =
+      this.rawJsxRenderingUsages.length > 0
+        ? this.rawJsxRenderingUsages
+        : undefined;
+
     return {
       externalFunctionCalls: deduplicatedExternalFunctionCalls,
       rootFunction,
@@ -5013,6 +5119,7 @@ export class ScopeDataStructure {
       conditionalEffects,
       compoundConditionals,
       childBoundaryGatingConditions,
+      jsxRenderingUsages,
     };
   }