@codeyam/codeyam-cli 0.1.0-bleeding-edge.8afd3ee → 0.1.0-staging.09652b8

package/analyzer-template/packages/ai/src/lib/dataStructure/ScopeDataStructure.ts

@@ -159,10 +159,12 @@ const ROOT_SCOPE_NAME = 'root';
 export let maxEquivalencyChainDepth = 0;
 export let addToSchemaCallCount = 0;
 export let followEquivalenciesCallCount = 0;
+export let addEquivalencyCallCount = 0;
 export function resetScopeDataStructureMetrics() {
   maxEquivalencyChainDepth = 0;
   addToSchemaCallCount = 0;
   followEquivalenciesCallCount = 0;
+  addEquivalencyCallCount = 0;
 }
 
 export class ScopeDataStructure {
@@ -188,6 +190,10 @@ export class ScopeDataStructure {
   private intermediatesOrderIndex: Map<string, EquivalencyDatabaseEntry> =
     new Map();
   private globalVisited: Set<string> = new Set();
+  // Separate visited set for addComplexSourcePathVariables - uses scope::path only (not value)
+  // This allows addComplexSourcePathVariables to run for each unique scope+path combo,
+  // while the main globalVisited uses scope+path+value for fine-grained deduplication
+  private complexSourceVisited: Set<string> = new Set();
 
   constructor(
     managers: EquivalencyManager[],
@@ -382,6 +388,18 @@ export class ScopeDataStructure {
     traceId?: number;
   }) {
     addToSchemaCallCount++;
+    // DEBUG: Detect infinite loops
+    if (addToSchemaCallCount > 500000) {
+      console.error('INFINITE LOOP DETECTED in addToSchema', {
+        callCount: addToSchemaCallCount,
+        path,
+        value,
+        scopeNodeName: scopeNode?.name,
+      });
+      throw new Error(
+        `Infinite loop detected: addToSchema called ${addToSchemaCallCount} times`,
+      );
+    }
     // Track max chain depth
     if (equivalencyValueChain.length > maxEquivalencyChainDepth) {
       maxEquivalencyChainDepth = equivalencyValueChain.length;
@@ -477,6 +495,7 @@ export class ScopeDataStructure {
       path,
     );
 
+    // Call addComplexSourcePathVariables like the original code (before visited check)
     this.addComplexSourcePathVariables(
       equivalenciesDatabaseEntry,
       scopeNode,
@@ -708,6 +727,21 @@ export class ScopeDataStructure {
     equivalencyValueChain?: EquivalencyValueChainItem[],
     traceId?: number,
   ) {
+    // DEBUG: Detect infinite loops
+    addEquivalencyCallCount++;
+    if (addEquivalencyCallCount > 50000) {
+      console.error('INFINITE LOOP DETECTED in addEquivalency', {
+        callCount: addEquivalencyCallCount,
+        path,
+        equivalentPath,
+        equivalentScopeName,
+        scopeNodeName: scopeNode.name,
+        equivalencyReason,
+      });
+      throw new Error(
+        `Infinite loop detected: addEquivalency called ${addEquivalencyCallCount} times`,
+      );
+    }
     // Temporary debugging to track down unnecessary equivalencies
     if (
       ![
@@ -736,6 +770,8 @@ export class ScopeDataStructure {
         'implicit parent equivalency - rerouted via useCallback', // 1 failure
         'Spread operator equivalency key update: implicit parent equivalency', // 1 failure
         'Array.from() equivalency',
+        'propagated sub-property equivalency',
+        'propagated function call return sub-property equivalency',
       ].includes(equivalencyReason)
     ) {
       if (
@@ -771,13 +807,47 @@ export class ScopeDataStructure {
     scopeNode.equivalencies[path] ||= [];
 
     const existing = scopeNode.equivalencies[path];
-    if (
-      existing.find(
-        (v) =>
-          v.schemaPath === equivalentPath &&
-          v.scopeNodeName === equivalentScopeName,
-      )
-    ) {
+    const existingEquivalency = existing.find(
+      (v) =>
+        v.schemaPath === equivalentPath &&
+        v.scopeNodeName === equivalentScopeName,
+    );
+    if (existingEquivalency) {
+      // During Phase 2 (onlyEquivalencies=false), we need to still process the equivalency
+      // to build the chain and add to the database, even if the equivalency already exists
+      if (!this.onlyEquivalencies) {
+        const equivalentScopeNode = this.getScopeOrFunctionCallInfo(
+          equivalentScopeName,
+        ) as ScopeNode;
+        if (equivalentScopeNode) {
+          // Extract function call name from path if it looks like a function call path
+          // e.g., "ChildComponent().signature[0].dataItem" -> "ChildComponent"
+          const pathMatch = path.match(/^([^().]+)\(\)/);
+          const previousPathScopeNodeName = pathMatch
+            ? pathMatch[1]
+            : scopeNode.name;
+
+          this.addToSchema({
+            path: equivalentPath,
+            value: 'unknown',
+            scopeNode: equivalentScopeNode,
+            equivalencyValueChain: [
+              ...(equivalencyValueChain ?? []),
+              {
+                id: existingEquivalency.id,
+                source: 'duplicate equivalency - Phase 2',
+                reason: equivalencyReason,
+                previousPath: {
+                  scopeNodeName: previousPathScopeNodeName,
+                  schemaPath: path,
+                  value: scopeNode.schema[path],
+                },
+              },
+            ],
+            traceId,
+          });
+        }
+      }
       return;
     }
 
@@ -1317,6 +1387,84 @@ export class ScopeDataStructure {
           scopeNode,
           'original equivalency',
         );
+
+        // Propagate sub-property equivalencies when the equivalentValue is a simple variable
+        // that has sub-properties defined in the isolatedEquivalentVariables.
+        // This handles cases like: dataItem={{ structure: completeDataStructure }}
+        // where completeDataStructure has sub-properties like completeDataStructure['Function Arguments']
+        // We need to propagate these to create: dataItem.structure['Function Arguments'] equivalencies
+        const isSimpleVariable =
+          !equivalentValue.startsWith('signature[') &&
+          !equivalentValue.includes('functionCallReturnValue') &&
+          !equivalentValue.includes('.') &&
+          !equivalentValue.includes('[');
+
+        if (isSimpleVariable) {
+          // Look in current scope and all parent scopes for sub-properties
+          const scopesToCheck = [scopeNode.name, ...scopeNode.tree];
+          for (const scopeName of scopesToCheck) {
+            const checkScope = this.scopeNodes[scopeName];
+            if (!checkScope?.analysis?.isolatedEquivalentVariables) continue;
+
+            for (const [subPath, subValue] of Object.entries(
+              checkScope.analysis.isolatedEquivalentVariables,
+            )) {
+              // Check if this is a sub-property of the equivalentValue variable
+              // e.g., completeDataStructure['Function Arguments'] or completeDataStructure.foo
+              const matchesDot = subPath.startsWith(equivalentValue + '.');
+              const matchesBracket = subPath.startsWith(equivalentValue + '[');
+              if (matchesDot || matchesBracket) {
+                const subPropertyPath = subPath.substring(
+                  equivalentValue.length,
+                );
+                const newPath = cleanPath(path + subPropertyPath, allPaths);
+                const newEquivalentValue = cleanPath(
+                  (subValue as string).replace(/::cyDuplicateKey\d+::/g, ''),
+                  allPaths,
+                );
+
+                if (
+                  newEquivalentValue &&
+                  this.isValidPath(newEquivalentValue)
+                ) {
+                  this.addEquivalency(
+                    newPath,
+                    newEquivalentValue,
+                    checkScope.name, // Use the scope where the sub-property was found
+                    scopeNode,
+                    'propagated sub-property equivalency',
+                  );
+                }
+              }
+
+              // Also check if equivalentValue itself maps to a functionCallReturnValue
+              // e.g., result = useMemo(...).functionCallReturnValue
+              if (
+                subPath === equivalentValue &&
+                typeof subValue === 'string' &&
+                subValue.endsWith('.functionCallReturnValue')
+              ) {
+                this.propagateFunctionCallReturnSubProperties(
+                  path,
+                  subValue,
+                  scopeNode,
+                  allPaths,
+                );
+              }
+            }
+          }
+        }
+
+        // Handle function call return values by propagating returnValue.* sub-properties
+        // from the callback scope to the usage path
+        if (equivalentValue.endsWith('.functionCallReturnValue')) {
+          this.propagateFunctionCallReturnSubProperties(
+            path,
+            equivalentValue,
+            scopeNode,
+            allPaths,
+          );
+        }
       }
     }
 
@@ -1356,6 +1504,257 @@ export class ScopeDataStructure {
     this.validateSchema(scopeNode, false, false);
   }
 
+  /**
+   * Propagates returnValue.* sub-properties from a callback scope to the usage path.
+   *
+   * When we have an equivalency like:
+   *   DataItemEditor().signature[0].structure -> getData().functionCallReturnValue
+   *
+   * And the callback scope for getData has:
+   *   returnValue.args -> dataStructure.arguments
+   *
+   * This method creates the transitive equivalency:
+   *   DataItemEditor().signature[0].structure.args -> signature[0].dataStructure.arguments
+   *
+   * It also resolves variable references through parent scopes (e.g., dataStructure -> signature[0].dataStructure)
+   * and ensures the database entry is updated with the usage path.
+   */
+  private propagateFunctionCallReturnSubProperties(
+    path: string,
+    equivalentValue: string,
+    scopeNode: ScopeNode,
+    allPaths: string[],
+  ) {
+    // Try to find the callback scope name from different patterns:
+    // 1. "getData().functionCallReturnValue" → look up getData variable to find scope
+    // 2. "useMemo(cyScope1(), [...]).functionCallReturnValue" → extract cyScope1 directly
+    let callbackScopeName: string | undefined;
+
+    // Pattern 1: Simple function call like getData().functionCallReturnValue
+    const simpleFunctionMatch = equivalentValue.match(
+      /^(\w+)\(\)\.functionCallReturnValue$/,
+    );
+    if (simpleFunctionMatch) {
+      const functionName = simpleFunctionMatch[1];
+      // Find the function reference in current or parent scopes
+      const scopesToCheck = [scopeNode.name, ...scopeNode.tree];
+      for (const scopeName of scopesToCheck) {
+        const checkScope = this.scopeNodes[scopeName];
+        if (!checkScope?.analysis?.isolatedEquivalentVariables) continue;
+
+        const functionRef =
+          checkScope.analysis.isolatedEquivalentVariables[functionName];
+        if (typeof functionRef === 'string' && functionRef.endsWith('F')) {
+          callbackScopeName = functionRef.slice(0, -1);
+          break;
+        }
+      }
+    }
+
+    // Pattern 2: useMemo/useCallback with callback scope
+    // e.g., "useMemo(cyScope1(), [deps]).functionCallReturnValue"
+    if (!callbackScopeName) {
+      const useMemoMatch = equivalentValue.match(
+        /^useMemo\((\w+)\(\),\s*\[.*\]\)\.functionCallReturnValue$/,
+      );
+      if (useMemoMatch) {
+        callbackScopeName = useMemoMatch[1];
+      }
+    }
+
+    if (!callbackScopeName) return;
+
+    const callbackScope = this.scopeNodes[callbackScopeName];
+    if (!callbackScope) return;
+
+    // Look for returnValue.* sub-properties in the callback scope
+    if (!callbackScope.analysis?.isolatedEquivalentVariables) return;
+
+    const isolatedVars = callbackScope.analysis.isolatedEquivalentVariables;
+
+    // First, check if returnValue is an alias to another variable (e.g., returnValue = intermediate)
+    // If so, we need to look for that variable's sub-properties too
+    const returnValueAlias =
+      typeof isolatedVars.returnValue === 'string' &&
+      !isolatedVars.returnValue.includes('.')
+        ? isolatedVars.returnValue
+        : undefined;
+
+    // Pattern 3: Object.keys(X).reduce() - the reduce result has the same sub-properties as X
+    // When returnValue = "Object.keys(source).reduce(...).functionCallReturnValue", look for source.* sub-properties
+    let reduceSourceVar: string | undefined;
+    if (typeof isolatedVars.returnValue === 'string') {
+      const reduceMatch = isolatedVars.returnValue.match(
+        /^Object\.keys\((\w+)\)\.reduce\(.*\)\.functionCallReturnValue$/,
+      );
+      if (reduceMatch) {
+        reduceSourceVar = reduceMatch[1];
+      }
+    }
+
+    for (const [subPath, subValue] of Object.entries(isolatedVars)) {
+      // Check for direct returnValue.* sub-properties
+      const isReturnValueSub =
+        subPath.startsWith('returnValue.') ||
+        subPath.startsWith('returnValue[');
+
+      // Also check for alias.* sub-properties (e.g., intermediate.args when returnValue = intermediate)
+      const isAliasSub =
+        returnValueAlias &&
+        (subPath.startsWith(returnValueAlias + '.') ||
+          subPath.startsWith(returnValueAlias + '['));
+
+      // Also check for reduce source.* sub-properties (e.g., source['Function Arguments'] when returnValue = Object.keys(source).reduce())
+      const isReduceSourceSub =
+        reduceSourceVar &&
+        (subPath.startsWith(reduceSourceVar + '.') ||
+          subPath.startsWith(reduceSourceVar + '['));
+
+      if (
+        typeof subValue !== 'string' ||
+        (!isReturnValueSub && !isAliasSub && !isReduceSourceSub)
+      )
+        continue;
+
+      // Convert alias/reduceSource paths to returnValue paths
+      let effectiveSubPath = subPath;
+      if (isAliasSub && !isReturnValueSub) {
+        // Replace the alias prefix with returnValue
+        effectiveSubPath =
+          'returnValue' + subPath.substring(returnValueAlias!.length);
+      } else if (isReduceSourceSub && !isReturnValueSub && !isAliasSub) {
+        // Replace the reduce source prefix with returnValue
+        effectiveSubPath =
+          'returnValue' + subPath.substring(reduceSourceVar!.length);
+      }
+      const subPropertyPath = effectiveSubPath.substring('returnValue'.length);
+      const newPath = cleanPath(path + subPropertyPath, allPaths);
+      let newEquivalentValue = cleanPath(
+        subValue.replace(/::cyDuplicateKey\d+::/g, ''),
+        allPaths,
+      );
+
+      // Resolve variable references through parent scope equivalencies
+      const resolved = this.resolveVariableThroughParentScopes(
+        newEquivalentValue,
+        callbackScope,
+        allPaths,
+      );
+      newEquivalentValue = resolved.resolvedPath;
+      const equivalentScopeName = resolved.scopeName;
+
+      if (!newEquivalentValue || !this.isValidPath(newEquivalentValue))
+        continue;
+
+      this.addEquivalency(
+        newPath,
+        newEquivalentValue,
+        equivalentScopeName,
+        scopeNode,
+        'propagated function call return sub-property equivalency',
+      );
+
+      // Ensure the database entry has the usage path
+      this.addUsageToEquivalencyDatabaseEntry(
+        newPath,
+        newEquivalentValue,
+        equivalentScopeName,
+        scopeNode.name,
+      );
+    }
+  }
+
+  /**
+   * Resolves a variable path through parent scope equivalencies.
+   *
+   * For example, if the path is "dataStructure.arguments" and the parent scope has
+   * dataStructure -> signature[0].dataStructure, this returns:
+   *   { resolvedPath: "signature[0].dataStructure.arguments", scopeName: "ParentScope" }
+   */
+  private resolveVariableThroughParentScopes(
+    path: string | undefined,
+    callbackScope: ScopeNode,
+    allPaths: string[],
+  ): { resolvedPath: string | undefined; scopeName: string } {
+    if (!path)
+      return { resolvedPath: undefined, scopeName: callbackScope.name };
+
+    // Extract the root variable name
+    const dotIndex = path.indexOf('.');
+    const bracketIndex = path.indexOf('[');
+    let firstDelimiter = -1;
+    if (dotIndex > -1 && bracketIndex > -1) {
+      firstDelimiter = Math.min(dotIndex, bracketIndex);
+    } else {
+      firstDelimiter = dotIndex > -1 ? dotIndex : bracketIndex;
+    }
+
+    if (firstDelimiter === -1)
+      return { resolvedPath: path, scopeName: callbackScope.name };
+
+    const rootVar = path.substring(0, firstDelimiter);
+    const restOfPath = path.substring(firstDelimiter);
+
+    // Look in parent scopes for the root variable's equivalency
+    for (const parentScopeName of callbackScope.tree || []) {
+      const parentScope = this.scopeNodes[parentScopeName];
+      if (!parentScope?.analysis?.isolatedEquivalentVariables) continue;
+
+      const rootEquiv =
+        parentScope.analysis.isolatedEquivalentVariables[rootVar];
+      if (typeof rootEquiv === 'string') {
+        return {
+          resolvedPath: cleanPath(rootEquiv + restOfPath, allPaths),
+          scopeName: parentScopeName,
+        };
+      }
+    }
+
+    return { resolvedPath: path, scopeName: callbackScope.name };
+  }
+
+  /**
+   * Adds a usage path to the equivalency database entry that has the given source.
+   *
+   * The addEquivalency call creates the equivalency but doesn't always properly
+   * link the usage in the database, so we need to do it explicitly.
+   */
+  private addUsageToEquivalencyDatabaseEntry(
+    usagePath: string,
+    sourcePath: string,
+    sourceScopeName: string,
+    fallbackScopeName: string,
+  ) {
+    const usageEntry = {
+      scopeNodeName: usagePath.match(/^([^().]+)\(\)/)
+        ? (usagePath.match(/^([^().]+)\(\)/)?.[1] ?? fallbackScopeName)
+        : fallbackScopeName,
+      schemaPath: usagePath,
+    };
+
+    const sourceKey = `${sourceScopeName}::${sourcePath}`;
+    for (const entry of this.equivalencyDatabase) {
+      if (
+        entry.intermediatesOrder[sourceKey] === 0 ||
+        entry.sourceCandidates.some(
+          (sc) =>
+            sc.scopeNodeName === sourceScopeName &&
+            sc.schemaPath === sourcePath,
+        )
+      ) {
+        const usageExists = entry.usages.some(
+          (u) =>
+            u.scopeNodeName === usageEntry.scopeNodeName &&
+            u.schemaPath === usageEntry.schemaPath,
+        );
+        if (!usageExists) {
+          entry.usages.push(usageEntry);
+        }
+        break;
+      }
+    }
+  }
+
   private checkForArrayItemPath(
     pathParts: string[],
     scopeNode: ScopeNode,
@@ -1635,6 +2034,14 @@ export class ScopeDataStructure {
             usageEquivalency.scopeNodeName,
           ) as ScopeNode;
 
+          // Guard against infinite recursion by tracking which paths we've already
+          // added from addComplexSourcePathVariables
+          const complexPathKey = `${usageScopeNode.name}::${newUsageEquivalentPath}`;
+          if (this.complexSourceVisited.has(complexPathKey)) {
+            continue;
+          }
+          this.complexSourceVisited.add(complexPathKey);
+
           this.addToSchema({
             path: newUsageEquivalentPath,
             value: 'unknown',
@@ -1785,6 +2192,7 @@ export class ScopeDataStructure {
 
     for (let i = 0; i < cleanEquivalencyValueChain.length; ++i) {
       const pathInfo = cleanEquivalencyValueChain[i].currentPath;
+      const previousPath = cleanEquivalencyValueChain[i].previousPath;
 
       const schemaPathParts = this.splitPath(pathInfo.schemaPath);
       const isSignaturePath =
@@ -1798,6 +2206,21 @@ export class ScopeDataStructure {
         databaseEntry.usages.push(pathInfo);
       }
 
+      // Also add previousPath as a usage if it matches the criteria
+      // This handles propagated sub-property equivalencies where the JSX prop path
+      // is in previousPath and should be tracked as a usage
+      if (previousPath) {
+        const prevSchemaPathParts = this.splitPath(previousPath.schemaPath);
+        const isPrevSignaturePath =
+          prevSchemaPathParts.findIndex((p) => p.startsWith('signature[')) > 0;
+        if (
+          prevSchemaPathParts[0].startsWith('returnValue') ||
+          isPrevSignaturePath
+        ) {
+          databaseEntry.usages.push(previousPath);
+        }
+      }
+
       const pathId = this.uniqueId(pathInfo);
       let intermediateIndex = cleanEquivalencyValueChain.length - i - 1;
       const existingIntermediateIndex =

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/fillInSchemaGapsAndUnknowns.ts

@@ -408,17 +408,26 @@ function checkIfObjectOrFunction(path: string, schema: ScopeNode['schema']) {
   }
 }
 
-// Methods that are ambiguous - used on both arrays and non-arrays (e.g., router.push, history.pop)
+// Methods that are ambiguous - used on both arrays and non-arrays
+// These methods exist on arrays but also on Map, Set, Headers, FormData, URLSearchParams, etc.
 // Don't infer array type from these alone without additional evidence
-const AMBIGUOUS_ARRAY_METHODS = new Set(['push', 'pop']);
+const AMBIGUOUS_ARRAY_METHODS = new Set([
+  'push',
+  'pop',
+  'entries',
+  'keys',
+  'values',
+  'forEach',
+]);
 
 // Methods that are strongly indicative of arrays - only arrays have these
+// Note: entries, keys, values, forEach are NOT in this list because they also exist
+// on Map, Set, Headers, FormData, URLSearchParams, and other iterable types
 const STRONG_ARRAY_METHODS = new Set([
   'map',
   'filter',
   'reduce',
   'reduceRight',
-  'forEach',
   'every',
   'some',
   'find',
@@ -429,9 +438,6 @@ const STRONG_ARRAY_METHODS = new Set([
   'flatMap',
   'fill',
   'copyWithin',
-  'entries',
-  'keys',
-  'values',
   'toReversed',
   'toSorted',
   'toSpliced',

package/analyzer-template/packages/ai/src/lib/generateChangesEntityDocumentation.ts

@@ -25,8 +25,7 @@ export default async function generateChangesEntityDocumentation({
 
   const { completion: documentation, stats } = await completionCall({
     type: 'entityDocumentation',
-    systemMessage:
-      'You are an expert coder and educator. You think a lot about how people can understand new things, recognizing that it is a mixture of direct facts and numerous examples from different perspectives.',
+    systemMessage: SYSTEM_MESSAGE,
     prompt,
     model,
     jsonResponse: false,
@@ -62,3 +61,22 @@ export default async function generateChangesEntityDocumentation({
 
   return { documentation, llmCall };
 }
+
+const SYSTEM_MESSAGE = `You are a technical documentation writer updating existing documentation after code changes.
+
+## When to Update
+- If existing documentation is still accurate: respond with exactly "No changes needed"
+- If code changed significantly: provide updated documentation
+
+## Style
+- Be concise—use bullets, not paragraphs
+- Provide diverse examples (no repetitive variations)
+- In Overview, describe intent directly without mentioning the function name
+
+## Required Sections (when updating)
+Return documentation with exactly these markdown headers:
+- ## Overview
+- ## Inputs
+- ## Output
+- ## Examples
+`;