@codeyam/codeyam-cli 0.1.0-staging.2a88920 → 0.1.0-staging.4684848

package/analyzer-template/packages/ai/src/lib/dataStructure/ScopeDataStructure.ts

@@ -397,6 +397,7 @@ const SILENTLY_IGNORED_EQUIVALENCY_REASONS = new Set([
   'transformed non-object function equivalency - implicit parent equivalency - rerouted via useCallback',
   'transformed non-object function equivalency - Array.from() equivalency',
   'Spread operator equivalency key update: Explicit array deconstruction equivalency value',
+  // 'transformed non-object function equivalency - Explicit array deconstruction equivalency value',
 ]);
 
 export class ScopeDataStructure {
@@ -799,6 +800,11 @@ export class ScopeDataStructure {
       return;
     }
 
+    // PERF: Early exit for paths with repeated function-call signature patterns
+    if (this.hasExcessivePatternRepetition(path)) {
+      return;
+    }
+
     // Update chain metadata for database tracking
     if (equivalencyValueChain.length > 0) {
       equivalencyValueChain[equivalencyValueChain.length - 1].addToSchemaId =
@@ -1486,6 +1492,15 @@ export class ScopeDataStructure {
 
           const bestValue = selectBestValue(value1, value2);
 
+          // PERF: Skip paths with repeated function-call signature patterns
+          // to prevent recursive type expansion (e.g., string.localeCompare returns string)
+          if (
+            this.hasExcessivePatternRepetition(schemaPath) ||
+            this.hasExcessivePatternRepetition(equivalentSchemaPath)
+          ) {
+            continue;
+          }
+
           scopeNode.schema[schemaPath] = bestValue;
           equivalentScopeNode.schema[equivalentSchemaPath] = bestValue;
         } else if (
@@ -1499,6 +1514,11 @@ export class ScopeDataStructure {
             ...remainingSchemaPathParts,
           ]);
 
+          // PERF: Skip paths with repeated function-call signature patterns
+          if (this.hasExcessivePatternRepetition(newEquivalentPath)) {
+            continue;
+          }
+
           equivalentScopeNode.schema[newEquivalentPath] =
             scopeNode.schema[schemaPath];
         }
@@ -1618,6 +1638,23 @@ export class ScopeDataStructure {
       }
     }
 
+    // Check for repeated function calls that indicate recursive type expansion.
+    // E.g., localeCompare(b[])...localeCompare(b[]) means string.localeCompare
+    // returns a type that again has localeCompare, causing infinite expansion.
+    // We extract all function call patterns like "funcName(args)" and check if
+    // the same normalized call appears more than once.
+    const funcCallPattern = /(?:^|\.)[^.([]+\([^)]*\)/g;
+    const funcCallMatches = path.match(funcCallPattern);
+    if (funcCallMatches && funcCallMatches.length > 1) {
+      const seen = new Set<string>();
+      for (const match of funcCallMatches) {
+        // Strip leading dot and normalize array indices
+        const normalized = match.replace(/^\./, '').replace(/\[\d+\]/g, '[]');
+        if (seen.has(normalized)) return true;
+        seen.add(normalized);
+      }
+    }
+
     // For longer paths, detect any repeated multi-part segments we haven't explicitly listed
     const pathParts = this.splitPath(path);
     if (pathParts.length <= 6) {
@@ -2574,6 +2611,7 @@ export class ScopeDataStructure {
         relevantSubPathParts.every((part, i) => part === schemaPathParts[i]) &&
         equivalentValue.scopeNodeName === scopeNode.name
       ) {
+        // DEBUG
         continue;
       }
 
@@ -2760,6 +2798,8 @@ export class ScopeDataStructure {
             usageEquivalency.scopeNodeName,
           ) as ScopeNode;
 
+          if (!usageScopeNode) continue;
+
           // Guard against infinite recursion by tracking which paths we've already
           // added from addComplexSourcePathVariables
           if (
@@ -2839,6 +2879,8 @@ export class ScopeDataStructure {
             usageEquivalency.scopeNodeName,
           ) as ScopeNode;
 
+          if (!usageScopeNode) continue;
+
           // This is put in place to avoid propagating array functions like 'filter' through complex equivalencies
           // but may cause problems if the funtion call is not on a known object (e.g. string or array)
           if (
@@ -3291,6 +3333,14 @@ export class ScopeDataStructure {
       }
     }
 
+    // Ensure parameter-to-signature equivalencies are fully propagated.
+    // When a parameter variable (e.g., `node`) is equivalenced to `signature[N]`,
+    // all sub-paths of that variable should also appear under `signature[N]`.
+    // This handles cases where the sub-path was added to the schema via a propagation
+    // chain that already included the variable↔signature equivalency, causing the
+    // cycle detection to prevent the reverse mapping.
+    this.propagateParameterToSignaturePaths(scopeNode);
+
     fillInSchemaGapsAndUnknowns(scopeNode, fillInUnknowns);
 
     if (final) {
@@ -3305,6 +3355,50 @@ export class ScopeDataStructure {
     }
   }
 
+  /**
+   * For each equivalency where a simple variable maps to signature[N],
+   * ensure all sub-paths of that variable are reflected under signature[N].
+   */
+  private propagateParameterToSignaturePaths(scopeNode: ScopeNode) {
+    // Find variable → signature[N] equivalencies
+    for (const [varName, equivalencies] of Object.entries(
+      scopeNode.equivalencies,
+    )) {
+      // Only process simple variable names (no dots, brackets, or parens)
+      if (
+        varName.includes('.') ||
+        varName.includes('[') ||
+        varName.includes('(')
+      ) {
+        continue;
+      }
+
+      for (const equiv of equivalencies) {
+        if (
+          equiv.scopeNodeName === scopeNode.name &&
+          equiv.schemaPath.startsWith('signature[')
+        ) {
+          const signaturePath = equiv.schemaPath;
+          const varPrefix = varName + '.';
+          const varBracketPrefix = varName + '[';
+
+          // Find all schema keys starting with the variable
+          for (const key in scopeNode.schema) {
+            if (key.startsWith(varPrefix) || key.startsWith(varBracketPrefix)) {
+              const suffix = key.slice(varName.length);
+              const sigKey = signaturePath + suffix;
+
+              // Only add if the signature path doesn't already exist
+              if (!scopeNode.schema[sigKey]) {
+                scopeNode.schema[sigKey] = scopeNode.schema[key];
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
   private filterAndConvertSchema({
     filterPath,
     newPath,
@@ -3391,6 +3485,9 @@ export class ScopeDataStructure {
                 equivalentValueSchemaPathParts.length,
               ),
             ]);
+            // PERF: Skip keys with repeated function-call signature patterns
+            // to prevent recursive type expansion (e.g., string.localeCompare returns string)
+            if (this.hasExcessivePatternRepetition(newKey)) continue;
             resolvedSchema[newKey] = value;
           }
         }
@@ -3413,6 +3510,8 @@ export class ScopeDataStructure {
         if (!subSchema) continue;
 
         for (const resolvedKey in subSchema) {
+          // PERF: Skip keys with repeated function-call signature patterns
+          if (this.hasExcessivePatternRepetition(resolvedKey)) continue;
           if (
             !resolvedSchema[resolvedKey] ||
             subSchema[resolvedKey] === 'unknown'
@@ -3804,25 +3903,116 @@ export class ScopeDataStructure {
       return [source];
     };
 
-    return entries.reduce(
-      (acc, entry) => {
-        if (entry.sourceCandidates.length === 0) return acc;
+    const acc = entries.reduce(
+      (result, entry) => {
+        if (entry.sourceCandidates.length === 0) return result;
         const usages = entry.usages.filter(usageMatchesScope);
         for (const usage of usages) {
-          acc[usage.schemaPath] ||= [];
+          result[usage.schemaPath] ||= [];
           // Resolve each source candidate through the equivalency chain
           for (const source of entry.sourceCandidates) {
             const resolvedSources = resolveToSignature(source, new Set());
-            acc[usage.schemaPath].push(...resolvedSources);
+            result[usage.schemaPath].push(...resolvedSources);
           }
         }
-        return acc;
+        return result;
       },
       {} as Record<
         string,
         Pick<ScopeVariable, 'scopeNodeName' | 'schemaPath'>[]
       >,
     );
+
+    // Post-processing: enrich useState-backed sources with co-located external
+    // function calls. When a useState value resolves to a setter variable that
+    // lives in the same scope as a fetch/API call, that fetch is a data source.
+    this.enrichUseStateSourcesWithCoLocatedCalls(acc);
+
+    return acc;
+  }
+
+  /**
+   * For each source that ends at a useState path, check if the setter was called
+   * from a scope that also contains external function calls (like fetch).
+   * If so, add those external calls as additional source candidates.
+   */
+  private enrichUseStateSourcesWithCoLocatedCalls(
+    acc: Record<string, Pick<ScopeVariable, 'scopeNodeName' | 'schemaPath'>[]>,
+  ) {
+    const rootScopeName = this.scopeTreeManager.getRootName();
+    const rootScope = this.scopeNodes[rootScopeName];
+    if (!rootScope) return;
+
+    // Collect all descendants for each scope node
+    const getAllDescendants = (
+      node: import('./helpers/ScopeTreeManager').ScopeTreeNode,
+    ): Set<string> => {
+      const names = new Set<string>([node.name]);
+      for (const child of node.children) {
+        for (const name of getAllDescendants(child)) {
+          names.add(name);
+        }
+      }
+      return names;
+    };
+
+    for (const [usagePath, sources] of Object.entries(acc)) {
+      const additionalSources: Pick<
+        ScopeVariable,
+        'scopeNodeName' | 'schemaPath'
+      >[] = [];
+
+      for (const source of sources) {
+        // Check if this source is a useState-related terminal path
+        // (e.g., useState(X).functionCallReturnValue[1] or useState(X).signature[0])
+        if (!source.schemaPath.match(/^useState\([^)]*\)\./)) continue;
+
+        // Find the useState call from the source path
+        const useStateCallMatch = source.schemaPath.match(
+          /^(useState\([^)]*\))\./,
+        );
+        if (!useStateCallMatch) continue;
+        const useStateCall = useStateCallMatch[1];
+
+        // Look in the root scope for the useState value equivalency
+        // which tells us where the setter was called from
+        const valuePath = `${useStateCall}.functionCallReturnValue[0]`;
+        const valueEquivs = rootScope.equivalencies[valuePath];
+        if (!valueEquivs) continue;
+
+        for (const equiv of valueEquivs) {
+          // Find the scope where the setter was called
+          const setterScopeName = equiv.scopeNodeName;
+          const setterScopeTree =
+            this.scopeTreeManager.findNode(setterScopeName);
+          if (!setterScopeTree) continue;
+
+          // Get all descendant scope names from the setter scope
+          const relatedScopes = getAllDescendants(setterScopeTree);
+
+          // Find external function calls in those scopes whose return values
+          // are actually consumed (assigned to a variable). This excludes
+          // fire-and-forget calls like analytics.track() or console.log().
+          const coLocatedCalls = this.externalFunctionCalls.filter(
+            (efc) =>
+              relatedScopes.has(efc.callScope) &&
+              efc.receivingVariableNames &&
+              efc.receivingVariableNames.length > 0,
+          );
+
+          for (const call of coLocatedCalls) {
+            additionalSources.push({
+              scopeNodeName: call.callScope,
+              schemaPath: `${call.callSignature}.functionCallReturnValue`,
+            });
+          }
+        }
+      }
+
+      if (additionalSources.length > 0) {
+        acc[usagePath].push(...additionalSources);
+      }
+    }
   }
 
   getUsageEquivalencies(functionName?: string) {
@@ -3927,6 +4117,54 @@ export class ScopeDataStructure {
       }
     }
 
+    // Enrich schema with deeply nested paths from internal function call scopes.
+    // When a function call like traverse(tree) exists, and traverse's scope has
+    // signature[0].children[path][entityName] (from propagateParameterToSignaturePaths),
+    // we need to map those paths back to the argument variable (tree) in this scope.
+    // This handles cases where cycle detection prevented the equivalency chain from
+    // propagating deep paths during Phase 2 batch queue processing.
+    for (const equivalenceKey in equivalencies ?? {}) {
+      // Look for keys matching function call pattern: funcName(...).signature[N]
+      const funcCallMatch = equivalenceKey.match(
+        /^([^(]+)\(.*?\)\.(signature\[\d+\])$/,
+      );
+      if (!funcCallMatch) continue;
+
+      const calledFunctionName = funcCallMatch[1];
+      const signatureParam = funcCallMatch[2]; // e.g., "signature[0]"
+
+      for (const equivalenceValue of equivalencies[equivalenceKey]) {
+        if (equivalenceValue.scopeNodeName !== scopeName) continue;
+
+        const targetVariable = equivalenceValue.schemaPath;
+
+        // Get the called function's schema (includes propagated parameter paths)
+        const childSchema = this.getSchema({
+          scopeName: calledFunctionName,
+        });
+        if (!childSchema) continue;
+
+        // Map child function's signature paths to parent variable paths
+        const sigPrefix = signatureParam + '.';
+        const sigBracketPrefix = signatureParam + '[';
+        for (const childKey in childSchema) {
+          let suffix: string | null = null;
+          if (childKey.startsWith(sigPrefix)) {
+            suffix = childKey.slice(signatureParam.length);
+          } else if (childKey.startsWith(sigBracketPrefix)) {
+            suffix = childKey.slice(signatureParam.length);
+          }
+
+          if (suffix !== null) {
+            const parentKey = targetVariable + suffix;
+            if (!schema[parentKey]) {
+              schema[parentKey] = childSchema[childKey];
+            }
+          }
+        }
+      }
+    }
+
     // Propagate nested paths from variables to their signature equivalents
     // e.g., if workouts = signature[0].workouts, then workouts[].title becomes
     // signature[0].workouts[].title
@@ -4281,6 +4519,15 @@ export class ScopeDataStructure {
           !equivalentValue.schemaPath.startsWith('signature[') && // not a signature path
           !equivalentValue.schemaPath.endsWith('.functionCallReturnValue') // not already handled above
         ) {
+          // Skip bare "returnValue" from child scopes — this is the child's return value,
+          // not a meaningful data source path in the parent scope
+          if (
+            equivalentValue.schemaPath === 'returnValue' &&
+            equivalentValue.scopeNodeName !==
+              this.scopeTreeManager.getRootName()
+          ) {
+            continue;
+          }
           // Add equivalency (will accumulate if multiple values for OR expressions)
           addEquivalency(path, equivalentValue.schemaPath);
         }
@@ -4977,6 +5224,10 @@ export class ScopeDataStructure {
   getEnrichedConditionalUsages(): Record<string, EnrichedConditionalUsage[]> {
     const enriched: Record<string, EnrichedConditionalUsage[]> = {};
 
+    console.log(
+      `[getEnrichedConditionalUsages] Processing ${Object.keys(this.rawConditionalUsages).length} conditional paths: [${Object.keys(this.rawConditionalUsages).join(', ')}]`,
+    );
+
     for (const [path, usages] of Object.entries(this.rawConditionalUsages)) {
       // Try to trace this path back to a data source
       // First, try the root scope
@@ -4985,10 +5236,69 @@ export class ScopeDataStructure {
 
       let sourceDataPath: string | undefined;
       if (explanation.source) {
-        // Build the full data path: scopeName.path
-        sourceDataPath = `${explanation.source.scope}.${explanation.source.path}`;
+        const { scope, path: sourcePath } = explanation.source;
+
+        // Build initial path — avoid redundant prefix when path already contains the scope call
+        let fullPath: string;
+        if (sourcePath.startsWith(`${scope}(`)) {
+          fullPath = sourcePath;
+        } else {
+          fullPath = `${scope}.${sourcePath}`;
+        }
+
+        sourceDataPath = fullPath;
+        console.log(
+          `[getEnrichedConditionalUsages] "${path}" explainPath → scope="${scope}", sourcePath="${sourcePath}" → sourceDataPath="${sourceDataPath}"`,
+        );
+      } else {
+        console.log(
+          `[getEnrichedConditionalUsages] "${path}" explainPath → no source found`,
+        );
       }
 
+      // If explainPath didn't find a useful external source (e.g., it traced to
+      // useState or just to the component scope itself), check sourceEquivalencies
+      // for an external function call source like a fetch call
+      const hasExternalSource = sourceDataPath?.includes(
+        '.functionCallReturnValue',
+      );
+      if (!hasExternalSource) {
+        console.log(
+          `[getEnrichedConditionalUsages] "${path}" no external source (sourceDataPath="${sourceDataPath}"), checking sourceEquivalencies fallback...`,
+        );
+        const sourceEquiv = this.getSourceEquivalencies();
+        const returnValueKey = `returnValue.${path}`;
+        const sources = sourceEquiv[returnValueKey];
+        if (sources) {
+          console.log(
+            `[getEnrichedConditionalUsages] "${path}" sourceEquivalencies["${returnValueKey}"] has ${sources.length} sources: [${sources.map((s: { schemaPath: string }) => s.schemaPath).join(', ')}]`,
+          );
+          const externalSource = sources.find(
+            (s: { schemaPath: string }) =>
+              s.schemaPath.includes('.functionCallReturnValue') &&
+              !s.schemaPath.startsWith('useState('),
+          );
+          if (externalSource) {
+            console.log(
+              `[getEnrichedConditionalUsages] "${path}" sourceEquivalencies fallback found external source: "${externalSource.schemaPath}"`,
+            );
+            sourceDataPath = externalSource.schemaPath;
+          } else {
+            console.log(
+              `[getEnrichedConditionalUsages] "${path}" sourceEquivalencies fallback found no external function call source`,
+            );
+          }
+        } else {
+          console.log(
+            `[getEnrichedConditionalUsages] "${path}" sourceEquivalencies["${returnValueKey}"] not found`,
+          );
+        }
+      }
+
+      console.log(
+        `[getEnrichedConditionalUsages] "${path}" FINAL sourceDataPath="${sourceDataPath ?? '(none)'}" (${usages.length} usages)`,
+      );
+
       enriched[path] = usages.map((usage) => ({
         ...usage,
         sourceDataPath,
@@ -5336,11 +5646,22 @@ export class ScopeDataStructure {
           }
         }
 
+        // Enrich the schema with inferred types by applying fillInSchemaGapsAndUnknowns.
+        // This ensures the serialized schema has the same type inference as getReturnValue().
+        // Without this, evidence like "entities[].analyses: array" becomes "unknown".
+        const enrichedSchema = { ...efc.schema };
+        const tempScopeNode = {
+          name: efc.name,
+          schema: enrichedSchema,
+          equivalencies: efc.equivalencies ?? {},
+        };
+        fillInSchemaGapsAndUnknowns(tempScopeNode, true);
+
         return {
           name: efc.name,
           callSignature: efc.callSignature,
           callScope: efc.callScope,
-          schema: efc.schema,
+          schema: enrichedSchema,
           equivalencies: efc.equivalencies
             ? Object.entries(efc.equivalencies).reduce(
                 (acc, [key, vars]) => {

package/analyzer-template/packages/ai/src/lib/dataStructure/equivalencyManagers/frameworks/JavascriptFrameworkManager.ts

@@ -72,7 +72,11 @@ export class JavascriptFrameworkManager implements EquivalencyManager {
   }
 
   finalize(scopeNode: ScopeNode, scopeDataStructure: ScopeDataStructure) {
-    cleanNonObjectFunctions(scopeNode, scopeDataStructure, true);
+    // Don't call cleanNonObjectFunctions with final=true here.
+    // The aggressive cleanup (transformKeyMapping, clearAttributes) removes method call
+    // evidence like .includes() before type inference can use it.
+    // Instead, defer final cleanup to toSerializable() when we're done with inference.
+    cleanNonObjectFunctions(scopeNode, scopeDataStructure, false);
     cleanKnownObjectFunctions(scopeNode, scopeDataStructure);
     this.handleUniformArrays(scopeNode, scopeDataStructure);
   }

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/cleanKnownObjectFunctions.ts

@@ -172,9 +172,18 @@ function scrub(
       const rawMethod = rawToken.split('(')[0];
 
       // Check if it's a prototype method (e.g., map, filter, toString)
+      // Only match when the token is actually a method reference/call:
+      // - Has parentheses (explicit call like map(), filter(x => x))
+      // - OR is typed as "function" (method reference like .split)
+      // Bare property names that happen to match method names
+      // (like .entries = "array", .values = "object") are preserved.
       if (protoMethodSets[rootKind].has(rawMethod)) {
-        onRemove(keyPath);
-        break;
+        const isMethodCall = rawToken.includes('(');
+        const isMethodReference = mapping[keyPath] === 'function';
+        if (isMethodCall || isMethodReference) {
+          onRemove(keyPath);
+          break;
+        }
       }
 
       // Check if it's a built-in property (e.g., length on arrays/strings)

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/cleanNonObjectFunctions.ts

@@ -510,7 +510,7 @@ function removeNonTransformingArrayMethods(variablePath: string): string {
     const part = keyParts[i];
 
     const arrayMethod = getMethodName(part);
-    if (isValidArrayMethod(arrayMethod)) {
+    if (isValidFunctionCall(part) && isValidArrayMethod(arrayMethod)) {
       const nextPart = keyParts[i + 1];
       if (
         !nonTransformingArrayMethodsSet.has(arrayMethod) ||
@@ -550,7 +550,7 @@ function removeIsolatingArrayMethods(
     const part = keyParts[i];
 
     const arrayMethod = getMethodName(part);
-    if (isValidArrayMethod(arrayMethod)) {
+    if (isValidFunctionCall(part) && isValidArrayMethod(arrayMethod)) {
       const nextPart = keyParts[i + 1];
       const prevPart = i > 1 ? keyParts[i - 1] : undefined;
       if (

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/coerceObjectsToPrimitivesBySchema.ts

@@ -0,0 +1,70 @@
+/**
+ * Coerces object/array values to primitive types when the schema expects a primitive.
+ *
+ * The LLM sometimes generates an object or array where the schema expects a primitive
+ * type like "string", "number", or "boolean". For example:
+ *   Schema: { body: "string" }
+ *   LLM output: { body: { "env": "production" } }
+ *
+ * This causes runtime errors like `TypeError: body.match is not a function` because
+ * the code expects a string but gets an object.
+ *
+ * Coercion rules:
+ * - object/array → string: JSON.stringify
+ * - object/array → number: 0
+ * - object/array → boolean: false
+ *
+ * @param data The mock data object to coerce (mutated in place)
+ * @param schema The schema describing expected types
+ */
+export default function coerceObjectsToPrimitivesBySchema<T>(
+  data: T,
+  schema: unknown,
+): T {
+  if (data === null || data === undefined) {
+    return data;
+  }
+
+  if (typeof data !== 'object') {
+    return data;
+  }
+
+  if (Array.isArray(data)) {
+    const itemSchema = Array.isArray(schema) ? schema[0] : undefined;
+    for (let i = 0; i < data.length; i++) {
+      data[i] = coerceObjectsToPrimitivesBySchema(data[i], itemSchema);
+    }
+    return data;
+  }
+
+  const dataObj = data as Record<string, unknown>;
+  const schemaObj = schema as Record<string, unknown> | undefined;
+
+  for (const key of Object.keys(dataObj)) {
+    const value = dataObj[key];
+    const fieldSchema = schemaObj?.[key];
+
+    if (value === null || value === undefined) {
+      continue;
+    }
+
+    if (typeof value === 'object' && typeof fieldSchema === 'string') {
+      // Schema expects a primitive but the value is an object/array.
+      // Coerce to the expected type.
+      const schemaType = fieldSchema.toLowerCase();
+
+      if (schemaType.includes('string')) {
+        dataObj[key] = JSON.stringify(value);
+      } else if (schemaType.includes('number')) {
+        dataObj[key] = 0;
+      } else if (schemaType.includes('boolean')) {
+        dataObj[key] = false;
+      }
+    } else if (typeof value === 'object' && value !== null) {
+      // Both value and schema are objects — recurse
+      coerceObjectsToPrimitivesBySchema(value, fieldSchema);
+    }
+  }
+
+  return data;
+}