@codeyam/codeyam-cli 0.1.0-staging.2a88920 → 0.1.0-staging.4813bf3

package/packages/ai/src/lib/dataStructure/ScopeDataStructure.js

@@ -181,6 +181,7 @@ const SILENTLY_IGNORED_EQUIVALENCY_REASONS = new Set([
     'transformed non-object function equivalency - implicit parent equivalency - rerouted via useCallback',
     'transformed non-object function equivalency - Array.from() equivalency',
     'Spread operator equivalency key update: Explicit array deconstruction equivalency value',
+    // 'transformed non-object function equivalency - Explicit array deconstruction equivalency value',
 ]);
 export class ScopeDataStructure {
     // Getter for backward compatibility - returns the tree structure
@@ -475,6 +476,10 @@ export class ScopeDataStructure {
             }
             return;
         }
+        // PERF: Early exit for paths with repeated function-call signature patterns
+        if (this.hasExcessivePatternRepetition(path)) {
+            return;
+        }
         // Update chain metadata for database tracking
         if (equivalencyValueChain.length > 0) {
             equivalencyValueChain[equivalencyValueChain.length - 1].addToSchemaId =
@@ -971,6 +976,12 @@ export class ScopeDataStructure {
                     const value1 = scopeNode.schema[schemaPath];
                     const value2 = equivalentScopeNode.schema[equivalentSchemaPath];
                     const bestValue = selectBestValue(value1, value2);
+                    // PERF: Skip paths with repeated function-call signature patterns
+                    // to prevent recursive type expansion (e.g., string.localeCompare returns string)
+                    if (this.hasExcessivePatternRepetition(schemaPath) ||
+                        this.hasExcessivePatternRepetition(equivalentSchemaPath)) {
+                        continue;
+                    }
                     scopeNode.schema[schemaPath] = bestValue;
                     equivalentScopeNode.schema[equivalentSchemaPath] = bestValue;
                 }
@@ -982,6 +993,10 @@ export class ScopeDataStructure {
                         equivalentPath,
                         ...remainingSchemaPathParts,
                     ]);
+                    // PERF: Skip paths with repeated function-call signature patterns
+                    if (this.hasExcessivePatternRepetition(newEquivalentPath)) {
+                        continue;
+                    }
                     equivalentScopeNode.schema[newEquivalentPath] =
                         scopeNode.schema[schemaPath];
                 }
@@ -1063,6 +1078,23 @@ export class ScopeDataStructure {
                 return true;
             }
         }
+        // Check for repeated function calls that indicate recursive type expansion.
+        // E.g., localeCompare(b[])...localeCompare(b[]) means string.localeCompare
+        // returns a type that again has localeCompare, causing infinite expansion.
+        // We extract all function call patterns like "funcName(args)" and check if
+        // the same normalized call appears more than once.
+        const funcCallPattern = /(?:^|\.)[^.([]+\([^)]*\)/g;
+        const funcCallMatches = path.match(funcCallPattern);
+        if (funcCallMatches && funcCallMatches.length > 1) {
+            const seen = new Set();
+            for (const match of funcCallMatches) {
+                // Strip leading dot and normalize array indices
+                const normalized = match.replace(/^\./, '').replace(/\[\d+\]/g, '[]');
+                if (seen.has(normalized))
+                    return true;
+                seen.add(normalized);
+            }
+        }
         // For longer paths, detect any repeated multi-part segments we haven't explicitly listed
         const pathParts = this.splitPath(path);
         if (pathParts.length <= 6) {
@@ -1752,6 +1784,7 @@ export class ScopeDataStructure {
             const remainingPath = this.joinPathParts(remainingPathParts);
             if (relevantSubPathParts.every((part, i) => part === schemaPathParts[i]) &&
                 equivalentValue.scopeNodeName === scopeNode.name) {
+                // DEBUG
                 continue;
             }
             const newEquivalentPath = this.joinPathParts([schemaPath, remainingPath]);
@@ -1893,6 +1926,8 @@ export class ScopeDataStructure {
                         return;
                     }
                     const usageScopeNode = this.getScopeOrFunctionCallInfo(usageEquivalency.scopeNodeName);
+                    if (!usageScopeNode)
+                        continue;
                     // Guard against infinite recursion by tracking which paths we've already
                     // added from addComplexSourcePathVariables
                     if (this.visitedTracker.checkAndMarkComplexSourceVisited(usageScopeNode.name, newUsageEquivalentPath)) {
@@ -1941,6 +1976,8 @@ export class ScopeDataStructure {
                         continue;
                     }
                     const usageScopeNode = this.getScopeOrFunctionCallInfo(usageEquivalency.scopeNodeName);
+                    if (!usageScopeNode)
+                        continue;
                     // This is put in place to avoid propagating array functions like 'filter' through complex equivalencies
                     // but may cause problems if the funtion call is not on a known object (e.g. string or array)
                     if (newUsageEquivalentPath.endsWith(')') ||
@@ -2246,6 +2283,13 @@ export class ScopeDataStructure {
                 delete scopeNode.schema[key];
             }
         }
+        // Ensure parameter-to-signature equivalencies are fully propagated.
+        // When a parameter variable (e.g., `node`) is equivalenced to `signature[N]`,
+        // all sub-paths of that variable should also appear under `signature[N]`.
+        // This handles cases where the sub-path was added to the schema via a propagation
+        // chain that already included the variable↔signature equivalency, causing the
+        // cycle detection to prevent the reverse mapping.
+        this.propagateParameterToSignaturePaths(scopeNode);
         fillInSchemaGapsAndUnknowns(scopeNode, fillInUnknowns);
         if (final) {
             for (const manager of this.equivalencyManagers) {
@@ -2257,6 +2301,40 @@ export class ScopeDataStructure {
             ensureSchemaConsistency(scopeNode.schema);
         }
     }
+    /**
+     * For each equivalency where a simple variable maps to signature[N],
+     * ensure all sub-paths of that variable are reflected under signature[N].
+     */
+    propagateParameterToSignaturePaths(scopeNode) {
+        // Find variable → signature[N] equivalencies
+        for (const [varName, equivalencies] of Object.entries(scopeNode.equivalencies)) {
+            // Only process simple variable names (no dots, brackets, or parens)
+            if (varName.includes('.') ||
+                varName.includes('[') ||
+                varName.includes('(')) {
+                continue;
+            }
+            for (const equiv of equivalencies) {
+                if (equiv.scopeNodeName === scopeNode.name &&
+                    equiv.schemaPath.startsWith('signature[')) {
+                    const signaturePath = equiv.schemaPath;
+                    const varPrefix = varName + '.';
+                    const varBracketPrefix = varName + '[';
+                    // Find all schema keys starting with the variable
+                    for (const key in scopeNode.schema) {
+                        if (key.startsWith(varPrefix) || key.startsWith(varBracketPrefix)) {
+                            const suffix = key.slice(varName.length);
+                            const sigKey = signaturePath + suffix;
+                            // Only add if the signature path doesn't already exist
+                            if (!scopeNode.schema[sigKey]) {
+                                scopeNode.schema[sigKey] = scopeNode.schema[key];
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
     filterAndConvertSchema({ filterPath, newPath, schema, }) {
         const filterPathParts = this.splitPath(filterPath);
         return Object.keys(schema).reduce((acc, key) => {
@@ -2316,6 +2394,10 @@ export class ScopeDataStructure {
                             path,
                             ...this.splitPath(key).slice(equivalentValueSchemaPathParts.length),
                         ]);
+                        // PERF: Skip keys with repeated function-call signature patterns
+                        // to prevent recursive type expansion (e.g., string.localeCompare returns string)
+                        if (this.hasExcessivePatternRepetition(newKey))
+                            continue;
                         resolvedSchema[newKey] = value;
                     }
                 }
@@ -2337,6 +2419,9 @@ export class ScopeDataStructure {
                 if (!subSchema)
                     continue;
                 for (const resolvedKey in subSchema) {
+                    // PERF: Skip keys with repeated function-call signature patterns
+                    if (this.hasExcessivePatternRepetition(resolvedKey))
+                        continue;
                     if (!resolvedSchema[resolvedKey] ||
                         subSchema[resolvedKey] === 'unknown') {
                         resolvedSchema[resolvedKey] = subSchema[resolvedKey];
@@ -2623,21 +2708,91 @@ export class ScopeDataStructure {
             }
             return [source];
         };
-        return entries.reduce((acc, entry) => {
+        const acc = entries.reduce((result, entry) => {
             var _a;
             if (entry.sourceCandidates.length === 0)
-                return acc;
+                return result;
             const usages = entry.usages.filter(usageMatchesScope);
             for (const usage of usages) {
-                acc[_a = usage.schemaPath] || (acc[_a] = []);
+                result[_a = usage.schemaPath] || (result[_a] = []);
                 // Resolve each source candidate through the equivalency chain
                 for (const source of entry.sourceCandidates) {
                     const resolvedSources = resolveToSignature(source, new Set());
-                    acc[usage.schemaPath].push(...resolvedSources);
+                    result[usage.schemaPath].push(...resolvedSources);
                 }
             }
-            return acc;
+            return result;
         }, {});
+        // Post-processing: enrich useState-backed sources with co-located external
+        // function calls. When a useState value resolves to a setter variable that
+        // lives in the same scope as a fetch/API call, that fetch is a data source.
+        this.enrichUseStateSourcesWithCoLocatedCalls(acc);
+        return acc;
+    }
+    /**
+     * For each source that ends at a useState path, check if the setter was called
+     * from a scope that also contains external function calls (like fetch).
+     * If so, add those external calls as additional source candidates.
+     */
+    enrichUseStateSourcesWithCoLocatedCalls(acc) {
+        const rootScopeName = this.scopeTreeManager.getRootName();
+        const rootScope = this.scopeNodes[rootScopeName];
+        if (!rootScope)
+            return;
+        // Collect all descendants for each scope node
+        const getAllDescendants = (node) => {
+            const names = new Set([node.name]);
+            for (const child of node.children) {
+                for (const name of getAllDescendants(child)) {
+                    names.add(name);
+                }
+            }
+            return names;
+        };
+        for (const [usagePath, sources] of Object.entries(acc)) {
+            const additionalSources = [];
+            for (const source of sources) {
+                // Check if this source is a useState-related terminal path
+                // (e.g., useState(X).functionCallReturnValue[1] or useState(X).signature[0])
+                if (!source.schemaPath.match(/^useState\([^)]*\)\./))
+                    continue;
+                // Find the useState call from the source path
+                const useStateCallMatch = source.schemaPath.match(/^(useState\([^)]*\))\./);
+                if (!useStateCallMatch)
+                    continue;
+                const useStateCall = useStateCallMatch[1];
+                // Look in the root scope for the useState value equivalency
+                // which tells us where the setter was called from
+                const valuePath = `${useStateCall}.functionCallReturnValue[0]`;
+                const valueEquivs = rootScope.equivalencies[valuePath];
+                if (!valueEquivs)
+                    continue;
+                for (const equiv of valueEquivs) {
+                    // Find the scope where the setter was called
+                    const setterScopeName = equiv.scopeNodeName;
+                    const setterScopeTree = this.scopeTreeManager.findNode(setterScopeName);
+                    if (!setterScopeTree)
+                        continue;
+                    // Get all descendant scope names from the setter scope
+                    const relatedScopes = getAllDescendants(setterScopeTree);
+                    // Find external function calls in those scopes whose return values
+                    // are actually consumed (assigned to a variable). This excludes
+                    // fire-and-forget calls like analytics.track() or console.log().
+                    const coLocatedCalls = this.externalFunctionCalls.filter((efc) => relatedScopes.has(efc.callScope) &&
+                        efc.receivingVariableNames &&
+                        efc.receivingVariableNames.length > 0);
+                    for (const call of coLocatedCalls) {
+                        additionalSources.push({
+                            scopeNodeName: call.callScope,
+                            schemaPath: `${call.callSignature}.functionCallReturnValue`,
+                        });
+                    }
+                }
+            }
+            if (additionalSources.length > 0) {
+                acc[usagePath].push(...additionalSources);
+            }
+        }
     }
     getUsageEquivalencies(functionName) {
         const scopeNode = this.getScopeOrFunctionCallInfo(functionName);
@@ -2702,6 +2857,49 @@ export class ScopeDataStructure {
                 }
             }
         }
+        // Enrich schema with deeply nested paths from internal function call scopes.
+        // When a function call like traverse(tree) exists, and traverse's scope has
+        // signature[0].children[path][entityName] (from propagateParameterToSignaturePaths),
+        // we need to map those paths back to the argument variable (tree) in this scope.
+        // This handles cases where cycle detection prevented the equivalency chain from
+        // propagating deep paths during Phase 2 batch queue processing.
+        for (const equivalenceKey in equivalencies ?? {}) {
+            // Look for keys matching function call pattern: funcName(...).signature[N]
+            const funcCallMatch = equivalenceKey.match(/^([^(]+)\(.*?\)\.(signature\[\d+\])$/);
+            if (!funcCallMatch)
+                continue;
+            const calledFunctionName = funcCallMatch[1];
+            const signatureParam = funcCallMatch[2]; // e.g., "signature[0]"
+            for (const equivalenceValue of equivalencies[equivalenceKey]) {
+                if (equivalenceValue.scopeNodeName !== scopeName)
+                    continue;
+                const targetVariable = equivalenceValue.schemaPath;
+                // Get the called function's schema (includes propagated parameter paths)
+                const childSchema = this.getSchema({
+                    scopeName: calledFunctionName,
+                });
+                if (!childSchema)
+                    continue;
+                // Map child function's signature paths to parent variable paths
+                const sigPrefix = signatureParam + '.';
+                const sigBracketPrefix = signatureParam + '[';
+                for (const childKey in childSchema) {
+                    let suffix = null;
+                    if (childKey.startsWith(sigPrefix)) {
+                        suffix = childKey.slice(signatureParam.length);
+                    }
+                    else if (childKey.startsWith(sigBracketPrefix)) {
+                        suffix = childKey.slice(signatureParam.length);
+                    }
+                    if (suffix !== null) {
+                        const parentKey = targetVariable + suffix;
+                        if (!schema[parentKey]) {
+                            schema[parentKey] = childSchema[childKey];
+                        }
+                    }
+                }
+            }
+        }
         // Propagate nested paths from variables to their signature equivalents
         // e.g., if workouts = signature[0].workouts, then workouts[].title becomes
         // signature[0].workouts[].title
@@ -2991,6 +3189,13 @@ export class ScopeDataStructure {
                     !equivalentValue.schemaPath.startsWith('signature[') && // not a signature path
                     !equivalentValue.schemaPath.endsWith('.functionCallReturnValue') // not already handled above
                 ) {
+                    // Skip bare "returnValue" from child scopes — this is the child's return value,
+                    // not a meaningful data source path in the parent scope
+                    if (equivalentValue.schemaPath === 'returnValue' &&
+                        equivalentValue.scopeNodeName !==
+                            this.scopeTreeManager.getRootName()) {
+                        continue;
+                    }
                     // Add equivalency (will accumulate if multiple values for OR expressions)
                     addEquivalency(path, equivalentValue.schemaPath);
                 }
@@ -3550,6 +3755,7 @@ export class ScopeDataStructure {
      */
     getEnrichedConditionalUsages() {
         const enriched = {};
+        console.log(`[getEnrichedConditionalUsages] Processing ${Object.keys(this.rawConditionalUsages).length} conditional paths: [${Object.keys(this.rawConditionalUsages).join(', ')}]`);
         for (const [path, usages] of Object.entries(this.rawConditionalUsages)) {
             // Try to trace this path back to a data source
             // First, try the root scope
@@ -3557,9 +3763,47 @@ export class ScopeDataStructure {
             const explanation = this.explainPath(rootScopeName, path);
             let sourceDataPath;
             if (explanation.source) {
-                // Build the full data path: scopeName.path
-                sourceDataPath = `${explanation.source.scope}.${explanation.source.path}`;
+                const { scope, path: sourcePath } = explanation.source;
+                // Build initial path — avoid redundant prefix when path already contains the scope call
+                let fullPath;
+                if (sourcePath.startsWith(`${scope}(`)) {
+                    fullPath = sourcePath;
+                }
+                else {
+                    fullPath = `${scope}.${sourcePath}`;
+                }
+                sourceDataPath = fullPath;
+                console.log(`[getEnrichedConditionalUsages] "${path}" explainPath → scope="${scope}", sourcePath="${sourcePath}" → sourceDataPath="${sourceDataPath}"`);
             }
+            else {
+                console.log(`[getEnrichedConditionalUsages] "${path}" explainPath → no source found`);
+            }
+            // If explainPath didn't find a useful external source (e.g., it traced to
+            // useState or just to the component scope itself), check sourceEquivalencies
+            // for an external function call source like a fetch call
+            const hasExternalSource = sourceDataPath?.includes('.functionCallReturnValue');
+            if (!hasExternalSource) {
+                console.log(`[getEnrichedConditionalUsages] "${path}" no external source (sourceDataPath="${sourceDataPath}"), checking sourceEquivalencies fallback...`);
+                const sourceEquiv = this.getSourceEquivalencies();
+                const returnValueKey = `returnValue.${path}`;
+                const sources = sourceEquiv[returnValueKey];
+                if (sources) {
+                    console.log(`[getEnrichedConditionalUsages] "${path}" sourceEquivalencies["${returnValueKey}"] has ${sources.length} sources: [${sources.map((s) => s.schemaPath).join(', ')}]`);
+                    const externalSource = sources.find((s) => s.schemaPath.includes('.functionCallReturnValue') &&
+                        !s.schemaPath.startsWith('useState('));
+                    if (externalSource) {
+                        console.log(`[getEnrichedConditionalUsages] "${path}" sourceEquivalencies fallback found external source: "${externalSource.schemaPath}"`);
+                        sourceDataPath = externalSource.schemaPath;
+                    }
+                    else {
+                        console.log(`[getEnrichedConditionalUsages] "${path}" sourceEquivalencies fallback found no external function call source`);
+                    }
+                }
+                else {
+                    console.log(`[getEnrichedConditionalUsages] "${path}" sourceEquivalencies["${returnValueKey}"] not found`);
+                }
+            }
+            console.log(`[getEnrichedConditionalUsages] "${path}" FINAL sourceDataPath="${sourceDataPath ?? '(none)'}" (${usages.length} usages)`);
             enriched[path] = usages.map((usage) => ({
                 ...usage,
                 sourceDataPath,
@@ -3818,11 +4062,21 @@ export class ScopeDataStructure {
                     perVariableSchemas = undefined;
                 }
             }
+            // Enrich the schema with inferred types by applying fillInSchemaGapsAndUnknowns.
+            // This ensures the serialized schema has the same type inference as getReturnValue().
+            // Without this, evidence like "entities[].analyses: array" becomes "unknown".
+            const enrichedSchema = { ...efc.schema };
+            const tempScopeNode = {
+                name: efc.name,
+                schema: enrichedSchema,
+                equivalencies: efc.equivalencies ?? {},
+            };
+            fillInSchemaGapsAndUnknowns(tempScopeNode, true);
             return {
                 name: efc.name,
                 callSignature: efc.callSignature,
                 callScope: efc.callScope,
-                schema: efc.schema,
+                schema: enrichedSchema,
                 equivalencies: efc.equivalencies
                     ? Object.entries(efc.equivalencies).reduce((acc, [key, vars]) => {
                         // Clean cyScope from the key as well as variable properties