@codeyam/codeyam-cli 0.1.20 → 0.1.22

package/packages/analyze/src/lib/files/scenarios/mergeInDependentDataStructure.js

@@ -25,6 +25,18 @@ function getTypeParameter(functionName) {
     }
     return null;
 }
+/**
+ * Check if a schema path contains a Set/Map collection method call.
+ * Paths like `.has(articleId)`, `.delete(articleId)`, `.add(articleId)` represent
+ * membership checks on Sets/Maps, not meaningful data flow for schema generation.
+ * These create massive combinatorial explosions when every filter field (filterRatings,
+ * filterPublications, filterAuthors, etc.) × every method (has, delete, add) gets
+ * tracked as a separate equivalency.
+ */
+const COLLECTION_METHOD_PATTERN = /\.(?:has|delete|add|clear|get|set)\(/;
+function isCollectionMethodPath(path) {
+    return COLLECTION_METHOD_PATTERN.test(path);
+}
 // Primitive types that should not have child paths
 const PRIMITIVE_TYPES = new Set([
     'number',
@@ -107,7 +119,27 @@ function bestValueFromOptions(options) {
     }
     return options[0] ?? 'unknown';
 }
-export default function mergeInDependentDataStructure({ importedExports, dependentAnalyses, rootScopeName, dataStructure, dependencySchemas, }) {
+/** Timeout (ms) for the merge operation. Throws DataStructureTimeoutError if exceeded.
+ *  All successful merges complete in <300ms. Anything exceeding 2s is pathological. */
+const MERGE_TIMEOUT_MS = 2000;
+export class DataStructureTimeoutError extends Error {
+    constructor(entityName, elapsedMs) {
+        super(`Data structure merge timed out for ${entityName} after ${Math.round(elapsedMs / 1000)}s (limit: ${MERGE_TIMEOUT_MS / 1000}s)`);
+        this.name = 'DataStructureTimeoutError';
+    }
+}
+export default function mergeInDependentDataStructure({ importedExports, dependentAnalyses, rootScopeName, dataStructure, dependencySchemas, timeoutMs = MERGE_TIMEOUT_MS, }) {
+    const mergeStartTime = Date.now();
+    const mergeDeadline = timeoutMs > 0 ? mergeStartTime + timeoutMs : 0;
+    /** Call in hot loops. Throws DataStructureTimeoutError if deadline exceeded.
+     *  Date.now() is ~20ns — negligible vs the ms-scale string ops in each iteration. */
+    const checkDeadline = () => {
+        if (!mergeDeadline)
+            return;
+        if (Date.now() > mergeDeadline) {
+            throw new DataStructureTimeoutError(rootScopeName, Date.now() - mergeStartTime);
+        }
+    };
     const mergedDataStructure = {
         signatureSchema: { ...(dataStructure.signatureSchema ?? {}) },
         returnValueSchema: { ...(dataStructure.returnValueSchema ?? {}) },
@@ -205,7 +237,16 @@ export default function mergeInDependentDataStructure({ importedExports, depende
     const cleanSchema = (schema, context) => {
         transformationTracer.traceSchemaTransform(rootScopeName, 'cleanKnownObjectFunctionsFromMapping', schema, cleanKnownObjectFunctionsFromMapping, context);
     };
+    // Cache translatePath results — the same path is often translated multiple times
+    // (once per equivalency entry that references it). Avoids redundant
+    // splitOutsideParenthesesAndArrays calls on long paths.
+    const translatePathCache = new Map();
     const translatePath = (path, dependencyName) => {
+        const cacheKey = `${dependencyName}\0${path}`;
+        const cached = translatePathCache.get(cacheKey);
+        if (cached !== undefined)
+            return cached;
+        let result = path;
         if (path.startsWith(dependencyName)) {
             const pathParts = splitOutsideParenthesesAndArrays(path);
             if (pathParts.length > 1) {
@@ -213,34 +254,27 @@ export default function mergeInDependentDataStructure({ importedExports, depende
                     // Check if this function has multiple DIFFERENT type parameters.
                     // If so, DON'T normalize to returnValue - keep the full path to avoid
                     // merging different type-parameterized variants together.
-                    // e.g., useFetcher<{ data: UserData }>().functionCallReturnValue.data
-                    // should NOT be merged with useFetcher<{ data: ConfigData }>().functionCallReturnValue.data
                     const baseName = cleanFunctionName(pathParts[0]);
-                    if (functionsWithMultipleTypeParams.has(baseName)) {
-                        return path; // Keep the original path with type parameters
+                    if (!functionsWithMultipleTypeParams.has(baseName)) {
+                        // functionCallReturnValue immediately follows - normalize to returnValue
+                        result = joinParenthesesAndArrays([
+                            'returnValue',
+                            ...pathParts.slice(2),
+                        ]);
                     }
-                    // functionCallReturnValue immediately follows - normalize to returnValue
-                    // e.g., useAuth().functionCallReturnValue.user -> returnValue.user
-                    return joinParenthesesAndArrays([
-                        'returnValue',
-                        ...pathParts.slice(2),
-                    ]);
                 }
                 else if (pathParts[0].endsWith(')') &&
                     pathParts[1].startsWith('signature[')) {
-                    // Hook-style with signature access (e.g., BranchChangesTab().signature[0]...)
-                    // Strip the function name for signature equivalency matching
-                    return joinParenthesesAndArrays(pathParts.slice(1));
+                    // Hook-style with signature access
+                    result = joinParenthesesAndArrays(pathParts.slice(1));
                 }
-                // For all other cases (object-style APIs like getSupabase().auth and
-                // direct object references like supabase.from), preserve the path as-is.
-                // The prefix must be kept for proper schema lookups in constructMockCode
-                // and gatherDataForMocks.
             }
         }
-        return path;
+        translatePathCache.set(cacheKey, result);
+        return result;
     };
     const gatherAllEquivalentSchemaPaths = (functionName, sourceAndUsageEquivalencies, dataStructure) => {
+        checkDeadline();
         if (!sourceAndUsageEquivalencies)
             return;
         const normalizedSchemaCache = new Map();
@@ -252,6 +286,7 @@ export default function mergeInDependentDataStructure({ importedExports, depende
                 return cached;
             const byFirstPart = new Map();
             for (const path in schema) {
+                checkDeadline();
                 let parts = splitOutsideParenthesesAndArrays(path);
                 if (parts[0].startsWith(functionName)) {
                     const baseName = cleanFunctionName(parts[0]);
@@ -300,6 +335,7 @@ export default function mergeInDependentDataStructure({ importedExports, depende
             // Use espIndex Map for O(1) lookup instead of O(E) linear search.
             // Falls back to linear search only when Map hit has a signature index conflict.
             for (const pathInfo of allPaths) {
+                checkDeadline();
                 if (equivalentSchemaPathsEntry)
                     break;
                 const candidate = espIndex.get(espIndexKey(pathInfo.path, pathInfo.functionName));
@@ -420,18 +456,40 @@ export default function mergeInDependentDataStructure({ importedExports, depende
         for (const equivalencies of allEquivalencies) {
             const schemaPathEntries = Object.entries(equivalencies);
             for (const [schemaPath, usages] of schemaPathEntries) {
+                checkDeadline();
+                // Skip equivalency entries whose source path is a Set/Map membership operation.
+                // Patterns like `.has(articleId)`, `.delete(articleId)`, `.add(articleId)` on
+                // Sets/Maps represent membership checks, not meaningful data flow for schema generation.
+                // In the Margo LibraryPage case, these account for 74% of all equivalency targets
+                // (19,444 of 26,340) and cause a combinatorial explosion in the merge.
+                if (isCollectionMethodPath(schemaPath))
+                    continue;
                 // First, check if the raw schemaPath starts with a function call to a dependency.
                 // If so, use that dependency name for translation (so translatePath can strip the prefix).
                 const extractedFuncName = extractFunctionNameFromPath(schemaPath);
                 const effectiveFunctionName = extractedFuncName || functionName;
                 const translatedPath = translatePath(schemaPath, effectiveFunctionName);
-                const allPaths = [
+                const allPathsRaw = [
                     { path: translatedPath, functionName: effectiveFunctionName },
-                    ...usages.map((u) => ({
+                    ...usages
+                        .filter((u) => !isCollectionMethodPath(u.schemaPath))
+                        .map((u) => ({
                         path: translatePath(u.schemaPath, u.scopeNodeName),
                         functionName: u.scopeNodeName,
                     })),
                 ].filter((pathInfo) => !pathInfo.path.includes('.map('));
+                // Deduplicate by translated path + function name.
+                // Multiple call variants (e.g., loadView(viewKey(null,null)) vs loadView(viewKey(newTag,newCol)))
+                // translate to the same path after stripping arguments. Processing duplicates
+                // creates O(n²) work in the schema matching loops below.
+                const seenPathKeys = new Set();
+                const allPaths = allPathsRaw.filter((p) => {
+                    const key = `${p.functionName ?? ''}::${p.path}`;
+                    if (seenPathKeys.has(key))
+                        return false;
+                    seenPathKeys.add(key);
+                    return true;
+                });
                 // Fix 38: Derive base paths from property access paths.
                 // When we have equivalent paths like:
                 //   Parent: signature[0].scenarios[].name
@@ -454,6 +512,7 @@ export default function mergeInDependentDataStructure({ importedExports, depende
                 const derivedBasePathSet = new Set();
                 // For each child path, find its equivalent parent path and derive bases
                 for (const childPathInfo of childPaths) {
+                    checkDeadline();
                     const childParts = splitOutsideParenthesesAndArrays(childPathInfo.path);
                     // Look for a parent path that shares a common suffix with this child path
                     for (const parentPathInfo of parentPaths) {
@@ -545,6 +604,7 @@ export default function mergeInDependentDataStructure({ importedExports, depende
                     });
                 }
                 for (const equivalentRoot of entry.equivalentRoots) {
+                    checkDeadline();
                     const dataStructures = equivalentRoot.function &&
                         equivalentRoot.function.name !== rootScopeName
                         ? [
@@ -582,6 +642,7 @@ export default function mergeInDependentDataStructure({ importedExports, depende
                             : lookupPart;
                         const candidates = schemaIndex.byFirstPart.get(lookupBase) || [];
                         for (const { path: schemaPath, parts: schemaPathParts, } of candidates) {
+                            checkDeadline();
                             if (schemaPathParts.length < pathParts.length)
                                 continue;
                             // Check if all path parts match (allowing function call variants)
@@ -676,6 +737,7 @@ export default function mergeInDependentDataStructure({ importedExports, depende
             // where we want both X().functionCallReturnValue and Y().functionCallReturnValue as bases
             const allBasePaths = new Set();
             for (const path of Object.keys(dataStructure.returnValueSchema)) {
+                checkDeadline();
                 const parts = splitOutsideParenthesesAndArrays(path);
                 // Find all positions of functionCallReturnValue and create base paths for each
                 for (let i = 0; i < parts.length; i++) {
@@ -702,6 +764,7 @@ export default function mergeInDependentDataStructure({ importedExports, depende
                 espIndex.set(espIndexKey(newRoot.schemaRootPath, newRootFuncName), entry);
                 const basePathParts = splitOutsideParenthesesAndArrays(basePath);
                 for (const schemaPath in dataStructure.returnValueSchema) {
+                    checkDeadline();
                     const schemaPathParts = splitOutsideParenthesesAndArrays(schemaPath);
                     if (schemaPathParts.length < basePathParts.length)
                         continue;
@@ -734,6 +797,7 @@ export default function mergeInDependentDataStructure({ importedExports, depende
         // We do this before the main merge to ensure the connection happens regardless
         // of processing order.
         for (const esp of equivalentSchemaPaths) {
+            checkDeadline();
             for (const root of esp.equivalentRoots) {
                 if (root.schemaRootPath.endsWith('[]')) {
                     // Find a matching parent entry with the base array path
@@ -746,6 +810,7 @@ export default function mergeInDependentDataStructure({ importedExports, depende
                         // Add transformed postfixes from child (array element) to parent (array)
                         // so they can be applied with [] prefix to parent paths
                         for (const [postfixPath, postfixValue] of Object.entries(esp.equivalentPostfixes)) {
+                            checkDeadline();
                             const transformedPostfix = joinParenthesesAndArrays(['[]', postfixPath].filter(Boolean));
                             if (!(transformedPostfix in parentEntry.equivalentPostfixes)) {
                                 parentEntry.equivalentPostfixes[transformedPostfix] =
@@ -798,6 +863,7 @@ export default function mergeInDependentDataStructure({ importedExports, depende
         const sortedEquivalentSchemaPaths = equivalentSchemaPaths.sort((a, b) => Math.max(...a.equivalentRoots.map((er) => splitOutsideParenthesesAndArrays(er.schemaRootPath).length)) -
             Math.max(...b.equivalentRoots.map((er) => splitOutsideParenthesesAndArrays(er.schemaRootPath).length)));
         for (const esp of sortedEquivalentSchemaPaths) {
+            checkDeadline();
             if (esp.equivalentRoots.length === 0)
                 continue;
             let bestCandidateLength;
@@ -857,6 +923,7 @@ export default function mergeInDependentDataStructure({ importedExports, depende
     // dependencySchemas contains usage information (how dependencies are called),
     // not internal implementation, so we want this for mocked dependencies too
     for (const dependency of importedExports) {
+        checkDeadline();
         const dependentDataStructure = dependencySchemas?.[dependency.filePath]?.[dependency.name];
         if (!dependentDataStructure)
             continue;
@@ -872,15 +939,28 @@ export default function mergeInDependentDataStructure({ importedExports, depende
             gatherAllEquivalentSchemaPaths(name, childMergedDataStructure);
         }
     }
+    const gatherElapsed = Date.now() - mergeStartTime;
     equivalentSchemaPaths = mergeAllEquivalentSchemaPaths();
+    const mergeEspElapsed = Date.now() - mergeStartTime;
     // Collect schemas that need cleaning — batch the calls for the end instead of
     // calling cleanSchema inside the inner root loop (which was O(roots * schemaSize)).
     const schemasToClean = new Set();
     for (const esp of equivalentSchemaPaths) {
+        checkDeadline();
         // Pre-compute which postfixes have children to avoid O(n²) lookups in the inner loop.
         // A postfix "has children" if there are other postfixes that extend it.
         const postfixesWithChildren = new Set();
         const postfixKeys = Object.keys(esp.equivalentPostfixes);
+        // Pre-parse ALL postfix paths once. These parsed parts are reused in:
+        // 1. The children detection loop below
+        // 2. The inner postfix application loop (lines that split postfixPath and equivalentRoot.postfix)
+        // This eliminates thousands of redundant splitOutsideParenthesesAndArrays calls.
+        const postfixPartsCache = new Map();
+        for (const postfixPath of postfixKeys) {
+            if (!postfixPath)
+                continue;
+            postfixPartsCache.set(postfixPath, splitOutsideParenthesesAndArrays(postfixPath));
+        }
         // Check for empty postfix having children (any other postfixes exist)
         if (postfixKeys.length > 1 && '' in esp.equivalentPostfixes) {
             postfixesWithChildren.add('');
@@ -892,7 +972,7 @@ export default function mergeInDependentDataStructure({ importedExports, depende
         for (const postfixPath of postfixKeys) {
             if (!postfixPath)
                 continue;
-            const parts = splitOutsideParenthesesAndArrays(postfixPath);
+            const parts = postfixPartsCache.get(postfixPath);
             for (let i = 1; i < parts.length; i++) {
                 postfixPrefixSet.add(joinParenthesesAndArrays(parts.slice(0, i)));
             }
@@ -912,7 +992,12 @@ export default function mergeInDependentDataStructure({ importedExports, depende
             seenRootKeys.add(key);
             return true;
         });
+        // Cap schema size to prevent combinatorial explosion.
+        // Successful merges produce <3K ret keys. Beyond 5K, further postfixes
+        // add noise but no useful data — they're cross-products of unrelated equivalencies.
+        const SCHEMA_KEY_CAP = 5000;
         for (const equivalentRoot of uniqueRoots) {
+            checkDeadline();
             let merged;
             if (equivalentRoot.function) {
                 merged = findOrCreateDependentSchemas(equivalentRoot.function);
@@ -925,7 +1010,11 @@ export default function mergeInDependentDataStructure({ importedExports, depende
             const schema = equivalentRoot.schemaRootPath.startsWith('signature[')
                 ? merged.signatureSchema
                 : merged.returnValueSchema;
+            // Skip if this schema has already grown past the cap
+            if (Object.keys(schema).length > SCHEMA_KEY_CAP)
+                continue;
             for (const [postfixPath, postfixValue] of Object.entries(esp.equivalentPostfixes)) {
+                checkDeadline();
                 let relevantPostfix = postfixPath;
                 if (equivalentRoot.postfix) {
                     // Check if postfixPath starts with equivalentRoot.postfix at a path boundary.
@@ -943,8 +1032,13 @@ export default function mergeInDependentDataStructure({ importedExports, depende
                         // This means "entity" matched "entityCode" which is wrong - they're different properties.
                         continue;
                     }
-                    const postFixPathParts = splitOutsideParenthesesAndArrays(postfixPath);
-                    const equivalentRootPostFixParts = splitOutsideParenthesesAndArrays(equivalentRoot.postfix);
+                    const postFixPathParts = postfixPartsCache.get(postfixPath) ??
+                        splitOutsideParenthesesAndArrays(postfixPath);
+                    // Cache equivalentRoot.postfix parts — same root reused across all postfixes
+                    if (!postfixPartsCache.has(equivalentRoot.postfix)) {
+                        postfixPartsCache.set(equivalentRoot.postfix, splitOutsideParenthesesAndArrays(equivalentRoot.postfix));
+                    }
+                    const equivalentRootPostFixParts = postfixPartsCache.get(equivalentRoot.postfix);
                     relevantPostfix = joinParenthesesAndArrays(postFixPathParts.slice(equivalentRootPostFixParts.length));
                 }
                 const newSchemaPath = joinParenthesesAndArrays([
@@ -1028,10 +1122,12 @@ export default function mergeInDependentDataStructure({ importedExports, depende
             schemasToClean.add(schema);
         }
     }
+    const postfixElapsed = Date.now() - mergeStartTime;
     // Batch-clean all modified schemas once (instead of once per root per ESP entry)
     for (const schema of schemasToClean) {
         cleanSchema(schema, { stage: 'afterMergePostfix' });
     }
+    const cleanElapsed = Date.now() - mergeStartTime;
     // Propagate equivalency-derived attributes to generic function call variants.
     // When attributes are traced via equivalencies (e.g., fileComparisons from buildDataMap.signature[2]),
     // they get written to non-generic paths (returnValue.data.x or funcName().functionCallReturnValue.data.x).
@@ -1050,6 +1146,7 @@ export default function mergeInDependentDataStructure({ importedExports, depende
             const genericVariants = new Set();
             const genericRegex = new RegExp(`^${depName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}<[^>]+>\\(\\)`);
             for (const path in schemaToSearchForGenericVariants) {
+                checkDeadline();
                 const match = path.match(genericRegex);
                 if (match) {
                     genericVariants.add(match[0]);
@@ -1061,6 +1158,7 @@ export default function mergeInDependentDataStructure({ importedExports, depende
             // create corresponding paths for each generic variant
             const pathsToAdd = [];
             for (const path in returnValueSchema) {
+                checkDeadline();
                 const value = returnValueSchema[path];
                 // Handle returnValue. paths
                 if (path.startsWith('returnValue.')) {
@@ -1107,6 +1205,7 @@ export default function mergeInDependentDataStructure({ importedExports, depende
         // This includes both returnValue. (dot) and returnValue[ (array) paths.
         const pathsToNormalize = [];
         for (const path in depSchema.returnValueSchema) {
+            checkDeadline();
             if (path === 'returnValue' ||
                 path.startsWith('returnValue.') ||
                 path.startsWith('returnValue[')) {
@@ -1140,6 +1239,7 @@ export default function mergeInDependentDataStructure({ importedExports, depende
         }
         // Now copy paths from the source schema (dependencySchemas)
         for (const path in srcSchema.returnValueSchema) {
+            checkDeadline();
             const value = srcSchema.returnValueSchema[path];
             // Normalize paths starting with 'returnValue' to use the standard format:
             // 'returnValue.foo' -> 'dependencyName().functionCallReturnValue.foo'
@@ -1300,9 +1400,11 @@ export default function mergeInDependentDataStructure({ importedExports, depende
             if (!existingSchema) {
                 const depSchema = findOrCreateDependentSchemas({ filePath, name });
                 for (const path in srcSchema.returnValueSchema) {
+                    checkDeadline();
                     depSchema.returnValueSchema[path] = srcSchema.returnValueSchema[path];
                 }
                 for (const path in srcSchema.signatureSchema) {
+                    checkDeadline();
                     depSchema.signatureSchema[path] = srcSchema.signatureSchema[path];
                 }
                 // Clean known object functions (like String.prototype.replace, Array.prototype.map)
@@ -1328,6 +1430,7 @@ export default function mergeInDependentDataStructure({ importedExports, depende
             const childSignatureSchema = childAnalysis?.metadata?.mergedDataStructure?.signatureSchema;
             if (childSignatureSchema) {
                 for (const path in depSchema.signatureSchema) {
+                    checkDeadline();
                     const parentType = depSchema.signatureSchema[path];
                     const childType = childSignatureSchema[path];
                     if (parentType && childType) {
@@ -1367,6 +1470,7 @@ export default function mergeInDependentDataStructure({ importedExports, depende
                     });
                     // Copy only paths that belong to this variant
                     for (const path in srcSchema.returnValueSchema) {
+                        checkDeadline();
                         if (path.startsWith(variant)) {
                             variantSchema.returnValueSchema[path] =
                                 srcSchema.returnValueSchema[path];
@@ -1388,6 +1492,7 @@ export default function mergeInDependentDataStructure({ importedExports, depende
     // EXCEPT: Skip mocked dependencies - we don't want their internal implementation details.
     for (const filePath in dependentAnalyses) {
         for (const name in dependentAnalyses[filePath]) {
+            checkDeadline();
             const dependentMergedDataStructure = dependentAnalyses[filePath][name].metadata?.mergedDataStructure;
             if (!dependentMergedDataStructure)
                 continue;
@@ -1400,6 +1505,7 @@ export default function mergeInDependentDataStructure({ importedExports, depende
                 // Copy over all paths from the dependent's returnValueSchema
                 // Only add paths that don't already exist (don't overwrite values set by equivalencies)
                 for (const path in dependentMergedDataStructure.returnValueSchema) {
+                    checkDeadline();
                     const translatedPath = translatePath(path, name);
                     if (!(translatedPath in depSchema.returnValueSchema)) {
                         depSchema.returnValueSchema[translatedPath] =
@@ -1408,6 +1514,7 @@ export default function mergeInDependentDataStructure({ importedExports, depende
                 }
                 // Copy over signature schema as well
                 for (const path in dependentMergedDataStructure.signatureSchema) {
+                    checkDeadline();
                     const translatedPath = translatePath(path, name);
                     if (!(translatedPath in depSchema.signatureSchema)) {
                         depSchema.signatureSchema[translatedPath] =
@@ -1427,12 +1534,14 @@ export default function mergeInDependentDataStructure({ importedExports, depende
                         });
                         // Merge in the nested dependency schemas
                         for (const path in nestedDepSchema.returnValueSchema) {
+                            checkDeadline();
                             if (!(path in targetDepSchema.returnValueSchema)) {
                                 const value = nestedDepSchema.returnValueSchema[path];
                                 targetDepSchema.returnValueSchema[path] = value;
                             }
                         }
                         for (const path in nestedDepSchema.signatureSchema) {
+                            checkDeadline();
                             if (!(path in targetDepSchema.signatureSchema)) {
                                 targetDepSchema.signatureSchema[path] =
                                     nestedDepSchema.signatureSchema[path];
@@ -1443,6 +1552,12 @@ export default function mergeInDependentDataStructure({ importedExports, depende
             }
         }
     }
+    const totalElapsed = Date.now() - mergeStartTime;
+    const retKeys = Object.keys(mergedDataStructure.returnValueSchema).length;
+    // Only log phase breakdown for slow merges (>2s)
+    if (totalElapsed > 2000) {
+        console.log(`CodeYam Log Level 2: ${rootScopeName} merge phases: gather=${gatherElapsed}ms mergeESP=${mergeEspElapsed - gatherElapsed}ms postfix=${postfixElapsed - mergeEspElapsed}ms clean=${cleanElapsed - postfixElapsed}ms depCopy=${totalElapsed - cleanElapsed}ms total=${totalElapsed}ms ret=${retKeys}`);
+    }
     return mergedDataStructure;
 }
 //# sourceMappingURL=mergeInDependentDataStructure.js.map