@codeyam/codeyam-cli 0.1.0-staging.dd216e0 → 0.1.0-staging.df25827

package/analyzer-template/packages/analyze/src/lib/files/scenarios/generateDataStructure.ts

@@ -9,12 +9,17 @@ import {
   fillInDirectSchemaGapsAndUnknowns,
   deduplicateFunctionSchemas,
   clearAttributesFromMapping,
+  mergeJsonTypeDefinitions,
 } from '~codeyam/ai';
-import mergeInDependentDataStructure from './mergeInDependentDataStructure';
-import { awsLog } from '~codeyam/utils';
+import mergeInDependentDataStructure, {
+  DataStructureTimeoutError,
+} from './mergeInDependentDataStructure';
+import { awsLog, awsLogDebugLevel } from '~codeyam/utils';
 import gatherDataForMocks from './gatherDataForMocks';
 import enrichArrayTypesFromChildSignatures from './enrichArrayTypesFromChildSignatures';
-
+import enrichUnknownTypesFromSourceEquivalencies from './enrichUnknownTypesFromSourceEquivalencies';
+import { transformationTracer } from './TransformationTracer';
+// import propagateArrayItemSchemas from './propagateArrayItemSchemas';
 export interface GenerateDataStructureArgs {
   entity: Entity;
   dependentAnalyses: ReadonlyAnalysisMap;
@@ -27,7 +32,9 @@ export default function generateDataStructure({
   analysis,
 }: GenerateDataStructureArgs) {
   try {
-    if (!['visual', 'library'].includes(entity.entityType)) return analysis;
+    if (!['visual', 'library'].includes(entity.entityType)) {
+      return analysis;
+    }
 
     awsLog('CodeYam: Generating data structure for', {
       filePath: entity.filePath,
@@ -159,6 +166,25 @@ export default function generateDataStructure({
         return acc;
       }, {} as ReadonlyAnalysisMap);
 
+    // Include same-file child components in nonMockedDependentAnalyses.
+    // Same-file components (e.g., AgentCard defined in the same file as AgentTranscriptsPage)
+    // are NOT in importedExports, so the reduce above never includes them.
+    // But they ARE in dependentAnalyses and their full signatureSchema (with all prop fields)
+    // needs to be merged into the parent's dependencySchemas for array enrichment to work.
+
+    const sameFileAnalyses = dependentAnalyses[entity.filePath];
+    if (sameFileAnalyses) {
+      for (const name in sameFileAnalyses) {
+        if (name === entity.name) continue;
+        if (nonMockedDependentAnalyses[entity.filePath]?.[name]) continue;
+        const childAnalysis = sameFileAnalyses[name];
+        if (childAnalysis?.entityType === 'visual') {
+          nonMockedDependentAnalyses[entity.filePath] ||= {};
+          nonMockedDependentAnalyses[entity.filePath][name] = childAnalysis;
+        }
+      }
+    }
+
     // For non-mocked child components (like visual components), include their mocked
     // dependencies in allImportedExports. This ensures that when a parent component
     // renders a child that uses the same hook with different destructure patterns,
@@ -174,6 +200,14 @@ export default function generateDataStructure({
     // child components (e.g., QuotesTableCompact inside AdminDashboard inside
     // DashboardPage) also have mocked dependencies that need mock data keys.
 
+    // Collect child dependency schemas to merge after mergeInDependentDataStructure
+    // This ensures that when we look up schemas for child imports in gatherDataForMocks,
+    // the child's schemas are available in mergedDataStructure.dependencySchemas.
+    type DependencySchemas = NonNullable<
+      Analysis['metadata']['mergedDataStructure']
+    >['dependencySchemas'];
+    const childDependencySchemas: DependencySchemas = {};
+
     // Helper function to add mocked imports from an entity to allImportedExports
     const addMockedImportsFromEntity = (childEntity: Entity | undefined) => {
       if (!childEntity?.metadata) return;
@@ -244,8 +278,10 @@ export default function generateDataStructure({
           );
 
           if (existingImport) {
-            // Merge callVariableNames if the same dependency exists with different patterns
-            if (imported.callVariableNames && imported.calls) {
+            // Merge calls from child into existing import
+            // Note: Check length > 0 because empty array [] is truthy but has no variable names
+            if (imported.callVariableNames?.length && imported.calls) {
+              // Case 1: Both calls and callVariableNames are set - merge as pairs
               existingImport.calls = existingImport.calls || [];
               existingImport.callVariableNames =
                 existingImport.callVariableNames || [];
@@ -262,6 +298,18 @@ export default function generateDataStructure({
                   existingImport.callVariableNames.push(varName);
                 }
               }
+            } else if (imported.calls) {
+              // Case 2: Only calls are set (no callVariableNames) - common with tRPC
+              // where destructuring like `const { data, isLoading } = trpc.xxx.useQuery()`
+              // doesn't create a variable name for the hook call itself
+              existingImport.calls = existingImport.calls || [];
+
+              // Add each call that doesn't already exist
+              for (const call of imported.calls) {
+                if (!existingImport.calls.includes(call)) {
+                  existingImport.calls.push(call);
+                }
+              }
             }
           } else {
             // Add new import
@@ -282,12 +330,23 @@ export default function generateDataStructure({
 
     // Recursive function to traverse the dependency tree
     const processedAnalyses = new Set<string>();
+    const traverseStart = Date.now();
+    const TRAVERSE_TIMEOUT_MS = 2_000;
+    let traverseCount = 0;
     const traverseDependencyTree = (analyses: ReadonlyAnalysisMap) => {
       for (const filePath in analyses) {
         for (const name in analyses[filePath]) {
           const analysisKey = `${filePath}:${name}`;
           if (processedAnalyses.has(analysisKey)) continue;
           processedAnalyses.add(analysisKey);
+          traverseCount++;
+
+          if (Date.now() - traverseStart > TRAVERSE_TIMEOUT_MS) {
+            throw new DataStructureTimeoutError(
+              entity.name,
+              Date.now() - traverseStart,
+            );
+          }
 
           const childAnalysis = analyses[filePath][name];
           const childEntity = childAnalysis?.entity;
@@ -299,6 +358,71 @@ export default function generateDataStructure({
           // via their mergedDataStructure's nested dependentAnalyses
           const childMergedDataStructure =
             childAnalysis?.metadata?.mergedDataStructure;
+
+          // Collect the child's dependency schemas for merging later.
+          // This ensures schemas for child imports are available in gatherDataForMocks.
+          if (childMergedDataStructure?.dependencySchemas) {
+            for (const schemaFilePath in childMergedDataStructure.dependencySchemas) {
+              for (const schemaName in childMergedDataStructure
+                .dependencySchemas[schemaFilePath]) {
+                childDependencySchemas[schemaFilePath] ||= {};
+                const existingSchema =
+                  childDependencySchemas[schemaFilePath][schemaName];
+                const newSchema =
+                  childMergedDataStructure.dependencySchemas[schemaFilePath][
+                    schemaName
+                  ];
+
+                if (existingSchema) {
+                  // Merge schemas if the same dependency exists from multiple children
+                  childDependencySchemas[schemaFilePath][schemaName] = {
+                    signatureSchema: {
+                      ...existingSchema.signatureSchema,
+                      ...newSchema.signatureSchema,
+                    },
+                    returnValueSchema: mergeJsonTypeDefinitions(
+                      existingSchema.returnValueSchema as Record<
+                        string,
+                        unknown
+                      >,
+                      newSchema.returnValueSchema as Record<string, unknown>,
+                    ) as Record<string, string>,
+                  };
+                } else {
+                  childDependencySchemas[schemaFilePath][schemaName] = {
+                    signatureSchema: { ...newSchema.signatureSchema },
+                    returnValueSchema: { ...newSchema.returnValueSchema },
+                  };
+                }
+              }
+            }
+          }
+
+          // Also collect the child's OWN signatureSchema for enrichment.
+          // This allows enrichArrayTypesFromChildSignatures to find what fields
+          // non-mocked child components expect on their props (e.g., SurveyCard
+          // expecting survey.createdAt, survey.updatedAt, etc.)
+          if (childMergedDataStructure?.signatureSchema) {
+            childDependencySchemas[filePath] ||= {};
+            const existingSchema = childDependencySchemas[filePath][name];
+            if (existingSchema) {
+              childDependencySchemas[filePath][name] = {
+                signatureSchema: {
+                  ...existingSchema.signatureSchema,
+                  ...childMergedDataStructure.signatureSchema,
+                },
+                returnValueSchema: existingSchema.returnValueSchema,
+              };
+            } else {
+              childDependencySchemas[filePath][name] = {
+                signatureSchema: {
+                  ...childMergedDataStructure.signatureSchema,
+                },
+                returnValueSchema: {},
+              };
+            }
+          }
+
           if (childMergedDataStructure?.dependencySchemas) {
             // We need to find non-mocked child components in the dependency tree
             // These would be in the child's importedExports that are NOT mocked
@@ -332,6 +456,25 @@ export default function generateDataStructure({
     // Start traversal from direct non-mocked children
     traverseDependencyTree(nonMockedDependentAnalyses);
 
+    const traverseElapsed = Date.now() - traverseStart;
+    const childSchemaKeyCount = Object.values(childDependencySchemas).reduce(
+      (sum, fileMap) =>
+        sum +
+        Object.values(fileMap).reduce(
+          (s, schema) =>
+            s +
+            Object.keys(schema.signatureSchema || {}).length +
+            Object.keys(schema.returnValueSchema || {}).length,
+          0,
+        ),
+      0,
+    );
+    awsLogDebugLevel(
+      1,
+      `Dependency tree traversal for ${entity.name}: ${traverseCount} analyses, ` +
+        `${childSchemaKeyCount} child schema keys, ${allImportedExports.length} imports, ${traverseElapsed}ms`,
+    );
+
     // console.info(
     //   'CODEYAM DEBUG: start merge',
     //   JSON.stringify(
@@ -350,6 +493,37 @@ export default function generateDataStructure({
 
     const { isolatedDataStructure } = entity.metadata;
 
+    if (!isolatedDataStructure) {
+      awsLog(
+        'CodeYam: Skipping data structure generation — isolatedDataStructure is undefined (scope analysis may have failed)',
+        {
+          entityName: entity.name,
+          filePath: entity.filePath,
+        },
+      );
+      return analysis;
+    }
+
+    // Transformation tracing: start entity and capture isolated data structure
+    transformationTracer.startEntity({
+      name: entity.name,
+      entityType: entity.entityType,
+      filePath: entity.filePath,
+    });
+    transformationTracer.snapshot(entity.name, 'isolated', {
+      signatureSchema: isolatedDataStructure.signatureSchema,
+      returnValueSchema: isolatedDataStructure.returnValueSchema,
+      dependencySchemas: isolatedDataStructure.dependencySchemas,
+    });
+
+    const mergeStart = Date.now();
+    // Shared deadline for all phases from merge onward
+    const genDeadline = mergeStart + 5_000; // 5s total budget for all phases
+    awsLogDebugLevel(
+      1,
+      `Starting mergeInDependentDataStructure for ${entity.name}...`,
+    );
+
     const mergedDataStructure = mergeInDependentDataStructure({
       importedExports: allImportedExports,
       dependentAnalyses: nonMockedDependentAnalyses,
@@ -364,9 +538,91 @@ export default function generateDataStructure({
       dependencySchemas: isolatedDataStructure?.dependencySchemas || {},
     });
 
+    const mergeTimedOut = !!(mergedDataStructure as any).timedOut;
+    awsLogDebugLevel(
+      1,
+      `mergeInDependentDataStructure ${mergeTimedOut ? 'PARTIAL' : 'complete'} for ${entity.name}: ` +
+        `${Object.keys(mergedDataStructure.signatureSchema).length} sig, ` +
+        `${Object.keys(mergedDataStructure.returnValueSchema).length} ret, ${Date.now() - mergeStart}ms`,
+    );
+
     mergedDataStructure.environmentVariables =
       isolatedDataStructure.environmentVariables || [];
 
+    // Transformation tracing: capture merged data structure
+    transformationTracer.snapshot(entity.name, 'merged', {
+      signatureSchema: mergedDataStructure.signatureSchema,
+      returnValueSchema: mergedDataStructure.returnValueSchema,
+      dependencySchemas: mergedDataStructure.dependencySchemas,
+    });
+
+    // Merge collected child dependency schemas into mergedDataStructure.
+    // This ensures that when gatherDataForMocks looks up schemas for child imports,
+    // the child's schemas are available.
+    const childMergeStart = Date.now();
+    for (const schemaFilePath in childDependencySchemas) {
+      for (const schemaName in childDependencySchemas[schemaFilePath]) {
+        mergedDataStructure.dependencySchemas[schemaFilePath] ||= {};
+        const existingSchema =
+          mergedDataStructure.dependencySchemas[schemaFilePath][schemaName];
+        const childSchema = childDependencySchemas[schemaFilePath][schemaName];
+
+        if (existingSchema) {
+          // Merge schemas - child schemas add to parent schemas
+          // But prefer more specific types (without | undefined) when both exist
+          const mergedSignatureSchema = { ...existingSchema.signatureSchema };
+          for (const path in childSchema.signatureSchema) {
+            const existingType = mergedSignatureSchema[path];
+            const childType = childSchema.signatureSchema[path];
+            if (!existingType) {
+              mergedSignatureSchema[path] = childType;
+            } else {
+              // Prefer the more specific (narrower) type
+              const existingIsOptional =
+                existingType.includes('| undefined') ||
+                existingType.includes('| null');
+              const childIsOptional =
+                childType.includes('| undefined') ||
+                childType.includes('| null');
+              if (childIsOptional && !existingIsOptional) {
+                // Keep existing - it's more specific
+              } else if (!childIsOptional && existingIsOptional) {
+                // Use child - it's more specific
+                mergedSignatureSchema[path] = childType;
+              } else {
+                // Same specificity - use child (original behavior)
+                mergedSignatureSchema[path] = childType;
+              }
+            }
+          }
+          mergedDataStructure.dependencySchemas[schemaFilePath][schemaName] = {
+            signatureSchema: mergedSignatureSchema,
+            returnValueSchema: mergeJsonTypeDefinitions(
+              existingSchema.returnValueSchema as Record<string, unknown>,
+              childSchema.returnValueSchema as Record<string, unknown>,
+            ) as Record<string, string>,
+          };
+        } else {
+          // No existing schema - just add the child's schema
+          mergedDataStructure.dependencySchemas[schemaFilePath][schemaName] =
+            childSchema;
+        }
+      }
+    }
+
+    awsLogDebugLevel(
+      1,
+      `Child schema merge for ${entity.name}: ${Date.now() - childMergeStart}ms`,
+    );
+
+    // Transformation tracing: capture after child schemas merged
+    const snapshotStart = Date.now();
+    transformationTracer.snapshot(entity.name, 'childSchemasMerged', {
+      signatureSchema: mergedDataStructure.signatureSchema,
+      returnValueSchema: mergedDataStructure.returnValueSchema,
+      dependencySchemas: mergedDataStructure.dependencySchemas,
+    });
+
     // console.info(
     //   'CODEYAM DEBUG: end merge',
     //   JSON.stringify(
@@ -387,95 +643,438 @@ export default function generateDataStructure({
     //   ),
     // );
 
+    awsLogDebugLevel(
+      1,
+      `Transformation snapshot for ${entity.name}: ${Date.now() - snapshotStart}ms`,
+    );
+
     // Deduplicate function schemas in the merged data structure
     // The merge process might introduce duplicates even if individual schemas were clean
-    mergedDataStructure.signatureSchema = deduplicateFunctionSchemas(
-      mergedDataStructure.signatureSchema,
-    );
-    mergedDataStructure.returnValueSchema = deduplicateFunctionSchemas(
+    const dedupStart = Date.now();
+
+    const sigKeyCount = Object.keys(mergedDataStructure.signatureSchema).length;
+    const retKeyCount = Object.keys(
       mergedDataStructure.returnValueSchema,
+    ).length;
+
+    // Skip dedup on very large schemas — the O(keys × variants × keyLength) cost
+    // is not worth it for schemas with 10K+ keys where function dedup rarely matters.
+    const DEDUP_KEY_THRESHOLD = 10_000;
+
+    if (sigKeyCount < DEDUP_KEY_THRESHOLD) {
+      const sigBefore = { ...mergedDataStructure.signatureSchema };
+      mergedDataStructure.signatureSchema = deduplicateFunctionSchemas(
+        mergedDataStructure.signatureSchema,
+      );
+      transformationTracer.traceSchemaTransformResult(
+        entity.name,
+        'deduplicateFunctionSchemas',
+        sigBefore,
+        mergedDataStructure.signatureSchema,
+        { schemaType: 'signature' },
+      );
+    } else {
+      awsLogDebugLevel(
+        2,
+        `${entity.name}: skipping sig dedup (${sigKeyCount} keys > ${DEDUP_KEY_THRESHOLD})`,
+      );
+    }
+
+    if (retKeyCount < DEDUP_KEY_THRESHOLD) {
+      const rvBefore = { ...mergedDataStructure.returnValueSchema };
+      mergedDataStructure.returnValueSchema = deduplicateFunctionSchemas(
+        mergedDataStructure.returnValueSchema,
+      );
+      transformationTracer.traceSchemaTransformResult(
+        entity.name,
+        'deduplicateFunctionSchemas',
+        rvBefore,
+        mergedDataStructure.returnValueSchema,
+        { schemaType: 'returnValue' },
+      );
+    } else {
+      awsLogDebugLevel(
+        2,
+        `${entity.name}: skipping ret dedup (${retKeyCount} keys > ${DEDUP_KEY_THRESHOLD})`,
+      );
+    }
+
+    awsLogDebugLevel(
+      1,
+      `${entity.name}: root schema dedup ${Date.now() - dedupStart}ms (sig=${sigKeyCount}, ret=${retKeyCount})`,
     );
 
-    // Also deduplicate dependency schemas
+    // Also deduplicate dependency schemas and clear known attributes
+    let depProcessed = 0;
+    const depStart = Date.now();
     for (const filePath in mergedDataStructure.dependencySchemas) {
-      for (const entityName in mergedDataStructure.dependencySchemas[
+      for (const depEntityName in mergedDataStructure.dependencySchemas[
         filePath
       ]) {
+        if (Date.now() > genDeadline) {
+          throw new DataStructureTimeoutError(
+            entity.name,
+            Date.now() - mergeStart,
+          );
+        }
+
         const depSchema =
-          mergedDataStructure.dependencySchemas[filePath][entityName];
-        depSchema.signatureSchema = deduplicateFunctionSchemas(
+          mergedDataStructure.dependencySchemas[filePath][depEntityName];
+        const depSigKeys = Object.keys(depSchema.signatureSchema || {}).length;
+        const depRetKeys = Object.keys(
+          depSchema.returnValueSchema || {},
+        ).length;
+
+        const depOpStart = Date.now();
+
+        if (depSigKeys < DEDUP_KEY_THRESHOLD) {
+          const depSigBefore = { ...depSchema.signatureSchema };
+          depSchema.signatureSchema = deduplicateFunctionSchemas(
+            depSchema.signatureSchema,
+          );
+          transformationTracer.traceSchemaTransformResult(
+            entity.name,
+            'deduplicateFunctionSchemas',
+            depSigBefore,
+            depSchema.signatureSchema,
+            {
+              filePath,
+              dependencyName: depEntityName,
+              schemaType: 'signature',
+            },
+          );
+        }
+
+        if (depRetKeys < DEDUP_KEY_THRESHOLD) {
+          const depRvBefore = { ...depSchema.returnValueSchema };
+          depSchema.returnValueSchema = deduplicateFunctionSchemas(
+            depSchema.returnValueSchema,
+          );
+          transformationTracer.traceSchemaTransformResult(
+            entity.name,
+            'deduplicateFunctionSchemas',
+            depRvBefore,
+            depSchema.returnValueSchema,
+            {
+              filePath,
+              dependencyName: depEntityName,
+              schemaType: 'returnValue',
+            },
+          );
+        }
+
+        const depOpElapsed = Date.now() - depOpStart;
+        if (depOpElapsed > 1000) {
+          awsLogDebugLevel(
+            2,
+            `${entity.name}: slow dep dedup ${depEntityName} sig=${depSigKeys} ret=${depRetKeys} ${depOpElapsed}ms`,
+          );
+        }
+        depProcessed++;
+
+        // Infer function types from child component usage BEFORE fillInSchemaGapsAndUnknowns.
+        // When a hook returns a property with type 'unknown' that is passed to a child
+        // component which calls it as a function (e.g., isEntityPending(entity)),
+        // promote it to 'function'. Without this, name heuristics like isLikelyBoolean
+        // would mistype "isEntityPending" as boolean because of the "is" prefix.
+        //
+        // sourceEquivalencies are in isolatedDataStructure.dependencySchemas (not in
+        // mergedDataStructure.dependencySchemas — they're dropped during the merge).
+        // The child's signatureSchema (with function call evidence) IS in the merged schemas.
+        {
+          const isolatedDepSchemas =
+            entity.metadata.isolatedDataStructure?.dependencySchemas;
+          if (isolatedDepSchemas) {
+            for (const childFilePath in isolatedDepSchemas) {
+              const childIsolated = isolatedDepSchemas[childFilePath];
+              for (const childName in childIsolated) {
+                const sourceEq = childIsolated[childName]?.sourceEquivalencies;
+                if (!sourceEq) continue;
+
+                // Get the child's signatureSchema from the MERGED schemas (richer)
+                const mergedChildSchema =
+                  mergedDataStructure.dependencySchemas[childFilePath]?.[
+                    childName
+                  ];
+                const sigSchema = mergedChildSchema?.signatureSchema;
+                if (!sigSchema) continue;
+
+                for (const childPath in sourceEq) {
+                  for (const source of sourceEq[childPath]) {
+                    const fcrvSuffix = '.functionCallReturnValue.';
+                    const fcrvIndex = source.schemaPath.indexOf(fcrvSuffix);
+                    if (fcrvIndex === -1) continue;
+
+                    const propName = source.schemaPath.slice(
+                      fcrvIndex + fcrvSuffix.length,
+                    );
+
+                    // Find the corresponding child signature path
+                    const entityCallPrefix = childName + '().';
+                    const sigPathBase = childPath.startsWith(entityCallPrefix)
+                      ? childPath.slice(entityCallPrefix.length)
+                      : childPath;
+
+                    // Check if the child's signatureSchema has a function call version
+                    let isFunction = false;
+                    for (const sigKey in sigSchema) {
+                      if (
+                        sigKey.startsWith(sigPathBase + '(') &&
+                        sigSchema[sigKey] === 'function'
+                      ) {
+                        isFunction = true;
+                        break;
+                      }
+                    }
+
+                    if (isFunction) {
+                      for (const rvKey in depSchema.returnValueSchema) {
+                        if (
+                          rvKey.endsWith('.' + propName) &&
+                          depSchema.returnValueSchema[rvKey] === 'unknown'
+                        ) {
+                          depSchema.returnValueSchema[rvKey] = 'function';
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+
+        // Fill in type gaps BEFORE clearing attributes, so evidence like .includes()
+        // is used for type inference before being deleted
+
+        const FILL_IN_KEY_THRESHOLD = 5_000;
+        const fillStart = Date.now();
+
+        if (depSigKeys < FILL_IN_KEY_THRESHOLD) {
+          depSchema.signatureSchema = fillInDirectSchemaGapsAndUnknowns({
+            schema: depSchema.signatureSchema,
+          });
+        }
+
+        if (depRetKeys < FILL_IN_KEY_THRESHOLD) {
+          depSchema.returnValueSchema = fillInDirectSchemaGapsAndUnknowns({
+            schema: depSchema.returnValueSchema,
+          });
+        } else {
+          awsLogDebugLevel(
+            2,
+            `${entity.name}: skipping fillInGaps for ${depEntityName} ret (${depRetKeys} keys > ${FILL_IN_KEY_THRESHOLD})`,
+          );
+        }
+
+        const fillElapsed = Date.now() - fillStart;
+        if (fillElapsed > 1000) {
+          awsLogDebugLevel(
+            2,
+            `${entity.name}: slow fillInGaps ${depEntityName} sig=${depSigKeys} ret=${depRetKeys} ${fillElapsed}ms`,
+          );
+        }
+
+        // Trace clearAttributesFromMapping on dependency schemas
+        transformationTracer.traceSchemaTransform(
+          entity.name,
+          'clearAttributesFromMapping',
           depSchema.signatureSchema,
+          clearAttributesFromMapping,
+          { filePath, dependencyName: depEntityName, schemaType: 'signature' },
         );
-        depSchema.returnValueSchema = deduplicateFunctionSchemas(
+
+        transformationTracer.traceSchemaTransform(
+          entity.name,
+          'clearAttributesFromMapping',
           depSchema.returnValueSchema,
+          clearAttributesFromMapping,
+          {
+            filePath,
+            dependencyName: depEntityName,
+            schemaType: 'returnValue',
+          },
         );
-
-        clearAttributesFromMapping(depSchema.signatureSchema);
-        clearAttributesFromMapping(depSchema.returnValueSchema);
       }
     }
 
+    awsLogDebugLevel(
+      1,
+      `${entity.name}: dep schema processing: ${depProcessed} deps in ${Date.now() - depStart}ms`,
+    );
+    awsLogDebugLevel(
+      1,
+      `Dedup + dep schema processing for ${entity.name}: ${Date.now() - dedupStart}ms`,
+    );
+
+    // Transformation tracing: capture after deduplication
+    const dedupSnapshotStart = Date.now();
+    transformationTracer.snapshot(entity.name, 'deduplicated', {
+      signatureSchema: mergedDataStructure.signatureSchema,
+      returnValueSchema: mergedDataStructure.returnValueSchema,
+      dependencySchemas: mergedDataStructure.dependencySchemas,
+    });
+
+    awsLogDebugLevel(
+      1,
+      `Dedup snapshot for ${entity.name}: ${Date.now() - dedupSnapshotStart}ms`,
+    );
+
     analysis.metadata ||= {};
     analysis.metadata.mergedDataStructure = mergedDataStructure;
+    if (mergeTimedOut) {
+      (analysis.metadata as any).mergeTimedOut = true;
+    }
+
+    const checkGenDeadline = (phase: string) => {
+      if (Date.now() > genDeadline) {
+        throw new DataStructureTimeoutError(
+          entity.name,
+          Date.now() - mergeStart,
+        );
+      }
+      awsLogDebugLevel(
+        1,
+        `${entity.name}: ${phase} complete (${Date.now() - mergeStart}ms total)`,
+      );
+    };
 
     // Enrich mocked dependency array types with fields from child component signatures
     // This ensures that when getSurveysAction returns { data: array }, and SurveyCard
-    // expects survey.updatedAt, the array element type includes updatedAt
-    enrichArrayTypesFromChildSignatures(
-      allImportedExports,
+    // expects survey.updatedAt, the array element type includes updatedAt.
+    // Uses usageEquivalencies to trace data flow and only enrich arrays with fields
+    // from children that actually receive elements from that specific array.
+    transformationTracer.traceDependencySchemaChanges(
+      entity.name,
+      'enrichArrayTypesFromChildSignatures',
       mergedDataStructure.dependencySchemas,
+      () =>
+        enrichArrayTypesFromChildSignatures(
+          allImportedExports,
+          mergedDataStructure.dependencySchemas,
+          mergedDataStructure.usageEquivalencies,
+        ),
     );
 
-    const dataForMocks = gatherDataForMocks(
-      allImportedExports,
+    checkGenDeadline('enrichArrayTypes');
+
+    // Transformation tracing: capture after array types enriched
+    transformationTracer.snapshot(entity.name, 'arrayTypesEnriched', {
+      signatureSchema: mergedDataStructure.signatureSchema,
+      returnValueSchema: mergedDataStructure.returnValueSchema,
+      dependencySchemas: mergedDataStructure.dependencySchemas,
+    });
+
+    // Enrich mocked dependency unknown types using sourceEquivalencies
+    // When a mocked dependency returns unknown, but that data flows to a child component
+    // with a known type, propagate the type back to the mocked dependency.
+    // This prevents fillInDirectSchemaGapsAndUnknowns from guessing incorrectly
+    // (e.g., 'data' becoming 'number' because pluralize thinks it's plural)
+    transformationTracer.traceDependencySchemaChanges(
+      entity.name,
+      'enrichUnknownTypesFromSourceEquivalencies',
       mergedDataStructure.dependencySchemas,
+      () =>
+        enrichUnknownTypesFromSourceEquivalencies(
+          isolatedDataStructure.dependencySchemas || {},
+          mergedDataStructure.dependencySchemas,
+          allImportedExports,
+        ),
     );
 
-    // console.info(
-    //   'CODEYAM DEBUG: dataForMocks',
-    //   JSON.stringify(
-    //     {
-    //       filePath: entity.filePath,
-    //       entityName: entity.name,
-    //       dataForMocks,
-    //     },
-    //     null,
-    //     2,
-    //   ),
-    // );
+    checkGenDeadline('enrichUnknownTypes');
+
+    // Transformation tracing: capture after unknown types enriched
+    transformationTracer.snapshot(entity.name, 'unknownTypesEnriched', {
+      signatureSchema: mergedDataStructure.signatureSchema,
+      returnValueSchema: mergedDataStructure.returnValueSchema,
+      dependencySchemas: mergedDataStructure.dependencySchemas,
+    });
 
-    // Log union types in signatureSchema before processing
-    const unionTypeEntries = Object.entries(
-      mergedDataStructure.signatureSchema,
-    ).filter(
-      ([_, value]) =>
-        typeof value === 'string' &&
-        value.includes("'") &&
-        value.includes(' | '),
+    // DISABLED: Testing if ternary fix eliminates the need for this heuristic
+    // Propagate array item schemas between arrays that likely have the same type.
+    // When one array (e.g., entities) has item schema traced but another similar array
+    // (e.g., currentEntities) doesn't (because it's only used in an OR expression fallback),
+    // copy the item schema from the rich array to the empty one.
+    // propagateArrayItemSchemas(mergedDataStructure.dependencySchemas);
+
+    let dataForMocks = gatherDataForMocks(
+      allImportedExports,
+      mergedDataStructure.dependencySchemas,
+      { entityName: entity.name },
     );
-    if (unionTypeEntries.length > 0) {
-      console.log(
-        `[UNION-TYPE] generateDataStructure for ${entity.name}: Found union types in signatureSchema:`,
-      );
-      unionTypeEntries.forEach(([key, value]) => {
-        console.log(`[UNION-TYPE]   "${key}": ${value}`);
-      });
-    }
 
-    const finalizedArgumentsSchema = fillInDirectSchemaGapsAndUnknowns({
-      schema: { ...mergedDataStructure.signatureSchema },
+    checkGenDeadline('gatherDataForMocks');
+
+    // Transformation tracing: capture after dataForMocks gathered
+    transformationTracer.snapshot(entity.name, 'dataForMocksGathered', {
+      signatureSchema: mergedDataStructure.signatureSchema,
+      returnValueSchema: mergedDataStructure.returnValueSchema,
+      dependencySchemas: mergedDataStructure.dependencySchemas,
+      dataForMocks,
     });
 
-    clearAttributesFromMapping(finalizedArgumentsSchema);
+    // Merge dataForMocks from non-mocked child analyses to include transitive dependency data.
+    // This handles cases where grandchild trpc calls aren't in allImportedExports because
+    // the grandchild's analysis wasn't in dependentAnalyses.
+    //
+    // Issue 19 fix: The child's dataForMocks already contains merged data from its
+    // dependencies (including grandchildren). By merging the child's dataForMocks,
+    // we transitively include all nested dependency data.
+    for (const filePath in nonMockedDependentAnalyses) {
+      for (const name in nonMockedDependentAnalyses[filePath]) {
+        const childAnalysis = nonMockedDependentAnalyses[filePath][name];
+        const childDataForMocks =
+          childAnalysis?.metadata?.scenariosDataStructure?.dataForMocks;
+        if (childDataForMocks && Object.keys(childDataForMocks).length > 0) {
+          dataForMocks = mergeJsonTypeDefinitions(
+            dataForMocks as Record<string, unknown>,
+            childDataForMocks as Record<string, unknown>,
+          ) as { [importAlias: string]: JsonTypeDefinition };
+        }
+      }
+    }
+
+    checkGenDeadline('mergeChildDataForMocks');
+
+    const finalizedArgumentsSchema = { ...mergedDataStructure.signatureSchema };
+
+    // Trace fillInDirectSchemaGapsAndUnknowns
+    transformationTracer.traceSchemaTransform(
+      entity.name,
+      'fillInDirectSchemaGapsAndUnknowns',
+      finalizedArgumentsSchema,
+      (schema) => fillInDirectSchemaGapsAndUnknowns({ schema }),
+      { stage: 'finalization' },
+    );
+
+    // Trace clearAttributesFromMapping
+    transformationTracer.traceSchemaTransform(
+      entity.name,
+      'clearAttributesFromMapping',
+      finalizedArgumentsSchema,
+      clearAttributesFromMapping,
+      { stage: 'finalization' },
+    );
+
+    checkGenDeadline('fillInGaps+clearAttributes');
 
     const argumentsSchema = convertDotNotation(finalizedArgumentsSchema)
       .signature as JsonTypeDefinition[];
 
+    checkGenDeadline('convertDotNotation');
+
     analysis.metadata.scenariosDataStructure = {
       arguments: argumentsSchema,
       dataForMocks,
     };
 
+    // Transformation tracing: capture final scenariosDataStructure
+    transformationTracer.snapshot(entity.name, 'finalized', {
+      signatureSchema: finalizedArgumentsSchema,
+      dataForMocks,
+    });
+
     return analysis;
   } catch (e) {
     awsLog('CodeYam Error: Error generating data structure', {