@codeyam/codeyam-cli 0.1.0-staging.1669d45 → 0.1.0-staging.2a88920

package/analyzer-template/packages/analyze/src/lib/files/scenarios/enrichArrayTypesFromChildSignatures.ts

@@ -9,8 +9,12 @@ import { DataStructure } from '~codeyam/types';
  * enriches the mocked dependency's returnValueSchema so the LLM knows what fields
  * to generate for array elements.
  *
+ * Uses usageEquivalencies to trace data flow and ONLY enrich arrays with fields
+ * from children that actually receive elements from that array.
+ *
  * Example:
  * - getSurveysAction returns { data: array }
+ * - usageEquivalencies shows: surveys[] -> SurveyCard().signature[0].survey
  * - SurveyCard expects signature[0].survey.updatedAt: date
  * - After enrichment: getSurveysAction().data[]: object, data[].updatedAt: date
  */
@@ -21,6 +25,7 @@ export default function enrichArrayTypesFromChildSignatures(
     isMocked?: boolean;
   }[],
   dependencySchemas: DataStructure['dependencySchemas'],
+  usageEquivalencies?: DataStructure['usageEquivalencies'],
 ): void {
   // Find all mocked dependencies with array return types
   const mockedArrayPaths: {
@@ -43,7 +48,8 @@ export default function enrichArrayTypesFromChildSignatures(
     //   - wrapThrows().data.createdBy (deeply nested)
     //   - getStatusOptions().functionCallReturnValue (direct array return - usually options/primitives)
     for (const [path, type] of Object.entries(schema)) {
-      if (type !== 'array') continue;
+      // Check if type is array (including nullable/optional variants like 'array | undefined')
+      if (!type.startsWith('array')) continue;
 
       // Skip paths with computed property patterns like [key], [i], [0], etc.
       // These represent dynamic access patterns, not simple data arrays
@@ -81,115 +87,222 @@ export default function enrichArrayTypesFromChildSignatures(
 
   if (mockedArrayPaths.length === 0) return;
 
-  // Find all non-mocked child components and extract their signature fields
-  // We check both direct imports AND all schemas in dependencySchemas because
-  // transitive dependencies (e.g., SurveyCard imported by SurveysList imported by SurveysPage)
-  // may have signature schemas that we need to use for enrichment.
-  const childSignatureFields: {
-    filePath: string;
-    entityName: string;
-    // Map of prop paths to types, e.g., { "survey.id": "string", "survey.updatedAt": "date" }
-    propFields: Record<string, string>;
-  }[] = [];
+  // Build a map from child component signature paths to their prop fields
+  // e.g., "SurveyCard().signature[0].survey" -> { "id": "string", "updatedAt": "date" }
+  const childSignatureFieldsByPath: Map<
+    string,
+    { propName: string; fields: Record<string, string> }
+  > = new Map();
 
-  // Helper to extract prop fields from a signature schema
-  const extractPropFields = (
+  // Helper to extract prop name and fields from a signature schema
+  const extractSignatureInfo = (
+    entityName: string,
     schema: Record<string, string>,
-  ): Record<string, string> => {
-    const propFields: Record<string, string> = {};
+  ): void => {
+    // Group by prop name (e.g., signature[0].survey.id, signature[0].survey.name -> "survey")
+    const propGroups: Record<string, Record<string, string>> = {};
+    // Also collect direct fields on signature[0] (e.g., signature[0].filePath -> "filePath")
+    const directFields: Record<string, string> = {};
+
     for (const [path, type] of Object.entries(schema)) {
-      // Match signature[0].propName.field patterns
-      const match = path.match(/^signature\[0\]\.(\w+)\.(.+)$/);
-      if (match) {
-        const [, , fieldPath] = match;
-        // Store as "fieldPath" without the prop name prefix
-        // e.g., signature[0].survey.updatedAt -> updatedAt
-        propFields[fieldPath] = type;
+      // Match signature[0].propName.field or signature[0].propName[].field patterns
+      // The (?:\[\])? makes array props optional, handling both:
+      // - signature[0].survey.updatedAt (object prop)
+      // - signature[0].questionAnswers[].question.questionText (array prop)
+      const nestedMatch = path.match(/^signature\[0\]\.(\w+)(?:\[\])?\.(.+)$/);
+      if (nestedMatch) {
+        const [, propName, fieldPath] = nestedMatch;
+        propGroups[propName] ||= {};
+        propGroups[propName][fieldPath] = type;
+        continue;
       }
-    }
-    return propFields;
-  };
-
-  // First, check direct imports (non-mocked)
-  const processedSchemas = new Set<string>();
-  for (const importedExport of importedExports) {
-    // Skip mocked dependencies - we want child components
-    if (importedExport.isMocked || !importedExport.filePath) continue;
 
-    const schemaKey = `${importedExport.filePath}::${importedExport.name}`;
-    processedSchemas.add(schemaKey);
+      // Also match direct fields: signature[0].fieldName (single level, no nested path)
+      // This handles cases like TruncatedFilePath where filePath is passed directly
+      const directMatch = path.match(/^signature\[0\]\.(\w+)$/);
+      if (directMatch) {
+        const [, fieldName] = directMatch;
+        directFields[fieldName] = type;
+      }
+    }
 
-    const schema =
-      dependencySchemas?.[importedExport.filePath]?.[importedExport.name]
-        ?.signatureSchema;
-    if (!schema) continue;
+    // Store nested prop groups with the full signature path as key
+    for (const [propName, fields] of Object.entries(propGroups)) {
+      if (Object.keys(fields).length > 0) {
+        const signaturePath = `${entityName}().signature[0].${propName}`;
+        childSignatureFieldsByPath.set(signaturePath, { propName, fields });
+      }
+    }
 
-    const propFields = extractPropFields(schema);
-    if (Object.keys(propFields).length > 0) {
-      childSignatureFields.push({
-        filePath: importedExport.filePath,
-        entityName: importedExport.name,
-        propFields,
+    // Store direct fields with signature[0] as key (for when entire object is passed)
+    if (Object.keys(directFields).length > 0) {
+      const signaturePath = `${entityName}().signature[0]`;
+      childSignatureFieldsByPath.set(signaturePath, {
+        propName: '',
+        fields: directFields,
       });
     }
-  }
+  };
 
-  // Also check all schemas in dependencySchemas for transitive child components
-  // that may not be in importedExports but have signature schemas with prop fields
+  // Process all schemas in dependencySchemas
   if (dependencySchemas) {
-    for (const [filePath, entities] of Object.entries(dependencySchemas)) {
+    for (const [, entities] of Object.entries(dependencySchemas)) {
       for (const [entityName, schemas] of Object.entries(entities)) {
-        const schemaKey = `${filePath}::${entityName}`;
-        // Skip already processed schemas
-        if (processedSchemas.has(schemaKey)) continue;
-
         const signatureSchema = schemas?.signatureSchema;
-        if (!signatureSchema) continue;
-
-        const propFields = extractPropFields(signatureSchema);
-        if (Object.keys(propFields).length > 0) {
-          childSignatureFields.push({
-            filePath,
-            entityName,
-            propFields,
-          });
+        if (signatureSchema) {
+          extractSignatureInfo(entityName, signatureSchema);
         }
       }
     }
   }
 
-  if (childSignatureFields.length === 0) return;
+  if (childSignatureFieldsByPath.size === 0) return;
+
+  // Helper to check if a local variable name could be derived from a mocked function name
+  // e.g., "getSurveys" -> "surveys", "fetchUsers" -> "users", "loadData" -> "data"
+  // e.g., "getSurveysAction" -> "surveys" (removes "Action" suffix too)
+  const couldBeDerivedFrom = (
+    localVarName: string,
+    mockedFunctionName: string,
+  ): boolean => {
+    const normalizedVar = localVarName.toLowerCase().replace(/\[\]$/, '');
+    let normalizedFunc = mockedFunctionName.toLowerCase();
 
-  // Enrich mocked dependency arrays with child signature fields
-  // Only enrich arrays that don't already have element types defined
-  // A more sophisticated approach would trace the data flow to match specific arrays to specific children
+    // Direct match (e.g., "surveys" from function "surveys")
+    if (normalizedFunc === normalizedVar) return true;
+
+    // Remove common prefixes like "get", "fetch", "load", "use"
+    const prefixPattern = /^(get|fetch|load|use)/i;
+    normalizedFunc = normalizedFunc.replace(prefixPattern, '');
+
+    // Remove common suffixes like "Action", "Query", "Mutation", "Handler"
+    const suffixPattern = /(action|query|mutation|handler)$/i;
+    normalizedFunc = normalizedFunc.replace(suffixPattern, '');
+
+    // Check if function name (without prefix/suffix) matches the variable name
+    if (normalizedFunc === normalizedVar) return true;
+
+    // Check for singular/plural relationship
+    // e.g., "survey" from "getSurveys" or "surveys" from "getSurvey"
+    if (
+      normalizedFunc.endsWith('s') &&
+      normalizedFunc.slice(0, -1) === normalizedVar
+    )
+      return true;
+    if (
+      normalizedVar.endsWith('s') &&
+      normalizedVar.slice(0, -1) === normalizedFunc
+    )
+      return true;
+
+    // Check if the array property name (like "data") is in the variable
+    // This handles cases like "const { data: surveys } = getSurveys()" where
+    // the local var might be the renamed destructured property
+    // Skip this for now as it's too loose
+
+    return false;
+  };
+
+  // For each mocked array, find which child signatures it flows to via usageEquivalencies
   for (const { filePath, entityName, arrayPath } of mockedArrayPaths) {
     const schema = dependencySchemas[filePath][entityName].returnValueSchema;
 
-    // Check if this array already has element type definitions
+    // Check if this array already has INDEXED element type definitions (e.g., [0]: null)
     // If so, skip it - it's likely a specialized array like [null] for refs
+    // But DON'T skip if it only has general [] element types - those can be enriched
     const elementPathPrefix = `${arrayPath}[`;
-    const hasExistingElementType = Object.keys(schema).some((path) =>
-      path.startsWith(elementPathPrefix),
-    );
+    const hasIndexedElementType = Object.keys(schema).some((path) => {
+      if (!path.startsWith(elementPathPrefix)) return false;
+      // Check if it's an indexed type like [0], [1], etc. vs general []
+      const afterPrefix = path.slice(elementPathPrefix.length);
+      // Indexed types start with a digit: [0], [1], etc.
+      return /^\d/.test(afterPrefix);
+    });
 
-    if (hasExistingElementType) {
-      // Array already has element type info (e.g., useAutoAnimate()[0]: null)
+    if (hasIndexedElementType) {
+      // Array already has indexed element type info (e.g., useAutoAnimate()[0]: null)
       // Don't override it with child signature fields
       continue;
     }
 
-    // Add array element type
+    // Find matching child signatures via usageEquivalencies
+    // Look for equivalencies where this array (or local variables derived from it)
+    // flows to a child component's signature
+    const matchedFields: Record<string, string> = {};
+
+    if (usageEquivalencies) {
+      // Check all equivalency entries for paths that could be derived from this array
+      for (const [sourcePath, targets] of Object.entries(usageEquivalencies)) {
+        // Check if this source path relates to our mocked array
+        // Direct match: "getSurveysAction().functionCallReturnValue.data[]"
+        // Indirect match: local variable like "surveys[]" that comes from the array
+        if (!sourcePath.endsWith('[]')) continue;
+
+        // Get the local variable name from the source path (e.g., "surveys[]" -> "surveys")
+        const localVarName = sourcePath.replace(/\[\]$/, '');
+
+        // Check if this local variable could have come from this mocked function
+        // Use heuristics: name matching (getSurveys -> surveys, getUsers -> users)
+        // Also check if the array path ends with the local variable name
+        // e.g., useLoaderData().functionCallReturnValue.entities ends with .entities
+        const arrayEndsWithLocalVar = arrayPath.endsWith(`.${localVarName}`);
+        if (
+          !couldBeDerivedFrom(localVarName, entityName) &&
+          !arrayEndsWithLocalVar
+        ) {
+          continue;
+        }
+
+        for (const target of targets) {
+          // Target should be something like "SurveyCard().signature[0].survey"
+          // or "SurveyCard().signature[0].surveys[]"
+          const { schemaPath } = target;
+
+          // Look for signature fields that match this target
+          // Handle both singular (survey) and plural (surveys[]) forms
+          const normalizedTarget = schemaPath.replace(/\[\]$/, '');
+
+          for (const [signaturePath, info] of childSignatureFieldsByPath) {
+            if (normalizedTarget === signaturePath) {
+              // Found a match! Add all fields from this child signature
+              for (const [fieldPath, type] of Object.entries(info.fields)) {
+                if (!matchedFields[fieldPath]) {
+                  matchedFields[fieldPath] = type;
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+
+    // If no equivalencies matched, skip this array entirely
+    // This prevents the bug where ALL child signatures get merged into ALL arrays
+    if (Object.keys(matchedFields).length === 0) {
+      continue;
+    }
+
+    // Add array element type and matched fields
     const elementPath = `${arrayPath}[]`;
     schema[elementPath] = 'object';
 
-    // Add fields from all child signatures to the array element
-    for (const { propFields } of childSignatureFields) {
-      for (const [fieldPath, type] of Object.entries(propFields)) {
-        const fullPath = `${elementPath}.${fieldPath}`;
-        if (!schema[fullPath]) {
+    for (const [fieldPath, type] of Object.entries(matchedFields)) {
+      const fullPath = `${elementPath}.${fieldPath}`;
+      const existingType = schema[fullPath];
+      if (!existingType) {
+        schema[fullPath] = type;
+      } else {
+        // Prefer the more specific (narrower) type
+        const existingIsOptional =
+          existingType.includes('| undefined') ||
+          existingType.includes('| null');
+        const newIsOptional =
+          type.includes('| undefined') || type.includes('| null');
+        if (existingIsOptional && !newIsOptional) {
+          // Use the new type - it's more specific
           schema[fullPath] = type;
         }
+        // Otherwise keep existing (same specificity or existing is more specific)
       }
     }
   }

package/analyzer-template/packages/analyze/src/lib/files/scenarios/gatherDataForMocks.ts

@@ -1,6 +1,8 @@
 import {
   convertDotNotation,
   fillInDirectSchemaGapsAndUnknowns,
+  buildSchemaIndexes,
+  type SchemaIndexes,
   joinParenthesesAndArrays,
   mergeJsonTypeDefinitions,
   splitOutsideParenthesesAndArrays,
@@ -165,6 +167,7 @@ function processCall(
   variableNameOccurrence: number | undefined,
   options: GatherDataForMocksOptions | undefined,
   hookCallIndex: number,
+  prebuiltIndexes?: SchemaIndexes,
 ): { key: string; value: JsonTypeDefinition } | null {
   const callName = importedExport.calls?.[callIndex] ?? importedExport.name;
 
@@ -465,6 +468,7 @@ function processCall(
   const filledSchema = fillInDirectSchemaGapsAndUnknowns({
     scopeName: importedExport.name,
     schema: relevantMergedDependencySchema,
+    prebuiltIndexes,
   });
 
   const schema = convertDotNotation(filledSchema);
@@ -568,6 +572,25 @@ export default function gatherDataForMocks(
   dependencySchemas: DataStructure['dependencySchemas'],
   options?: GatherDataForMocksOptions,
 ) {
+  // Build schema indexes ONCE from all dependency schemas combined.
+  // This avoids rebuilding indexes (~1-2 seconds for 18k keys) for each processCall.
+  const allSchemaKeys: Record<string, string> = {};
+  if (dependencySchemas) {
+    for (const filePath in dependencySchemas) {
+      for (const entityName in dependencySchemas[filePath]) {
+        const schema =
+          dependencySchemas[filePath][entityName]?.returnValueSchema;
+        if (schema) {
+          Object.assign(allSchemaKeys, schema);
+        }
+      }
+    }
+  }
+  const prebuiltIndexes =
+    Object.keys(allSchemaKeys).length > 0
+      ? buildSchemaIndexes(allSchemaKeys)
+      : undefined;
+
   // Track the global index for each hook name across all imports
   // This enables canonical keys like EntityName::useLoaderData::0, EntityName::useFetcher::0, etc.
   const hookCallIndices: Record<string, number> = {};
@@ -653,6 +676,7 @@ export default function gatherDataForMocks(
             undefined,
             options,
             hookCallIndex,
+            prebuiltIndexes,
           );
 
           if (processResult) {
@@ -717,6 +741,7 @@ export default function gatherDataForMocks(
             occurrence,
             options,
             hookCallIndex,
+            prebuiltIndexes,
           );
           if (processResult) {
             addResultToAccumulator(acc, processResult.key, processResult.value);
@@ -749,6 +774,7 @@ export default function gatherDataForMocks(
             undefined,
             options,
             hookCallIndex,
+            prebuiltIndexes,
           );
 
           if (processResult) {
@@ -769,6 +795,7 @@ export default function gatherDataForMocks(
           undefined,
           options,
           hookCallIndex,
+          prebuiltIndexes,
         );
 
         // Check if we got a meaningful result (not empty object or null)
@@ -813,6 +840,7 @@ export default function gatherDataForMocks(
                 0, // Use 0 to trigger variable-qualified lookup
                 options,
                 hookCallIndex,
+                prebuiltIndexes,
               );
               if (varQualifiedResult && varQualifiedResult.value) {
                 mergedValue = mergeJsonTypeDefinitions(
@@ -849,6 +877,7 @@ export default function gatherDataForMocks(
               0, // Use 0 to trigger variable-qualified lookup
               options,
               hookCallIndex,
+              prebuiltIndexes,
             );
             if (varQualifiedResult && varQualifiedResult.value) {
               // Deep merge the value into the accumulated result

package/analyzer-template/packages/analyze/src/lib/files/scenarios/generateDataStructure.ts

@@ -16,7 +16,7 @@ import { awsLog } from '~codeyam/utils';
 import gatherDataForMocks from './gatherDataForMocks';
 import enrichArrayTypesFromChildSignatures from './enrichArrayTypesFromChildSignatures';
 import enrichUnknownTypesFromSourceEquivalencies from './enrichUnknownTypesFromSourceEquivalencies';
-
+// import propagateArrayItemSchemas from './propagateArrayItemSchemas';
 export interface GenerateDataStructureArgs {
   entity: Entity;
   dependentAnalyses: ReadonlyAnalysisMap;
@@ -256,8 +256,10 @@ export default function generateDataStructure({
           );
 
           if (existingImport) {
-            // Merge callVariableNames if the same dependency exists with different patterns
-            if (imported.callVariableNames && imported.calls) {
+            // Merge calls from child into existing import
+            // Note: Check length > 0 because empty array [] is truthy but has no variable names
+            if (imported.callVariableNames?.length && imported.calls) {
+              // Case 1: Both calls and callVariableNames are set - merge as pairs
               existingImport.calls = existingImport.calls || [];
               existingImport.callVariableNames =
                 existingImport.callVariableNames || [];
@@ -274,6 +276,18 @@ export default function generateDataStructure({
                   existingImport.callVariableNames.push(varName);
                 }
               }
+            } else if (imported.calls) {
+              // Case 2: Only calls are set (no callVariableNames) - common with tRPC
+              // where destructuring like `const { data, isLoading } = trpc.xxx.useQuery()`
+              // doesn't create a variable name for the hook call itself
+              existingImport.calls = existingImport.calls || [];
+
+              // Add each call that doesn't already exist
+              for (const call of imported.calls) {
+                if (!existingImport.calls.includes(call)) {
+                  existingImport.calls.push(call);
+                }
+              }
             }
           } else {
             // Add new import
@@ -456,11 +470,34 @@ export default function generateDataStructure({
 
         if (existingSchema) {
           // Merge schemas - child schemas add to parent schemas
+          // But prefer more specific types (without | undefined) when both exist
+          const mergedSignatureSchema = { ...existingSchema.signatureSchema };
+          for (const path in childSchema.signatureSchema) {
+            const existingType = mergedSignatureSchema[path];
+            const childType = childSchema.signatureSchema[path];
+            if (!existingType) {
+              mergedSignatureSchema[path] = childType;
+            } else {
+              // Prefer the more specific (narrower) type
+              const existingIsOptional =
+                existingType.includes('| undefined') ||
+                existingType.includes('| null');
+              const childIsOptional =
+                childType.includes('| undefined') ||
+                childType.includes('| null');
+              if (childIsOptional && !existingIsOptional) {
+                // Keep existing - it's more specific
+              } else if (!childIsOptional && existingIsOptional) {
+                // Use child - it's more specific
+                mergedSignatureSchema[path] = childType;
+              } else {
+                // Same specificity - use child (original behavior)
+                mergedSignatureSchema[path] = childType;
+              }
+            }
+          }
           mergedDataStructure.dependencySchemas[schemaFilePath][schemaName] = {
-            signatureSchema: {
-              ...existingSchema.signatureSchema,
-              ...childSchema.signatureSchema,
-            },
+            signatureSchema: mergedSignatureSchema,
             returnValueSchema: mergeJsonTypeDefinitions(
               existingSchema.returnValueSchema as Record<string, unknown>,
               childSchema.returnValueSchema as Record<string, unknown>,
@@ -527,10 +564,13 @@ export default function generateDataStructure({
 
     // Enrich mocked dependency array types with fields from child component signatures
     // This ensures that when getSurveysAction returns { data: array }, and SurveyCard
-    // expects survey.updatedAt, the array element type includes updatedAt
+    // expects survey.updatedAt, the array element type includes updatedAt.
+    // Uses usageEquivalencies to trace data flow and only enrich arrays with fields
+    // from children that actually receive elements from that specific array.
     enrichArrayTypesFromChildSignatures(
       allImportedExports,
       mergedDataStructure.dependencySchemas,
+      mergedDataStructure.usageEquivalencies,
     );
 
     // Enrich mocked dependency unknown types using sourceEquivalencies
@@ -544,12 +584,40 @@ export default function generateDataStructure({
       allImportedExports,
     );
 
-    const dataForMocks = gatherDataForMocks(
+    // DISABLED: Testing if ternary fix eliminates the need for this heuristic
+    // Propagate array item schemas between arrays that likely have the same type.
+    // When one array (e.g., entities) has item schema traced but another similar array
+    // (e.g., currentEntities) doesn't (because it's only used in an OR expression fallback),
+    // copy the item schema from the rich array to the empty one.
+    // propagateArrayItemSchemas(mergedDataStructure.dependencySchemas);
+
+    let dataForMocks = gatherDataForMocks(
       allImportedExports,
       mergedDataStructure.dependencySchemas,
       { entityName: entity.name },
     );
 
+    // Merge dataForMocks from non-mocked child analyses to include transitive dependency data.
+    // This handles cases where grandchild trpc calls aren't in allImportedExports because
+    // the grandchild's analysis wasn't in dependentAnalyses.
+    //
+    // Issue 19 fix: The child's dataForMocks already contains merged data from its
+    // dependencies (including grandchildren). By merging the child's dataForMocks,
+    // we transitively include all nested dependency data.
+    for (const filePath in nonMockedDependentAnalyses) {
+      for (const name in nonMockedDependentAnalyses[filePath]) {
+        const childAnalysis = nonMockedDependentAnalyses[filePath][name];
+        const childDataForMocks =
+          childAnalysis?.metadata?.scenariosDataStructure?.dataForMocks;
+        if (childDataForMocks && Object.keys(childDataForMocks).length > 0) {
+          dataForMocks = mergeJsonTypeDefinitions(
+            dataForMocks as Record<string, unknown>,
+            childDataForMocks as Record<string, unknown>,
+          ) as { [importAlias: string]: JsonTypeDefinition };
+        }
+      }
+    }
+
     const finalizedArgumentsSchema = fillInDirectSchemaGapsAndUnknowns({
       schema: { ...mergedDataStructure.signatureSchema },
     });