@codeyam/codeyam-cli 0.1.0-staging.323686 → 0.1.0-staging.4813bf3

package/analyzer-template/packages/analyze/src/lib/files/scenarios/enrichArrayTypesFromChildSignatures.ts

@@ -101,27 +101,47 @@ export default function enrichArrayTypesFromChildSignatures(
   ): void => {
     // Group by prop name (e.g., signature[0].survey.id, signature[0].survey.name -> "survey")
     const propGroups: Record<string, Record<string, string>> = {};
+    // Also collect direct fields on signature[0] (e.g., signature[0].filePath -> "filePath")
+    const directFields: Record<string, string> = {};
 
     for (const [path, type] of Object.entries(schema)) {
       // Match signature[0].propName.field or signature[0].propName[].field patterns
       // The (?:\[\])? makes array props optional, handling both:
       // - signature[0].survey.updatedAt (object prop)
       // - signature[0].questionAnswers[].question.questionText (array prop)
-      const match = path.match(/^signature\[0\]\.(\w+)(?:\[\])?\.(.+)$/);
-      if (match) {
-        const [, propName, fieldPath] = match;
+      const nestedMatch = path.match(/^signature\[0\]\.(\w+)(?:\[\])?\.(.+)$/);
+      if (nestedMatch) {
+        const [, propName, fieldPath] = nestedMatch;
         propGroups[propName] ||= {};
         propGroups[propName][fieldPath] = type;
+        continue;
+      }
+
+      // Also match direct fields: signature[0].fieldName (single level, no nested path)
+      // This handles cases like TruncatedFilePath where filePath is passed directly
+      const directMatch = path.match(/^signature\[0\]\.(\w+)$/);
+      if (directMatch) {
+        const [, fieldName] = directMatch;
+        directFields[fieldName] = type;
       }
     }
 
-    // Store with the full signature path as key
+    // Store nested prop groups with the full signature path as key
     for (const [propName, fields] of Object.entries(propGroups)) {
       if (Object.keys(fields).length > 0) {
         const signaturePath = `${entityName}().signature[0].${propName}`;
         childSignatureFieldsByPath.set(signaturePath, { propName, fields });
       }
     }
+
+    // Store direct fields with signature[0] as key (for when entire object is passed)
+    if (Object.keys(directFields).length > 0) {
+      const signaturePath = `${entityName}().signature[0]`;
+      childSignatureFieldsByPath.set(signaturePath, {
+        propName: '',
+        fields: directFields,
+      });
+    }
   };
 
   // Process all schemas in dependencySchemas
@@ -187,15 +207,20 @@ export default function enrichArrayTypesFromChildSignatures(
   for (const { filePath, entityName, arrayPath } of mockedArrayPaths) {
     const schema = dependencySchemas[filePath][entityName].returnValueSchema;
 
-    // Check if this array already has element type definitions
+    // Check if this array already has INDEXED element type definitions (e.g., [0]: null)
     // If so, skip it - it's likely a specialized array like [null] for refs
+    // But DON'T skip if it only has general [] element types - those can be enriched
     const elementPathPrefix = `${arrayPath}[`;
-    const hasExistingElementType = Object.keys(schema).some((path) =>
-      path.startsWith(elementPathPrefix),
-    );
-
-    if (hasExistingElementType) {
-      // Array already has element type info (e.g., useAutoAnimate()[0]: null)
+    const hasIndexedElementType = Object.keys(schema).some((path) => {
+      if (!path.startsWith(elementPathPrefix)) return false;
+      // Check if it's an indexed type like [0], [1], etc. vs general []
+      const afterPrefix = path.slice(elementPathPrefix.length);
+      // Indexed types start with a digit: [0], [1], etc.
+      return /^\d/.test(afterPrefix);
+    });
+
+    if (hasIndexedElementType) {
+      // Array already has indexed element type info (e.g., useAutoAnimate()[0]: null)
       // Don't override it with child signature fields
       continue;
     }
@@ -218,7 +243,17 @@ export default function enrichArrayTypesFromChildSignatures(
 
         // Check if this local variable could have come from this mocked function
         // Use heuristics: name matching (getSurveys -> surveys, getUsers -> users)
-        if (!couldBeDerivedFrom(localVarName, entityName)) {
+        // Also check if the array path ends with the local variable name
+        // e.g., useLoaderData().functionCallReturnValue.entities ends with .entities
+        const arrayEndsWithLocalVar = arrayPath.endsWith(`.${localVarName}`);
+        // Full-path usageEquivalencies (e.g., from same-file child components) use
+        // the complete call path as the local var name, which matches arrayPath exactly
+        const exactPathMatch = localVarName === arrayPath;
+        if (
+          !exactPathMatch &&
+          !couldBeDerivedFrom(localVarName, entityName) &&
+          !arrayEndsWithLocalVar
+        ) {
           continue;
         }
 
@@ -257,8 +292,21 @@ export default function enrichArrayTypesFromChildSignatures(
 
     for (const [fieldPath, type] of Object.entries(matchedFields)) {
       const fullPath = `${elementPath}.${fieldPath}`;
-      if (!schema[fullPath]) {
+      const existingType = schema[fullPath];
+      if (!existingType) {
         schema[fullPath] = type;
+      } else {
+        // Prefer the more specific (narrower) type
+        const existingIsOptional =
+          existingType.includes('| undefined') ||
+          existingType.includes('| null');
+        const newIsOptional =
+          type.includes('| undefined') || type.includes('| null');
+        if (existingIsOptional && !newIsOptional) {
+          // Use the new type - it's more specific
+          schema[fullPath] = type;
+        }
+        // Otherwise keep existing (same specificity or existing is more specific)
       }
     }
   }

package/analyzer-template/packages/analyze/src/lib/files/scenarios/gatherDataForMocks.ts

@@ -1,6 +1,8 @@
 import {
   convertDotNotation,
   fillInDirectSchemaGapsAndUnknowns,
+  buildSchemaIndexes,
+  type SchemaIndexes,
   joinParenthesesAndArrays,
   mergeJsonTypeDefinitions,
   splitOutsideParenthesesAndArrays,
@@ -165,6 +167,7 @@ function processCall(
   variableNameOccurrence: number | undefined,
   options: GatherDataForMocksOptions | undefined,
   hookCallIndex: number,
+  prebuiltIndexes?: SchemaIndexes,
 ): { key: string; value: JsonTypeDefinition } | null {
   const callName = importedExport.calls?.[callIndex] ?? importedExport.name;
 
@@ -465,6 +468,7 @@ function processCall(
   const filledSchema = fillInDirectSchemaGapsAndUnknowns({
     scopeName: importedExport.name,
     schema: relevantMergedDependencySchema,
+    prebuiltIndexes,
   });
 
   const schema = convertDotNotation(filledSchema);
@@ -568,6 +572,33 @@ export default function gatherDataForMocks(
   dependencySchemas: DataStructure['dependencySchemas'],
   options?: GatherDataForMocksOptions,
 ) {
+  // Build schema indexes ONCE from all dependency schemas combined.
+  // This avoids rebuilding indexes (~1-2 seconds for 18k keys) for each processCall.
+  //
+  // IMPORTANT: We must preprocess schemas BEFORE building indexes!
+  // The schemas contain .functionCallReturnValue paths that get stripped during processing.
+  // If we build indexes from original paths, lookups with preprocessed paths will fail.
+  // Example: 'hook().functionCallReturnValue.x.length' → index maps 'hook().functionCallReturnValue.x' → ['length']
+  //          But preprocessed path 'hook().x.length' looks for 'hook().x' which doesn't exist in the index.
+  const allSchemaKeys: Record<string, string> = {};
+  if (dependencySchemas) {
+    for (const filePath in dependencySchemas) {
+      for (const entityName in dependencySchemas[filePath]) {
+        const schema =
+          dependencySchemas[filePath][entityName]?.returnValueSchema;
+        if (schema) {
+          // Preprocess each schema to remove .functionCallReturnValue before collecting
+          const preprocessed = preprocessSchemaForMocks(schema);
+          Object.assign(allSchemaKeys, preprocessed);
+        }
+      }
+    }
+  }
+  const prebuiltIndexes =
+    Object.keys(allSchemaKeys).length > 0
+      ? buildSchemaIndexes(allSchemaKeys)
+      : undefined;
+
   // Track the global index for each hook name across all imports
   // This enables canonical keys like EntityName::useLoaderData::0, EntityName::useFetcher::0, etc.
   const hookCallIndices: Record<string, number> = {};
@@ -653,6 +684,7 @@ export default function gatherDataForMocks(
             undefined,
             options,
             hookCallIndex,
+            prebuiltIndexes,
           );
 
           if (processResult) {
@@ -717,6 +749,7 @@ export default function gatherDataForMocks(
             occurrence,
             options,
             hookCallIndex,
+            prebuiltIndexes,
           );
           if (processResult) {
             addResultToAccumulator(acc, processResult.key, processResult.value);
@@ -749,6 +782,7 @@ export default function gatherDataForMocks(
             undefined,
             options,
             hookCallIndex,
+            prebuiltIndexes,
           );
 
           if (processResult) {
@@ -769,6 +803,7 @@ export default function gatherDataForMocks(
           undefined,
           options,
           hookCallIndex,
+          prebuiltIndexes,
         );
 
         // Check if we got a meaningful result (not empty object or null)
@@ -813,6 +848,7 @@ export default function gatherDataForMocks(
                 0, // Use 0 to trigger variable-qualified lookup
                 options,
                 hookCallIndex,
+                prebuiltIndexes,
               );
               if (varQualifiedResult && varQualifiedResult.value) {
                 mergedValue = mergeJsonTypeDefinitions(
@@ -849,6 +885,7 @@ export default function gatherDataForMocks(
               0, // Use 0 to trigger variable-qualified lookup
               options,
               hookCallIndex,
+              prebuiltIndexes,
             );
             if (varQualifiedResult && varQualifiedResult.value) {
               // Deep merge the value into the accumulated result

package/analyzer-template/packages/analyze/src/lib/files/scenarios/generateDataStructure.ts

@@ -16,6 +16,8 @@ import { awsLog } from '~codeyam/utils';
 import gatherDataForMocks from './gatherDataForMocks';
 import enrichArrayTypesFromChildSignatures from './enrichArrayTypesFromChildSignatures';
 import enrichUnknownTypesFromSourceEquivalencies from './enrichUnknownTypesFromSourceEquivalencies';
+import { transformationTracer } from './TransformationTracer';
+// import propagateArrayItemSchemas from './propagateArrayItemSchemas';
 export interface GenerateDataStructureArgs {
   entity: Entity;
   dependentAnalyses: ReadonlyAnalysisMap;
@@ -162,6 +164,25 @@ export default function generateDataStructure({
         return acc;
       }, {} as ReadonlyAnalysisMap);
 
+    // Include same-file child components in nonMockedDependentAnalyses.
+    // Same-file components (e.g., AgentCard defined in the same file as AgentTranscriptsPage)
+    // are NOT in importedExports, so the reduce above never includes them.
+    // But they ARE in dependentAnalyses and their full signatureSchema (with all prop fields)
+    // needs to be merged into the parent's dependencySchemas for array enrichment to work.
+
+    const sameFileAnalyses = dependentAnalyses[entity.filePath];
+    if (sameFileAnalyses) {
+      for (const name in sameFileAnalyses) {
+        if (name === entity.name) continue;
+        if (nonMockedDependentAnalyses[entity.filePath]?.[name]) continue;
+        const childAnalysis = sameFileAnalyses[name];
+        if (childAnalysis?.entityType === 'visual') {
+          nonMockedDependentAnalyses[entity.filePath] ||= {};
+          nonMockedDependentAnalyses[entity.filePath][name] = childAnalysis;
+        }
+      }
+    }
+
     // For non-mocked child components (like visual components), include their mocked
     // dependencies in allImportedExports. This ensures that when a parent component
     // renders a child that uses the same hook with different destructure patterns,
@@ -440,6 +461,18 @@ export default function generateDataStructure({
 
     const { isolatedDataStructure } = entity.metadata;
 
+    // Transformation tracing: start entity and capture isolated data structure
+    transformationTracer.startEntity({
+      name: entity.name,
+      entityType: entity.entityType,
+      filePath: entity.filePath,
+    });
+    transformationTracer.snapshot(entity.name, 'isolated', {
+      signatureSchema: isolatedDataStructure.signatureSchema,
+      returnValueSchema: isolatedDataStructure.returnValueSchema,
+      dependencySchemas: isolatedDataStructure.dependencySchemas,
+    });
+
     const mergedDataStructure = mergeInDependentDataStructure({
       importedExports: allImportedExports,
       dependentAnalyses: nonMockedDependentAnalyses,
@@ -457,6 +490,13 @@ export default function generateDataStructure({
     mergedDataStructure.environmentVariables =
       isolatedDataStructure.environmentVariables || [];
 
+    // Transformation tracing: capture merged data structure
+    transformationTracer.snapshot(entity.name, 'merged', {
+      signatureSchema: mergedDataStructure.signatureSchema,
+      returnValueSchema: mergedDataStructure.returnValueSchema,
+      dependencySchemas: mergedDataStructure.dependencySchemas,
+    });
+
     // Merge collected child dependency schemas into mergedDataStructure.
     // This ensures that when gatherDataForMocks looks up schemas for child imports,
     // the child's schemas are available.
@@ -469,11 +509,34 @@ export default function generateDataStructure({
 
         if (existingSchema) {
           // Merge schemas - child schemas add to parent schemas
+          // But prefer more specific types (without | undefined) when both exist
+          const mergedSignatureSchema = { ...existingSchema.signatureSchema };
+          for (const path in childSchema.signatureSchema) {
+            const existingType = mergedSignatureSchema[path];
+            const childType = childSchema.signatureSchema[path];
+            if (!existingType) {
+              mergedSignatureSchema[path] = childType;
+            } else {
+              // Prefer the more specific (narrower) type
+              const existingIsOptional =
+                existingType.includes('| undefined') ||
+                existingType.includes('| null');
+              const childIsOptional =
+                childType.includes('| undefined') ||
+                childType.includes('| null');
+              if (childIsOptional && !existingIsOptional) {
+                // Keep existing - it's more specific
+              } else if (!childIsOptional && existingIsOptional) {
+                // Use child - it's more specific
+                mergedSignatureSchema[path] = childType;
+              } else {
+                // Same specificity - use child (original behavior)
+                mergedSignatureSchema[path] = childType;
+              }
+            }
+          }
           mergedDataStructure.dependencySchemas[schemaFilePath][schemaName] = {
-            signatureSchema: {
-              ...existingSchema.signatureSchema,
-              ...childSchema.signatureSchema,
-            },
+            signatureSchema: mergedSignatureSchema,
             returnValueSchema: mergeJsonTypeDefinitions(
               existingSchema.returnValueSchema as Record<string, unknown>,
               childSchema.returnValueSchema as Record<string, unknown>,
@@ -487,6 +550,13 @@ export default function generateDataStructure({
       }
     }
 
+    // Transformation tracing: capture after child schemas merged
+    transformationTracer.snapshot(entity.name, 'childSchemasMerged', {
+      signatureSchema: mergedDataStructure.signatureSchema,
+      returnValueSchema: mergedDataStructure.returnValueSchema,
+      dependencySchemas: mergedDataStructure.dependencySchemas,
+    });
+
     // console.info(
     //   'CODEYAM DEBUG: end merge',
     //   JSON.stringify(
@@ -509,32 +579,111 @@ export default function generateDataStructure({
 
     // Deduplicate function schemas in the merged data structure
     // The merge process might introduce duplicates even if individual schemas were clean
+    // Trace deduplication of signatureSchema
+    const sigBefore = { ...mergedDataStructure.signatureSchema };
     mergedDataStructure.signatureSchema = deduplicateFunctionSchemas(
       mergedDataStructure.signatureSchema,
     );
+    transformationTracer.traceSchemaTransformResult(
+      entity.name,
+      'deduplicateFunctionSchemas',
+      sigBefore,
+      mergedDataStructure.signatureSchema,
+      { schemaType: 'signature' },
+    );
+
+    // Trace deduplication of returnValueSchema
+    const rvBefore = { ...mergedDataStructure.returnValueSchema };
     mergedDataStructure.returnValueSchema = deduplicateFunctionSchemas(
       mergedDataStructure.returnValueSchema,
     );
+    transformationTracer.traceSchemaTransformResult(
+      entity.name,
+      'deduplicateFunctionSchemas',
+      rvBefore,
+      mergedDataStructure.returnValueSchema,
+      { schemaType: 'returnValue' },
+    );
 
-    // Also deduplicate dependency schemas
+    // Also deduplicate dependency schemas and clear known attributes
     for (const filePath in mergedDataStructure.dependencySchemas) {
-      for (const entityName in mergedDataStructure.dependencySchemas[
+      for (const depEntityName in mergedDataStructure.dependencySchemas[
         filePath
       ]) {
         const depSchema =
-          mergedDataStructure.dependencySchemas[filePath][entityName];
+          mergedDataStructure.dependencySchemas[filePath][depEntityName];
+
+        // Trace deduplication of dependency signatureSchema
+        const depSigBefore = { ...depSchema.signatureSchema };
         depSchema.signatureSchema = deduplicateFunctionSchemas(
           depSchema.signatureSchema,
         );
+        transformationTracer.traceSchemaTransformResult(
+          entity.name,
+          'deduplicateFunctionSchemas',
+          depSigBefore,
+          depSchema.signatureSchema,
+          { filePath, dependencyName: depEntityName, schemaType: 'signature' },
+        );
+
+        // Trace deduplication of dependency returnValueSchema
+        const depRvBefore = { ...depSchema.returnValueSchema };
         depSchema.returnValueSchema = deduplicateFunctionSchemas(
           depSchema.returnValueSchema,
         );
+        transformationTracer.traceSchemaTransformResult(
+          entity.name,
+          'deduplicateFunctionSchemas',
+          depRvBefore,
+          depSchema.returnValueSchema,
+          {
+            filePath,
+            dependencyName: depEntityName,
+            schemaType: 'returnValue',
+          },
+        );
 
-        clearAttributesFromMapping(depSchema.signatureSchema);
-        clearAttributesFromMapping(depSchema.returnValueSchema);
+        // Fill in type gaps BEFORE clearing attributes, so evidence like .includes()
+        // is used for type inference before being deleted
+
+        depSchema.signatureSchema = fillInDirectSchemaGapsAndUnknowns({
+          schema: depSchema.signatureSchema,
+        });
+
+        depSchema.returnValueSchema = fillInDirectSchemaGapsAndUnknowns({
+          schema: depSchema.returnValueSchema,
+        });
+
+        // Trace clearAttributesFromMapping on dependency schemas
+        transformationTracer.traceSchemaTransform(
+          entity.name,
+          'clearAttributesFromMapping',
+          depSchema.signatureSchema,
+          clearAttributesFromMapping,
+          { filePath, dependencyName: depEntityName, schemaType: 'signature' },
+        );
+
+        transformationTracer.traceSchemaTransform(
+          entity.name,
+          'clearAttributesFromMapping',
+          depSchema.returnValueSchema,
+          clearAttributesFromMapping,
+          {
+            filePath,
+            dependencyName: depEntityName,
+            schemaType: 'returnValue',
+          },
+        );
       }
     }
 
+    // Transformation tracing: capture after deduplication
+    transformationTracer.snapshot(entity.name, 'deduplicated', {
+      signatureSchema: mergedDataStructure.signatureSchema,
+      returnValueSchema: mergedDataStructure.returnValueSchema,
+      dependencySchemas: mergedDataStructure.dependencySchemas,
+    });
+
     analysis.metadata ||= {};
     analysis.metadata.mergedDataStructure = mergedDataStructure;
 
@@ -543,29 +692,70 @@ export default function generateDataStructure({
     // expects survey.updatedAt, the array element type includes updatedAt.
     // Uses usageEquivalencies to trace data flow and only enrich arrays with fields
     // from children that actually receive elements from that specific array.
-    enrichArrayTypesFromChildSignatures(
-      allImportedExports,
+    transformationTracer.traceDependencySchemaChanges(
+      entity.name,
+      'enrichArrayTypesFromChildSignatures',
       mergedDataStructure.dependencySchemas,
-      mergedDataStructure.usageEquivalencies,
+      () =>
+        enrichArrayTypesFromChildSignatures(
+          allImportedExports,
+          mergedDataStructure.dependencySchemas,
+          mergedDataStructure.usageEquivalencies,
+        ),
     );
 
+    // Transformation tracing: capture after array types enriched
+    transformationTracer.snapshot(entity.name, 'arrayTypesEnriched', {
+      signatureSchema: mergedDataStructure.signatureSchema,
+      returnValueSchema: mergedDataStructure.returnValueSchema,
+      dependencySchemas: mergedDataStructure.dependencySchemas,
+    });
+
     // Enrich mocked dependency unknown types using sourceEquivalencies
     // When a mocked dependency returns unknown, but that data flows to a child component
     // with a known type, propagate the type back to the mocked dependency.
     // This prevents fillInDirectSchemaGapsAndUnknowns from guessing incorrectly
     // (e.g., 'data' becoming 'number' because pluralize thinks it's plural)
-    enrichUnknownTypesFromSourceEquivalencies(
-      isolatedDataStructure.dependencySchemas || {},
+    transformationTracer.traceDependencySchemaChanges(
+      entity.name,
+      'enrichUnknownTypesFromSourceEquivalencies',
       mergedDataStructure.dependencySchemas,
-      allImportedExports,
+      () =>
+        enrichUnknownTypesFromSourceEquivalencies(
+          isolatedDataStructure.dependencySchemas || {},
+          mergedDataStructure.dependencySchemas,
+          allImportedExports,
+        ),
     );
 
+    // Transformation tracing: capture after unknown types enriched
+    transformationTracer.snapshot(entity.name, 'unknownTypesEnriched', {
+      signatureSchema: mergedDataStructure.signatureSchema,
+      returnValueSchema: mergedDataStructure.returnValueSchema,
+      dependencySchemas: mergedDataStructure.dependencySchemas,
+    });
+
+    // DISABLED: Testing if ternary fix eliminates the need for this heuristic
+    // Propagate array item schemas between arrays that likely have the same type.
+    // When one array (e.g., entities) has item schema traced but another similar array
+    // (e.g., currentEntities) doesn't (because it's only used in an OR expression fallback),
+    // copy the item schema from the rich array to the empty one.
+    // propagateArrayItemSchemas(mergedDataStructure.dependencySchemas);
+
     let dataForMocks = gatherDataForMocks(
       allImportedExports,
       mergedDataStructure.dependencySchemas,
       { entityName: entity.name },
     );
 
+    // Transformation tracing: capture after dataForMocks gathered
+    transformationTracer.snapshot(entity.name, 'dataForMocksGathered', {
+      signatureSchema: mergedDataStructure.signatureSchema,
+      returnValueSchema: mergedDataStructure.returnValueSchema,
+      dependencySchemas: mergedDataStructure.dependencySchemas,
+      dataForMocks,
+    });
+
     // Merge dataForMocks from non-mocked child analyses to include transitive dependency data.
     // This handles cases where grandchild trpc calls aren't in allImportedExports because
     // the grandchild's analysis wasn't in dependentAnalyses.
@@ -587,11 +777,25 @@ export default function generateDataStructure({
       }
     }
 
-    const finalizedArgumentsSchema = fillInDirectSchemaGapsAndUnknowns({
-      schema: { ...mergedDataStructure.signatureSchema },
-    });
+    const finalizedArgumentsSchema = { ...mergedDataStructure.signatureSchema };
 
-    clearAttributesFromMapping(finalizedArgumentsSchema);
+    // Trace fillInDirectSchemaGapsAndUnknowns
+    transformationTracer.traceSchemaTransform(
+      entity.name,
+      'fillInDirectSchemaGapsAndUnknowns',
+      finalizedArgumentsSchema,
+      (schema) => fillInDirectSchemaGapsAndUnknowns({ schema }),
+      { stage: 'finalization' },
+    );
+
+    // Trace clearAttributesFromMapping
+    transformationTracer.traceSchemaTransform(
+      entity.name,
+      'clearAttributesFromMapping',
+      finalizedArgumentsSchema,
+      clearAttributesFromMapping,
+      { stage: 'finalization' },
+    );
 
     const argumentsSchema = convertDotNotation(finalizedArgumentsSchema)
       .signature as JsonTypeDefinition[];
@@ -601,6 +805,12 @@ export default function generateDataStructure({
       dataForMocks,
     };
 
+    // Transformation tracing: capture final scenariosDataStructure
+    transformationTracer.snapshot(entity.name, 'finalized', {
+      signatureSchema: finalizedArgumentsSchema,
+      dataForMocks,
+    });
+
     return analysis;
   } catch (e) {
     awsLog('CodeYam Error: Error generating data structure', {