@codeyam/codeyam-cli 0.1.0-staging.2a88920 → 0.1.0-staging.4057bd2

package/analyzer-template/packages/analyze/src/lib/files/scenarios/enrichArrayTypesFromChildSignatures.ts

@@ -246,7 +246,11 @@ export default function enrichArrayTypesFromChildSignatures(
         // Also check if the array path ends with the local variable name
         // e.g., useLoaderData().functionCallReturnValue.entities ends with .entities
         const arrayEndsWithLocalVar = arrayPath.endsWith(`.${localVarName}`);
+        // Full-path usageEquivalencies (e.g., from same-file child components) use
+        // the complete call path as the local var name, which matches arrayPath exactly
+        const exactPathMatch = localVarName === arrayPath;
         if (
+          !exactPathMatch &&
           !couldBeDerivedFrom(localVarName, entityName) &&
           !arrayEndsWithLocalVar
         ) {

package/analyzer-template/packages/analyze/src/lib/files/scenarios/gatherDataForMocks.ts

@@ -353,8 +353,6 @@ function processCall(
   // Preprocess to filter signatures and extract functionCallReturnValue
   let preprocessedSchema = preprocessSchemaForMocks(mergedSchema);
 
-  const functionNames: string[] = [];
-
   const isFunctionNameWithoutFunction = (
     path: string,
     functionName: string,
@@ -439,31 +437,58 @@ function processCall(
     }
   }
 
-  const relevantMergedDependencySchema = Object.keys(preprocessedSchema).reduce(
-    (acc, path) => {
-      if (path.endsWith(')')) {
-        const pathParts = splitOutsideParenthesesAndArrays(path);
-        const functionName = joinParenthesesAndArrays([
-          ...pathParts.slice(0, -1),
-          pathParts[pathParts.length - 1].split('(')[0],
-        ]);
-        functionNames.push(functionName);
-        for (const existingPath in acc) {
-          if (isFunctionNameWithoutFunction(existingPath, functionName)) {
-            delete acc[existingPath];
-          }
-        }
-      }
+  // Two-pass function name filtering to avoid O(k²) retroactive deletion.
+  // Pass 1: Discover all unique function names from paths ending with ')'
+  const functionNameSet = new Set<string>();
+  const schemaKeys = Object.keys(preprocessedSchema);
+  for (const path of schemaKeys) {
+    if (path.endsWith(')')) {
+      const pathParts = splitOutsideParenthesesAndArrays(path);
+      const functionName = joinParenthesesAndArrays([
+        ...pathParts.slice(0, -1),
+        pathParts[pathParts.length - 1].split('(')[0],
+      ]);
+      functionNameSet.add(functionName);
+    }
+  }
 
-      if (functionNames.some((fn) => isFunctionNameWithoutFunction(path, fn))) {
-        return acc;
-      }
+  // Index function names by first segment for O(1) bucket lookup instead of
+  // checking all function names for every path. Most paths only need to check
+  // 1-2 function names instead of all F.
+  const fnByFirstSegment = new Map<string, string[]>();
+  for (const fn of functionNameSet) {
+    const dotIdx = fn.indexOf('.');
+    const firstSeg = dotIdx >= 0 ? fn.slice(0, dotIdx) : fn;
+    let bucket = fnByFirstSegment.get(firstSeg);
+    if (!bucket) {
+      bucket = [];
+      fnByFirstSegment.set(firstSeg, bucket);
+    }
+    bucket.push(fn);
+  }
 
-      acc[path] = preprocessedSchema[path];
-      return acc;
-    },
-    {} as Record<string, string>,
-  );
+  // Pass 2: Filter paths that match any discovered function name prefix.
+  // Use indexed lookup: extract path's first segment, check only matching function names.
+  const relevantMergedDependencySchema: Record<string, string> = {};
+  for (const path of schemaKeys) {
+    // Extract first segment of path (before first '.', '(', or '<')
+    const dotIdx = path.indexOf('.');
+    const parenIdx = path.indexOf('(');
+    const angleIdx = path.indexOf('<');
+    let minIdx = path.length;
+    if (dotIdx >= 0 && dotIdx < minIdx) minIdx = dotIdx;
+    if (parenIdx >= 0 && parenIdx < minIdx) minIdx = parenIdx;
+    if (angleIdx >= 0 && angleIdx < minIdx) minIdx = angleIdx;
+    const firstSeg = path.slice(0, minIdx);
+
+    const candidates = fnByFirstSegment.get(firstSeg);
+    if (
+      !candidates ||
+      !candidates.some((fn) => isFunctionNameWithoutFunction(path, fn))
+    ) {
+      relevantMergedDependencySchema[path] = preprocessedSchema[path];
+    }
+  }
 
   const filledSchema = fillInDirectSchemaGapsAndUnknowns({
     scopeName: importedExport.name,
@@ -574,6 +599,12 @@ export default function gatherDataForMocks(
 ) {
   // Build schema indexes ONCE from all dependency schemas combined.
   // This avoids rebuilding indexes (~1-2 seconds for 18k keys) for each processCall.
+  //
+  // IMPORTANT: We must preprocess schemas BEFORE building indexes!
+  // The schemas contain .functionCallReturnValue paths that get stripped during processing.
+  // If we build indexes from original paths, lookups with preprocessed paths will fail.
+  // Example: 'hook().functionCallReturnValue.x.length' → index maps 'hook().functionCallReturnValue.x' → ['length']
+  //          But preprocessed path 'hook().x.length' looks for 'hook().x' which doesn't exist in the index.
   const allSchemaKeys: Record<string, string> = {};
   if (dependencySchemas) {
     for (const filePath in dependencySchemas) {
@@ -581,7 +612,9 @@ export default function gatherDataForMocks(
         const schema =
           dependencySchemas[filePath][entityName]?.returnValueSchema;
         if (schema) {
-          Object.assign(allSchemaKeys, schema);
+          // Preprocess each schema to remove .functionCallReturnValue before collecting
+          const preprocessed = preprocessSchemaForMocks(schema);
+          Object.assign(allSchemaKeys, preprocessed);
         }
       }
     }

package/analyzer-template/packages/analyze/src/lib/files/scenarios/generateDataStructure.ts

@@ -16,6 +16,7 @@ import { awsLog } from '~codeyam/utils';
 import gatherDataForMocks from './gatherDataForMocks';
 import enrichArrayTypesFromChildSignatures from './enrichArrayTypesFromChildSignatures';
 import enrichUnknownTypesFromSourceEquivalencies from './enrichUnknownTypesFromSourceEquivalencies';
+import { transformationTracer } from './TransformationTracer';
 // import propagateArrayItemSchemas from './propagateArrayItemSchemas';
 export interface GenerateDataStructureArgs {
   entity: Entity;
@@ -163,6 +164,25 @@ export default function generateDataStructure({
         return acc;
       }, {} as ReadonlyAnalysisMap);
 
+    // Include same-file child components in nonMockedDependentAnalyses.
+    // Same-file components (e.g., AgentCard defined in the same file as AgentTranscriptsPage)
+    // are NOT in importedExports, so the reduce above never includes them.
+    // But they ARE in dependentAnalyses and their full signatureSchema (with all prop fields)
+    // needs to be merged into the parent's dependencySchemas for array enrichment to work.
+
+    const sameFileAnalyses = dependentAnalyses[entity.filePath];
+    if (sameFileAnalyses) {
+      for (const name in sameFileAnalyses) {
+        if (name === entity.name) continue;
+        if (nonMockedDependentAnalyses[entity.filePath]?.[name]) continue;
+        const childAnalysis = sameFileAnalyses[name];
+        if (childAnalysis?.entityType === 'visual') {
+          nonMockedDependentAnalyses[entity.filePath] ||= {};
+          nonMockedDependentAnalyses[entity.filePath][name] = childAnalysis;
+        }
+      }
+    }
+
     // For non-mocked child components (like visual components), include their mocked
     // dependencies in allImportedExports. This ensures that when a parent component
     // renders a child that uses the same hook with different destructure patterns,
@@ -441,6 +461,29 @@ export default function generateDataStructure({
 
     const { isolatedDataStructure } = entity.metadata;
 
+    if (!isolatedDataStructure) {
+      awsLog(
+        'CodeYam: Skipping data structure generation — isolatedDataStructure is undefined (scope analysis may have failed)',
+        {
+          entityName: entity.name,
+          filePath: entity.filePath,
+        },
+      );
+      return analysis;
+    }
+
+    // Transformation tracing: start entity and capture isolated data structure
+    transformationTracer.startEntity({
+      name: entity.name,
+      entityType: entity.entityType,
+      filePath: entity.filePath,
+    });
+    transformationTracer.snapshot(entity.name, 'isolated', {
+      signatureSchema: isolatedDataStructure.signatureSchema,
+      returnValueSchema: isolatedDataStructure.returnValueSchema,
+      dependencySchemas: isolatedDataStructure.dependencySchemas,
+    });
+
     const mergedDataStructure = mergeInDependentDataStructure({
       importedExports: allImportedExports,
       dependentAnalyses: nonMockedDependentAnalyses,
@@ -458,6 +501,13 @@ export default function generateDataStructure({
     mergedDataStructure.environmentVariables =
       isolatedDataStructure.environmentVariables || [];
 
+    // Transformation tracing: capture merged data structure
+    transformationTracer.snapshot(entity.name, 'merged', {
+      signatureSchema: mergedDataStructure.signatureSchema,
+      returnValueSchema: mergedDataStructure.returnValueSchema,
+      dependencySchemas: mergedDataStructure.dependencySchemas,
+    });
+
     // Merge collected child dependency schemas into mergedDataStructure.
     // This ensures that when gatherDataForMocks looks up schemas for child imports,
     // the child's schemas are available.
@@ -511,6 +561,13 @@ export default function generateDataStructure({
       }
     }
 
+    // Transformation tracing: capture after child schemas merged
+    transformationTracer.snapshot(entity.name, 'childSchemasMerged', {
+      signatureSchema: mergedDataStructure.signatureSchema,
+      returnValueSchema: mergedDataStructure.returnValueSchema,
+      dependencySchemas: mergedDataStructure.dependencySchemas,
+    });
+
     // console.info(
     //   'CODEYAM DEBUG: end merge',
     //   JSON.stringify(
@@ -533,32 +590,183 @@ export default function generateDataStructure({
 
     // Deduplicate function schemas in the merged data structure
     // The merge process might introduce duplicates even if individual schemas were clean
+    // Trace deduplication of signatureSchema
+    const sigBefore = { ...mergedDataStructure.signatureSchema };
     mergedDataStructure.signatureSchema = deduplicateFunctionSchemas(
       mergedDataStructure.signatureSchema,
     );
+    transformationTracer.traceSchemaTransformResult(
+      entity.name,
+      'deduplicateFunctionSchemas',
+      sigBefore,
+      mergedDataStructure.signatureSchema,
+      { schemaType: 'signature' },
+    );
+
+    // Trace deduplication of returnValueSchema
+    const rvBefore = { ...mergedDataStructure.returnValueSchema };
     mergedDataStructure.returnValueSchema = deduplicateFunctionSchemas(
       mergedDataStructure.returnValueSchema,
     );
+    transformationTracer.traceSchemaTransformResult(
+      entity.name,
+      'deduplicateFunctionSchemas',
+      rvBefore,
+      mergedDataStructure.returnValueSchema,
+      { schemaType: 'returnValue' },
+    );
 
-    // Also deduplicate dependency schemas
+    // Also deduplicate dependency schemas and clear known attributes
     for (const filePath in mergedDataStructure.dependencySchemas) {
-      for (const entityName in mergedDataStructure.dependencySchemas[
+      for (const depEntityName in mergedDataStructure.dependencySchemas[
         filePath
       ]) {
         const depSchema =
-          mergedDataStructure.dependencySchemas[filePath][entityName];
+          mergedDataStructure.dependencySchemas[filePath][depEntityName];
+
+        // Trace deduplication of dependency signatureSchema
+        const depSigBefore = { ...depSchema.signatureSchema };
         depSchema.signatureSchema = deduplicateFunctionSchemas(
           depSchema.signatureSchema,
         );
+        transformationTracer.traceSchemaTransformResult(
+          entity.name,
+          'deduplicateFunctionSchemas',
+          depSigBefore,
+          depSchema.signatureSchema,
+          { filePath, dependencyName: depEntityName, schemaType: 'signature' },
+        );
+
+        // Trace deduplication of dependency returnValueSchema
+        const depRvBefore = { ...depSchema.returnValueSchema };
         depSchema.returnValueSchema = deduplicateFunctionSchemas(
           depSchema.returnValueSchema,
         );
+        transformationTracer.traceSchemaTransformResult(
+          entity.name,
+          'deduplicateFunctionSchemas',
+          depRvBefore,
+          depSchema.returnValueSchema,
+          {
+            filePath,
+            dependencyName: depEntityName,
+            schemaType: 'returnValue',
+          },
+        );
+
+        // Infer function types from child component usage BEFORE fillInSchemaGapsAndUnknowns.
+        // When a hook returns a property with type 'unknown' that is passed to a child
+        // component which calls it as a function (e.g., isEntityPending(entity)),
+        // promote it to 'function'. Without this, name heuristics like isLikelyBoolean
+        // would mistype "isEntityPending" as boolean because of the "is" prefix.
+        //
+        // sourceEquivalencies are in isolatedDataStructure.dependencySchemas (not in
+        // mergedDataStructure.dependencySchemas — they're dropped during the merge).
+        // The child's signatureSchema (with function call evidence) IS in the merged schemas.
+        {
+          const isolatedDepSchemas =
+            entity.metadata.isolatedDataStructure?.dependencySchemas;
+          if (isolatedDepSchemas) {
+            for (const childFilePath in isolatedDepSchemas) {
+              const childIsolated = isolatedDepSchemas[childFilePath];
+              for (const childName in childIsolated) {
+                const sourceEq = childIsolated[childName]?.sourceEquivalencies;
+                if (!sourceEq) continue;
+
+                // Get the child's signatureSchema from the MERGED schemas (richer)
+                const mergedChildSchema =
+                  mergedDataStructure.dependencySchemas[childFilePath]?.[
+                    childName
+                  ];
+                const sigSchema = mergedChildSchema?.signatureSchema;
+                if (!sigSchema) continue;
+
+                for (const childPath in sourceEq) {
+                  for (const source of sourceEq[childPath]) {
+                    const fcrvSuffix = '.functionCallReturnValue.';
+                    const fcrvIndex = source.schemaPath.indexOf(fcrvSuffix);
+                    if (fcrvIndex === -1) continue;
+
+                    const propName = source.schemaPath.slice(
+                      fcrvIndex + fcrvSuffix.length,
+                    );
+
+                    // Find the corresponding child signature path
+                    const entityCallPrefix = childName + '().';
+                    const sigPathBase = childPath.startsWith(entityCallPrefix)
+                      ? childPath.slice(entityCallPrefix.length)
+                      : childPath;
+
+                    // Check if the child's signatureSchema has a function call version
+                    let isFunction = false;
+                    for (const sigKey in sigSchema) {
+                      if (
+                        sigKey.startsWith(sigPathBase + '(') &&
+                        sigSchema[sigKey] === 'function'
+                      ) {
+                        isFunction = true;
+                        break;
+                      }
+                    }
+
+                    if (isFunction) {
+                      for (const rvKey in depSchema.returnValueSchema) {
+                        if (
+                          rvKey.endsWith('.' + propName) &&
+                          depSchema.returnValueSchema[rvKey] === 'unknown'
+                        ) {
+                          depSchema.returnValueSchema[rvKey] = 'function';
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+
+        // Fill in type gaps BEFORE clearing attributes, so evidence like .includes()
+        // is used for type inference before being deleted
+
+        depSchema.signatureSchema = fillInDirectSchemaGapsAndUnknowns({
+          schema: depSchema.signatureSchema,
+        });
+
+        depSchema.returnValueSchema = fillInDirectSchemaGapsAndUnknowns({
+          schema: depSchema.returnValueSchema,
+        });
+
+        // Trace clearAttributesFromMapping on dependency schemas
+        transformationTracer.traceSchemaTransform(
+          entity.name,
+          'clearAttributesFromMapping',
+          depSchema.signatureSchema,
+          clearAttributesFromMapping,
+          { filePath, dependencyName: depEntityName, schemaType: 'signature' },
+        );
 
-        clearAttributesFromMapping(depSchema.signatureSchema);
-        clearAttributesFromMapping(depSchema.returnValueSchema);
+        transformationTracer.traceSchemaTransform(
+          entity.name,
+          'clearAttributesFromMapping',
+          depSchema.returnValueSchema,
+          clearAttributesFromMapping,
+          {
+            filePath,
+            dependencyName: depEntityName,
+            schemaType: 'returnValue',
+          },
+        );
       }
     }
 
+    // Transformation tracing: capture after deduplication
+    transformationTracer.snapshot(entity.name, 'deduplicated', {
+      signatureSchema: mergedDataStructure.signatureSchema,
+      returnValueSchema: mergedDataStructure.returnValueSchema,
+      dependencySchemas: mergedDataStructure.dependencySchemas,
+    });
+
     analysis.metadata ||= {};
     analysis.metadata.mergedDataStructure = mergedDataStructure;
 
@@ -567,23 +775,49 @@ export default function generateDataStructure({
     // expects survey.updatedAt, the array element type includes updatedAt.
     // Uses usageEquivalencies to trace data flow and only enrich arrays with fields
     // from children that actually receive elements from that specific array.
-    enrichArrayTypesFromChildSignatures(
-      allImportedExports,
+    transformationTracer.traceDependencySchemaChanges(
+      entity.name,
+      'enrichArrayTypesFromChildSignatures',
       mergedDataStructure.dependencySchemas,
-      mergedDataStructure.usageEquivalencies,
+      () =>
+        enrichArrayTypesFromChildSignatures(
+          allImportedExports,
+          mergedDataStructure.dependencySchemas,
+          mergedDataStructure.usageEquivalencies,
+        ),
     );
 
+    // Transformation tracing: capture after array types enriched
+    transformationTracer.snapshot(entity.name, 'arrayTypesEnriched', {
+      signatureSchema: mergedDataStructure.signatureSchema,
+      returnValueSchema: mergedDataStructure.returnValueSchema,
+      dependencySchemas: mergedDataStructure.dependencySchemas,
+    });
+
     // Enrich mocked dependency unknown types using sourceEquivalencies
     // When a mocked dependency returns unknown, but that data flows to a child component
     // with a known type, propagate the type back to the mocked dependency.
     // This prevents fillInDirectSchemaGapsAndUnknowns from guessing incorrectly
     // (e.g., 'data' becoming 'number' because pluralize thinks it's plural)
-    enrichUnknownTypesFromSourceEquivalencies(
-      isolatedDataStructure.dependencySchemas || {},
+    transformationTracer.traceDependencySchemaChanges(
+      entity.name,
+      'enrichUnknownTypesFromSourceEquivalencies',
       mergedDataStructure.dependencySchemas,
-      allImportedExports,
+      () =>
+        enrichUnknownTypesFromSourceEquivalencies(
+          isolatedDataStructure.dependencySchemas || {},
+          mergedDataStructure.dependencySchemas,
+          allImportedExports,
+        ),
     );
 
+    // Transformation tracing: capture after unknown types enriched
+    transformationTracer.snapshot(entity.name, 'unknownTypesEnriched', {
+      signatureSchema: mergedDataStructure.signatureSchema,
+      returnValueSchema: mergedDataStructure.returnValueSchema,
+      dependencySchemas: mergedDataStructure.dependencySchemas,
+    });
+
     // DISABLED: Testing if ternary fix eliminates the need for this heuristic
     // Propagate array item schemas between arrays that likely have the same type.
     // When one array (e.g., entities) has item schema traced but another similar array
@@ -597,6 +831,14 @@ export default function generateDataStructure({
       { entityName: entity.name },
     );
 
+    // Transformation tracing: capture after dataForMocks gathered
+    transformationTracer.snapshot(entity.name, 'dataForMocksGathered', {
+      signatureSchema: mergedDataStructure.signatureSchema,
+      returnValueSchema: mergedDataStructure.returnValueSchema,
+      dependencySchemas: mergedDataStructure.dependencySchemas,
+      dataForMocks,
+    });
+
     // Merge dataForMocks from non-mocked child analyses to include transitive dependency data.
     // This handles cases where grandchild trpc calls aren't in allImportedExports because
     // the grandchild's analysis wasn't in dependentAnalyses.
@@ -618,11 +860,25 @@ export default function generateDataStructure({
       }
     }
 
-    const finalizedArgumentsSchema = fillInDirectSchemaGapsAndUnknowns({
-      schema: { ...mergedDataStructure.signatureSchema },
-    });
+    const finalizedArgumentsSchema = { ...mergedDataStructure.signatureSchema };
+
+    // Trace fillInDirectSchemaGapsAndUnknowns
+    transformationTracer.traceSchemaTransform(
+      entity.name,
+      'fillInDirectSchemaGapsAndUnknowns',
+      finalizedArgumentsSchema,
+      (schema) => fillInDirectSchemaGapsAndUnknowns({ schema }),
+      { stage: 'finalization' },
+    );
 
-    clearAttributesFromMapping(finalizedArgumentsSchema);
+    // Trace clearAttributesFromMapping
+    transformationTracer.traceSchemaTransform(
+      entity.name,
+      'clearAttributesFromMapping',
+      finalizedArgumentsSchema,
+      clearAttributesFromMapping,
+      { stage: 'finalization' },
+    );
 
     const argumentsSchema = convertDotNotation(finalizedArgumentsSchema)
       .signature as JsonTypeDefinition[];
@@ -632,6 +888,12 @@ export default function generateDataStructure({
       dataForMocks,
     };
 
+    // Transformation tracing: capture final scenariosDataStructure
+    transformationTracer.snapshot(entity.name, 'finalized', {
+      signatureSchema: finalizedArgumentsSchema,
+      dataForMocks,
+    });
+
     return analysis;
   } catch (e) {
     awsLog('CodeYam Error: Error generating data structure', {