@codeyam/codeyam-cli 0.1.0-staging.28f73cf → 0.1.0-staging.76566f9

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/convertNullToUndefinedBySchema.ts

@@ -0,0 +1,98 @@
+/**
+ * Converts null values to undefined (removes them) based on schema type constraints.
+ *
+ * When the LLM generates mock data, it uses `null` to represent "no value" because
+ * JSON doesn't support `undefined`. However, TypeScript types like `string | undefined`
+ * don't accept `null`.
+ *
+ * This function traverses data and schema together:
+ * - If schema type is "T | undefined" (but NOT "T | null"), convert null → undefined (delete key)
+ * - If schema type includes "null" (e.g., "T | null" or "T | null | undefined"), preserve null
+ * - If field not in schema, preserve as-is
+ *
+ * @param data The data object with potential null values
+ * @param schema The schema describing expected types
+ * @returns The data with null→undefined conversions applied (mutates original)
+ */
+export default function convertNullToUndefinedBySchema<T>(
+  data: T,
+  schema: unknown,
+): T {
+  // Handle null/undefined data
+  if (data === null || data === undefined) {
+    return data;
+  }
+
+  // Only process objects
+  if (typeof data !== 'object') {
+    return data;
+  }
+
+  // Handle arrays
+  if (Array.isArray(data)) {
+    // For arrays, the schema should be an array with one element describing the item type
+    const itemSchema = Array.isArray(schema) ? schema[0] : undefined;
+    for (let i = 0; i < data.length; i++) {
+      data[i] = convertNullToUndefinedBySchema(data[i], itemSchema);
+    }
+    return data;
+  }
+
+  // Handle objects
+  const dataObj = data as Record<string, unknown>;
+  const schemaObj = schema as Record<string, unknown> | undefined;
+
+  for (const key of Object.keys(dataObj)) {
+    const value = dataObj[key];
+    const fieldSchema = schemaObj?.[key];
+
+    if (value === null) {
+      // Check if schema allows null
+      if (shouldConvertNullToUndefined(fieldSchema)) {
+        // Delete the key (converts null to undefined)
+        delete dataObj[key];
+      }
+      // If schema allows null, preserve it
+    } else if (typeof value === 'object' && value !== null) {
+      // Recursively process nested objects/arrays
+      convertNullToUndefinedBySchema(value, fieldSchema);
+    }
+  }
+
+  return data;
+}
+
+/**
+ * Determines if a null value should be converted to undefined based on schema type.
+ *
+ * @param fieldSchema The schema type string or object for the field
+ * @returns true if null should be converted to undefined, false if null should be preserved
+ */
+function shouldConvertNullToUndefined(fieldSchema: unknown): boolean {
+  // If no schema for this field, preserve null (be conservative)
+  if (fieldSchema === undefined || fieldSchema === null) {
+    return false;
+  }
+
+  // If schema is a string type definition, check if it allows null
+  if (typeof fieldSchema === 'string') {
+    // Schema types that include "null" should preserve null values
+    // e.g., "string | null", "number | null | undefined", "null"
+    const allowsNull = fieldSchema.includes('null');
+    const allowsUndefined = fieldSchema.includes('undefined');
+
+    // If type allows undefined but NOT null, convert null to undefined
+    // e.g., "string | undefined" → convert null to undefined
+    // e.g., "string | null" → preserve null
+    // e.g., "string | null | undefined" → preserve null
+    if (allowsUndefined && !allowsNull) {
+      return true;
+    }
+
+    return false;
+  }
+
+  // If schema is an object or array, it's a nested structure
+  // Don't convert the container itself, just recurse into it
+  return false;
+}

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/fillInSchemaGapsAndUnknowns.ts

@@ -208,10 +208,12 @@ export default function fillInSchemaGapsAndUnknowns(
 export function fillInDirectSchemaGapsAndUnknowns({
   scopeName,
   schema,
+  mergedSchema,
   attempts = 0,
 }: {
   scopeName?: string;
   schema: Record<string, string>;
+  mergedSchema?: Record<string, string>;
   attempts?: number;
 }) {
   try {
@@ -245,7 +247,9 @@ export function fillInDirectSchemaGapsAndUnknowns({
     const guessTypeForPath = (path: string, defaultType = 'string') => {
       let knownType = checkIfKnownType(path, functionKeysMapping);
       if (!knownType) {
-        knownType = checkIfObjectOrFunction(path, schema);
+        // Use mergedSchema for type inference if available (has more complete nested paths)
+        // Fall back to schema if mergedSchema is not provided
+        knownType = checkIfObjectOrFunction(path, mergedSchema ?? schema);
       }
 
       if (knownType && schema[path] !== knownType) {
@@ -373,10 +377,39 @@ export function fillInDirectSchemaGapsAndUnknowns({
       fillInDirectSchemaGapsAndUnknowns({
         scopeName,
         schema,
+        mergedSchema,
         attempts: ++attempts,
       });
     }
 
+    // Remove .length paths when the parent is typed as 'array' or 'string' (or nullable variants).
+    // These are built-in properties that were used for type inference but shouldn't appear
+    // in the final schema (they create { length: N } objects instead of proper arrays).
+    // However, if the parent could be an 'object' or other type, keep .length as it may be
+    // a custom property.
+    for (const key of Object.keys(schema)) {
+      const parts = splitOutsideParenthesesAndArrays(key);
+      if (parts[parts.length - 1] === 'length') {
+        const parentPath = joinParenthesesAndArrays(parts.slice(0, -1));
+        const parentType = schema[parentPath];
+        if (parentType) {
+          // Split union types and filter out nullable parts
+          const typeParts = parentType.split(' | ').map((t) => t.trim());
+          const nonNullableTypes = typeParts.filter(
+            (t) => t !== 'undefined' && t !== 'null',
+          );
+          // Only remove if ALL non-nullable types are array or string
+          // This ensures we don't remove .length from potential custom objects
+          if (
+            nonNullableTypes.length > 0 &&
+            nonNullableTypes.every((t) => t === 'array' || t === 'string')
+          ) {
+            delete schema[key];
+          }
+        }
+      }
+    }
+
     return schema;
   } catch (error) {
     console.info(
@@ -454,6 +487,8 @@ function checkIfKnownType(
   let isString: boolean | undefined;
   let hasStrongArrayEvidence = false;
   let hasOnlyAmbiguousArrayMethods = true;
+  let hasLengthAccess = false;
+  let hasArrayStringAmbiguousMethod = false;
 
   for (const functionOrAttributeName of functionAndAttributesKeysMapping[key] ??
     []) {
@@ -467,11 +502,18 @@ function checkIfKnownType(
       knownMethodCalls(functionOrAttributeName).includes('string') ||
       functionOrAttributeName === 'length';
 
-    // Check if this is strong evidence of an array
-    if (
-      STRONG_ARRAY_METHODS.has(methodName) ||
-      functionOrAttributeName === 'length'
-    ) {
+    // Track .length access
+    if (functionOrAttributeName === 'length') {
+      hasLengthAccess = true;
+    }
+
+    // Track methods that exist on both arrays and strings (like includes, indexOf)
+    if (couldBeArray && couldBeString && functionOrAttributeName !== 'length') {
+      hasArrayStringAmbiguousMethod = true;
+    }
+
+    // Check if this is strong evidence of an array (unambiguous array methods)
+    if (STRONG_ARRAY_METHODS.has(methodName)) {
       hasStrongArrayEvidence = true;
     }
 
@@ -495,10 +537,24 @@ function checkIfKnownType(
     }
   }
 
+  // Strong array evidence (unambiguous methods like .map, .filter) -> definitely array
+  if (isArray && hasStrongArrayEvidence) {
+    return 'array';
+  }
+
+  // When we have BOTH .length AND an ambiguous method like .includes(),
+  // prefer array. This pattern (checking .length AND calling .includes)
+  // is very common with arrays (e.g., arr.length > 0 && arr.includes(x))
+  // but rare with strings (you'd typically just call str.includes(x))
+  if (isArray && hasLengthAccess && hasArrayStringAmbiguousMethod) {
+    return 'array';
+  }
+
   if (isString) return 'string';
+
   // Only infer array if we have strong evidence OR not only ambiguous methods
   // This prevents router.push() from being incorrectly inferred as an array
-  if (isArray && (hasStrongArrayEvidence || !hasOnlyAmbiguousArrayMethods)) {
+  if (isArray && !hasOnlyAmbiguousArrayMethods) {
     return 'array';
   }
 }

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/getFunctionCallRoot.ts

@@ -1,11 +1,41 @@
 import { splitOutsideParenthesesAndArrays } from '../../splitOutsideParentheses';
 
+/**
+ * Find the position of the first '(' that is NOT inside angle brackets.
+ * Returns -1 if not found.
+ */
+function findFirstParenOutsideTypeParams(signature: string): number {
+  let angleDepth = 0;
+
+  for (let i = 0; i < signature.length; i++) {
+    const char = signature[i];
+
+    if (char === '<') {
+      angleDepth++;
+    } else if (char === '>') {
+      angleDepth--;
+    } else if (char === '(' && angleDepth === 0) {
+      return i;
+    }
+  }
+
+  return -1;
+}
+
 export default function getFunctionCallRoot(functionCallSignature: string) {
   const functionCallSignatureParts = splitOutsideParenthesesAndArrays(
     functionCallSignature,
   );
 
-  return functionCallSignatureParts[0]
-    .replace(/(cyScope\d+)F/, '$1')
-    .split('(')[0];
+  const firstPart = functionCallSignatureParts[0];
+
+  // Find the first '(' that's not inside type parameters (angle brackets)
+  // This handles cases like "useFetcher<{ data: UserData | null }>()"
+  // which should return "useFetcher<{ data: UserData | null }>"
+  // (preserving type params for scope node matching)
+  const parenPos = findFirstParenOutsideTypeParams(firstPart);
+
+  const result = parenPos >= 0 ? firstPart.substring(0, parenPos) : firstPart;
+
+  return result.replace(/(cyScope\d+)F/, '$1');
 }

package/analyzer-template/packages/ai/src/lib/gatherRelevantDependentKeyAttributes.ts

@@ -1,4 +1,9 @@
-import { Entity, ReadonlyAnalysis, ReadonlyAnalysisMap } from '~codeyam/types';
+import {
+  DataStructure,
+  Entity,
+  ReadonlyAnalysis,
+  ReadonlyAnalysisMap,
+} from '~codeyam/types';
 import {
   joinParenthesesAndArrays,
   splitOutsideParenthesesAndArrays,
@@ -7,6 +12,7 @@ import { gatherAttributesMap } from './promptGenerators/gatherAttributesMap';
 
 export default function gatherRelevantDependentKeyAttributes(
   entity: Pick<Entity, 'name' | 'metadata'>,
+  mergedDataStructure: Omit<DataStructure, 'equivalentSignatureVariables'>,
   dependentAnalyses: ReadonlyAnalysisMap,
 ) {
   // console.info(
@@ -52,10 +58,16 @@ export default function gatherRelevantDependentKeyAttributes(
   //   ),
   // );
 
-  const { associationMap } = gatherAttributesMap(entity);
+  const isolatedDataStructure = entity.metadata?.isolatedDataStructure;
+  const equivalentSignatureVariables =
+    isolatedDataStructure?.equivalentSignatureVariables ?? {};
+  const { associationMap } = gatherAttributesMap(
+    isolatedDataStructure ?? mergedDataStructure,
+    mergedDataStructure,
+    equivalentSignatureVariables,
+  );
 
-  const dependencySchemas =
-    entity.metadata?.isolatedDataStructure?.dependencySchemas;
+  const dependencySchemas = mergedDataStructure?.dependencySchemas;
   const relevantDependentKeyAttributes: ReadonlyAnalysis['metadata']['keyAttributes'][0][] =
     [];
   Object.keys(dependentAnalyses).forEach((path) => {
@@ -70,37 +82,178 @@ export default function gatherRelevantDependentKeyAttributes(
       const signatureSchema = dependencySchema?.signatureSchema;
       if (!signatureSchema) return;
 
-      const sourceEquivalencies = dependencySchema?.sourceEquivalencies ?? {};
+      // Get sourceEquivalencies from the dependent analysis's mergedDataStructure,
+      // not from dependencySchemas (which doesn't store them)
+      const sourceEquivalencies =
+        dependentAnalysis?.metadata?.mergedDataStructure?.sourceEquivalencies ??
+        {};
+
+      // Build a fallback mapping from parent's usageEquivalencies
+      // This helps when the child's sourceEquivalencies don't have a mapping
+      // For example, if parent passes `entity` to child's `signature[0]`,
+      // usageEquivalencies might have: entity -> [{schemaPath: 'signature[0]', scopeNodeName: entityName}]
+      const usageEquivalencies = mergedDataStructure?.usageEquivalencies ?? {};
+      const signatureToParentPath: Record<string, string> = {};
+
+      for (const parentPath in usageEquivalencies) {
+        const usages = usageEquivalencies[parentPath];
+        for (const usage of usages) {
+          if (usage.scopeNodeName !== entityName) continue;
+
+          // Extract signature path from schemaPath
+          // Format can be:
+          // 1. "functionName(args).signature[N]" or "functionName(args).signature[N].property" (real data)
+          // 2. "signature[N]" or "signature[N].property" (simplified/test format)
+          let signaturePath: string;
+          const signatureIndex = usage.schemaPath.indexOf('.signature[');
+          if (signatureIndex !== -1) {
+            // Full format: extract "signature[N]" or "signature[N].property" after the dot
+            signaturePath = usage.schemaPath.slice(signatureIndex + 1);
+          } else if (usage.schemaPath.startsWith('signature[')) {
+            // Simplified format: already starts with signature[
+            signaturePath = usage.schemaPath;
+          } else {
+            continue;
+          }
+
+          // Map child's signature path to parent's path
+          // e.g., 'signature[0]' -> 'entity'
+          signatureToParentPath[signaturePath] = parentPath;
+        }
+      }
 
       // If sourceEquivalencies is empty but the child has mocked imports,
-      // propagate key attributes directly since they affect the parent's visual output
+      // only propagate key attributes that come from mocked data sources
+      // (not props that the parent passes to the child)
       if (Object.keys(sourceEquivalencies).length === 0) {
-        // Check both importedExports and nodeModuleImports for mocked imports
+        // Collect names of mocked imports
         const importedExports =
           dependentAnalysis.entity?.metadata?.importedExports ?? [];
         const nodeModuleImports =
           dependentAnalysis.entity?.metadata?.nodeModuleImports ?? {};
 
-        const hasMockedImportedExports = importedExports.some(
-          (imp: { isMocked?: boolean }) => imp.isMocked,
-        );
-        const hasMockedNodeModuleImports = Object.values(
-          nodeModuleImports,
-        ).some((imports: any[]) =>
-          imports?.some((imp: { isMocked?: boolean }) => imp.isMocked),
-        );
+        const mockedImportNames = new Set<string>();
+
+        // Collect mocked names from importedExports
+        for (const imp of importedExports) {
+          if ((imp as { isMocked?: boolean }).isMocked) {
+            mockedImportNames.add((imp as { name: string }).name);
+          }
+        }
+
+        // Collect mocked names from nodeModuleImports
+        for (const imports of Object.values(nodeModuleImports)) {
+          for (const imp of imports as any[]) {
+            if ((imp as { isMocked?: boolean }).isMocked) {
+              mockedImportNames.add((imp as { name: string }).name);
+            }
+          }
+        }
+
+        if (mockedImportNames.size > 0) {
+          for (const dependentKeyAttribute of dependentKeyAtttributes) {
+            // Include key attributes that come from controllable data sources:
+            // 1. Mocked data sources (like getScenarioDisplayState())
+            // 2. Arguments/props passed to the child (signature[N].*)
+            // Exclude attributes that are clearly just the prop name itself (like "size")
+            const externalPath = dependentKeyAttribute.externalPath || '';
+            const dataStructurePath =
+              dependentKeyAttribute.dataStructurePath || '';
 
-        const childHasMockedImports =
-          hasMockedImportedExports || hasMockedNodeModuleImports;
+            const comesFromMockedSource = Array.from(mockedImportNames).some(
+              (mockedName) =>
+                externalPath.includes(`${mockedName}(`) ||
+                externalPath.includes(`${mockedName}.`) ||
+                dataStructurePath.includes(`${mockedName}(`) ||
+                dataStructurePath.includes(`${mockedName}.`) ||
+                dataStructurePath.startsWith(mockedName),
+            );
+
+            // Check if it comes from an argument (signature[N].something.deeper)
+            // but not just a simple prop like "size" or "isOutdated"
+            const comesFromArgument =
+              externalPath.startsWith('signature[') &&
+              // Must have nested path (at least one dot after the prop name)
+              // e.g., signature[0].scenario.metadata.* is OK
+              // but signature[0].size is not (it's just a simple prop)
+              (externalPath.match(/\./g) || []).length >= 2;
+
+            if (comesFromMockedSource || comesFromArgument) {
+              relevantDependentKeyAttributes.push({
+                ...dependentKeyAttribute,
+                dependentEntityName: entityName,
+                originalEntityName: entityName,
+                filePath: path,
+              } as any);
+            }
+          }
+          return;
+        }
 
-        if (childHasMockedImports) {
+        // No mocked imports, try usageEquivalencies fallback for direct argument usage
+        if (Object.keys(signatureToParentPath).length > 0) {
           for (const dependentKeyAttribute of dependentKeyAtttributes) {
-            relevantDependentKeyAttributes.push({
-              ...dependentKeyAttribute,
-              dependentEntityName: entityName,
-              originalEntityName: entityName,
-              filePath: path,
-            } as any);
+            if (!dependentKeyAttribute.externalPath?.startsWith('signature['))
+              continue;
+
+            const dependentKeyAttributeExternalPathParts =
+              splitOutsideParenthesesAndArrays(
+                dependentKeyAttribute.externalPath,
+              );
+
+            // Try to find a matching signature path
+            for (
+              let i = dependentKeyAttributeExternalPathParts.length;
+              i > 0;
+              i--
+            ) {
+              const subExternalPath = joinParenthesesAndArrays(
+                dependentKeyAttributeExternalPathParts.slice(0, i),
+              );
+
+              if (signatureToParentPath[subExternalPath]) {
+                const parentPath = signatureToParentPath[subExternalPath];
+
+                // parentPath is already in parent's schema format (e.g., "signature[0].entity" or just "entity")
+                // We need to handle both cases
+                const parentPathStartsWithSignature =
+                  parentPath.startsWith('signature[');
+                let externalPath = parentPathStartsWithSignature
+                  ? parentPath
+                  : `signature[0].${parentPath}`;
+
+                // internalPath is parentPath without the signature[N]. prefix
+                let internalPath: string | undefined =
+                  parentPathStartsWithSignature
+                    ? parentPath.replace(/^signature\[\d+\]\./, '')
+                    : parentPath;
+
+                if (i < dependentKeyAttributeExternalPathParts.length) {
+                  const remainingPath = joinParenthesesAndArrays(
+                    dependentKeyAttributeExternalPathParts.slice(i),
+                  );
+                  externalPath = `${externalPath}.${remainingPath}`;
+                  internalPath = `${internalPath}.${remainingPath}`;
+                }
+
+                // Look up in associationMap for proper internal path
+                if (associationMap[externalPath]) {
+                  internalPath = associationMap[externalPath];
+                }
+
+                relevantDependentKeyAttributes.push({
+                  ...dependentKeyAttribute,
+                  dependentEntityName: entityName,
+                  originalEntityName: entityName,
+                  filePath: path,
+                  internalPath,
+                  externalPath,
+                  dataStructurePath: internalPath,
+                } as any);
+
+                break;
+              }
+            }
           }
         }
         return;
@@ -111,6 +264,8 @@ export default function gatherRelevantDependentKeyAttributes(
           splitOutsideParenthesesAndArrays(dependentKeyAttribute.externalPath);
 
         let matchFound = false;
+
+        // First, try to match using child's sourceEquivalencies
         for (
           let i = dependentKeyAttributeExternalPathParts.length;
           i > 0;
@@ -120,8 +275,8 @@ export default function gatherRelevantDependentKeyAttributes(
             dependentKeyAttributeExternalPathParts.slice(0, i),
           );
 
-          for (const sourceEquivalencyPath in dependencySchema?.sourceEquivalencies ??
-            {}) {
+          // Use the sourceEquivalencies from the child's analysis
+          for (const sourceEquivalencyPath in sourceEquivalencies) {
             const sourceEquivalencyPathParts = splitOutsideParenthesesAndArrays(
               sourceEquivalencyPath,
             );
@@ -133,8 +288,7 @@ export default function gatherRelevantDependentKeyAttributes(
             if (relevantSourceEquivalencyPath !== subExternalPath) continue;
 
             const sourcePath =
-              dependencySchema?.sourceEquivalencies[sourceEquivalencyPath][0]
-                ?.schemaPath;
+              sourceEquivalencies[sourceEquivalencyPath][0]?.schemaPath;
 
             if (!sourcePath) continue;
 
@@ -169,6 +323,8 @@ export default function gatherRelevantDependentKeyAttributes(
               filePath: path, // File path for restructuring into nested format
               internalPath,
               externalPath,
+              // Transform dataStructurePath to parent's path structure
+              dataStructurePath: internalPath,
             } as any);
 
             matchFound = true;
@@ -177,6 +333,74 @@ export default function gatherRelevantDependentKeyAttributes(
 
           if (matchFound) break;
         }
+
+        // Fallback: if no sourceEquivalency match, try using parent's usageEquivalencies
+        // This handles direct argument usage like entity.entityType
+        if (
+          !matchFound &&
+          dependentKeyAttribute.externalPath?.startsWith('signature[')
+        ) {
+          // Try to find a matching signature path in the usageEquivalencies-derived map
+          for (
+            let i = dependentKeyAttributeExternalPathParts.length;
+            i > 0;
+            i--
+          ) {
+            const subExternalPath = joinParenthesesAndArrays(
+              dependentKeyAttributeExternalPathParts.slice(0, i),
+            );
+
+            if (signatureToParentPath[subExternalPath]) {
+              const parentPath = signatureToParentPath[subExternalPath];
+
+              // parentPath is already in parent's schema format (e.g., "signature[0].entity" or just "entity")
+              // We need to handle both cases
+              const parentPathStartsWithSignature =
+                parentPath.startsWith('signature[');
+              let externalPath = parentPathStartsWithSignature
+                ? parentPath
+                : `signature[0].${parentPath}`;
+
+              // internalPath is parentPath without the signature[N]. prefix
+              let internalPath: string | undefined =
+                parentPathStartsWithSignature
+                  ? parentPath.replace(/^signature\[\d+\]\./, '')
+                  : parentPath;
+
+              if (i < dependentKeyAttributeExternalPathParts.length) {
+                const remainingPath = joinParenthesesAndArrays(
+                  dependentKeyAttributeExternalPathParts.slice(i),
+                );
+                externalPath = `${externalPath}.${remainingPath}`;
+                internalPath = `${internalPath}.${remainingPath}`;
+              }
+
+              // Look up the full external path in associationMap for proper internal path
+              const fullExternalPath = externalPath;
+              if (associationMap[fullExternalPath]) {
+                internalPath = associationMap[fullExternalPath];
+              }
+
+              const dependentEntityName =
+                dependentKeyAttribute.dependentEntityName
+                  ? `${dependentKeyAttribute.dependentEntityName} (through ${entityName})`
+                  : entityName;
+
+              relevantDependentKeyAttributes.push({
+                ...dependentKeyAttribute,
+                dependentEntityName,
+                originalEntityName: entityName,
+                filePath: path,
+                internalPath,
+                externalPath,
+                dataStructurePath: internalPath,
+              } as any);
+
+              matchFound = true;
+              break;
+            }
+          }
+        }
       }
     });
   });