@codeyam/codeyam-cli 0.1.0-staging.596f0eb → 0.1.0-staging.62d4615

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/cleanNonObjectFunctions.ts

@@ -41,6 +41,19 @@ const isolatingArrayMethodsSet = new Set([
 // Methods that are ambiguous - used on both arrays and non-arrays (e.g., router.push, history.pop)
 // Don't infer array type from these alone without additional evidence
 const ambiguousArrayMethodsSet = new Set(['push', 'pop']);
+
+// Methods that return primitives (boolean, number) - their return values are not mockable
+// These should have their .functionCallReturnValue paths removed from the schema
+const primitiveReturningMethodsSet = new Set([
+  'some', // Returns boolean
+  'every', // Returns boolean
+  'includes', // Returns boolean
+  'indexOf', // Returns number (index)
+  'lastIndexOf', // Returns number (index from end)
+  'findIndex', // Returns number (index)
+  'findLastIndex', // Returns number (index from end)
+]);
+
 export const transformingArrayMethodsSet = new Set([
   'map',
   'reduce',
@@ -106,6 +119,17 @@ const stringMethodsSet = new Set([
 ]);
 export const allMethodsSet = new Set([...arrayMethodsSet, ...stringMethodsSet]);
 
+// Methods that exist in both stringMethodsSet and arrayMethodsSet.
+// When ONLY these methods are called on an object, we can't determine if it's a string or array.
+// Don't infer 'string' type from these alone - leave as 'unknown' for other type resolution.
+const stringArrayAmbiguousMethodsSet = new Set([
+  'includes', // exists in both String.includes() and Array.includes()
+  'indexOf', // exists in both String.indexOf() and Array.indexOf()
+  'lastIndexOf', // exists in both String.lastIndexOf() and Array.lastIndexOf()
+  'concat', // exists in both String.concat() and Array.concat()
+  'slice', // exists in both String.slice() and Array.slice()
+]);
+
 const signatureKeyWord = 'signature';
 const returnValueKeyWord = 'functionCallReturnValue';
 
@@ -115,31 +139,75 @@ const signatureRegExp = new RegExp(`^${signatureKeyWord}\\[\\d+\\]$`);
 
 // --- Caching splitOutsideParentheses ---
 // Instead of splitting the same string repeatedly, we cache the result.
+// IMPORTANT: Do NOT strip ::cyDuplicateKey:: markers - they are critical for
+// distinguishing variable reassignments (e.g., fetcher vs fetcher::cyDuplicateKey1::).
+// Stripping them causes schema contamination where fields from one type-parameterized
+// call get incorrectly attributed to another.
 const splitCache = new Map<string, string[]>();
 function cachedSplit(str: string): string[] {
   if (splitCache.has(str)) return splitCache.get(str)!;
-  const parts = splitOutsideParentheses(str).map((p) =>
-    p.replace(/::cyDuplicateKey\d+::/g, ''),
-  );
+  const parts = splitOutsideParentheses(str);
   splitCache.set(str, parts);
   return parts;
 }
 
+/**
+ * Clear the module-level cache.
+ * Call this between entity analyses to prevent unbounded memory growth.
+ */
+export function clearCleanNonObjectFunctionsCache(): {
+  count: number;
+  estimatedBytes: number;
+} {
+  let totalBytes = 0;
+  for (const [key, value] of splitCache) {
+    // Key is a string, value is string[] - estimate UTF-16 encoding (2 bytes per char)
+    totalBytes += key.length * 2;
+    for (const part of value) {
+      totalBytes += part.length * 2;
+    }
+    // Add overhead for array and Map entry (~50 bytes per entry)
+    totalBytes += 50;
+  }
+  const count = splitCache.size;
+  splitCache.clear();
+  return { count, estimatedBytes: totalBytes };
+}
+
 // --- Prefix Index for O(1) lookup in subPathIsFromKnownObject ---
 // Maps parent prefix -> array of last parts (method names without parens)
 type PrefixIndex = Map<string, string[]>;
 
+function buildPrefixPartsWithRoot(parts: string[]): string[] {
+  const prefixes = new Array(parts.length + 1);
+  prefixes[0] = '';
+  let current = '';
+  for (let i = 0; i < parts.length; i++) {
+    const part = parts[i];
+    if (i === 0) {
+      current = part;
+    } else if (part.startsWith('[') || part.startsWith('(')) {
+      current += part;
+    } else {
+      current += `.${part}`;
+    }
+    prefixes[i + 1] = current;
+  }
+  return prefixes;
+}
+
 function buildPrefixIndex(allPaths: string[]): PrefixIndex {
   const index: PrefixIndex = new Map();
 
   for (const path of allPaths) {
     const parts = cachedSplit(path);
     if (parts.length < 1) continue;
+    const prefixes = buildPrefixPartsWithRoot(parts);
 
     // For each possible prefix length, add the last part to that prefix's entry
     // This allows lookup at any depth
     for (let prefixLen = 0; prefixLen < parts.length; prefixLen++) {
-      const prefix = joinParenthesesAndArrays(parts.slice(0, prefixLen));
+      const prefix = prefixes[prefixLen];
       const lastPart = parts[prefixLen]?.split('(')?.[0];
       if (lastPart) {
         let existing = index.get(prefix);
@@ -442,7 +510,7 @@ function removeNonTransformingArrayMethods(variablePath: string): string {
     const part = keyParts[i];
 
     const arrayMethod = getMethodName(part);
-    if (isValidArrayMethod(arrayMethod)) {
+    if (isValidFunctionCall(part) && isValidArrayMethod(arrayMethod)) {
       const nextPart = keyParts[i + 1];
       if (
         !nonTransformingArrayMethodsSet.has(arrayMethod) ||
@@ -482,7 +550,7 @@ function removeIsolatingArrayMethods(
     const part = keyParts[i];
 
     const arrayMethod = getMethodName(part);
-    if (isValidArrayMethod(arrayMethod)) {
+    if (isValidFunctionCall(part) && isValidArrayMethod(arrayMethod)) {
       const nextPart = keyParts[i + 1];
       const prevPart = i > 1 ? keyParts[i - 1] : undefined;
       if (
@@ -682,7 +750,14 @@ function handleFunctionMapping(
         allProperties.every((prop) => stringMethodsSet.has(getMethodName(prop)))
       ) {
         if (allProperties.every((prop) => isValidStringMethod(prop))) {
-          keyMapping[previousPath] = 'string';
+          // Don't infer string if ALL methods are ambiguous between string and array
+          // (like includes, indexOf, concat, slice - these exist on both types)
+          const hasOnlyStringArrayAmbiguousMethods = allProperties.every(
+            (prop) => stringArrayAmbiguousMethodsSet.has(getMethodName(prop)),
+          );
+          if (!hasOnlyStringArrayAmbiguousMethods) {
+            keyMapping[previousPath] = 'string';
+          }
         } else {
           return false;
         }
@@ -703,19 +778,36 @@ function handleFunctionMapping(
 
     // Don't add [] element for ambiguous methods without other array evidence
     // Also don't add [] if the path was already identified as a string type
+    // Also don't add [] if the method is ambiguous between string and array (like includes)
     const methodName = getMethodName(lastPart);
     const isAmbiguousMethod = ambiguousArrayMethodsSet.has(methodName);
+    const isStringArrayAmbiguous =
+      stringArrayAmbiguousMethodsSet.has(methodName);
     const isStringType = keyMapping[previousPath] === 'string';
     if (
       !isValidFunctionCall(previousPart) &&
       arrayMethodsSet.has(methodName) &&
       !keyMapping[previousPath + '[]'] &&
       !isAmbiguousMethod &&
+      !isStringArrayAmbiguous &&
       !isStringType
     ) {
       keyMapping[previousPath + '[]'] = 'unknown';
     }
 
+    // For primitive-returning methods (like includes, indexOf, some, every),
+    // also delete the .functionCallReturnValue children since the return value
+    // is just a primitive (boolean/number) and not useful data.
+    // For other methods (like filter, map), keep children for transformation.
+    const isPrimitiveReturning = primitiveReturningMethodsSet.has(methodName);
+    if (isPrimitiveReturning) {
+      for (const otherPath of Object.keys(keyMapping)) {
+        if (otherPath.startsWith(path + '.')) {
+          delete keyMapping[otherPath];
+        }
+      }
+    }
+
     delete keyMapping[path];
     return true;
   }
@@ -941,14 +1033,34 @@ function clearAttributes(
     }
   }
 
+  // Track paths that have primitive-returning method calls (like includes, some, every)
+  // These need cleanup even if we can't determine the base type
+  const pathsWithPrimitiveReturningMethods = new Set<string>();
+
   for (const key in functionCallsOnObjects) {
     const functionCalls = functionCallsOnObjects[key];
+
+    // Check if any method calls are primitive-returning
+    const hasPrimitiveReturningMethods = functionCalls.some((f) =>
+      primitiveReturningMethodsSet.has(getMethodName(f)),
+    );
+    if (hasPrimitiveReturningMethods) {
+      pathsWithPrimitiveReturningMethods.add(key);
+    }
+
     if (
       functionCalls.every(
         (f) => stringMethodsSet.has(getMethodName(f)) && isValidStringMethod(f),
       )
     ) {
-      mapping[key] = 'string';
+      // Don't infer string if ALL methods are ambiguous between string and array
+      // (like includes, indexOf, concat, slice - these exist on both types)
+      const hasOnlyStringArrayAmbiguousMethods = functionCalls.every((f) =>
+        stringArrayAmbiguousMethodsSet.has(getMethodName(f)),
+      );
+      if (!hasOnlyStringArrayAmbiguousMethods) {
+        mapping[key] = 'string';
+      }
     } else if (
       functionCalls.every(
         (f) => arrayMethodsSet.has(getMethodName(f)) && isValidArrayMethod(f),
@@ -970,25 +1082,55 @@ function clearAttributes(
   for (const key in mapping) {
     const value = mapping[key];
     if (knownTypes.has(value)) {
+      // Don't add string-typed array element paths (ending with []) to knownBasePaths.
+      // When an array element is typed as 'string' but has child paths (like data[].id),
+      // it's actually an object - the 'string' type is incorrect. The child paths
+      // describe the array element structure and should not be deleted.
+      //
+      // However, array-typed [] paths (like matrix[] where matrix is array of arrays)
+      // should still be in knownBasePaths so their method calls (filter, map, etc.)
+      // get cleaned up properly.
+      if (key.endsWith('[]') && value === 'string') {
+        continue;
+      }
       knownBasePaths.add(key);
     }
   }
 
+  // Also add paths with primitive-returning method calls (like includes, some, every)
+  // to knownBasePaths so their method call paths get cleaned up, even if we couldn't
+  // determine the base type (e.g., when only ambiguous methods like includes are used)
+  for (const path of pathsWithPrimitiveReturningMethods) {
+    knownBasePaths.add(path);
+  }
+
+  // OPTIMIZATION: Instead of O(keys × basePaths), use O(keys × keyDepth) where keyDepth is small
+  // For each key, find all its prefixes and check if any is a known basePath (O(1) Set lookup)
   for (const key in mapping) {
     const keyParts = cachedSplit(key);
-    for (const basePath of knownBasePaths) {
-      const basePathParts = cachedSplit(basePath);
-      if (keyParts.length <= basePathParts.length) continue;
-      if (
-        joinParenthesesAndArrays(keyParts.slice(0, basePathParts.length)) !==
-        basePath
-      )
-        continue;
+    if (keyParts.length <= 1) continue; // No possible parent basePath
+
+    // Check each prefix of the key to see if it's a known basePath
+    // Start from shortest prefix (most likely to match) for early exit
+    for (let prefixLen = 1; prefixLen < keyParts.length; prefixLen++) {
+      const prefix = joinParenthesesAndArrays(keyParts.slice(0, prefixLen));
+      if (!knownBasePaths.has(prefix)) continue;
+
+      // Found a matching basePath - now check the conditions
+      const basePath = prefix;
+
+      // Preserve functionCallReturnValue for non-string types, UNLESS
+      // the method is a primitive-returning method (like some, every, includes, indexOf)
+      // whose return values are not mockable
+      const methodPart = keyParts[prefixLen];
+      const methodName = methodPart?.split('(')[0];
+      const isPrimitiveReturning = primitiveReturningMethodsSet.has(
+        methodName ?? '',
+      );
       if (
         mapping[basePath] !== 'string' &&
-        keyParts[basePathParts.length + 1]?.startsWith(
-          'functionCallReturnValue',
-        )
+        keyParts[prefixLen + 1]?.startsWith('functionCallReturnValue') &&
+        !isPrimitiveReturning
       )
         continue;
 

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/coerceObjectsToPrimitivesBySchema.ts

@@ -0,0 +1,70 @@
+/**
+ * Coerces object/array values to primitive types when the schema expects a primitive.
+ *
+ * The LLM sometimes generates an object or array where the schema expects a primitive
+ * type like "string", "number", or "boolean". For example:
+ *   Schema: { body: "string" }
+ *   LLM output: { body: { "env": "production" } }
+ *
+ * This causes runtime errors like `TypeError: body.match is not a function` because
+ * the code expects a string but gets an object.
+ *
+ * Coercion rules:
+ * - object/array → string: JSON.stringify
+ * - object/array → number: 0
+ * - object/array → boolean: false
+ *
+ * @param data The mock data object to coerce (mutated in place)
+ * @param schema The schema describing expected types
+ */
+export default function coerceObjectsToPrimitivesBySchema<T>(
+  data: T,
+  schema: unknown,
+): T {
+  if (data === null || data === undefined) {
+    return data;
+  }
+
+  if (typeof data !== 'object') {
+    return data;
+  }
+
+  if (Array.isArray(data)) {
+    const itemSchema = Array.isArray(schema) ? schema[0] : undefined;
+    for (let i = 0; i < data.length; i++) {
+      data[i] = coerceObjectsToPrimitivesBySchema(data[i], itemSchema);
+    }
+    return data;
+  }
+
+  const dataObj = data as Record<string, unknown>;
+  const schemaObj = schema as Record<string, unknown> | undefined;
+
+  for (const key of Object.keys(dataObj)) {
+    const value = dataObj[key];
+    const fieldSchema = schemaObj?.[key];
+
+    if (value === null || value === undefined) {
+      continue;
+    }
+
+    if (typeof value === 'object' && typeof fieldSchema === 'string') {
+      // Schema expects a primitive but the value is an object/array.
+      // Coerce to the expected type.
+      const schemaType = fieldSchema.toLowerCase();
+
+      if (schemaType.includes('string')) {
+        dataObj[key] = JSON.stringify(value);
+      } else if (schemaType.includes('number')) {
+        dataObj[key] = 0;
+      } else if (schemaType.includes('boolean')) {
+        dataObj[key] = false;
+      }
+    } else if (typeof value === 'object' && value !== null) {
+      // Both value and schema are objects — recurse
+      coerceObjectsToPrimitivesBySchema(value, fieldSchema);
+    }
+  }
+
+  return data;
+}

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/convertDotNotation.ts

@@ -6,6 +6,14 @@ const STANDALONE_INDEX_RE = /^\[(\d+)\]$/;
 const isExplicitIndex = (s?: string) => !!s && INDEXED_RE.test(s);
 const isStandaloneIndex = (s?: string) => !!s && STANDALONE_INDEX_RE.test(s);
 
+// Detect .length properties that come from dynamic iteration patterns
+// These aren't useful for mock data - arrays work without explicit length,
+// and objects shouldn't have length. We filter .length that appears:
+// 1. After array notation: metadata[].length, items[][].length
+// 2. On object paths that likely come from iteration: metadata.length (when metadata has other props)
+// The regex matches any path ending with .length that has [] somewhere before it
+const DYNAMIC_LENGTH_RE = /\[\].*\.length$/;
+
 // Treat these as structural placeholders (don't commit them as concrete leaves)
 function isSkippableLeafType(t: string) {
   // 'unknown' by itself is a placeholder (but 'boolean | unknown' is not)
@@ -17,12 +25,74 @@ function isSkippableLeafType(t: string) {
   );
 }
 
+// Matches paths containing [][] — e.g., "items[][]" or "items[][].text"
+const DOUBLE_ARRAY_RE = /\[\]\[\]/;
+
+// Matches a [].property segment — e.g., "items[].name" (but not "items[][]")
+const ARRAY_OBJECT_PROPERTY_RE = /\[\]\.[^[\]]/;
+
+/**
+ * When scope analysis produces both `prefix[].property` (array of objects) and
+ * `prefix[][]` (array of arrays) for the same prefix, there's a conflict about
+ * what the array elements are. Object-property paths are more specific and come
+ * from actual property access; double-array paths come from mixed iteration
+ * patterns in the scope analysis. Drop the double-array paths when both exist.
+ */
+function resolveDoubleArrayConflicts(
+  schema: Record<string, string>,
+): Record<string, string> {
+  // Collect prefixes that end in [] and have object property children
+  const objectArrayPrefixes = new Set<string>();
+  for (const path of Object.keys(schema)) {
+    if (!ARRAY_OBJECT_PROPERTY_RE.test(path)) continue;
+    // Extract every "...[]" prefix that is followed by ".prop"
+    let idx = 0;
+    while ((idx = path.indexOf('[].', idx)) !== -1) {
+      objectArrayPrefixes.add(path.slice(0, idx + 2)); // includes the "[]"
+      idx += 3;
+    }
+  }
+
+  if (objectArrayPrefixes.size === 0) return schema;
+
+  // Check if any double-array paths actually conflict
+  let hasConflict = false;
+  for (const path of Object.keys(schema)) {
+    if (!DOUBLE_ARRAY_RE.test(path)) continue;
+    const idx = path.indexOf('[][]');
+    if (idx !== -1 && objectArrayPrefixes.has(path.slice(0, idx + 2))) {
+      hasConflict = true;
+      break;
+    }
+  }
+
+  if (!hasConflict) return schema;
+
+  // Filter out conflicting double-array paths
+  const resolved: Record<string, string> = {};
+  for (const [path, value] of Object.entries(schema)) {
+    if (DOUBLE_ARRAY_RE.test(path)) {
+      const idx = path.indexOf('[][]');
+      if (idx !== -1 && objectArrayPrefixes.has(path.slice(0, idx + 2))) {
+        continue; // drop this conflicting path
+      }
+    }
+    resolved[path] = value;
+  }
+  return resolved;
+}
+
 export default function convertDotNotation(
   schema: Record<string, string>,
 ): JsonTypeDefinition {
+  const resolved = resolveDoubleArrayConflicts(schema);
   const result: JsonTypeDefinition = {};
 
-  for (const [rawPath, typ] of Object.entries(schema)) {
+  for (const [rawPath, typ] of Object.entries(resolved)) {
+    // Skip paths with .length after array notation from dynamic iteration
+    // e.g., metadata[].length, metadata[][].length
+    if (DYNAMIC_LENGTH_RE.test(rawPath)) continue;
+
     const parts = splitOutsideParenthesesAndArrays(rawPath);
     if (!parts || parts.length === 0) continue;
 
@@ -187,6 +257,17 @@ export default function convertDotNotation(
           i++; // consume '[]'
         } else {
           // Pattern like: "items", "[]", "id" -> items: [ { id: ... } ]
+          // But if cursor[key] is already a non-array object (from paths like "metadata: object"),
+          // don't overwrite it with an array - the object type should take precedence.
+          // This handles spurious "metadata[]" paths from dynamic iteration (Object.keys, for...in).
+          if (
+            cursor[key] !== undefined &&
+            typeof cursor[key] === 'object' &&
+            !Array.isArray(cursor[key])
+          ) {
+            // Skip this path - it's trying to treat an object as an array
+            break;
+          }
           if (!Array.isArray(cursor[key])) cursor[key] = [];
 
           // Look ahead to see what comes after '[]'
@@ -195,6 +276,17 @@ export default function convertDotNotation(
           // If the next segment after '[]' is another '[]' or a standalone index,
           // we need an array at cursor[key][0], not an object (for patterns like "key[][]" or "key[][0]")
           if (afterArray === '[]' || isStandaloneIndex(afterArray)) {
+            // But don't overwrite an existing populated object — object properties
+            // (from paths like key[].property) take precedence over double-array hints
+            // (from paths like key[][]) which come from mixed access patterns in scope analysis
+            if (
+              cursor[key][0] &&
+              typeof cursor[key][0] === 'object' &&
+              !Array.isArray(cursor[key][0]) &&
+              Object.keys(cursor[key][0]).length > 0
+            ) {
+              break;
+            }
             if (!Array.isArray(cursor[key][0])) cursor[key][0] = [];
           } else {
             // Default case: create an object

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/convertNullToUndefinedBySchema.ts

@@ -0,0 +1,98 @@
+/**
+ * Converts null values to undefined (removes them) based on schema type constraints.
+ *
+ * When the LLM generates mock data, it uses `null` to represent "no value" because
+ * JSON doesn't support `undefined`. However, TypeScript types like `string | undefined`
+ * don't accept `null`.
+ *
+ * This function traverses data and schema together:
+ * - If schema type is "T | undefined" (but NOT "T | null"), convert null → undefined (delete key)
+ * - If schema type includes "null" (e.g., "T | null" or "T | null | undefined"), preserve null
+ * - If field not in schema, preserve as-is
+ *
+ * @param data The data object with potential null values
+ * @param schema The schema describing expected types
+ * @returns The data with null→undefined conversions applied (mutates original)
+ */
+export default function convertNullToUndefinedBySchema<T>(
+  data: T,
+  schema: unknown,
+): T {
+  // Handle null/undefined data
+  if (data === null || data === undefined) {
+    return data;
+  }
+
+  // Only process objects
+  if (typeof data !== 'object') {
+    return data;
+  }
+
+  // Handle arrays
+  if (Array.isArray(data)) {
+    // For arrays, the schema should be an array with one element describing the item type
+    const itemSchema = Array.isArray(schema) ? schema[0] : undefined;
+    for (let i = 0; i < data.length; i++) {
+      data[i] = convertNullToUndefinedBySchema(data[i], itemSchema);
+    }
+    return data;
+  }
+
+  // Handle objects
+  const dataObj = data as Record<string, unknown>;
+  const schemaObj = schema as Record<string, unknown> | undefined;
+
+  for (const key of Object.keys(dataObj)) {
+    const value = dataObj[key];
+    const fieldSchema = schemaObj?.[key];
+
+    if (value === null) {
+      // Check if schema allows null
+      if (shouldConvertNullToUndefined(fieldSchema)) {
+        // Delete the key (converts null to undefined)
+        delete dataObj[key];
+      }
+      // If schema allows null, preserve it
+    } else if (typeof value === 'object' && value !== null) {
+      // Recursively process nested objects/arrays
+      convertNullToUndefinedBySchema(value, fieldSchema);
+    }
+  }
+
+  return data;
+}
+
+/**
+ * Determines if a null value should be converted to undefined based on schema type.
+ *
+ * @param fieldSchema The schema type string or object for the field
+ * @returns true if null should be converted to undefined, false if null should be preserved
+ */
+function shouldConvertNullToUndefined(fieldSchema: unknown): boolean {
+  // If no schema for this field, preserve null (be conservative)
+  if (fieldSchema === undefined || fieldSchema === null) {
+    return false;
+  }
+
+  // If schema is a string type definition, check if it allows null
+  if (typeof fieldSchema === 'string') {
+    // Schema types that include "null" should preserve null values
+    // e.g., "string | null", "number | null | undefined", "null"
+    const allowsNull = fieldSchema.includes('null');
+    const allowsUndefined = fieldSchema.includes('undefined');
+
+    // If type allows undefined but NOT null, convert null to undefined
+    // e.g., "string | undefined" → convert null to undefined
+    // e.g., "string | null" → preserve null
+    // e.g., "string | null | undefined" → preserve null
+    if (allowsUndefined && !allowsNull) {
+      return true;
+    }
+
+    return false;
+  }
+
+  // If schema is an object or array, it's a nested structure
+  // Don't convert the container itself, just recurse into it
+  return false;
+}