@codeyam/codeyam-cli 0.1.0-staging.76566f9 → 0.1.0-staging.79ef713

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/cleanNonObjectFunctions.ts

@@ -119,6 +119,17 @@ const stringMethodsSet = new Set([
 ]);
 export const allMethodsSet = new Set([...arrayMethodsSet, ...stringMethodsSet]);
 
+// Methods that exist in both stringMethodsSet and arrayMethodsSet.
+// When ONLY these methods are called on an object, we can't determine if it's a string or array.
+// Don't infer 'string' type from these alone - leave as 'unknown' for other type resolution.
+const stringArrayAmbiguousMethodsSet = new Set([
+  'includes', // exists in both String.includes() and Array.includes()
+  'indexOf', // exists in both String.indexOf() and Array.indexOf()
+  'lastIndexOf', // exists in both String.lastIndexOf() and Array.lastIndexOf()
+  'concat', // exists in both String.concat() and Array.concat()
+  'slice', // exists in both String.slice() and Array.slice()
+]);
+
 const signatureKeyWord = 'signature';
 const returnValueKeyWord = 'functionCallReturnValue';
 
@@ -128,31 +139,75 @@ const signatureRegExp = new RegExp(`^${signatureKeyWord}\\[\\d+\\]$`);
 
 // --- Caching splitOutsideParentheses ---
 // Instead of splitting the same string repeatedly, we cache the result.
+// IMPORTANT: Do NOT strip ::cyDuplicateKey:: markers - they are critical for
+// distinguishing variable reassignments (e.g., fetcher vs fetcher::cyDuplicateKey1::).
+// Stripping them causes schema contamination where fields from one type-parameterized
+// call get incorrectly attributed to another.
 const splitCache = new Map<string, string[]>();
 function cachedSplit(str: string): string[] {
   if (splitCache.has(str)) return splitCache.get(str)!;
-  const parts = splitOutsideParentheses(str).map((p) =>
-    p.replace(/::cyDuplicateKey\d+::/g, ''),
-  );
+  const parts = splitOutsideParentheses(str);
   splitCache.set(str, parts);
   return parts;
 }
 
+/**
+ * Clear the module-level cache.
+ * Call this between entity analyses to prevent unbounded memory growth.
+ */
+export function clearCleanNonObjectFunctionsCache(): {
+  count: number;
+  estimatedBytes: number;
+} {
+  let totalBytes = 0;
+  for (const [key, value] of splitCache) {
+    // Key is a string, value is string[] - estimate UTF-16 encoding (2 bytes per char)
+    totalBytes += key.length * 2;
+    for (const part of value) {
+      totalBytes += part.length * 2;
+    }
+    // Add overhead for array and Map entry (~50 bytes per entry)
+    totalBytes += 50;
+  }
+  const count = splitCache.size;
+  splitCache.clear();
+  return { count, estimatedBytes: totalBytes };
+}
+
 // --- Prefix Index for O(1) lookup in subPathIsFromKnownObject ---
 // Maps parent prefix -> array of last parts (method names without parens)
 type PrefixIndex = Map<string, string[]>;
 
+function buildPrefixPartsWithRoot(parts: string[]): string[] {
+  const prefixes = new Array(parts.length + 1);
+  prefixes[0] = '';
+  let current = '';
+  for (let i = 0; i < parts.length; i++) {
+    const part = parts[i];
+    if (i === 0) {
+      current = part;
+    } else if (part.startsWith('[') || part.startsWith('(')) {
+      current += part;
+    } else {
+      current += `.${part}`;
+    }
+    prefixes[i + 1] = current;
+  }
+  return prefixes;
+}
+
 function buildPrefixIndex(allPaths: string[]): PrefixIndex {
   const index: PrefixIndex = new Map();
 
   for (const path of allPaths) {
     const parts = cachedSplit(path);
     if (parts.length < 1) continue;
+    const prefixes = buildPrefixPartsWithRoot(parts);
 
     // For each possible prefix length, add the last part to that prefix's entry
     // This allows lookup at any depth
     for (let prefixLen = 0; prefixLen < parts.length; prefixLen++) {
-      const prefix = joinParenthesesAndArrays(parts.slice(0, prefixLen));
+      const prefix = prefixes[prefixLen];
       const lastPart = parts[prefixLen]?.split('(')?.[0];
       if (lastPart) {
         let existing = index.get(prefix);
@@ -455,7 +510,7 @@ function removeNonTransformingArrayMethods(variablePath: string): string {
     const part = keyParts[i];
 
     const arrayMethod = getMethodName(part);
-    if (isValidArrayMethod(arrayMethod)) {
+    if (isValidFunctionCall(part) && isValidArrayMethod(arrayMethod)) {
       const nextPart = keyParts[i + 1];
       if (
         !nonTransformingArrayMethodsSet.has(arrayMethod) ||
@@ -495,7 +550,7 @@ function removeIsolatingArrayMethods(
     const part = keyParts[i];
 
     const arrayMethod = getMethodName(part);
-    if (isValidArrayMethod(arrayMethod)) {
+    if (isValidFunctionCall(part) && isValidArrayMethod(arrayMethod)) {
       const nextPart = keyParts[i + 1];
       const prevPart = i > 1 ? keyParts[i - 1] : undefined;
       if (
@@ -695,7 +750,14 @@ function handleFunctionMapping(
         allProperties.every((prop) => stringMethodsSet.has(getMethodName(prop)))
       ) {
         if (allProperties.every((prop) => isValidStringMethod(prop))) {
-          keyMapping[previousPath] = 'string';
+          // Don't infer string if ALL methods are ambiguous between string and array
+          // (like includes, indexOf, concat, slice - these exist on both types)
+          const hasOnlyStringArrayAmbiguousMethods = allProperties.every(
+            (prop) => stringArrayAmbiguousMethodsSet.has(getMethodName(prop)),
+          );
+          if (!hasOnlyStringArrayAmbiguousMethods) {
+            keyMapping[previousPath] = 'string';
+          }
         } else {
           return false;
         }
@@ -716,19 +778,36 @@ function handleFunctionMapping(
 
     // Don't add [] element for ambiguous methods without other array evidence
     // Also don't add [] if the path was already identified as a string type
+    // Also don't add [] if the method is ambiguous between string and array (like includes)
     const methodName = getMethodName(lastPart);
     const isAmbiguousMethod = ambiguousArrayMethodsSet.has(methodName);
+    const isStringArrayAmbiguous =
+      stringArrayAmbiguousMethodsSet.has(methodName);
     const isStringType = keyMapping[previousPath] === 'string';
     if (
       !isValidFunctionCall(previousPart) &&
       arrayMethodsSet.has(methodName) &&
       !keyMapping[previousPath + '[]'] &&
       !isAmbiguousMethod &&
+      !isStringArrayAmbiguous &&
       !isStringType
     ) {
       keyMapping[previousPath + '[]'] = 'unknown';
     }
 
+    // For primitive-returning methods (like includes, indexOf, some, every),
+    // also delete the .functionCallReturnValue children since the return value
+    // is just a primitive (boolean/number) and not useful data.
+    // For other methods (like filter, map), keep children for transformation.
+    const isPrimitiveReturning = primitiveReturningMethodsSet.has(methodName);
+    if (isPrimitiveReturning) {
+      for (const otherPath of Object.keys(keyMapping)) {
+        if (otherPath.startsWith(path + '.')) {
+          delete keyMapping[otherPath];
+        }
+      }
+    }
+
     delete keyMapping[path];
     return true;
   }
@@ -954,14 +1033,34 @@ function clearAttributes(
     }
   }
 
+  // Track paths that have primitive-returning method calls (like includes, some, every)
+  // These need cleanup even if we can't determine the base type
+  const pathsWithPrimitiveReturningMethods = new Set<string>();
+
   for (const key in functionCallsOnObjects) {
     const functionCalls = functionCallsOnObjects[key];
+
+    // Check if any method calls are primitive-returning
+    const hasPrimitiveReturningMethods = functionCalls.some((f) =>
+      primitiveReturningMethodsSet.has(getMethodName(f)),
+    );
+    if (hasPrimitiveReturningMethods) {
+      pathsWithPrimitiveReturningMethods.add(key);
+    }
+
     if (
       functionCalls.every(
         (f) => stringMethodsSet.has(getMethodName(f)) && isValidStringMethod(f),
       )
     ) {
-      mapping[key] = 'string';
+      // Don't infer string if ALL methods are ambiguous between string and array
+      // (like includes, indexOf, concat, slice - these exist on both types)
+      const hasOnlyStringArrayAmbiguousMethods = functionCalls.every((f) =>
+        stringArrayAmbiguousMethodsSet.has(getMethodName(f)),
+      );
+      if (!hasOnlyStringArrayAmbiguousMethods) {
+        mapping[key] = 'string';
+      }
     } else if (
       functionCalls.every(
         (f) => arrayMethodsSet.has(getMethodName(f)) && isValidArrayMethod(f),
@@ -998,29 +1097,39 @@ function clearAttributes(
     }
   }
 
+  // Also add paths with primitive-returning method calls (like includes, some, every)
+  // to knownBasePaths so their method call paths get cleaned up, even if we couldn't
+  // determine the base type (e.g., when only ambiguous methods like includes are used)
+  for (const path of pathsWithPrimitiveReturningMethods) {
+    knownBasePaths.add(path);
+  }
+
+  // OPTIMIZATION: Instead of O(keys × basePaths), use O(keys × keyDepth) where keyDepth is small
+  // For each key, find all its prefixes and check if any is a known basePath (O(1) Set lookup)
   for (const key in mapping) {
     const keyParts = cachedSplit(key);
-    for (const basePath of knownBasePaths) {
-      const basePathParts = cachedSplit(basePath);
-      if (keyParts.length <= basePathParts.length) continue;
-      if (
-        joinParenthesesAndArrays(keyParts.slice(0, basePathParts.length)) !==
-        basePath
-      )
-        continue;
+    if (keyParts.length <= 1) continue; // No possible parent basePath
+
+    // Check each prefix of the key to see if it's a known basePath
+    // Start from shortest prefix (most likely to match) for early exit
+    for (let prefixLen = 1; prefixLen < keyParts.length; prefixLen++) {
+      const prefix = joinParenthesesAndArrays(keyParts.slice(0, prefixLen));
+      if (!knownBasePaths.has(prefix)) continue;
+
+      // Found a matching basePath - now check the conditions
+      const basePath = prefix;
+
       // Preserve functionCallReturnValue for non-string types, UNLESS
       // the method is a primitive-returning method (like some, every, includes, indexOf)
       // whose return values are not mockable
-      const methodPart = keyParts[basePathParts.length];
+      const methodPart = keyParts[prefixLen];
       const methodName = methodPart?.split('(')[0];
       const isPrimitiveReturning = primitiveReturningMethodsSet.has(
         methodName ?? '',
       );
       if (
         mapping[basePath] !== 'string' &&
-        keyParts[basePathParts.length + 1]?.startsWith(
-          'functionCallReturnValue',
-        ) &&
+        keyParts[prefixLen + 1]?.startsWith('functionCallReturnValue') &&
         !isPrimitiveReturning
       )
         continue;

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/coerceObjectsToPrimitivesBySchema.ts

@@ -0,0 +1,70 @@
+/**
+ * Coerces object/array values to primitive types when the schema expects a primitive.
+ *
+ * The LLM sometimes generates an object or array where the schema expects a primitive
+ * type like "string", "number", or "boolean". For example:
+ *   Schema: { body: "string" }
+ *   LLM output: { body: { "env": "production" } }
+ *
+ * This causes runtime errors like `TypeError: body.match is not a function` because
+ * the code expects a string but gets an object.
+ *
+ * Coercion rules:
+ * - object/array → string: JSON.stringify
+ * - object/array → number: 0
+ * - object/array → boolean: false
+ *
+ * @param data The mock data object to coerce (mutated in place)
+ * @param schema The schema describing expected types
+ */
+export default function coerceObjectsToPrimitivesBySchema<T>(
+  data: T,
+  schema: unknown,
+): T {
+  if (data === null || data === undefined) {
+    return data;
+  }
+
+  if (typeof data !== 'object') {
+    return data;
+  }
+
+  if (Array.isArray(data)) {
+    const itemSchema = Array.isArray(schema) ? schema[0] : undefined;
+    for (let i = 0; i < data.length; i++) {
+      data[i] = coerceObjectsToPrimitivesBySchema(data[i], itemSchema);
+    }
+    return data;
+  }
+
+  const dataObj = data as Record<string, unknown>;
+  const schemaObj = schema as Record<string, unknown> | undefined;
+
+  for (const key of Object.keys(dataObj)) {
+    const value = dataObj[key];
+    const fieldSchema = schemaObj?.[key];
+
+    if (value === null || value === undefined) {
+      continue;
+    }
+
+    if (typeof value === 'object' && typeof fieldSchema === 'string') {
+      // Schema expects a primitive but the value is an object/array.
+      // Coerce to the expected type.
+      const schemaType = fieldSchema.toLowerCase();
+
+      if (schemaType.includes('string')) {
+        dataObj[key] = JSON.stringify(value);
+      } else if (schemaType.includes('number')) {
+        dataObj[key] = 0;
+      } else if (schemaType.includes('boolean')) {
+        dataObj[key] = false;
+      }
+    } else if (typeof value === 'object' && value !== null) {
+      // Both value and schema are objects — recurse
+      coerceObjectsToPrimitivesBySchema(value, fieldSchema);
+    }
+  }
+
+  return data;
+}

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/convertDotNotation.ts

@@ -17,20 +17,84 @@ const DYNAMIC_LENGTH_RE = /\[\].*\.length$/;
 // Treat these as structural placeholders (don't commit them as concrete leaves)
 function isSkippableLeafType(t: string) {
   // 'unknown' by itself is a placeholder (but 'boolean | unknown' is not)
+  // Also handles optional variants like 'object | undefined', 'object | null',
+  // 'array | undefined', etc. — these are still structural placeholders that
+  // should not overwrite already-populated structures.
+  const parts = t.split('|').map((s) => s.trim());
+  const base = parts.filter((s) => s !== 'undefined' && s !== 'null');
   return (
-    t === 'object' ||
-    t === 'array' ||
-    t === 'function' ||
-    t.trim() === 'unknown'
+    base.length === 1 &&
+    (base[0] === 'object' ||
+      base[0] === 'array' ||
+      base[0] === 'function' ||
+      base[0] === 'unknown')
   );
 }
 
+// Matches paths containing [][] — e.g., "items[][]" or "items[][].text"
+const DOUBLE_ARRAY_RE = /\[\]\[\]/;
+
+// Matches a [].property segment — e.g., "items[].name" (but not "items[][]")
+const ARRAY_OBJECT_PROPERTY_RE = /\[\]\.[^[\]]/;
+
+/**
+ * When scope analysis produces both `prefix[].property` (array of objects) and
+ * `prefix[][]` (array of arrays) for the same prefix, there's a conflict about
+ * what the array elements are. Object-property paths are more specific and come
+ * from actual property access; double-array paths come from mixed iteration
+ * patterns in the scope analysis. Drop the double-array paths when both exist.
+ */
+function resolveDoubleArrayConflicts(
+  schema: Record<string, string>,
+): Record<string, string> {
+  // Collect prefixes that end in [] and have object property children
+  const objectArrayPrefixes = new Set<string>();
+  for (const path of Object.keys(schema)) {
+    if (!ARRAY_OBJECT_PROPERTY_RE.test(path)) continue;
+    // Extract every "...[]" prefix that is followed by ".prop"
+    let idx = 0;
+    while ((idx = path.indexOf('[].', idx)) !== -1) {
+      objectArrayPrefixes.add(path.slice(0, idx + 2)); // includes the "[]"
+      idx += 3;
+    }
+  }
+
+  if (objectArrayPrefixes.size === 0) return schema;
+
+  // Check if any double-array paths actually conflict
+  let hasConflict = false;
+  for (const path of Object.keys(schema)) {
+    if (!DOUBLE_ARRAY_RE.test(path)) continue;
+    const idx = path.indexOf('[][]');
+    if (idx !== -1 && objectArrayPrefixes.has(path.slice(0, idx + 2))) {
+      hasConflict = true;
+      break;
+    }
+  }
+
+  if (!hasConflict) return schema;
+
+  // Filter out conflicting double-array paths
+  const resolved: Record<string, string> = {};
+  for (const [path, value] of Object.entries(schema)) {
+    if (DOUBLE_ARRAY_RE.test(path)) {
+      const idx = path.indexOf('[][]');
+      if (idx !== -1 && objectArrayPrefixes.has(path.slice(0, idx + 2))) {
+        continue; // drop this conflicting path
+      }
+    }
+    resolved[path] = value;
+  }
+  return resolved;
+}
+
 export default function convertDotNotation(
   schema: Record<string, string>,
 ): JsonTypeDefinition {
+  const resolved = resolveDoubleArrayConflicts(schema);
   const result: JsonTypeDefinition = {};
 
-  for (const [rawPath, typ] of Object.entries(schema)) {
+  for (const [rawPath, typ] of Object.entries(resolved)) {
     // Skip paths with .length after array notation from dynamic iteration
     // e.g., metadata[].length, metadata[][].length
     if (DYNAMIC_LENGTH_RE.test(rawPath)) continue;
@@ -218,6 +282,17 @@ export default function convertDotNotation(
           // If the next segment after '[]' is another '[]' or a standalone index,
           // we need an array at cursor[key][0], not an object (for patterns like "key[][]" or "key[][0]")
           if (afterArray === '[]' || isStandaloneIndex(afterArray)) {
+            // But don't overwrite an existing populated object — object properties
+            // (from paths like key[].property) take precedence over double-array hints
+            // (from paths like key[][]) which come from mixed access patterns in scope analysis
+            if (
+              cursor[key][0] &&
+              typeof cursor[key][0] === 'object' &&
+              !Array.isArray(cursor[key][0]) &&
+              Object.keys(cursor[key][0]).length > 0
+            ) {
+              break;
+            }
             if (!Array.isArray(cursor[key][0])) cursor[key][0] = [];
           } else {
             // Default case: create an object

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/convertTypeAnnotationsToValues.ts

@@ -0,0 +1,179 @@
+/**
+ * Converts type annotation strings that appear as values in mockData to actual values.
+ *
+ * The LLM sometimes echoes the schema type annotation as the actual value.
+ * For example, if the schema says { filePath: "string | undefined" },
+ * the LLM might return { filePath: "string | undefined" } instead of
+ * generating an actual value like { filePath: "/path/to/file.tsx" }.
+ *
+ * This causes runtime errors when the code tries to use these "values"
+ * as actual data (e.g., calling .length on the string "string | undefined").
+ *
+ * This function detects type annotation strings and converts them to
+ * appropriate default values:
+ * - "string" → ""
+ * - "number" → 0
+ * - "boolean" → false
+ * - "function" → () => {}
+ * - "Date" → new Date()
+ * - "T | undefined" → undefined
+ * - "T | null" → null
+ * - "string[]" → []
+ */
+
+// Pattern to detect if a string looks like a TypeScript type annotation
+// This matches common type patterns:
+// - Primitive types: string, number, boolean, function, Date, object, any, unknown
+// - Union types: string | undefined, number | null, T | null | undefined
+// - Array types: string[], number[], T[]
+// - Optional types: T?
+const TYPE_ANNOTATION_PATTERN =
+  /^(string|number|boolean|function|Date|object|any|unknown|void|never|null|undefined|array)(\s*\|\s*(string|number|boolean|function|Date|object|any|unknown|void|never|null|undefined|array))*(\[\])?$/i;
+
+/**
+ * Check if a string value looks like a TypeScript type annotation.
+ */
+function isTypeAnnotation(value: string): boolean {
+  if (typeof value !== 'string') return false;
+
+  const trimmed = value.trim();
+
+  // Quick check: type annotations are typically short
+  if (trimmed.length > 100) return false;
+
+  // Check against the pattern
+  if (TYPE_ANNOTATION_PATTERN.test(trimmed)) return true;
+
+  // Also check for common patterns that the regex might miss
+  const commonTypePatterns = [
+    'string',
+    'number',
+    'boolean',
+    'function',
+    'object',
+    'array',
+    'Date',
+    'any',
+    'unknown',
+    'void',
+    'never',
+    'null',
+    'undefined',
+    'string | undefined',
+    'string | null',
+    'string | null | undefined',
+    'number | undefined',
+    'number | null',
+    'number | null | undefined',
+    'boolean | undefined',
+    'boolean | null',
+    'boolean | null | undefined',
+    'Date | undefined',
+    'Date | null',
+    'function | undefined',
+    'object | undefined',
+    'object | null',
+    'string[]',
+    'number[]',
+    'boolean[]',
+    'any[]',
+    'string | number',
+    'string | number | undefined',
+  ];
+
+  if (commonTypePatterns.includes(trimmed)) return true;
+
+  // Check for arrow function type annotations like "() => void", "(event: MouseEvent) => void"
+  // These contain '=>' but the body is a type keyword, not a JS expression
+  if (trimmed.includes('=>')) {
+    const arrowIndex = trimmed.indexOf('=>');
+    const body = trimmed.slice(arrowIndex + 2).trim();
+    if (
+      /^(void|string|number|boolean|any|unknown|never|null|undefined|object|Date|Promise<\w+>)$/i.test(
+        body,
+      )
+    ) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+/**
+ * Convert a type annotation string to an appropriate default value.
+ */
+function typeAnnotationToValue(typeAnnotation: string): unknown {
+  const trimmed = typeAnnotation.trim().toLowerCase();
+
+  // Handle undefined types first (they take precedence)
+  if (trimmed.includes('| undefined') || trimmed.includes('undefined |')) {
+    return undefined;
+  }
+
+  // Handle null types
+  if (trimmed.includes('| null') || trimmed.includes('null |')) {
+    return null;
+  }
+
+  // Handle array types
+  if (trimmed.endsWith('[]') || trimmed === 'array') {
+    return [];
+  }
+
+  // Handle arrow function type annotations like "() => void"
+  if (trimmed.includes('=>')) return () => {};
+
+  // Handle primitive types
+  if (trimmed === 'string') return '';
+  if (trimmed === 'number') return 0;
+  if (trimmed === 'boolean') return false;
+  if (trimmed === 'function') return () => {};
+  if (trimmed === 'date') return new Date();
+  if (trimmed === 'object') return {};
+  if (trimmed === 'null') return null;
+  if (trimmed === 'undefined') return undefined;
+  if (trimmed === 'any' || trimmed === 'unknown') return undefined;
+  if (trimmed === 'void' || trimmed === 'never') return undefined;
+
+  // Handle union types - prefer the first concrete type
+  if (trimmed.includes(' | ')) {
+    const parts = trimmed.split(' | ').map((p) => p.trim());
+    for (const part of parts) {
+      if (part === 'string') return '';
+      if (part === 'number') return 0;
+      if (part === 'boolean') return false;
+    }
+  }
+
+  // Fallback: return undefined for unknown type patterns
+  return undefined;
+}
+
+/**
+ * Recursively convert type annotation strings to values in a mockData object.
+ * Mutates the object in place.
+ */
+export default function convertTypeAnnotationsToValues(
+  mockData: Record<string, unknown>,
+): void {
+  for (const [key, value] of Object.entries(mockData)) {
+    if (typeof value === 'string' && isTypeAnnotation(value)) {
+      mockData[key] = typeAnnotationToValue(value);
+    } else if (Array.isArray(value)) {
+      // Recursively process array elements
+      for (const item of value) {
+        if (item !== null && typeof item === 'object' && !Array.isArray(item)) {
+          convertTypeAnnotationsToValues(item as Record<string, unknown>);
+        }
+      }
+    } else if (
+      value !== null &&
+      typeof value === 'object' &&
+      !Array.isArray(value)
+    ) {
+      // Recursively process nested objects
+      convertTypeAnnotationsToValues(value as Record<string, unknown>);
+    }
+  }
+}