@codeyam/codeyam-cli 0.1.0-staging.dd216e0 → 0.1.0-staging.e2d4438

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/convertDotNotation.ts

@@ -6,23 +6,124 @@ const STANDALONE_INDEX_RE = /^\[(\d+)\]$/;
 const isExplicitIndex = (s?: string) => !!s && INDEXED_RE.test(s);
 const isStandaloneIndex = (s?: string) => !!s && STANDALONE_INDEX_RE.test(s);
 
+// Detect .length properties that come from dynamic iteration patterns
+// These aren't useful for mock data - arrays work without explicit length,
+// and objects shouldn't have length. We filter .length that appears:
+// 1. After array notation: metadata[].length, items[][].length
+// 2. On object paths that likely come from iteration: metadata.length (when metadata has other props)
+// The regex matches any path ending with .length that has [] somewhere before it
+const DYNAMIC_LENGTH_RE = /\[\].*\.length$/;
+
+// Cache for type string analysis to avoid repeated split/filter operations.
+// These functions are called multiple times per path segment across thousands of paths.
+const typeAnalysisCache = new Map<
+  string,
+  { isSkippable: boolean; baseType: string; isNullable: boolean }
+>();
+
+function getTypeAnalysis(t: string) {
+  const cached = typeAnalysisCache.get(t);
+  if (cached) return cached;
+  const parts = t.split('|').map((s) => s.trim());
+  const base = parts.filter((s) => s !== 'undefined' && s !== 'null');
+  const result = {
+    isSkippable:
+      base.length === 1 &&
+      (base[0] === 'object' ||
+        base[0] === 'array' ||
+        base[0] === 'function' ||
+        base[0] === 'unknown'),
+    baseType: base[0],
+    isNullable: parts.includes('undefined') || parts.includes('null'),
+  };
+  typeAnalysisCache.set(t, result);
+  return result;
+}
+
 // Treat these as structural placeholders (don't commit them as concrete leaves)
 function isSkippableLeafType(t: string) {
-  // 'unknown' by itself is a placeholder (but 'boolean | unknown' is not)
-  return (
-    t === 'object' ||
-    t === 'array' ||
-    t === 'function' ||
-    t.trim() === 'unknown'
-  );
+  return getTypeAnalysis(t).isSkippable;
+}
+
+// Extract the base structural type from a potentially nullable type string.
+// e.g., 'object | undefined' → 'object', 'array | null' → 'array'
+function getBaseSkippableType(t: string): string {
+  return getTypeAnalysis(t).baseType;
+}
+
+// Check if a type string has nullable annotations (| undefined or | null)
+function isNullableType(t: string): boolean {
+  return getTypeAnalysis(t).isNullable;
+}
+
+// Matches paths containing [][] — e.g., "items[][]" or "items[][].text"
+const DOUBLE_ARRAY_RE = /\[\]\[\]/;
+
+// Matches a [].property segment — e.g., "items[].name" (but not "items[][]")
+const ARRAY_OBJECT_PROPERTY_RE = /\[\]\.[^[\]]/;
+
+/**
+ * When scope analysis produces both `prefix[].property` (array of objects) and
+ * `prefix[][]` (array of arrays) for the same prefix, there's a conflict about
+ * what the array elements are. Object-property paths are more specific and come
+ * from actual property access; double-array paths come from mixed iteration
+ * patterns in the scope analysis. Drop the double-array paths when both exist.
+ */
+function resolveDoubleArrayConflicts(
+  schema: Record<string, string>,
+): Record<string, string> {
+  // Collect prefixes that end in [] and have object property children
+  const objectArrayPrefixes = new Set<string>();
+  for (const path of Object.keys(schema)) {
+    if (!ARRAY_OBJECT_PROPERTY_RE.test(path)) continue;
+    // Extract every "...[]" prefix that is followed by ".prop"
+    let idx = 0;
+    while ((idx = path.indexOf('[].', idx)) !== -1) {
+      objectArrayPrefixes.add(path.slice(0, idx + 2)); // includes the "[]"
+      idx += 3;
+    }
+  }
+
+  if (objectArrayPrefixes.size === 0) return schema;
+
+  // Check if any double-array paths actually conflict
+  let hasConflict = false;
+  for (const path of Object.keys(schema)) {
+    if (!DOUBLE_ARRAY_RE.test(path)) continue;
+    const idx = path.indexOf('[][]');
+    if (idx !== -1 && objectArrayPrefixes.has(path.slice(0, idx + 2))) {
+      hasConflict = true;
+      break;
+    }
+  }
+
+  if (!hasConflict) return schema;
+
+  // Filter out conflicting double-array paths
+  const resolved: Record<string, string> = {};
+  for (const [path, value] of Object.entries(schema)) {
+    if (DOUBLE_ARRAY_RE.test(path)) {
+      const idx = path.indexOf('[][]');
+      if (idx !== -1 && objectArrayPrefixes.has(path.slice(0, idx + 2))) {
+        continue; // drop this conflicting path
+      }
+    }
+    resolved[path] = value;
+  }
+  return resolved;
 }
 
 export default function convertDotNotation(
   schema: Record<string, string>,
 ): JsonTypeDefinition {
+  const resolved = resolveDoubleArrayConflicts(schema);
   const result: JsonTypeDefinition = {};
 
-  for (const [rawPath, typ] of Object.entries(schema)) {
+  for (const [rawPath, typ] of Object.entries(resolved)) {
+    // Skip paths with .length after array notation from dynamic iteration
+    // e.g., metadata[].length, metadata[][].length
+    if (DYNAMIC_LENGTH_RE.test(rawPath)) continue;
+
     const parts = splitOutsideParenthesesAndArrays(rawPath);
     if (!parts || parts.length === 0) continue;
 
@@ -187,6 +288,17 @@ export default function convertDotNotation(
           i++; // consume '[]'
         } else {
           // Pattern like: "items", "[]", "id" -> items: [ { id: ... } ]
+          // But if cursor[key] is already a non-array object (from paths like "metadata: object"),
+          // don't overwrite it with an array - the object type should take precedence.
+          // This handles spurious "metadata[]" paths from dynamic iteration (Object.keys, for...in).
+          if (
+            cursor[key] !== undefined &&
+            typeof cursor[key] === 'object' &&
+            !Array.isArray(cursor[key])
+          ) {
+            // Skip this path - it's trying to treat an object as an array
+            break;
+          }
           if (!Array.isArray(cursor[key])) cursor[key] = [];
 
           // Look ahead to see what comes after '[]'
@@ -195,6 +307,17 @@ export default function convertDotNotation(
           // If the next segment after '[]' is another '[]' or a standalone index,
           // we need an array at cursor[key][0], not an object (for patterns like "key[][]" or "key[][0]")
           if (afterArray === '[]' || isStandaloneIndex(afterArray)) {
+            // But don't overwrite an existing populated object — object properties
+            // (from paths like key[].property) take precedence over double-array hints
+            // (from paths like key[][]) which come from mixed access patterns in scope analysis
+            if (
+              cursor[key][0] &&
+              typeof cursor[key][0] === 'object' &&
+              !Array.isArray(cursor[key][0]) &&
+              Object.keys(cursor[key][0]).length > 0
+            ) {
+              break;
+            }
             if (!Array.isArray(cursor[key][0])) cursor[key][0] = [];
           } else {
             // Default case: create an object
@@ -219,15 +342,41 @@ export default function convertDotNotation(
           cursor[key] = typ;
         } else {
           // Structural/placeholder terminal
-          if (typ === 'array') {
+          const nullable = isNullableType(typ);
+          const baseType = getBaseSkippableType(typ);
+
+          if (baseType === 'array') {
             if (!Array.isArray(cursor[key])) cursor[key] = [];
+            if (nullable) {
+              (cursor[key] as any)._nullable = true;
+            }
           } else if (
-            typ === 'object' ||
-            typ === 'function' ||
-            typ.trim() === 'unknown'
+            baseType === 'object' ||
+            baseType === 'function' ||
+            baseType === 'unknown'
           ) {
-            if (cursor[key] === undefined) {
-              cursor[key] = typ;
+            if (nullable) {
+              // Nullable object: ensure it's an actual object (not a string
+              // placeholder) so _nullable can be set and child paths can
+              // populate properties on it.
+              if (
+                cursor[key] === undefined ||
+                typeof cursor[key] === 'string'
+              ) {
+                cursor[key] = {};
+              }
+              if (
+                typeof cursor[key] === 'object' &&
+                cursor[key] !== null &&
+                !Array.isArray(cursor[key])
+              ) {
+                (cursor[key] as any)._nullable = true;
+              }
+            } else {
+              // Non-nullable: preserve existing behavior (string placeholder)
+              if (cursor[key] === undefined) {
+                cursor[key] = typ;
+              }
             }
           }
         }

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/convertNullToUndefinedBySchema.ts

@@ -0,0 +1,98 @@
+/**
+ * Converts null values to undefined (removes them) based on schema type constraints.
+ *
+ * When the LLM generates mock data, it uses `null` to represent "no value" because
+ * JSON doesn't support `undefined`. However, TypeScript types like `string | undefined`
+ * don't accept `null`.
+ *
+ * This function traverses data and schema together:
+ * - If schema type is "T | undefined" (but NOT "T | null"), convert null → undefined (delete key)
+ * - If schema type includes "null" (e.g., "T | null" or "T | null | undefined"), preserve null
+ * - If field not in schema, preserve as-is
+ *
+ * @param data The data object with potential null values
+ * @param schema The schema describing expected types
+ * @returns The data with null→undefined conversions applied (mutates original)
+ */
+export default function convertNullToUndefinedBySchema<T>(
+  data: T,
+  schema: unknown,
+): T {
+  // Handle null/undefined data
+  if (data === null || data === undefined) {
+    return data;
+  }
+
+  // Only process objects
+  if (typeof data !== 'object') {
+    return data;
+  }
+
+  // Handle arrays
+  if (Array.isArray(data)) {
+    // For arrays, the schema should be an array with one element describing the item type
+    const itemSchema = Array.isArray(schema) ? schema[0] : undefined;
+    for (let i = 0; i < data.length; i++) {
+      data[i] = convertNullToUndefinedBySchema(data[i], itemSchema);
+    }
+    return data;
+  }
+
+  // Handle objects
+  const dataObj = data as Record<string, unknown>;
+  const schemaObj = schema as Record<string, unknown> | undefined;
+
+  for (const key of Object.keys(dataObj)) {
+    const value = dataObj[key];
+    const fieldSchema = schemaObj?.[key];
+
+    if (value === null) {
+      // Check if schema allows null
+      if (shouldConvertNullToUndefined(fieldSchema)) {
+        // Delete the key (converts null to undefined)
+        delete dataObj[key];
+      }
+      // If schema allows null, preserve it
+    } else if (typeof value === 'object' && value !== null) {
+      // Recursively process nested objects/arrays
+      convertNullToUndefinedBySchema(value, fieldSchema);
+    }
+  }
+
+  return data;
+}
+
+/**
+ * Determines if a null value should be converted to undefined based on schema type.
+ *
+ * @param fieldSchema The schema type string or object for the field
+ * @returns true if null should be converted to undefined, false if null should be preserved
+ */
+function shouldConvertNullToUndefined(fieldSchema: unknown): boolean {
+  // If no schema for this field, preserve null (be conservative)
+  if (fieldSchema === undefined || fieldSchema === null) {
+    return false;
+  }
+
+  // If schema is a string type definition, check if it allows null
+  if (typeof fieldSchema === 'string') {
+    // Schema types that include "null" should preserve null values
+    // e.g., "string | null", "number | null | undefined", "null"
+    const allowsNull = fieldSchema.includes('null');
+    const allowsUndefined = fieldSchema.includes('undefined');
+
+    // If type allows undefined but NOT null, convert null to undefined
+    // e.g., "string | undefined" → convert null to undefined
+    // e.g., "string | null" → preserve null
+    // e.g., "string | null | undefined" → preserve null
+    if (allowsUndefined && !allowsNull) {
+      return true;
+    }
+
+    return false;
+  }
+
+  // If schema is an object or array, it's a nested structure
+  // Don't convert the container itself, just recurse into it
+  return false;
+}

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/convertTypeAnnotationsToValues.ts

@@ -0,0 +1,179 @@
+/**
+ * Converts type annotation strings that appear as values in mockData to actual values.
+ *
+ * The LLM sometimes echoes the schema type annotation as the actual value.
+ * For example, if the schema says { filePath: "string | undefined" },
+ * the LLM might return { filePath: "string | undefined" } instead of
+ * generating an actual value like { filePath: "/path/to/file.tsx" }.
+ *
+ * This causes runtime errors when the code tries to use these "values"
+ * as actual data (e.g., calling .length on the string "string | undefined").
+ *
+ * This function detects type annotation strings and converts them to
+ * appropriate default values:
+ * - "string" → ""
+ * - "number" → 0
+ * - "boolean" → false
+ * - "function" → () => {}
+ * - "Date" → new Date()
+ * - "T | undefined" → undefined
+ * - "T | null" → null
+ * - "string[]" → []
+ */
+
+// Pattern to detect if a string looks like a TypeScript type annotation
+// This matches common type patterns:
+// - Primitive types: string, number, boolean, function, Date, object, any, unknown
+// - Union types: string | undefined, number | null, T | null | undefined
+// - Array types: string[], number[], T[]
+// - Optional types: T?
+const TYPE_ANNOTATION_PATTERN =
+  /^(string|number|boolean|function|Date|object|any|unknown|void|never|null|undefined|array)(\s*\|\s*(string|number|boolean|function|Date|object|any|unknown|void|never|null|undefined|array))*(\[\])?$/i;
+
+/**
+ * Check if a string value looks like a TypeScript type annotation.
+ */
+function isTypeAnnotation(value: string): boolean {
+  if (typeof value !== 'string') return false;
+
+  const trimmed = value.trim();
+
+  // Quick check: type annotations are typically short
+  if (trimmed.length > 100) return false;
+
+  // Check against the pattern
+  if (TYPE_ANNOTATION_PATTERN.test(trimmed)) return true;
+
+  // Also check for common patterns that the regex might miss
+  const commonTypePatterns = [
+    'string',
+    'number',
+    'boolean',
+    'function',
+    'object',
+    'array',
+    'Date',
+    'any',
+    'unknown',
+    'void',
+    'never',
+    'null',
+    'undefined',
+    'string | undefined',
+    'string | null',
+    'string | null | undefined',
+    'number | undefined',
+    'number | null',
+    'number | null | undefined',
+    'boolean | undefined',
+    'boolean | null',
+    'boolean | null | undefined',
+    'Date | undefined',
+    'Date | null',
+    'function | undefined',
+    'object | undefined',
+    'object | null',
+    'string[]',
+    'number[]',
+    'boolean[]',
+    'any[]',
+    'string | number',
+    'string | number | undefined',
+  ];
+
+  if (commonTypePatterns.includes(trimmed)) return true;
+
+  // Check for arrow function type annotations like "() => void", "(event: MouseEvent) => void"
+  // These contain '=>' but the body is a type keyword, not a JS expression
+  if (trimmed.includes('=>')) {
+    const arrowIndex = trimmed.indexOf('=>');
+    const body = trimmed.slice(arrowIndex + 2).trim();
+    if (
+      /^(void|string|number|boolean|any|unknown|never|null|undefined|object|Date|Promise<\w+>)$/i.test(
+        body,
+      )
+    ) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+/**
+ * Convert a type annotation string to an appropriate default value.
+ */
+function typeAnnotationToValue(typeAnnotation: string): unknown {
+  const trimmed = typeAnnotation.trim().toLowerCase();
+
+  // Handle undefined types first (they take precedence)
+  if (trimmed.includes('| undefined') || trimmed.includes('undefined |')) {
+    return undefined;
+  }
+
+  // Handle null types
+  if (trimmed.includes('| null') || trimmed.includes('null |')) {
+    return null;
+  }
+
+  // Handle array types
+  if (trimmed.endsWith('[]') || trimmed === 'array') {
+    return [];
+  }
+
+  // Handle arrow function type annotations like "() => void"
+  if (trimmed.includes('=>')) return () => {};
+
+  // Handle primitive types
+  if (trimmed === 'string') return '';
+  if (trimmed === 'number') return 0;
+  if (trimmed === 'boolean') return false;
+  if (trimmed === 'function') return () => {};
+  if (trimmed === 'date') return new Date();
+  if (trimmed === 'object') return {};
+  if (trimmed === 'null') return null;
+  if (trimmed === 'undefined') return undefined;
+  if (trimmed === 'any' || trimmed === 'unknown') return undefined;
+  if (trimmed === 'void' || trimmed === 'never') return undefined;
+
+  // Handle union types - prefer the first concrete type
+  if (trimmed.includes(' | ')) {
+    const parts = trimmed.split(' | ').map((p) => p.trim());
+    for (const part of parts) {
+      if (part === 'string') return '';
+      if (part === 'number') return 0;
+      if (part === 'boolean') return false;
+    }
+  }
+
+  // Fallback: return undefined for unknown type patterns
+  return undefined;
+}
+
+/**
+ * Recursively convert type annotation strings to values in a mockData object.
+ * Mutates the object in place.
+ */
+export default function convertTypeAnnotationsToValues(
+  mockData: Record<string, unknown>,
+): void {
+  for (const [key, value] of Object.entries(mockData)) {
+    if (typeof value === 'string' && isTypeAnnotation(value)) {
+      mockData[key] = typeAnnotationToValue(value);
+    } else if (Array.isArray(value)) {
+      // Recursively process array elements
+      for (const item of value) {
+        if (item !== null && typeof item === 'object' && !Array.isArray(item)) {
+          convertTypeAnnotationsToValues(item as Record<string, unknown>);
+        }
+      }
+    } else if (
+      value !== null &&
+      typeof value === 'object' &&
+      !Array.isArray(value)
+    ) {
+      // Recursively process nested objects
+      convertTypeAnnotationsToValues(value as Record<string, unknown>);
+    }
+  }
+}

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/deduplicateFunctionSchemas.ts

@@ -77,42 +77,52 @@ export default function deduplicateFunctionSchemas(
 
       // If multiple variants with arguments exist, find the one with the most robust structure
       if (withArgs.length > 1) {
-        // Count the number of related schema keys for each variant
-        const variantDepths = withArgs.map((variant) => {
-          // Find the end of the function call (the closing parenthesis)
+        // Pre-compute function call prefixes for all variants
+        const variantPrefixes = withArgs.map((variant) => {
           const closingParenIndex = variant.lastIndexOf(')');
-          const functionCallPrefix =
-            closingParenIndex > -1
-              ? variant.slice(0, closingParenIndex + 1)
-              : variant;
-
-          const relatedKeys = Object.keys(schema).filter((k) => {
-            // Check if this key is a child of the variant (starts with the function call)
-            return (
-              k.startsWith(functionCallPrefix + '.') ||
-              k.startsWith(functionCallPrefix + '[')
-            );
-          });
-
           return {
             key: variant,
-            depth: relatedKeys.length,
-            maxDepth: Math.max(
-              0,
-              ...relatedKeys.map((k) => {
-                // Count the nesting depth by counting dots and brackets after the variant
-                const remaining = k.slice(functionCallPrefix.length);
-                return (
-                  (remaining.match(/\./g) || []).length +
-                  (remaining.match(/\[/g) || []).length
-                );
-              }),
-            ),
+            prefix:
+              closingParenIndex > -1
+                ? variant.slice(0, closingParenIndex + 1)
+                : variant,
+            depth: 0,
+            maxDepth: 0,
           };
         });
 
+        // Single pass through schema keys to count related keys for ALL variants at once.
+        // This is O(schemaSize × variants) but done in one pass instead of
+        // O(variants × schemaSize) with separate passes per variant.
+        const schemaKeys = Object.keys(schema);
+        for (const k of schemaKeys) {
+          for (const variantInfo of variantPrefixes) {
+            // Check if key starts with prefix followed by . or [
+            const prefixLen = variantInfo.prefix.length;
+            if (k.length > prefixLen) {
+              const nextChar = k[prefixLen];
+              if (
+                (nextChar === '.' || nextChar === '[') &&
+                k.startsWith(variantInfo.prefix)
+              ) {
+                variantInfo.depth++;
+                // Count nesting depth by counting . and [ characters (faster than regex)
+                const remaining = k.slice(prefixLen);
+                let nestingDepth = 0;
+                for (let i = 0; i < remaining.length; i++) {
+                  const c = remaining[i];
+                  if (c === '.' || c === '[') nestingDepth++;
+                }
+                if (nestingDepth > variantInfo.maxDepth) {
+                  variantInfo.maxDepth = nestingDepth;
+                }
+              }
+            }
+          }
+        }
+
         // Sort by depth (number of related keys) and then by max depth (nesting level)
-        variantDepths.sort((a, b) => {
+        variantPrefixes.sort((a, b) => {
           if (b.depth !== a.depth) {
             return b.depth - a.depth;
           }
@@ -120,7 +130,7 @@ export default function deduplicateFunctionSchemas(
         });
 
         // Keep the one with the most robust structure, remove others
-        const keepVariant = variantDepths[0].key;
+        const keepVariant = variantPrefixes[0].key;
         withArgs.forEach((variant) => {
           if (variant !== keepVariant) {
             keysToRemove.add(variant);