@codeyam/codeyam-cli 0.1.0-staging.6e699e5 → 0.1.0-staging.73a4bf4

package/analyzer-template/packages/ai/src/lib/dataStructure/equivalencyManagers/frameworks/JavascriptFrameworkManager.ts

@@ -72,7 +72,11 @@ export class JavascriptFrameworkManager implements EquivalencyManager {
   }
 
   finalize(scopeNode: ScopeNode, scopeDataStructure: ScopeDataStructure) {
-    cleanNonObjectFunctions(scopeNode, scopeDataStructure, true);
+    // Don't call cleanNonObjectFunctions with final=true here.
+    // The aggressive cleanup (transformKeyMapping, clearAttributes) removes method call
+    // evidence like .includes() before type inference can use it.
+    // Instead, defer final cleanup to toSerializable() when we're done with inference.
+    cleanNonObjectFunctions(scopeNode, scopeDataStructure, false);
     cleanKnownObjectFunctions(scopeNode, scopeDataStructure);
     this.handleUniformArrays(scopeNode, scopeDataStructure);
   }

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/BatchSchemaProcessor.ts

@@ -41,6 +41,7 @@ export interface BatchProcessorStats {
 
 export class BatchSchemaProcessor {
   private workQueue: WorkItem[] = [];
+  private queueIndex = 0;
   private stats: BatchProcessorStats = {
     itemsProcessed: 0,
     itemsSkipped: 0, // Kept for API compatibility, always 0
@@ -71,17 +72,28 @@ export class BatchSchemaProcessor {
    * Check if there are more items to process.
    */
   hasWork(): boolean {
-    return this.workQueue.length > 0;
+    return this.queueIndex < this.workQueue.length;
   }
 
   /**
    * Get the next work item from the queue.
    */
   getNextWork(): WorkItem | undefined {
-    const item = this.workQueue.shift();
+    if (this.queueIndex >= this.workQueue.length) {
+      return undefined;
+    }
+
+    const item = this.workQueue[this.queueIndex++];
     if (item) {
       this.stats.itemsProcessed++;
     }
+
+    // Once we've consumed the queue, reset to avoid unbounded growth
+    if (this.queueIndex >= this.workQueue.length) {
+      this.workQueue = [];
+      this.queueIndex = 0;
+    }
+
     return item;
   }
 
@@ -122,6 +134,7 @@ export class BatchSchemaProcessor {
    */
   reset(): void {
     this.workQueue = [];
+    this.queueIndex = 0;
     this.stats = {
       itemsProcessed: 0,
       itemsSkipped: 0,
@@ -134,6 +147,6 @@ export class BatchSchemaProcessor {
    * Get current queue size (for debugging/monitoring).
    */
   getQueueSize(): number {
-    return this.workQueue.length;
+    return this.workQueue.length - this.queueIndex;
   }
 }

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/ScopeTreeManager.ts

@@ -117,9 +117,10 @@ export class ScopeTreeManager {
    */
   findNode(name: string): ScopeTreeNode | undefined {
     const queue: ScopeTreeNode[] = [this.tree];
+    let index = 0;
 
-    while (queue.length > 0) {
-      const node = queue.shift()!;
+    while (index < queue.length) {
+      const node = queue[index++]!;
       if (node.name === name) {
         return node;
       }
@@ -135,9 +136,10 @@ export class ScopeTreeManager {
   getAllNames(): string[] {
     const names: string[] = [];
     const queue: ScopeTreeNode[] = [this.tree];
+    let index = 0;
 
-    while (queue.length > 0) {
-      const node = queue.shift()!;
+    while (index < queue.length) {
+      const node = queue[index++]!;
       names.push(node.name);
       queue.push(...node.children);
     }

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/cleanKnownObjectFunctions.ts

@@ -35,6 +35,47 @@ const getParts = (path: string): string[] => {
   return parts;
 };
 
+function buildPrefixParts(parts: string[]): string[] {
+  if (parts.length === 0) return [];
+  const prefixes = new Array(parts.length);
+  let current = '';
+  for (let i = 0; i < parts.length; i++) {
+    const part = parts[i];
+    if (i === 0) {
+      current = part;
+    } else if (part.startsWith('[') || part.startsWith('(')) {
+      current += part;
+    } else {
+      current += `.${part}`;
+    }
+    prefixes[i] = current;
+  }
+  return prefixes;
+}
+
+/**
+ * Clear the module-level cache.
+ * Call this between entity analyses to prevent unbounded memory growth.
+ */
+export function clearCleanKnownObjectFunctionsCache(): {
+  count: number;
+  estimatedBytes: number;
+} {
+  let totalBytes = 0;
+  for (const [key, value] of partsCache) {
+    // Key is a string, value is string[] - estimate UTF-16 encoding (2 bytes per char)
+    totalBytes += key.length * 2;
+    for (const part of value) {
+      totalBytes += part.length * 2;
+    }
+    // Add overhead for array and Map entry (~50 bytes per entry)
+    totalBytes += 50;
+  }
+  const count = partsCache.size;
+  partsCache.clear();
+  return { count, estimatedBytes: totalBytes };
+}
+
 /** pre-compute the prototype-method sets once per module load */
 const protoMethodSets: Record<PrimitiveKind, Set<string>> = {
   array: new Set(
@@ -110,6 +151,7 @@ function scrub(
 
   for (const keyPath of sortedKeys) {
     const keyParts = getParts(keyPath);
+    const prefixParts = buildPrefixParts(keyParts);
 
     if (
       IGNORE_CLASSES_AND_OBJECTS.find(
@@ -122,7 +164,7 @@ function scrub(
 
     // walk back through the prefixes: foo.bar.baz.qux → foo.bar.baz → foo.bar …
     for (let cut = keyParts.length - 1; cut > 0; --cut) {
-      const prefix = joinParenthesesAndArrays(keyParts.slice(0, cut));
+      const prefix = prefixParts[cut - 1];
       const rootKind = rootTable.get(prefix);
       if (!rootKind) continue;
 
@@ -130,9 +172,18 @@ function scrub(
       const rawMethod = rawToken.split('(')[0];
 
       // Check if it's a prototype method (e.g., map, filter, toString)
+      // Only match when the token is actually a method reference/call:
+      // - Has parentheses (explicit call like map(), filter(x => x))
+      // - OR is typed as "function" (method reference like .split)
+      // Bare property names that happen to match method names
+      // (like .entries = "array", .values = "object") are preserved.
       if (protoMethodSets[rootKind].has(rawMethod)) {
-        onRemove(keyPath);
-        break;
+        const isMethodCall = rawToken.includes('(');
+        const isMethodReference = mapping[keyPath] === 'function';
+        if (isMethodCall || isMethodReference) {
+          onRemove(keyPath);
+          break;
+        }
       }
 
       // Check if it's a built-in property (e.g., length on arrays/strings)

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/cleanNonObjectFunctions.ts

@@ -151,21 +151,63 @@ function cachedSplit(str: string): string[] {
   return parts;
 }
 
+/**
+ * Clear the module-level cache.
+ * Call this between entity analyses to prevent unbounded memory growth.
+ */
+export function clearCleanNonObjectFunctionsCache(): {
+  count: number;
+  estimatedBytes: number;
+} {
+  let totalBytes = 0;
+  for (const [key, value] of splitCache) {
+    // Key is a string, value is string[] - estimate UTF-16 encoding (2 bytes per char)
+    totalBytes += key.length * 2;
+    for (const part of value) {
+      totalBytes += part.length * 2;
+    }
+    // Add overhead for array and Map entry (~50 bytes per entry)
+    totalBytes += 50;
+  }
+  const count = splitCache.size;
+  splitCache.clear();
+  return { count, estimatedBytes: totalBytes };
+}
+
 // --- Prefix Index for O(1) lookup in subPathIsFromKnownObject ---
 // Maps parent prefix -> array of last parts (method names without parens)
 type PrefixIndex = Map<string, string[]>;
 
+function buildPrefixPartsWithRoot(parts: string[]): string[] {
+  const prefixes = new Array(parts.length + 1);
+  prefixes[0] = '';
+  let current = '';
+  for (let i = 0; i < parts.length; i++) {
+    const part = parts[i];
+    if (i === 0) {
+      current = part;
+    } else if (part.startsWith('[') || part.startsWith('(')) {
+      current += part;
+    } else {
+      current += `.${part}`;
+    }
+    prefixes[i + 1] = current;
+  }
+  return prefixes;
+}
+
 function buildPrefixIndex(allPaths: string[]): PrefixIndex {
   const index: PrefixIndex = new Map();
 
   for (const path of allPaths) {
     const parts = cachedSplit(path);
     if (parts.length < 1) continue;
+    const prefixes = buildPrefixPartsWithRoot(parts);
 
     // For each possible prefix length, add the last part to that prefix's entry
     // This allows lookup at any depth
     for (let prefixLen = 0; prefixLen < parts.length; prefixLen++) {
-      const prefix = joinParenthesesAndArrays(parts.slice(0, prefixLen));
+      const prefix = prefixes[prefixLen];
       const lastPart = parts[prefixLen]?.split('(')?.[0];
       if (lastPart) {
         let existing = index.get(prefix);
@@ -468,7 +510,7 @@ function removeNonTransformingArrayMethods(variablePath: string): string {
     const part = keyParts[i];
 
     const arrayMethod = getMethodName(part);
-    if (isValidArrayMethod(arrayMethod)) {
+    if (isValidFunctionCall(part) && isValidArrayMethod(arrayMethod)) {
       const nextPart = keyParts[i + 1];
       if (
         !nonTransformingArrayMethodsSet.has(arrayMethod) ||
@@ -508,7 +550,7 @@ function removeIsolatingArrayMethods(
     const part = keyParts[i];
 
     const arrayMethod = getMethodName(part);
-    if (isValidArrayMethod(arrayMethod)) {
+    if (isValidFunctionCall(part) && isValidArrayMethod(arrayMethod)) {
       const nextPart = keyParts[i + 1];
       const prevPart = i > 1 ? keyParts[i - 1] : undefined;
       if (
@@ -1062,29 +1104,32 @@ function clearAttributes(
     knownBasePaths.add(path);
   }
 
+  // OPTIMIZATION: Instead of O(keys × basePaths), use O(keys × keyDepth) where keyDepth is small
+  // For each key, find all its prefixes and check if any is a known basePath (O(1) Set lookup)
   for (const key in mapping) {
     const keyParts = cachedSplit(key);
-    for (const basePath of knownBasePaths) {
-      const basePathParts = cachedSplit(basePath);
-      if (keyParts.length <= basePathParts.length) continue;
-      if (
-        joinParenthesesAndArrays(keyParts.slice(0, basePathParts.length)) !==
-        basePath
-      )
-        continue;
+    if (keyParts.length <= 1) continue; // No possible parent basePath
+
+    // Check each prefix of the key to see if it's a known basePath
+    // Start from shortest prefix (most likely to match) for early exit
+    for (let prefixLen = 1; prefixLen < keyParts.length; prefixLen++) {
+      const prefix = joinParenthesesAndArrays(keyParts.slice(0, prefixLen));
+      if (!knownBasePaths.has(prefix)) continue;
+
+      // Found a matching basePath - now check the conditions
+      const basePath = prefix;
+
       // Preserve functionCallReturnValue for non-string types, UNLESS
       // the method is a primitive-returning method (like some, every, includes, indexOf)
       // whose return values are not mockable
-      const methodPart = keyParts[basePathParts.length];
+      const methodPart = keyParts[prefixLen];
       const methodName = methodPart?.split('(')[0];
       const isPrimitiveReturning = primitiveReturningMethodsSet.has(
         methodName ?? '',
       );
       if (
         mapping[basePath] !== 'string' &&
-        keyParts[basePathParts.length + 1]?.startsWith(
-          'functionCallReturnValue',
-        ) &&
+        keyParts[prefixLen + 1]?.startsWith('functionCallReturnValue') &&
         !isPrimitiveReturning
       )
         continue;

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/coerceObjectsToPrimitivesBySchema.ts

@@ -0,0 +1,70 @@
+/**
+ * Coerces object/array values to primitive types when the schema expects a primitive.
+ *
+ * The LLM sometimes generates an object or array where the schema expects a primitive
+ * type like "string", "number", or "boolean". For example:
+ *   Schema: { body: "string" }
+ *   LLM output: { body: { "env": "production" } }
+ *
+ * This causes runtime errors like `TypeError: body.match is not a function` because
+ * the code expects a string but gets an object.
+ *
+ * Coercion rules:
+ * - object/array → string: JSON.stringify
+ * - object/array → number: 0
+ * - object/array → boolean: false
+ *
+ * @param data The mock data object to coerce (mutated in place)
+ * @param schema The schema describing expected types
+ */
+export default function coerceObjectsToPrimitivesBySchema<T>(
+  data: T,
+  schema: unknown,
+): T {
+  if (data === null || data === undefined) {
+    return data;
+  }
+
+  if (typeof data !== 'object') {
+    return data;
+  }
+
+  if (Array.isArray(data)) {
+    const itemSchema = Array.isArray(schema) ? schema[0] : undefined;
+    for (let i = 0; i < data.length; i++) {
+      data[i] = coerceObjectsToPrimitivesBySchema(data[i], itemSchema);
+    }
+    return data;
+  }
+
+  const dataObj = data as Record<string, unknown>;
+  const schemaObj = schema as Record<string, unknown> | undefined;
+
+  for (const key of Object.keys(dataObj)) {
+    const value = dataObj[key];
+    const fieldSchema = schemaObj?.[key];
+
+    if (value === null || value === undefined) {
+      continue;
+    }
+
+    if (typeof value === 'object' && typeof fieldSchema === 'string') {
+      // Schema expects a primitive but the value is an object/array.
+      // Coerce to the expected type.
+      const schemaType = fieldSchema.toLowerCase();
+
+      if (schemaType.includes('string')) {
+        dataObj[key] = JSON.stringify(value);
+      } else if (schemaType.includes('number')) {
+        dataObj[key] = 0;
+      } else if (schemaType.includes('boolean')) {
+        dataObj[key] = false;
+      }
+    } else if (typeof value === 'object' && value !== null) {
+      // Both value and schema are objects — recurse
+      coerceObjectsToPrimitivesBySchema(value, fieldSchema);
+    }
+  }
+
+  return data;
+}

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/convertDotNotation.ts

@@ -14,23 +14,112 @@ const isStandaloneIndex = (s?: string) => !!s && STANDALONE_INDEX_RE.test(s);
 // The regex matches any path ending with .length that has [] somewhere before it
 const DYNAMIC_LENGTH_RE = /\[\].*\.length$/;
 
+// Cache for type string analysis to avoid repeated split/filter operations.
+// These functions are called multiple times per path segment across thousands of paths.
+const typeAnalysisCache = new Map<
+  string,
+  { isSkippable: boolean; baseType: string; isNullable: boolean }
+>();
+
+function getTypeAnalysis(t: string) {
+  const cached = typeAnalysisCache.get(t);
+  if (cached) return cached;
+  const parts = t.split('|').map((s) => s.trim());
+  const base = parts.filter((s) => s !== 'undefined' && s !== 'null');
+  const result = {
+    isSkippable:
+      base.length === 1 &&
+      (base[0] === 'object' ||
+        base[0] === 'array' ||
+        base[0] === 'function' ||
+        base[0] === 'unknown'),
+    baseType: base[0],
+    isNullable: parts.includes('undefined') || parts.includes('null'),
+  };
+  typeAnalysisCache.set(t, result);
+  return result;
+}
+
 // Treat these as structural placeholders (don't commit them as concrete leaves)
 function isSkippableLeafType(t: string) {
-  // 'unknown' by itself is a placeholder (but 'boolean | unknown' is not)
-  return (
-    t === 'object' ||
-    t === 'array' ||
-    t === 'function' ||
-    t.trim() === 'unknown'
-  );
+  return getTypeAnalysis(t).isSkippable;
+}
+
+// Extract the base structural type from a potentially nullable type string.
+// e.g., 'object | undefined' → 'object', 'array | null' → 'array'
+function getBaseSkippableType(t: string): string {
+  return getTypeAnalysis(t).baseType;
+}
+
+// Check if a type string has nullable annotations (| undefined or | null)
+function isNullableType(t: string): boolean {
+  return getTypeAnalysis(t).isNullable;
+}
+
+// Matches paths containing [][] — e.g., "items[][]" or "items[][].text"
+const DOUBLE_ARRAY_RE = /\[\]\[\]/;
+
+// Matches a [].property segment — e.g., "items[].name" (but not "items[][]")
+const ARRAY_OBJECT_PROPERTY_RE = /\[\]\.[^[\]]/;
+
+/**
+ * When scope analysis produces both `prefix[].property` (array of objects) and
+ * `prefix[][]` (array of arrays) for the same prefix, there's a conflict about
+ * what the array elements are. Object-property paths are more specific and come
+ * from actual property access; double-array paths come from mixed iteration
+ * patterns in the scope analysis. Drop the double-array paths when both exist.
+ */
+function resolveDoubleArrayConflicts(
+  schema: Record<string, string>,
+): Record<string, string> {
+  // Collect prefixes that end in [] and have object property children
+  const objectArrayPrefixes = new Set<string>();
+  for (const path of Object.keys(schema)) {
+    if (!ARRAY_OBJECT_PROPERTY_RE.test(path)) continue;
+    // Extract every "...[]" prefix that is followed by ".prop"
+    let idx = 0;
+    while ((idx = path.indexOf('[].', idx)) !== -1) {
+      objectArrayPrefixes.add(path.slice(0, idx + 2)); // includes the "[]"
+      idx += 3;
+    }
+  }
+
+  if (objectArrayPrefixes.size === 0) return schema;
+
+  // Check if any double-array paths actually conflict
+  let hasConflict = false;
+  for (const path of Object.keys(schema)) {
+    if (!DOUBLE_ARRAY_RE.test(path)) continue;
+    const idx = path.indexOf('[][]');
+    if (idx !== -1 && objectArrayPrefixes.has(path.slice(0, idx + 2))) {
+      hasConflict = true;
+      break;
+    }
+  }
+
+  if (!hasConflict) return schema;
+
+  // Filter out conflicting double-array paths
+  const resolved: Record<string, string> = {};
+  for (const [path, value] of Object.entries(schema)) {
+    if (DOUBLE_ARRAY_RE.test(path)) {
+      const idx = path.indexOf('[][]');
+      if (idx !== -1 && objectArrayPrefixes.has(path.slice(0, idx + 2))) {
+        continue; // drop this conflicting path
+      }
+    }
+    resolved[path] = value;
+  }
+  return resolved;
 }
 
 export default function convertDotNotation(
   schema: Record<string, string>,
 ): JsonTypeDefinition {
+  const resolved = resolveDoubleArrayConflicts(schema);
   const result: JsonTypeDefinition = {};
 
-  for (const [rawPath, typ] of Object.entries(schema)) {
+  for (const [rawPath, typ] of Object.entries(resolved)) {
     // Skip paths with .length after array notation from dynamic iteration
     // e.g., metadata[].length, metadata[][].length
     if (DYNAMIC_LENGTH_RE.test(rawPath)) continue;
@@ -218,6 +307,17 @@ export default function convertDotNotation(
           // If the next segment after '[]' is another '[]' or a standalone index,
           // we need an array at cursor[key][0], not an object (for patterns like "key[][]" or "key[][0]")
           if (afterArray === '[]' || isStandaloneIndex(afterArray)) {
+            // But don't overwrite an existing populated object — object properties
+            // (from paths like key[].property) take precedence over double-array hints
+            // (from paths like key[][]) which come from mixed access patterns in scope analysis
+            if (
+              cursor[key][0] &&
+              typeof cursor[key][0] === 'object' &&
+              !Array.isArray(cursor[key][0]) &&
+              Object.keys(cursor[key][0]).length > 0
+            ) {
+              break;
+            }
             if (!Array.isArray(cursor[key][0])) cursor[key][0] = [];
           } else {
             // Default case: create an object
@@ -242,15 +342,41 @@ export default function convertDotNotation(
           cursor[key] = typ;
         } else {
           // Structural/placeholder terminal
-          if (typ === 'array') {
+          const nullable = isNullableType(typ);
+          const baseType = getBaseSkippableType(typ);
+
+          if (baseType === 'array') {
             if (!Array.isArray(cursor[key])) cursor[key] = [];
+            if (nullable) {
+              (cursor[key] as any)._nullable = true;
+            }
           } else if (
-            typ === 'object' ||
-            typ === 'function' ||
-            typ.trim() === 'unknown'
+            baseType === 'object' ||
+            baseType === 'function' ||
+            baseType === 'unknown'
           ) {
-            if (cursor[key] === undefined) {
-              cursor[key] = typ;
+            if (nullable) {
+              // Nullable object: ensure it's an actual object (not a string
+              // placeholder) so _nullable can be set and child paths can
+              // populate properties on it.
+              if (
+                cursor[key] === undefined ||
+                typeof cursor[key] === 'string'
+              ) {
+                cursor[key] = {};
+              }
+              if (
+                typeof cursor[key] === 'object' &&
+                cursor[key] !== null &&
+                !Array.isArray(cursor[key])
+              ) {
+                (cursor[key] as any)._nullable = true;
+              }
+            } else {
+              // Non-nullable: preserve existing behavior (string placeholder)
+              if (cursor[key] === undefined) {
+                cursor[key] = typ;
+              }
             }
           }
         }