@codeyam/codeyam-cli 0.1.0-staging.6e699e5 → 0.1.0-staging.7481014

package/analyzer-template/packages/ai/src/lib/dataStructure/equivalencyManagers/ParentScopeManager.ts

@@ -97,6 +97,16 @@ export default class ParentScopeManager implements EquivalencyManager {
       }
 
       if (instantiatedInParent || returnValueFromChildScope) {
+        // Skip if we've already propagated this variable to the parent scope.
+        // Without this guard, nested scopes trigger redundant addEquivalency calls
+        // for the same variable on every addToSchema pass (e.g., 'a' and 'b' in sort
+        // callbacks get propagated 30+ times per scope instead of once).
+        this.addedToParent[scopeNode.name] ||= new Set<string>();
+        if (this.addedToParent[scopeNode.name].has(rootPath)) {
+          return false;
+        }
+        this.addedToParent[scopeNode.name].add(rootPath);
+
         if (traceId && debugLevel > 0) {
           console.info(
             'Debug Propagation: parentScope executed',
@@ -114,9 +124,6 @@ export default class ParentScopeManager implements EquivalencyManager {
           );
         }
 
-        this.addedToParent[scopeNode.name] ||= new Set<string>();
-        this.addedToParent[scopeNode.name].add(rootPath);
-
         scopeDataStructure.addEquivalency(
           rootPath,
           rootPath,

package/analyzer-template/packages/ai/src/lib/dataStructure/equivalencyManagers/frameworks/JavascriptFrameworkManager.ts

@@ -72,7 +72,11 @@ export class JavascriptFrameworkManager implements EquivalencyManager {
   }
 
   finalize(scopeNode: ScopeNode, scopeDataStructure: ScopeDataStructure) {
-    cleanNonObjectFunctions(scopeNode, scopeDataStructure, true);
+    // Don't call cleanNonObjectFunctions with final=true here.
+    // The aggressive cleanup (transformKeyMapping, clearAttributes) removes method call
+    // evidence like .includes() before type inference can use it.
+    // Instead, defer final cleanup to toSerializable() when we're done with inference.
+    cleanNonObjectFunctions(scopeNode, scopeDataStructure, false);
     cleanKnownObjectFunctions(scopeNode, scopeDataStructure);
     this.handleUniformArrays(scopeNode, scopeDataStructure);
   }

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/BatchSchemaProcessor.ts

@@ -41,6 +41,7 @@ export interface BatchProcessorStats {
 
 export class BatchSchemaProcessor {
   private workQueue: WorkItem[] = [];
+  private queueIndex = 0;
   private stats: BatchProcessorStats = {
     itemsProcessed: 0,
     itemsSkipped: 0, // Kept for API compatibility, always 0
@@ -71,17 +72,28 @@ export class BatchSchemaProcessor {
    * Check if there are more items to process.
    */
   hasWork(): boolean {
-    return this.workQueue.length > 0;
+    return this.queueIndex < this.workQueue.length;
   }
 
   /**
    * Get the next work item from the queue.
    */
   getNextWork(): WorkItem | undefined {
-    const item = this.workQueue.shift();
+    if (this.queueIndex >= this.workQueue.length) {
+      return undefined;
+    }
+
+    const item = this.workQueue[this.queueIndex++];
     if (item) {
       this.stats.itemsProcessed++;
     }
+
+    // Once we've consumed the queue, reset to avoid unbounded growth
+    if (this.queueIndex >= this.workQueue.length) {
+      this.workQueue = [];
+      this.queueIndex = 0;
+    }
+
     return item;
   }
 
@@ -122,6 +134,7 @@ export class BatchSchemaProcessor {
    */
   reset(): void {
     this.workQueue = [];
+    this.queueIndex = 0;
     this.stats = {
       itemsProcessed: 0,
       itemsSkipped: 0,
@@ -134,6 +147,6 @@ export class BatchSchemaProcessor {
    * Get current queue size (for debugging/monitoring).
    */
   getQueueSize(): number {
-    return this.workQueue.length;
+    return this.workQueue.length - this.queueIndex;
   }
 }

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/ScopeTreeManager.ts

@@ -117,9 +117,10 @@ export class ScopeTreeManager {
    */
   findNode(name: string): ScopeTreeNode | undefined {
     const queue: ScopeTreeNode[] = [this.tree];
+    let index = 0;
 
-    while (queue.length > 0) {
-      const node = queue.shift()!;
+    while (index < queue.length) {
+      const node = queue[index++]!;
       if (node.name === name) {
         return node;
       }
@@ -135,9 +136,10 @@ export class ScopeTreeManager {
   getAllNames(): string[] {
     const names: string[] = [];
     const queue: ScopeTreeNode[] = [this.tree];
+    let index = 0;
 
-    while (queue.length > 0) {
-      const node = queue.shift()!;
+    while (index < queue.length) {
+      const node = queue[index++]!;
       names.push(node.name);
       queue.push(...node.children);
     }

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/cleanKnownObjectFunctions.ts

@@ -6,7 +6,7 @@ import { ScopeNode, ScopeDataStructure } from '../ScopeDataStructure';
 
 type PrimitiveKind = 'array' | 'string' | 'number' | 'boolean' | 'date';
 
-const IGNORE_CLASSES_AND_OBJECTS = [
+const IGNORE_CLASSES_AND_OBJECTS_SET = new Set([
   'Math',
   'JSON',
   'Reflect',
@@ -22,7 +22,7 @@ const IGNORE_CLASSES_AND_OBJECTS = [
   'useContext',
   'useReducer',
   'useLayoutEffect',
-];
+]);
 
 /** cache the tokenised representation of every path we touch */
 const partsCache = new Map<string, string[]>();
@@ -35,6 +35,47 @@ const getParts = (path: string): string[] => {
   return parts;
 };
 
+function buildPrefixParts(parts: string[]): string[] {
+  if (parts.length === 0) return [];
+  const prefixes = new Array(parts.length);
+  let current = '';
+  for (let i = 0; i < parts.length; i++) {
+    const part = parts[i];
+    if (i === 0) {
+      current = part;
+    } else if (part.startsWith('[') || part.startsWith('(')) {
+      current += part;
+    } else {
+      current += `.${part}`;
+    }
+    prefixes[i] = current;
+  }
+  return prefixes;
+}
+
+/**
+ * Clear the module-level cache.
+ * Call this between entity analyses to prevent unbounded memory growth.
+ */
+export function clearCleanKnownObjectFunctionsCache(): {
+  count: number;
+  estimatedBytes: number;
+} {
+  let totalBytes = 0;
+  for (const [key, value] of partsCache) {
+    // Key is a string, value is string[] - estimate UTF-16 encoding (2 bytes per char)
+    totalBytes += key.length * 2;
+    for (const part of value) {
+      totalBytes += part.length * 2;
+    }
+    // Add overhead for array and Map entry (~50 bytes per entry)
+    totalBytes += 50;
+  }
+  const count = partsCache.size;
+  partsCache.clear();
+  return { count, estimatedBytes: totalBytes };
+}
+
 /** pre-compute the prototype-method sets once per module load */
 const protoMethodSets: Record<PrimitiveKind, Set<string>> = {
   array: new Set(
@@ -106,15 +147,26 @@ function scrub(
   rootTable: Map<string, PrimitiveKind>,
   onRemove: (key: string) => void,
 ): void {
-  const sortedKeys = Object.keys(mapping).sort((a, b) => a.localeCompare(b));
+  // Early exit: if no primitive types exist in the schema, there are no
+  // built-in methods to clean. Saves sorting + walking 10K+ keys.
+  if (rootTable.size === 0) return;
+
+  // Use direct comparison instead of localeCompare (3-5x faster for large arrays)
+  const sortedKeys = Object.keys(mapping).sort((a, b) =>
+    a < b ? -1 : a > b ? 1 : 0,
+  );
 
   for (const keyPath of sortedKeys) {
     const keyParts = getParts(keyPath);
+    const prefixParts = buildPrefixParts(keyParts);
 
+    const firstPart = keyParts[0];
+    const firstPartBase = firstPart.includes('(')
+      ? firstPart.split('(')[0]
+      : firstPart;
     if (
-      IGNORE_CLASSES_AND_OBJECTS.find(
-        (m) => keyParts[0] === m || keyParts[0].startsWith(`${m}(`),
-      )
+      IGNORE_CLASSES_AND_OBJECTS_SET.has(firstPart) ||
+      IGNORE_CLASSES_AND_OBJECTS_SET.has(firstPartBase)
     ) {
       delete mapping[keyPath];
       continue;
@@ -122,7 +174,7 @@ function scrub(
 
     // walk back through the prefixes: foo.bar.baz.qux → foo.bar.baz → foo.bar …
     for (let cut = keyParts.length - 1; cut > 0; --cut) {
-      const prefix = joinParenthesesAndArrays(keyParts.slice(0, cut));
+      const prefix = prefixParts[cut - 1];
       const rootKind = rootTable.get(prefix);
       if (!rootKind) continue;
 
@@ -130,9 +182,18 @@ function scrub(
       const rawMethod = rawToken.split('(')[0];
 
       // Check if it's a prototype method (e.g., map, filter, toString)
+      // Only match when the token is actually a method reference/call:
+      // - Has parentheses (explicit call like map(), filter(x => x))
+      // - OR is typed as "function" (method reference like .split)
+      // Bare property names that happen to match method names
+      // (like .entries = "array", .values = "object") are preserved.
       if (protoMethodSets[rootKind].has(rawMethod)) {
-        onRemove(keyPath);
-        break;
+        const isMethodCall = rawToken.includes('(');
+        const isMethodReference = mapping[keyPath] === 'function';
+        if (isMethodCall || isMethodReference) {
+          onRemove(keyPath);
+          break;
+        }
       }
 
       // Check if it's a built-in property (e.g., length on arrays/strings)

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/cleanNonObjectFunctions.ts

@@ -151,21 +151,63 @@ function cachedSplit(str: string): string[] {
   return parts;
 }
 
+/**
+ * Clear the module-level cache.
+ * Call this between entity analyses to prevent unbounded memory growth.
+ */
+export function clearCleanNonObjectFunctionsCache(): {
+  count: number;
+  estimatedBytes: number;
+} {
+  let totalBytes = 0;
+  for (const [key, value] of splitCache) {
+    // Key is a string, value is string[] - estimate UTF-16 encoding (2 bytes per char)
+    totalBytes += key.length * 2;
+    for (const part of value) {
+      totalBytes += part.length * 2;
+    }
+    // Add overhead for array and Map entry (~50 bytes per entry)
+    totalBytes += 50;
+  }
+  const count = splitCache.size;
+  splitCache.clear();
+  return { count, estimatedBytes: totalBytes };
+}
+
 // --- Prefix Index for O(1) lookup in subPathIsFromKnownObject ---
 // Maps parent prefix -> array of last parts (method names without parens)
 type PrefixIndex = Map<string, string[]>;
 
+function buildPrefixPartsWithRoot(parts: string[]): string[] {
+  const prefixes = new Array(parts.length + 1);
+  prefixes[0] = '';
+  let current = '';
+  for (let i = 0; i < parts.length; i++) {
+    const part = parts[i];
+    if (i === 0) {
+      current = part;
+    } else if (part.startsWith('[') || part.startsWith('(')) {
+      current += part;
+    } else {
+      current += `.${part}`;
+    }
+    prefixes[i + 1] = current;
+  }
+  return prefixes;
+}
+
 function buildPrefixIndex(allPaths: string[]): PrefixIndex {
   const index: PrefixIndex = new Map();
 
   for (const path of allPaths) {
     const parts = cachedSplit(path);
     if (parts.length < 1) continue;
+    const prefixes = buildPrefixPartsWithRoot(parts);
 
     // For each possible prefix length, add the last part to that prefix's entry
     // This allows lookup at any depth
     for (let prefixLen = 0; prefixLen < parts.length; prefixLen++) {
-      const prefix = joinParenthesesAndArrays(parts.slice(0, prefixLen));
+      const prefix = prefixes[prefixLen];
       const lastPart = parts[prefixLen]?.split('(')?.[0];
       if (lastPart) {
         let existing = index.get(prefix);
@@ -468,7 +510,7 @@ function removeNonTransformingArrayMethods(variablePath: string): string {
     const part = keyParts[i];
 
     const arrayMethod = getMethodName(part);
-    if (isValidArrayMethod(arrayMethod)) {
+    if (isValidFunctionCall(part) && isValidArrayMethod(arrayMethod)) {
       const nextPart = keyParts[i + 1];
       if (
         !nonTransformingArrayMethodsSet.has(arrayMethod) ||
@@ -508,7 +550,7 @@ function removeIsolatingArrayMethods(
     const part = keyParts[i];
 
     const arrayMethod = getMethodName(part);
-    if (isValidArrayMethod(arrayMethod)) {
+    if (isValidFunctionCall(part) && isValidArrayMethod(arrayMethod)) {
       const nextPart = keyParts[i + 1];
       const prevPart = i > 1 ? keyParts[i - 1] : undefined;
       if (
@@ -1062,29 +1104,32 @@ function clearAttributes(
     knownBasePaths.add(path);
   }
 
+  // OPTIMIZATION: Instead of O(keys × basePaths), use O(keys × keyDepth) where keyDepth is small
+  // For each key, find all its prefixes and check if any is a known basePath (O(1) Set lookup)
   for (const key in mapping) {
     const keyParts = cachedSplit(key);
-    for (const basePath of knownBasePaths) {
-      const basePathParts = cachedSplit(basePath);
-      if (keyParts.length <= basePathParts.length) continue;
-      if (
-        joinParenthesesAndArrays(keyParts.slice(0, basePathParts.length)) !==
-        basePath
-      )
-        continue;
+    if (keyParts.length <= 1) continue; // No possible parent basePath
+
+    // Check each prefix of the key to see if it's a known basePath
+    // Start from shortest prefix (most likely to match) for early exit
+    for (let prefixLen = 1; prefixLen < keyParts.length; prefixLen++) {
+      const prefix = joinParenthesesAndArrays(keyParts.slice(0, prefixLen));
+      if (!knownBasePaths.has(prefix)) continue;
+
+      // Found a matching basePath - now check the conditions
+      const basePath = prefix;
+
       // Preserve functionCallReturnValue for non-string types, UNLESS
       // the method is a primitive-returning method (like some, every, includes, indexOf)
       // whose return values are not mockable
-      const methodPart = keyParts[basePathParts.length];
+      const methodPart = keyParts[prefixLen];
       const methodName = methodPart?.split('(')[0];
       const isPrimitiveReturning = primitiveReturningMethodsSet.has(
         methodName ?? '',
       );
       if (
         mapping[basePath] !== 'string' &&
-        keyParts[basePathParts.length + 1]?.startsWith(
-          'functionCallReturnValue',
-        ) &&
+        keyParts[prefixLen + 1]?.startsWith('functionCallReturnValue') &&
         !isPrimitiveReturning
       )
         continue;

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/coerceObjectsToPrimitivesBySchema.ts

@@ -0,0 +1,70 @@
+/**
+ * Coerces object/array values to primitive types when the schema expects a primitive.
+ *
+ * The LLM sometimes generates an object or array where the schema expects a primitive
+ * type like "string", "number", or "boolean". For example:
+ *   Schema: { body: "string" }
+ *   LLM output: { body: { "env": "production" } }
+ *
+ * This causes runtime errors like `TypeError: body.match is not a function` because
+ * the code expects a string but gets an object.
+ *
+ * Coercion rules:
+ * - object/array → string: JSON.stringify
+ * - object/array → number: 0
+ * - object/array → boolean: false
+ *
+ * @param data The mock data object to coerce (mutated in place)
+ * @param schema The schema describing expected types
+ */
+export default function coerceObjectsToPrimitivesBySchema<T>(
+  data: T,
+  schema: unknown,
+): T {
+  if (data === null || data === undefined) {
+    return data;
+  }
+
+  if (typeof data !== 'object') {
+    return data;
+  }
+
+  if (Array.isArray(data)) {
+    const itemSchema = Array.isArray(schema) ? schema[0] : undefined;
+    for (let i = 0; i < data.length; i++) {
+      data[i] = coerceObjectsToPrimitivesBySchema(data[i], itemSchema);
+    }
+    return data;
+  }
+
+  const dataObj = data as Record<string, unknown>;
+  const schemaObj = schema as Record<string, unknown> | undefined;
+
+  for (const key of Object.keys(dataObj)) {
+    const value = dataObj[key];
+    const fieldSchema = schemaObj?.[key];
+
+    if (value === null || value === undefined) {
+      continue;
+    }
+
+    if (typeof value === 'object' && typeof fieldSchema === 'string') {
+      // Schema expects a primitive but the value is an object/array.
+      // Coerce to the expected type.
+      const schemaType = fieldSchema.toLowerCase();
+
+      if (schemaType.includes('string')) {
+        dataObj[key] = JSON.stringify(value);
+      } else if (schemaType.includes('number')) {
+        dataObj[key] = 0;
+      } else if (schemaType.includes('boolean')) {
+        dataObj[key] = false;
+      }
+    } else if (typeof value === 'object' && value !== null) {
+      // Both value and schema are objects — recurse
+      coerceObjectsToPrimitivesBySchema(value, fieldSchema);
+    }
+  }
+
+  return data;
+}

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/coercePrimitivesToArraysBySchema.ts

@@ -0,0 +1,62 @@
+/**
+ * Coerces primitive values to empty arrays when the schema expects an array.
+ *
+ * The LLM sometimes generates a primitive (boolean, number, string) where the
+ * schema expects an array. For example:
+ *   Schema: { webapps: [{ name: "string" }] }
+ *   LLM output: { webapps: true }
+ *
+ * This causes runtime errors like `TypeError: config.webapps?.forEach is not a function`
+ * because the code expects an array but gets a boolean.
+ *
+ * Coercion rule:
+ * - primitive → array: replace with []
+ *
+ * Null and undefined values are preserved (they may be intentionally nullable).
+ *
+ * @param data The mock data object to coerce (mutated in place)
+ * @param schema The schema describing expected types
+ */
+export default function coercePrimitivesToArraysBySchema<T>(
+  data: T,
+  schema: unknown,
+): T {
+  if (data === null || data === undefined) {
+    return data;
+  }
+
+  if (typeof data !== 'object') {
+    return data;
+  }
+
+  if (Array.isArray(data)) {
+    const itemSchema = Array.isArray(schema) ? schema[0] : undefined;
+    for (let i = 0; i < data.length; i++) {
+      data[i] = coercePrimitivesToArraysBySchema(data[i], itemSchema);
+    }
+    return data;
+  }
+
+  const dataObj = data as Record<string, unknown>;
+  const schemaObj = schema as Record<string, unknown> | undefined;
+
+  for (const key of Object.keys(dataObj)) {
+    const value = dataObj[key];
+    const fieldSchema = schemaObj?.[key];
+
+    if (value === null || value === undefined) {
+      continue;
+    }
+
+    if (typeof value !== 'object' && Array.isArray(fieldSchema)) {
+      // Schema expects an array but the value is a primitive.
+      // Replace with an empty array.
+      dataObj[key] = [];
+    } else if (typeof value === 'object' && value !== null) {
+      // Both value and schema are objects — recurse
+      coercePrimitivesToArraysBySchema(value, fieldSchema);
+    }
+  }
+
+  return data;
+}