@codeyam/codeyam-cli 0.1.0-staging.4c0c3c9 → 0.1.0-staging.5370992

package/analyzer-template/packages/ai/src/lib/dataStructure/ScopeDataStructure.ts

@@ -398,6 +398,7 @@ const SILENTLY_IGNORED_EQUIVALENCY_REASONS = new Set([
   'transformed non-object function equivalency - Array.from() equivalency',
   'Spread operator equivalency key update: Explicit array deconstruction equivalency value',
   // 'transformed non-object function equivalency - Explicit array deconstruction equivalency value',
+  'useState setter call equivalency (direct)',
 ]);
 
 export class ScopeDataStructure {
@@ -1048,9 +1049,38 @@ export class ScopeDataStructure {
     equivalencyValueChain?: EquivalencyValueChainItem[],
     traceId?: number,
   ) {
+    // Fast early exits that don't count toward the limit.
+    // These are "no-op" calls that would be caught further below anyway,
+    // but checking them first prevents counter inflation from redundant calls
+    // generated by managers and followEquivalencies during scope processing.
+
+    // Skip silently-ignored reasons before counting
+    if (SILENTLY_IGNORED_EQUIVALENCY_REASONS.has(equivalencyReason)) {
+      return;
+    }
+
+    // Skip exact duplicates before counting (Phase 1 only — Phase 2 needs
+    // to process duplicates for database chain tracking)
+    if (this.onlyEquivalencies) {
+      const existing = scopeNode.equivalencies[path];
+      if (
+        existing?.some(
+          (v) =>
+            v.schemaPath === equivalentPath &&
+            v.scopeNodeName === equivalentScopeName,
+        )
+      ) {
+        return;
+      }
+    }
+
     addEquivalencyCallCount++;
 
-    if (addEquivalencyCallCount > 50000) {
+    const limit =
+      typeof (globalThis as any).__CODEYAM_EQUIVALENCY_LIMIT === 'number'
+        ? (globalThis as any).__CODEYAM_EQUIVALENCY_LIMIT
+        : 50000;
+    if (addEquivalencyCallCount > limit) {
       console.error('INFINITE LOOP DETECTED in addEquivalency', {
         callCount: addEquivalencyCallCount,
         path,
@@ -1065,14 +1095,11 @@ export class ScopeDataStructure {
     }
     // Filter equivalency reasons - use pre-computed Sets for O(1) lookup
     if (!ALLOWED_EQUIVALENCY_REASONS.has(equivalencyReason)) {
-      if (SILENTLY_IGNORED_EQUIVALENCY_REASONS.has(equivalencyReason)) {
-        return;
-      } else {
-        // Log and skip - if an equivalency reason isn't in ALLOWED or SILENTLY_IGNORED,
-        // it shouldn't be stored (was previously missing the return)
-        console.info('Not tracked equivalency reason', { equivalencyReason });
-        return;
-      }
+      // Silently-ignored reasons already handled above
+      // Log and skip - if an equivalency reason isn't in ALLOWED or SILENTLY_IGNORED,
+      // it shouldn't be stored (was previously missing the return)
+      console.info('Not tracked equivalency reason', { equivalencyReason });
+      return;
     }
 
     // Skip self-referential equivalencies that would create infinite loops.
@@ -1108,6 +1135,33 @@ export class ScopeDataStructure {
       return;
     }
 
+    // Case 3: Circular reference through scope-suffixed names (____cyScope pattern)
+    // When a named arrow function is defined inside a scope (e.g., useEffect callback):
+    //   const identifyUser = async () => { ... };
+    //   identifyUser();
+    // This creates a variable "identifyUser" and a scope "identifyUser____cyScope9F".
+    // Mutual equivalencies between these cause infinite loops in Phase 2 because
+    // processing one triggers addToSchema → followEquivalencies → addEquivalency
+    // on the reverse, which repeats indefinitely.
+    // Only block when the REVERSE direction already exists (creating a cycle).
+    // The initial one-directional equivalency is necessary for scope resolution.
+    if (
+      path &&
+      equivalentPath &&
+      (equivalentPath.startsWith(path + '____') ||
+        path.startsWith(equivalentPath + '____'))
+    ) {
+      // Check if the reverse equivalency already exists
+      const reverseEquivalencies =
+        scopeNode.equivalencies[equivalentPath] || [];
+      const reverseExists = reverseEquivalencies.some(
+        (v) => v.schemaPath === path,
+      );
+      if (reverseExists) {
+        return;
+      }
+    }
+
     if (!equivalentScopeName) {
       console.error(
         'CodeYam Error: Missing equivalent scope name - FULL CONTEXT:',
@@ -3360,6 +3414,22 @@ export class ScopeDataStructure {
    * ensure all sub-paths of that variable are reflected under signature[N].
    */
   private propagateParameterToSignaturePaths(scopeNode: ScopeNode) {
+    // Helper: check if a type is a concrete scalar that cannot have sub-properties.
+    const SCALAR_TYPES = new Set([
+      'string',
+      'number',
+      'boolean',
+      'bigint',
+      'symbol',
+      'void',
+      'never',
+    ]);
+    const isDefinitelyScalar = (type: string): boolean => {
+      const parts = type.split('|').map((s) => s.trim());
+      const base = parts.filter((s) => s !== 'undefined' && s !== 'null');
+      return base.length > 0 && base.every((b) => SCALAR_TYPES.has(b));
+    };
+
     // Find variable → signature[N] equivalencies
     for (const [varName, equivalencies] of Object.entries(
       scopeNode.equivalencies,
@@ -3390,7 +3460,38 @@ export class ScopeDataStructure {
 
               // Only add if the signature path doesn't already exist
               if (!scopeNode.schema[sigKey]) {
-                scopeNode.schema[sigKey] = scopeNode.schema[key];
+                // Check if this path represents variable conflation:
+                // When a standalone variable (e.g., showWorkoutForm from useState)
+                // appears as a sub-property of a scalar-typed ancestor (e.g.,
+                // activity_type = "string"), it's from scope conflation, not real
+                // property access. Block these while allowing legitimate built-in
+                // accesses like string.length or string.slice.
+                let isConflatedPath = false;
+                let checkPos = signaturePath.length;
+                while (true) {
+                  checkPos = sigKey.indexOf('.', checkPos + 1);
+                  if (checkPos === -1) break;
+                  const ancestorPath = sigKey.substring(0, checkPos);
+                  const ancestorType = scopeNode.schema[ancestorPath];
+                  if (ancestorType && isDefinitelyScalar(ancestorType)) {
+                    // Ancestor is scalar — check if the immediate sub-property
+                    // is also a standalone variable (indicating conflation)
+                    const afterDot = sigKey.substring(checkPos + 1);
+                    const nextSep = afterDot.search(/[.\[]/);
+                    const subPropName =
+                      nextSep === -1
+                        ? afterDot
+                        : afterDot.substring(0, nextSep);
+                    if (scopeNode.schema[subPropName] !== undefined) {
+                      isConflatedPath = true;
+                      break;
+                    }
+                  }
+                }
+
+                if (!isConflatedPath) {
+                  scopeNode.schema[sigKey] = scopeNode.schema[key];
+                }
               }
             }
           }
@@ -4165,6 +4266,24 @@ export class ScopeDataStructure {
       }
     }
 
+    // Helper: check if a type is a concrete scalar that cannot have sub-properties.
+    // e.g., "string", "number | undefined", "boolean | null" are scalar.
+    // "object", "array", "function", "unknown", "Workout", etc. are NOT scalar.
+    const SCALAR_TYPES = new Set([
+      'string',
+      'number',
+      'boolean',
+      'bigint',
+      'symbol',
+      'void',
+      'never',
+    ]);
+    const isDefinitelyScalarType = (type: string): boolean => {
+      const parts = type.split('|').map((s) => s.trim());
+      const base = parts.filter((s) => s !== 'undefined' && s !== 'null');
+      return base.length > 0 && base.every((b) => SCALAR_TYPES.has(b));
+    };
+
     // Propagate nested paths from variables to their signature equivalents
     // e.g., if workouts = signature[0].workouts, then workouts[].title becomes
     // signature[0].workouts[].title
@@ -4189,7 +4308,69 @@ export class ScopeDataStructure {
 
           // Add to schema if not already present
           if (!tempScopeNode.schema[signatureKey]) {
-            tempScopeNode.schema[signatureKey] = schema[schemaKey];
+            // Check if this path represents variable conflation:
+            // When a standalone variable (e.g., showWorkoutForm from useState)
+            // appears as a sub-property of a scalar-typed ancestor (e.g.,
+            // activity_type = "string"), it's from scope conflation, not real
+            // property access. Block these while allowing legitimate built-in
+            // accesses like string.length or string.slice.
+            let isConflatedPath = false;
+            let checkPos = signaturePath.length;
+            while (true) {
+              checkPos = signatureKey.indexOf('.', checkPos + 1);
+              if (checkPos === -1) break;
+              const ancestorPath = signatureKey.substring(0, checkPos);
+              const ancestorType = tempScopeNode.schema[ancestorPath];
+              if (ancestorType && isDefinitelyScalarType(ancestorType)) {
+                // Ancestor is scalar — check if the immediate sub-property
+                // is also a standalone variable (indicating conflation)
+                const afterDot = signatureKey.substring(checkPos + 1);
+                const nextSep = afterDot.search(/[.\[]/);
+                const subPropName =
+                  nextSep === -1 ? afterDot : afterDot.substring(0, nextSep);
+                if (schema[subPropName] !== undefined) {
+                  isConflatedPath = true;
+                  break;
+                }
+              }
+            }
+
+            if (!isConflatedPath) {
+              tempScopeNode.schema[signatureKey] = schema[schemaKey];
+            }
+          }
+        }
+      }
+    }
+
+    // Post-process: filter out conflated signature paths.
+    // During phase 2 scope analysis, useState(false) conflation can create
+    // bad paths like signature[0].mockWorkouts[].activity_type.showWorkoutForm
+    // directly in scopeNode.schema. These flow through signatureInSchema into
+    // tempScopeNode.schema without any guard. Filter them out here by checking:
+    //   1. An ancestor in the path has a concrete scalar type (string, number, boolean, etc.)
+    //   2. The immediate sub-property of that scalar ancestor is also a standalone
+    //      variable in the schema (indicating conflation, not a real property access)
+    for (const key of Object.keys(tempScopeNode.schema)) {
+      if (!key.startsWith('signature[')) continue;
+
+      // Walk through the path looking for scalar-typed ancestors
+      let pos = 0;
+      while (true) {
+        pos = key.indexOf('.', pos + 1);
+        if (pos === -1) break;
+        const ancestorPath = key.substring(0, pos);
+        const ancestorType = tempScopeNode.schema[ancestorPath];
+        if (ancestorType && isDefinitelyScalarType(ancestorType)) {
+          // Found a scalar ancestor — check if the sub-property name
+          // is a standalone variable in the getSchema() result
+          const afterDot = key.substring(pos + 1);
+          const nextSep = afterDot.search(/[.\[]/);
+          const subPropName =
+            nextSep === -1 ? afterDot : afterDot.substring(0, nextSep);
+          if (schema[subPropName] !== undefined) {
+            delete tempScopeNode.schema[key];
+            break;
           }
         }
       }
@@ -4929,9 +5110,109 @@ export class ScopeDataStructure {
     // Replace cyScope placeholders in all external function call data
     // This ensures call signatures and schema paths use actual callback text
     // instead of internal cyScope names, preventing mock data merge conflicts.
-    return this.externalFunctionCalls.map((efc) =>
-      this.cleanCyScopeFromFunctionCallInfo(efc),
-    );
+    const rootScopeName = this.scopeTreeManager.getRootName();
+    const rootSchema = this.scopeNodes[rootScopeName]?.schema ?? {};
+
+    return this.externalFunctionCalls.map((efc) => {
+      const cleaned = this.cleanCyScopeFromFunctionCallInfo(efc);
+      return this.filterConflatedExternalPaths(cleaned, rootSchema);
+    });
+  }
+
+  /**
+   * Filters out conflated paths from external function call schemas.
+   *
+   * When multiple useState(false) calls create equivalency conflation during
+   * Phase 1 analysis, standalone boolean state variables (like showWorkoutForm,
+   * showGoalForm) can bleed into external function call schemas as sub-properties
+   * of unrelated data fields (like data[].activity_type.showWorkoutForm).
+   *
+   * Detection: group sub-properties by parent path. If 2+ sub-properties of
+   * the same parent all match standalone root scope variable names, treat them
+   * as conflation artifacts and remove them.
+   */
+  private filterConflatedExternalPaths(
+    efc: FunctionCallInfo,
+    rootSchema: Record<string, string>,
+  ): FunctionCallInfo {
+    // Build a set of top-level root scope variable names (simple names, no dots/brackets)
+    const topLevelRootVars = new Set<string>();
+    for (const key of Object.keys(rootSchema)) {
+      if (!key.includes('.') && !key.includes('[')) {
+        topLevelRootVars.add(key);
+      }
+    }
+
+    if (topLevelRootVars.size === 0) return efc;
+
+    // Group sub-property matches by their parent path.
+    // For a path like "...data[].activity_type.showWorkoutForm",
+    // parent = "...data[].activity_type", child = "showWorkoutForm"
+    const parentToConflatedKeys = new Map<string, string[]>();
+
+    for (const key of Object.keys(efc.schema)) {
+      const lastDot = key.lastIndexOf('.');
+      if (lastDot === -1) continue;
+
+      const parent = key.substring(0, lastDot);
+      const child = key.substring(lastDot + 1);
+
+      // Skip array access or function call patterns
+      if (child.includes('[') || child.includes('(')) continue;
+
+      // Only consider paths inside array element chains (contains []).
+      // Direct children of functionCallReturnValue are legitimate destructured
+      // return values, not conflation. Conflation happens deeper in the chain
+      // when array element fields get corrupted sub-properties.
+      if (!parent.includes('[')) continue;
+
+      if (topLevelRootVars.has(child)) {
+        if (!parentToConflatedKeys.has(parent)) {
+          parentToConflatedKeys.set(parent, []);
+        }
+        parentToConflatedKeys.get(parent)!.push(key);
+      }
+    }
+
+    // Only filter when 2+ sub-properties of the same parent match root scope vars.
+    // This threshold avoids false positives from coincidental name matches.
+    const keysToRemove = new Set<string>();
+    const parentsToRestore = new Set<string>();
+
+    for (const [parent, conflatedKeys] of parentToConflatedKeys) {
+      if (conflatedKeys.length >= 2) {
+        for (const key of conflatedKeys) {
+          keysToRemove.add(key);
+        }
+        parentsToRestore.add(parent);
+      }
+    }
+
+    if (keysToRemove.size === 0) return efc;
+
+    // Create a new schema without the conflated paths
+    const newSchema: Record<string, string> = {};
+    for (const [key, value] of Object.entries(efc.schema)) {
+      if (keysToRemove.has(key)) continue;
+
+      // Restore parent type: if it was changed to "object" because of conflated
+      // sub-properties, and now all those sub-properties are removed, change it
+      // back to "unknown" (we don't know the original type)
+      if (parentsToRestore.has(key) && value === 'object') {
+        // Check if there are any remaining sub-properties
+        const hasRemainingSubProps = Object.keys(efc.schema).some(
+          (k) =>
+            !keysToRemove.has(k) &&
+            k !== key &&
+            (k.startsWith(key + '.') || k.startsWith(key + '[')),
+        );
+        newSchema[key] = hasRemainingSubProps ? value : 'unknown';
+      } else {
+        newSchema[key] = value;
+      }
+    }
+
+    return { ...efc, schema: newSchema };
   }
 
   /**

package/analyzer-template/packages/ai/src/lib/dataStructure/equivalencyManagers/ParentScopeManager.ts

@@ -97,6 +97,16 @@ export default class ParentScopeManager implements EquivalencyManager {
       }
 
       if (instantiatedInParent || returnValueFromChildScope) {
+        // Skip if we've already propagated this variable to the parent scope.
+        // Without this guard, nested scopes trigger redundant addEquivalency calls
+        // for the same variable on every addToSchema pass (e.g., 'a' and 'b' in sort
+        // callbacks get propagated 30+ times per scope instead of once).
+        this.addedToParent[scopeNode.name] ||= new Set<string>();
+        if (this.addedToParent[scopeNode.name].has(rootPath)) {
+          return false;
+        }
+        this.addedToParent[scopeNode.name].add(rootPath);
+
         if (traceId && debugLevel > 0) {
           console.info(
             'Debug Propagation: parentScope executed',
@@ -114,9 +124,6 @@ export default class ParentScopeManager implements EquivalencyManager {
           );
         }
 
-        this.addedToParent[scopeNode.name] ||= new Set<string>();
-        this.addedToParent[scopeNode.name].add(rootPath);
-
         scopeDataStructure.addEquivalency(
           rootPath,
           rootPath,

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/cleanKnownObjectFunctions.ts

@@ -6,7 +6,7 @@ import { ScopeNode, ScopeDataStructure } from '../ScopeDataStructure';
 
 type PrimitiveKind = 'array' | 'string' | 'number' | 'boolean' | 'date';
 
-const IGNORE_CLASSES_AND_OBJECTS = [
+const IGNORE_CLASSES_AND_OBJECTS_SET = new Set([
   'Math',
   'JSON',
   'Reflect',
@@ -22,7 +22,7 @@ const IGNORE_CLASSES_AND_OBJECTS = [
   'useContext',
   'useReducer',
   'useLayoutEffect',
-];
+]);
 
 /** cache the tokenised representation of every path we touch */
 const partsCache = new Map<string, string[]>();
@@ -147,16 +147,26 @@ function scrub(
   rootTable: Map<string, PrimitiveKind>,
   onRemove: (key: string) => void,
 ): void {
-  const sortedKeys = Object.keys(mapping).sort((a, b) => a.localeCompare(b));
+  // Early exit: if no primitive types exist in the schema, there are no
+  // built-in methods to clean. Saves sorting + walking 10K+ keys.
+  if (rootTable.size === 0) return;
+
+  // Use direct comparison instead of localeCompare (3-5x faster for large arrays)
+  const sortedKeys = Object.keys(mapping).sort((a, b) =>
+    a < b ? -1 : a > b ? 1 : 0,
+  );
 
   for (const keyPath of sortedKeys) {
     const keyParts = getParts(keyPath);
     const prefixParts = buildPrefixParts(keyParts);
 
+    const firstPart = keyParts[0];
+    const firstPartBase = firstPart.includes('(')
+      ? firstPart.split('(')[0]
+      : firstPart;
     if (
-      IGNORE_CLASSES_AND_OBJECTS.find(
-        (m) => keyParts[0] === m || keyParts[0].startsWith(`${m}(`),
-      )
+      IGNORE_CLASSES_AND_OBJECTS_SET.has(firstPart) ||
+      IGNORE_CLASSES_AND_OBJECTS_SET.has(firstPartBase)
     ) {
       delete mapping[keyPath];
       continue;

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/coercePrimitivesToArraysBySchema.ts

@@ -0,0 +1,62 @@
+/**
+ * Coerces primitive values to empty arrays when the schema expects an array.
+ *
+ * The LLM sometimes generates a primitive (boolean, number, string) where the
+ * schema expects an array. For example:
+ *   Schema: { webapps: [{ name: "string" }] }
+ *   LLM output: { webapps: true }
+ *
+ * This causes runtime errors like `TypeError: config.webapps?.forEach is not a function`
+ * because the code expects an array but gets a boolean.
+ *
+ * Coercion rule:
+ * - primitive → array: replace with []
+ *
+ * Null and undefined values are preserved (they may be intentionally nullable).
+ *
+ * @param data The mock data object to coerce (mutated in place)
+ * @param schema The schema describing expected types
+ */
+export default function coercePrimitivesToArraysBySchema<T>(
+  data: T,
+  schema: unknown,
+): T {
+  if (data === null || data === undefined) {
+    return data;
+  }
+
+  if (typeof data !== 'object') {
+    return data;
+  }
+
+  if (Array.isArray(data)) {
+    const itemSchema = Array.isArray(schema) ? schema[0] : undefined;
+    for (let i = 0; i < data.length; i++) {
+      data[i] = coercePrimitivesToArraysBySchema(data[i], itemSchema);
+    }
+    return data;
+  }
+
+  const dataObj = data as Record<string, unknown>;
+  const schemaObj = schema as Record<string, unknown> | undefined;
+
+  for (const key of Object.keys(dataObj)) {
+    const value = dataObj[key];
+    const fieldSchema = schemaObj?.[key];
+
+    if (value === null || value === undefined) {
+      continue;
+    }
+
+    if (typeof value !== 'object' && Array.isArray(fieldSchema)) {
+      // Schema expects an array but the value is a primitive.
+      // Replace with an empty array.
+      dataObj[key] = [];
+    } else if (typeof value === 'object' && value !== null) {
+      // Both value and schema are objects — recurse
+      coercePrimitivesToArraysBySchema(value, fieldSchema);
+    }
+  }
+
+  return data;
+}

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/stripNullableMarkers.ts

@@ -0,0 +1,35 @@
+/**
+ * Recursively strips `_nullable` keys from LLM-generated mock data.
+ *
+ * The `_nullable` marker is an internal CodeYam concept used in type definitions
+ * to indicate that a field can be null/undefined. The LLM sometimes includes
+ * these markers in its generated scenario data, which causes runtime errors
+ * when code iterates over object keys (e.g., `Object.keys(importedBy)` picks
+ * up `_nullable` as a key alongside real data).
+ *
+ * This function mutates the input object in-place, consistent with other
+ * post-processing helpers like `convertNullToUndefinedBySchema`.
+ */
+export default function stripNullableMarkers(
+  data: Record<string, unknown>,
+): void {
+  if (data == null || typeof data !== 'object') return;
+
+  // Delete _nullable from this level
+  if ('_nullable' in data) {
+    delete data._nullable;
+  }
+
+  // Recurse into nested objects and arrays
+  for (const value of Object.values(data)) {
+    if (Array.isArray(value)) {
+      for (const item of value) {
+        if (item !== null && typeof item === 'object' && !Array.isArray(item)) {
+          stripNullableMarkers(item as Record<string, unknown>);
+        }
+      }
+    } else if (value !== null && typeof value === 'object') {
+      stripNullableMarkers(value as Record<string, unknown>);
+    }
+  }
+}

package/analyzer-template/packages/ai/src/lib/dataStructureChunking.ts

@@ -4,10 +4,10 @@ import type { ExecutionFlow, ScenariosDataStructure } from '~codeyam/types';
 type RequiredValue = NonNullable<ExecutionFlow['requiredValues']>[number];
 
 const DEFAULT_MAX_CHUNK_SIZE = 10_000; // ~10K chars per chunk
+const DEFAULT_MAX_KEYS_PER_CHUNK = 20;
 
 export interface ChunkOptions {
   maxChunkSize?: number;
-  /** @deprecated Use maxChunkSize instead. Kept for backward compatibility in tests. */
   maxKeysPerChunk?: number;
 }
 
@@ -33,16 +33,19 @@ export function chunkDataStructure(
     return [dataForMocks];
   }
 
-  const { maxChunkSize = DEFAULT_MAX_CHUNK_SIZE } = options;
+  const {
+    maxChunkSize = DEFAULT_MAX_CHUNK_SIZE,
+    maxKeysPerChunk = DEFAULT_MAX_KEYS_PER_CHUNK,
+  } = options;
   const keys = Object.keys(dataForMocks);
 
   // Calculate total size to see if chunking is needed
   const totalSize = JSON.stringify(dataForMocks).length;
-  if (totalSize <= maxChunkSize) {
+  if (totalSize <= maxChunkSize && keys.length <= maxKeysPerChunk) {
     return [dataForMocks];
   }
 
-  // Greedily pack keys into chunks by cumulative size
+  // Greedily pack keys into chunks by cumulative size and key count
   const chunks: Array<ScenariosDataStructure['dataForMocks']> = [];
   let currentChunk: Record<string, unknown> = {};
   let currentSize = 0;
@@ -52,8 +55,14 @@ export function chunkDataStructure(
       (dataForMocks as Record<string, unknown>)[key],
     ).length;
 
-    // If adding this key would exceed the limit AND chunk isn't empty, start a new chunk
-    if (currentSize > 0 && currentSize + keySize > maxChunkSize) {
+    const currentKeyCount = Object.keys(currentChunk).length;
+
+    // If adding this key would exceed size or key count limit AND chunk isn't empty, start a new chunk
+    if (
+      currentSize > 0 &&
+      (currentSize + keySize > maxChunkSize ||
+        currentKeyCount >= maxKeysPerChunk)
+    ) {
       chunks.push(currentChunk as ScenariosDataStructure['dataForMocks']);
       currentChunk = {};
       currentSize = 0;