@codeyam/codeyam-cli 0.1.0-staging.036391e → 0.1.0-staging.07d8f7b

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/convertDotNotation.ts

@@ -14,21 +14,46 @@ const isStandaloneIndex = (s?: string) => !!s && STANDALONE_INDEX_RE.test(s);
 // The regex matches any path ending with .length that has [] somewhere before it
 const DYNAMIC_LENGTH_RE = /\[\].*\.length$/;
 
-// Treat these as structural placeholders (don't commit them as concrete leaves)
-function isSkippableLeafType(t: string) {
-  // 'unknown' by itself is a placeholder (but 'boolean | unknown' is not)
-  // Also handles optional variants like 'object | undefined', 'object | null',
-  // 'array | undefined', etc. — these are still structural placeholders that
-  // should not overwrite already-populated structures.
+// Cache for type string analysis to avoid repeated split/filter operations.
+// These functions are called multiple times per path segment across thousands of paths.
+const typeAnalysisCache = new Map<
+  string,
+  { isSkippable: boolean; baseType: string; isNullable: boolean }
+>();
+
+function getTypeAnalysis(t: string) {
+  const cached = typeAnalysisCache.get(t);
+  if (cached) return cached;
   const parts = t.split('|').map((s) => s.trim());
   const base = parts.filter((s) => s !== 'undefined' && s !== 'null');
-  return (
-    base.length === 1 &&
-    (base[0] === 'object' ||
-      base[0] === 'array' ||
-      base[0] === 'function' ||
-      base[0] === 'unknown')
-  );
+  const result = {
+    isSkippable:
+      base.length === 1 &&
+      (base[0] === 'object' ||
+        base[0] === 'array' ||
+        base[0] === 'function' ||
+        base[0] === 'unknown'),
+    baseType: base[0],
+    isNullable: parts.includes('undefined') || parts.includes('null'),
+  };
+  typeAnalysisCache.set(t, result);
+  return result;
+}
+
+// Treat these as structural placeholders (don't commit them as concrete leaves)
+function isSkippableLeafType(t: string) {
+  return getTypeAnalysis(t).isSkippable;
+}
+
+// Extract the base structural type from a potentially nullable type string.
+// e.g., 'object | undefined' → 'object', 'array | null' → 'array'
+function getBaseSkippableType(t: string): string {
+  return getTypeAnalysis(t).baseType;
+}
+
+// Check if a type string has nullable annotations (| undefined or | null)
+function isNullableType(t: string): boolean {
+  return getTypeAnalysis(t).isNullable;
 }
 
 // Matches paths containing [][] — e.g., "items[][]" or "items[][].text"
@@ -317,15 +342,41 @@ export default function convertDotNotation(
           cursor[key] = typ;
         } else {
           // Structural/placeholder terminal
-          if (typ === 'array') {
+          const nullable = isNullableType(typ);
+          const baseType = getBaseSkippableType(typ);
+
+          if (baseType === 'array') {
             if (!Array.isArray(cursor[key])) cursor[key] = [];
+            if (nullable) {
+              (cursor[key] as any)._nullable = true;
+            }
           } else if (
-            typ === 'object' ||
-            typ === 'function' ||
-            typ.trim() === 'unknown'
+            baseType === 'object' ||
+            baseType === 'function' ||
+            baseType === 'unknown'
           ) {
-            if (cursor[key] === undefined) {
-              cursor[key] = typ;
+            if (nullable) {
+              // Nullable object: ensure it's an actual object (not a string
+              // placeholder) so _nullable can be set and child paths can
+              // populate properties on it.
+              if (
+                cursor[key] === undefined ||
+                typeof cursor[key] === 'string'
+              ) {
+                cursor[key] = {};
+              }
+              if (
+                typeof cursor[key] === 'object' &&
+                cursor[key] !== null &&
+                !Array.isArray(cursor[key])
+              ) {
+                (cursor[key] as any)._nullable = true;
+              }
+            } else {
+              // Non-nullable: preserve existing behavior (string placeholder)
+              if (cursor[key] === undefined) {
+                cursor[key] = typ;
+              }
             }
           }
         }

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/stripNullableMarkers.ts

@@ -0,0 +1,35 @@
+/**
+ * Recursively strips `_nullable` keys from LLM-generated mock data.
+ *
+ * The `_nullable` marker is an internal CodeYam concept used in type definitions
+ * to indicate that a field can be null/undefined. The LLM sometimes includes
+ * these markers in its generated scenario data, which causes runtime errors
+ * when code iterates over object keys (e.g., `Object.keys(importedBy)` picks
+ * up `_nullable` as a key alongside real data).
+ *
+ * This function mutates the input object in-place, consistent with other
+ * post-processing helpers like `convertNullToUndefinedBySchema`.
+ */
+export default function stripNullableMarkers(
+  data: Record<string, unknown>,
+): void {
+  if (data == null || typeof data !== 'object') return;
+
+  // Delete _nullable from this level
+  if ('_nullable' in data) {
+    delete data._nullable;
+  }
+
+  // Recurse into nested objects and arrays
+  for (const value of Object.values(data)) {
+    if (Array.isArray(value)) {
+      for (const item of value) {
+        if (item !== null && typeof item === 'object' && !Array.isArray(item)) {
+          stripNullableMarkers(item as Record<string, unknown>);
+        }
+      }
+    } else if (value !== null && typeof value === 'object') {
+      stripNullableMarkers(value as Record<string, unknown>);
+    }
+  }
+}

package/analyzer-template/packages/ai/src/lib/dataStructureChunking.ts

@@ -3,9 +3,11 @@ import type { ExecutionFlow, ScenariosDataStructure } from '~codeyam/types';
 // Type for a single required value from ExecutionFlow
 type RequiredValue = NonNullable<ExecutionFlow['requiredValues']>[number];
 
-const DEFAULT_MAX_KEYS_PER_CHUNK = 8;
+const DEFAULT_MAX_CHUNK_SIZE = 10_000; // ~10K chars per chunk
+const DEFAULT_MAX_KEYS_PER_CHUNK = 20;
 
 export interface ChunkOptions {
+  maxChunkSize?: number;
   maxKeysPerChunk?: number;
 }
 
@@ -15,8 +17,12 @@ export interface ChunkOptions {
  * Large schemas overwhelm LLMs, causing them to make mistakes on some keys.
  * By processing smaller chunks, each key gets more focused attention.
  *
+ * Uses cumulative JSON size to decide chunk boundaries rather than a fixed
+ * key count, so a single oversized key gets its own chunk while many small
+ * keys are grouped together efficiently.
+ *
  * @param dataForMocks - The full data structure schema
- * @param options - Chunking options (maxKeysPerChunk defaults to 8)
+ * @param options - Chunking options (maxChunkSize defaults to 10_000)
  * @returns Array of smaller data structure chunks
  */
 export function chunkDataStructure(
@@ -27,27 +33,48 @@ export function chunkDataStructure(
     return [dataForMocks];
   }
 
-  const { maxKeysPerChunk = DEFAULT_MAX_KEYS_PER_CHUNK } = options;
+  const {
+    maxChunkSize = DEFAULT_MAX_CHUNK_SIZE,
+    maxKeysPerChunk = DEFAULT_MAX_KEYS_PER_CHUNK,
+  } = options;
   const keys = Object.keys(dataForMocks);
 
-  // Don't chunk if already small enough
-  if (keys.length <= maxKeysPerChunk) {
+  // Calculate total size to see if chunking is needed
+  const totalSize = JSON.stringify(dataForMocks).length;
+  if (totalSize <= maxChunkSize && keys.length <= maxKeysPerChunk) {
     return [dataForMocks];
   }
 
+  // Greedily pack keys into chunks by cumulative size and key count
   const chunks: Array<ScenariosDataStructure['dataForMocks']> = [];
+  let currentChunk: Record<string, unknown> = {};
+  let currentSize = 0;
+
+  for (const key of keys) {
+    const keySize = JSON.stringify(
+      (dataForMocks as Record<string, unknown>)[key],
+    ).length;
 
-  for (let i = 0; i < keys.length; i += maxKeysPerChunk) {
-    const chunkKeys = keys.slice(i, i + maxKeysPerChunk);
-    const chunk: ScenariosDataStructure['dataForMocks'] = {};
+    const currentKeyCount = Object.keys(currentChunk).length;
 
-    for (const key of chunkKeys) {
-      (chunk as Record<string, unknown>)[key] = (
-        dataForMocks as Record<string, unknown>
-      )[key];
+    // If adding this key would exceed size or key count limit AND chunk isn't empty, start a new chunk
+    if (
+      currentSize > 0 &&
+      (currentSize + keySize > maxChunkSize ||
+        currentKeyCount >= maxKeysPerChunk)
+    ) {
+      chunks.push(currentChunk as ScenariosDataStructure['dataForMocks']);
+      currentChunk = {};
+      currentSize = 0;
     }
 
-    chunks.push(chunk);
+    currentChunk[key] = (dataForMocks as Record<string, unknown>)[key];
+    currentSize += keySize;
+  }
+
+  // Push the last chunk
+  if (Object.keys(currentChunk).length > 0) {
+    chunks.push(currentChunk as ScenariosDataStructure['dataForMocks']);
   }
 
   return chunks;

package/analyzer-template/packages/ai/src/lib/generateEntityScenarioData.ts

@@ -19,6 +19,8 @@ import convertNullToUndefinedBySchema from './dataStructure/helpers/convertNullT
 import convertTypeAnnotationsToValues from './dataStructure/helpers/convertTypeAnnotationsToValues';
 import fixNullIdsBySchema from './dataStructure/helpers/fixNullIdsBySchema';
 import coerceObjectsToPrimitivesBySchema from './dataStructure/helpers/coerceObjectsToPrimitivesBySchema';
+import coercePrimitivesToArraysBySchema from './dataStructure/helpers/coercePrimitivesToArraysBySchema';
+import stripNullableMarkers from './dataStructure/helpers/stripNullableMarkers';
 import { JsonTypeDefinition } from '~codeyam/types';
 import { deepMerge } from '~codeyam/generate';
 import {
@@ -172,6 +174,46 @@ function findKeyPath(
   return null;
 }
 
+/**
+ * Strip primitive values from `source` when the `destination` already has an
+ * array or object at the same key. Returns a cleaned copy of source (or null
+ * if nothing remains after stripping).
+ *
+ * This prevents misplaced LLM data (e.g., `{ webapps: true }`) from
+ * overwriting correct complex values (e.g., `{ webapps: [...] }`) during
+ * deep merge in relocateMisplacedNestedKeys.
+ */
+function stripPrimitivesOverwritingComplexValues(
+  destination: Record<string, unknown>,
+  source: unknown,
+): Record<string, unknown> | null {
+  if (typeof source !== 'object' || source === null || Array.isArray(source)) {
+    return null;
+  }
+
+  const result: Record<string, unknown> = {};
+  let hasKeys = false;
+
+  for (const key of Object.keys(source as Record<string, unknown>)) {
+    const srcVal = (source as Record<string, unknown>)[key];
+    const dstVal = destination[key];
+
+    // If destination has an array or object but source has a primitive, skip it
+    if (
+      typeof dstVal === 'object' &&
+      dstVal !== null &&
+      typeof srcVal !== 'object'
+    ) {
+      continue;
+    }
+
+    result[key] = srcVal;
+    hasKeys = true;
+  }
+
+  return hasKeys ? result : null;
+}
+
 /**
  * Relocate misplaced nested keys in mockData to their correct position
  * based on the dataForMocks structure.
@@ -233,12 +275,22 @@ function relocateMisplacedNestedKeys(
     }
 
     // Deep merge the value into the correct location
-    // Use deep merge to preserve existing data at that location
+    // Use deep merge to preserve existing data at that location.
+    // Before merging, strip primitives from the misplaced value that would
+    // overwrite arrays/objects at the destination. The misplaced data is lower
+    // quality — the LLM put it in the wrong place — so primitives like `true`
+    // should not overwrite correct complex values like arrays.
     if (current[key] !== undefined && typeof current[key] === 'object') {
-      current[key] = deepMerge(
+      const safeValue = stripPrimitivesOverwritingComplexValues(
         current[key] as Record<string, unknown>,
         value as Record<string, unknown>,
       );
+      if (safeValue !== null) {
+        current[key] = deepMerge(
+          current[key] as Record<string, unknown>,
+          safeValue,
+        );
+      }
     } else {
       current[key] = value;
     }
@@ -294,11 +346,13 @@ function generateDefaultForSchemaType(schemaType: unknown): unknown {
     if (schemaType.includes('| null')) return null;
     return schemaType; // Return the type as a string placeholder
   }
-  if (
-    typeof schemaType === 'object' &&
-    schemaType !== null &&
-    !Array.isArray(schemaType)
-  ) {
+  if (Array.isArray(schemaType)) {
+    if (schemaType.length === 0) return [];
+    // Generate a single default element based on the first element's schema
+    const elementDefault = generateDefaultForSchemaType(schemaType[0]);
+    return elementDefault !== undefined ? [elementDefault] : [];
+  }
+  if (typeof schemaType === 'object' && schemaType !== null) {
     // Recursively generate defaults for nested objects
     const result: Record<string, unknown> = {};
     for (const [key, value] of Object.entries(schemaType)) {
@@ -426,6 +480,8 @@ function fillMissingMockDataKeysWithDefaults(
   const missingKeys: string[] = [];
 
   for (const key of Object.keys(dataForMocks)) {
+    if (key === '_nullable') continue; // Internal marker, not a data key
+
     const fullPath = pathPrefix ? `${pathPrefix}.${key}` : key;
 
     if (mockData[key] === undefined) {
@@ -841,6 +897,29 @@ export async function generateDataForScenario({
         }
       }
 
+      // Detect keys that were lost from failed or partial chunk responses
+      // and fill them with schema-based defaults so they aren't permanently lost.
+      const allChunkedKeys = chunks.flatMap((c) => Object.keys(c || {}));
+      const returnedKeys = new Set(Object.keys(chunkedMockData));
+      const missingChunkKeys = allChunkedKeys.filter(
+        (k) => !returnedKeys.has(k),
+      );
+
+      if (missingChunkKeys.length > 0) {
+        awsLog(
+          `Chunked processing: ${missingChunkKeys.length} key(s) missing from chunk results, filling with defaults: ${missingChunkKeys.join(', ')}`,
+        );
+        const dataForMocksRecord = structure.dataForMocks as Record<
+          string,
+          unknown
+        >;
+        for (const key of missingChunkKeys) {
+          chunkedMockData[key] = generateDefaultForSchemaType(
+            dataForMocksRecord[key],
+          );
+        }
+      }
+
       awsLog(
         `Chunked processing complete. Generated ${Object.keys(chunkedMockData).length} keys total`,
       );
@@ -1036,6 +1115,18 @@ export async function generateDataForScenario({
     );
   }
 
+  // Strip _nullable markers from LLM-generated mock data.
+  // The _nullable marker is an internal CodeYam concept used in type definitions
+  // to indicate that a field can be null/undefined. The LLM sometimes includes
+  // these markers in its generated data, which causes runtime errors when code
+  // iterates over object keys (e.g., Object.keys(importedBy) picks up "_nullable",
+  // then Object.keys(importedBy["_nullable"]) calls Object.keys(null) and throws).
+  if (fullScenarioData.data.mockData) {
+    stripNullableMarkers(
+      fullScenarioData.data.mockData as Record<string, unknown>,
+    );
+  }
+
   // Relocate misplaced nested keys to their correct position.
   // The LLM sometimes places nested keys at root level instead of inside their
   // parent object (e.g., 'fastener' at root instead of inside 'trpc').
@@ -1071,6 +1162,18 @@ export async function generateDataForScenario({
     );
   }
 
+  // Coerce primitives to empty arrays when the schema expects an array.
+  // The LLM sometimes generates a primitive (e.g., `webapps: true`) where the
+  // schema expects an array (e.g., `webapps: [{ name: "string" }]`).
+  // This causes runtime errors like "TypeError: config.webapps?.forEach is not a function".
+  // Must run AFTER coerceObjectsToPrimitivesBySchema and BEFORE convertCommaSeparatedStringsToArrays.
+  if (structure.dataForMocks && fullScenarioData.data.mockData) {
+    coercePrimitivesToArraysBySchema(
+      fullScenarioData.data.mockData,
+      structure.dataForMocks,
+    );
+  }
+
   // Convert comma-separated strings to arrays when appropriate.
   // The LLM sometimes generates strings like "color,size" instead of arrays
   // like ["color", "size"] when the schema type is incorrectly inferred as

package/analyzer-template/packages/ai/src/lib/generateExecutionFlows.ts

@@ -32,6 +32,30 @@ import resolvePathToControllable from './resolvePathToControllable';
 import { AI } from '~codeyam/ai';
 import type { JsxRenderingUsage, ConditionalUsage } from './astScopes/types';
 
+/**
+ * Patterns that indicate a child component is a modal/overlay/blocking element.
+ * Same patterns used in generateExecutionFlowsFromConditionalEffects.ts for
+ * state variable names — here applied to child component names.
+ */
+const BLOCKING_CHILD_PATTERNS = [
+  /modal/i,
+  /overlay/i,
+  /dialog/i,
+  /popup/i,
+  /drawer/i,
+  /sheet/i,
+  /lightbox/i,
+  /backdrop/i,
+];
+
+/**
+ * Check if a child component name suggests it's a blocking/modal element.
+ * Exported for testing.
+ */
+export function isBlockingChildComponent(componentName: string): boolean {
+  return BLOCKING_CHILD_PATTERNS.some((pattern) => pattern.test(componentName));
+}
+
 interface GenerateExecutionFlowsArgs {
   entity: Pick<Entity, 'sha' | 'name' | 'filePath' | 'code' | 'metadata'>;
   mergedDataStructure: Omit<DataStructure, 'equivalentSignatureVariables'>;
@@ -105,6 +129,19 @@ export default function generateExecutionFlows({
     fullToShortPathMap[fullPath] = shortPath;
   }
 
+  console.log(
+    `[generateExecutionFlows] ${entity.name}: attributesMap has ${Object.keys(resolvedAttributesMap).length} keys, fullToShortPathMap has ${Object.keys(fullToShortPathMap).length} entries`,
+  );
+  console.log(
+    `[generateExecutionFlows] ${entity.name}: attributesMap keys: [${Object.keys(resolvedAttributesMap).join(', ')}]`,
+  );
+  console.log(
+    `[generateExecutionFlows] ${entity.name}: fullToShortPathMap: ${JSON.stringify(fullToShortPathMap)}`,
+  );
+  console.log(
+    `[generateExecutionFlows] ${entity.name}: equivalentSignatureVariables: ${JSON.stringify(equivalentSignatureVariables)}`,
+  );
+
   // Get pre-computed conditional data from metadata
   const conditionalUsages: Record<string, ConditionalUsage[]> =
     entity.metadata?.isolatedDataStructure?.conditionalUsages ?? {};
@@ -149,6 +186,26 @@ export default function generateExecutionFlows({
     }
   }
 
+  console.log(
+    `[generateExecutionFlows] ${entity.name}: conditionalUsages keys: [${Object.keys(conditionalUsages).join(', ')}]`,
+  );
+  console.log(
+    `[generateExecutionFlows] ${entity.name}: conditionalUsages sourceDataPaths: ${JSON.stringify(
+      Object.fromEntries(
+        Object.entries(conditionalUsages).map(([k, v]) => [
+          k,
+          (v as any[]).map((u: any) => u.sourceDataPath ?? '(none)'),
+        ]),
+      ),
+    )}`,
+  );
+  console.log(
+    `[generateExecutionFlows] ${entity.name}: compoundConditionals count: ${compoundConditionals.length}`,
+  );
+  console.log(
+    `[generateExecutionFlows] ${entity.name}: sourceEquivalencies keys count: ${Object.keys(mergedDataStructure.sourceEquivalencies ?? {}).length}`,
+  );
+
   const conditionalsBasedFlows = generateExecutionFlowsFromConditionals({
     conditionalUsages,
     compoundConditionals,
@@ -240,6 +297,11 @@ export default function generateExecutionFlows({
     calculateFlowPriority(flow, gatingPathToChildren);
   }
 
+  // Assign exclusiveGroup for truthy/falsy flow pairs that control child components.
+  // When the same gating path produces both a truthy flow (show component) and a
+  // falsy flow (hide component), they are mutually exclusive and should be grouped.
+  assignExclusiveGroupsForGatingPairs(allFlows);
+
   // Sort flows by priority (highest first)
   allFlows.sort((a, b) => (b.priority ?? 0) - (a.priority ?? 0));
 
@@ -328,6 +390,60 @@ function buildGatingPathToChildrenMap(
   return gatingPathToChildren;
 }
 
+/**
+ * Assign exclusiveGroup for truthy/falsy flow pairs that control child components.
+ *
+ * When the same attribute path produces both a truthy and falsy flow (e.g.,
+ * {diffView && <DiffModal/>}), they are mutually exclusive — only one can be
+ * active at a time. Grouping them ensures the scenario generator treats them
+ * as alternative states rather than combining them.
+ */
+function assignExclusiveGroupsForGatingPairs(flows: ExecutionFlow[]): void {
+  // Build a map: attributePath -> flows with truthy or falsy comparison for that path
+  const pathToFlows = new Map<
+    string,
+    { truthy: ExecutionFlow[]; falsy: ExecutionFlow[] }
+  >();
+
+  for (const flow of flows) {
+    // Only consider flows that control child components
+    if (!flow.childComponentsControlled?.length) continue;
+
+    for (const rv of flow.requiredValues) {
+      if (rv.comparison === 'truthy' || rv.comparison === 'falsy') {
+        if (!pathToFlows.has(rv.attributePath)) {
+          pathToFlows.set(rv.attributePath, { truthy: [], falsy: [] });
+        }
+        pathToFlows.get(rv.attributePath)![rv.comparison].push(flow);
+      }
+    }
+  }
+
+  // Also check falsy flows that DON'T have childComponentsControlled —
+  // they still pair with truthy flows that do
+  for (const flow of flows) {
+    if (flow.childComponentsControlled?.length) continue; // Already processed
+
+    for (const rv of flow.requiredValues) {
+      if (rv.comparison === 'falsy' && pathToFlows.has(rv.attributePath)) {
+        pathToFlows.get(rv.attributePath)!.falsy.push(flow);
+      }
+    }
+  }
+
+  // Assign exclusiveGroup where both truthy and falsy flows exist
+  for (const [attributePath, { truthy, falsy }] of pathToFlows) {
+    if (truthy.length > 0 && falsy.length > 0) {
+      const groupName = `gating:${attributePath}`;
+      for (const flow of [...truthy, ...falsy]) {
+        if (!flow.exclusiveGroup) {
+          flow.exclusiveGroup = groupName;
+        }
+      }
+    }
+  }
+}
+
 /**
  * Calculate priority and childComponentsControlled for a flow.
  *
@@ -382,6 +498,19 @@ function calculateFlowPriority(
     if (flow.impact !== 'high') {
       flow.impact = 'high';
     }
+
+    // Check if any controlled child component is a modal/dialog/overlay.
+    // Only truthy flows (which SHOW the component) should be marked as blocking.
+    // Falsy flows (which HIDE the component) should not be blocking.
+    const isTruthyFlow = flow.requiredValues.some(
+      (rv) => rv.comparison === 'truthy' || rv.comparison === 'equals',
+    );
+    if (
+      isTruthyFlow &&
+      Array.from(childComponentsControlled).some(isBlockingChildComponent)
+    ) {
+      flow.blocksOtherFlows = true;
+    }
   }
 
   // Calculate priority score
@@ -503,8 +632,11 @@ function buildValueGroupMap(flows: ExecutionFlow[]): Map<string, string[]> {
   for (const [, flowValues] of pathToValues) {
     // Only create groups if there are multiple different values for the same path
     const uniqueValues = new Set(flowValues.map((fv) => fv.value));
-    if (uniqueValues.size > 1) {
-      // This attribute has multiple values, so all flows are siblings
+    if (uniqueValues.size > 1 && flowValues.length <= 10) {
+      // This attribute has multiple values, so all flows are siblings.
+      // Skip large groups (>10 flows) — these are typically shared gating conditions
+      // (like route params) that would dominate the selection by pulling in too many
+      // siblings and displacing higher-priority flows.
       const siblingIds = flowValues.map((fv) => fv.flowId);
       for (const fv of flowValues) {
         valueGroupMap.set(