@codeyam/codeyam-cli 0.1.0-staging.2a88920 → 0.1.0-staging.4057bd2

package/analyzer-template/packages/ai/src/lib/dataStructureChunking.ts

@@ -3,9 +3,11 @@ import type { ExecutionFlow, ScenariosDataStructure } from '~codeyam/types';
 // Type for a single required value from ExecutionFlow
 type RequiredValue = NonNullable<ExecutionFlow['requiredValues']>[number];
 
-const DEFAULT_MAX_KEYS_PER_CHUNK = 8;
+const DEFAULT_MAX_CHUNK_SIZE = 10_000; // ~10K chars per chunk
+const DEFAULT_MAX_KEYS_PER_CHUNK = 20;
 
 export interface ChunkOptions {
+  maxChunkSize?: number;
   maxKeysPerChunk?: number;
 }
 
@@ -15,8 +17,12 @@ export interface ChunkOptions {
  * Large schemas overwhelm LLMs, causing them to make mistakes on some keys.
  * By processing smaller chunks, each key gets more focused attention.
  *
+ * Uses cumulative JSON size to decide chunk boundaries rather than a fixed
+ * key count, so a single oversized key gets its own chunk while many small
+ * keys are grouped together efficiently.
+ *
  * @param dataForMocks - The full data structure schema
- * @param options - Chunking options (maxKeysPerChunk defaults to 8)
+ * @param options - Chunking options (maxChunkSize defaults to 10_000)
  * @returns Array of smaller data structure chunks
  */
 export function chunkDataStructure(
@@ -27,27 +33,48 @@ export function chunkDataStructure(
     return [dataForMocks];
   }
 
-  const { maxKeysPerChunk = DEFAULT_MAX_KEYS_PER_CHUNK } = options;
+  const {
+    maxChunkSize = DEFAULT_MAX_CHUNK_SIZE,
+    maxKeysPerChunk = DEFAULT_MAX_KEYS_PER_CHUNK,
+  } = options;
   const keys = Object.keys(dataForMocks);
 
-  // Don't chunk if already small enough
-  if (keys.length <= maxKeysPerChunk) {
+  // Calculate total size to see if chunking is needed
+  const totalSize = JSON.stringify(dataForMocks).length;
+  if (totalSize <= maxChunkSize && keys.length <= maxKeysPerChunk) {
     return [dataForMocks];
   }
 
+  // Greedily pack keys into chunks by cumulative size and key count
   const chunks: Array<ScenariosDataStructure['dataForMocks']> = [];
+  let currentChunk: Record<string, unknown> = {};
+  let currentSize = 0;
+
+  for (const key of keys) {
+    const keySize = JSON.stringify(
+      (dataForMocks as Record<string, unknown>)[key],
+    ).length;
 
-  for (let i = 0; i < keys.length; i += maxKeysPerChunk) {
-    const chunkKeys = keys.slice(i, i + maxKeysPerChunk);
-    const chunk: ScenariosDataStructure['dataForMocks'] = {};
+    const currentKeyCount = Object.keys(currentChunk).length;
 
-    for (const key of chunkKeys) {
-      (chunk as Record<string, unknown>)[key] = (
-        dataForMocks as Record<string, unknown>
-      )[key];
+    // If adding this key would exceed size or key count limit AND chunk isn't empty, start a new chunk
+    if (
+      currentSize > 0 &&
+      (currentSize + keySize > maxChunkSize ||
+        currentKeyCount >= maxKeysPerChunk)
+    ) {
+      chunks.push(currentChunk as ScenariosDataStructure['dataForMocks']);
+      currentChunk = {};
+      currentSize = 0;
     }
 
-    chunks.push(chunk);
+    currentChunk[key] = (dataForMocks as Record<string, unknown>)[key];
+    currentSize += keySize;
+  }
+
+  // Push the last chunk
+  if (Object.keys(currentChunk).length > 0) {
+    chunks.push(currentChunk as ScenariosDataStructure['dataForMocks']);
   }
 
   return chunks;

package/analyzer-template/packages/ai/src/lib/generateEntityDataStructure.ts

@@ -1,9 +1,12 @@
 import { Entity } from '~codeyam/types';
 import isolateScopes from './isolateScopes';
 import analyzeScope from './analyzeScope';
-import { FileAnalyzer } from '~codeyam/analyze';
+import { FileAnalyzer, transformationTracer } from '~codeyam/analyze';
 import { AI, SerializableDataStructure } from '~codeyam/ai';
-import { resetScopeDataStructureMetrics } from './dataStructure/ScopeDataStructure';
+import {
+  resetScopeDataStructureMetrics,
+  ScopeDataStructure,
+} from './dataStructure/ScopeDataStructure';
 
 // import { awsLog } from '~codeyam/utils';
 
@@ -91,8 +94,60 @@ export default async function generateEntityDataStructure({
   }
 
   // Inspect the runtime type of dataStructure to see if it must be serialized
+  // Note: When worker threads are used, dataStructure is already serialized (no toSerializable).
+  // Workers are disabled when CODEYAM_TRACE_TRANSFORMS=1 to enable full tracing.
   if ('toSerializable' in entityScope.dataStructure) {
-    return entityScope.dataStructure.toSerializable();
+    const scopeDataStructure =
+      entityScope.dataStructure as unknown as ScopeDataStructure;
+
+    // Trace the internal state BEFORE serialization - captures what methods return
+    if (transformationTracer.isEnabled()) {
+      transformationTracer.startEntity({
+        name: entity.name,
+        entityType: entity.entityType ?? 'unknown',
+        filePath: entity.filePath,
+      });
+
+      // Capture internal state - this is what getReturnValue/getFunctionSignature return
+      transformationTracer.snapshot(entity.name, 'scopeDataStructure', {
+        signatureSchema: scopeDataStructure.getFunctionSignature({
+          fillInUnknowns: true,
+        }),
+        returnValueSchema: scopeDataStructure.getReturnValue({
+          fillInUnknowns: true,
+        }),
+      });
+    }
+
+    const serialized = scopeDataStructure.toSerializable();
+
+    // Trace AFTER serialization - shows what gets output
+    if (transformationTracer.isEnabled()) {
+      // Build schema from external function calls for comparison
+      const externalSchemas: Record<string, string> = {};
+      for (const efc of serialized.externalFunctionCalls ?? []) {
+        for (const [path, type] of Object.entries(efc.schema ?? {})) {
+          externalSchemas[path] = type;
+        }
+      }
+
+      transformationTracer.snapshot(entity.name, 'serialized', {
+        signatureSchema:
+          serialized.functionResults?.[entity.name]?.signatureWithUnknowns,
+        returnValueSchema:
+          serialized.functionResults?.[entity.name]?.returnValueWithUnknowns,
+        // Include external function schemas to detect serialization gaps
+        dependencySchemas: {
+          externalFunctions: {
+            combined: {
+              returnValueSchema: externalSchemas,
+            },
+          },
+        },
+      });
+    }
+
+    return serialized;
   }
   return entityScope.dataStructure;
 }

package/analyzer-template/packages/ai/src/lib/generateEntityScenarioData.ts

@@ -18,6 +18,9 @@ import { AI, parseJsonSafe } from '~codeyam/ai';
 import convertNullToUndefinedBySchema from './dataStructure/helpers/convertNullToUndefinedBySchema';
 import convertTypeAnnotationsToValues from './dataStructure/helpers/convertTypeAnnotationsToValues';
 import fixNullIdsBySchema from './dataStructure/helpers/fixNullIdsBySchema';
+import coerceObjectsToPrimitivesBySchema from './dataStructure/helpers/coerceObjectsToPrimitivesBySchema';
+import coercePrimitivesToArraysBySchema from './dataStructure/helpers/coercePrimitivesToArraysBySchema';
+import stripNullableMarkers from './dataStructure/helpers/stripNullableMarkers';
 import { JsonTypeDefinition } from '~codeyam/types';
 import { deepMerge } from '~codeyam/generate';
 import {
@@ -171,6 +174,46 @@ function findKeyPath(
   return null;
 }
 
+/**
+ * Strip primitive values from `source` when the `destination` already has an
+ * array or object at the same key. Returns a cleaned copy of source (or null
+ * if nothing remains after stripping).
+ *
+ * This prevents misplaced LLM data (e.g., `{ webapps: true }`) from
+ * overwriting correct complex values (e.g., `{ webapps: [...] }`) during
+ * deep merge in relocateMisplacedNestedKeys.
+ */
+function stripPrimitivesOverwritingComplexValues(
+  destination: Record<string, unknown>,
+  source: unknown,
+): Record<string, unknown> | null {
+  if (typeof source !== 'object' || source === null || Array.isArray(source)) {
+    return null;
+  }
+
+  const result: Record<string, unknown> = {};
+  let hasKeys = false;
+
+  for (const key of Object.keys(source as Record<string, unknown>)) {
+    const srcVal = (source as Record<string, unknown>)[key];
+    const dstVal = destination[key];
+
+    // If destination has an array or object but source has a primitive, skip it
+    if (
+      typeof dstVal === 'object' &&
+      dstVal !== null &&
+      typeof srcVal !== 'object'
+    ) {
+      continue;
+    }
+
+    result[key] = srcVal;
+    hasKeys = true;
+  }
+
+  return hasKeys ? result : null;
+}
+
 /**
  * Relocate misplaced nested keys in mockData to their correct position
  * based on the dataForMocks structure.
@@ -232,12 +275,22 @@ function relocateMisplacedNestedKeys(
     }
 
     // Deep merge the value into the correct location
-    // Use deep merge to preserve existing data at that location
+    // Use deep merge to preserve existing data at that location.
+    // Before merging, strip primitives from the misplaced value that would
+    // overwrite arrays/objects at the destination. The misplaced data is lower
+    // quality — the LLM put it in the wrong place — so primitives like `true`
+    // should not overwrite correct complex values like arrays.
     if (current[key] !== undefined && typeof current[key] === 'object') {
-      current[key] = deepMerge(
+      const safeValue = stripPrimitivesOverwritingComplexValues(
         current[key] as Record<string, unknown>,
         value as Record<string, unknown>,
       );
+      if (safeValue !== null) {
+        current[key] = deepMerge(
+          current[key] as Record<string, unknown>,
+          safeValue,
+        );
+      }
     } else {
       current[key] = value;
     }
@@ -293,11 +346,13 @@ function generateDefaultForSchemaType(schemaType: unknown): unknown {
     if (schemaType.includes('| null')) return null;
     return schemaType; // Return the type as a string placeholder
   }
-  if (
-    typeof schemaType === 'object' &&
-    schemaType !== null &&
-    !Array.isArray(schemaType)
-  ) {
+  if (Array.isArray(schemaType)) {
+    if (schemaType.length === 0) return [];
+    // Generate a single default element based on the first element's schema
+    const elementDefault = generateDefaultForSchemaType(schemaType[0]);
+    return elementDefault !== undefined ? [elementDefault] : [];
+  }
+  if (typeof schemaType === 'object' && schemaType !== null) {
     // Recursively generate defaults for nested objects
     const result: Record<string, unknown> = {};
     for (const [key, value] of Object.entries(schemaType)) {
@@ -425,6 +480,8 @@ function fillMissingMockDataKeysWithDefaults(
   const missingKeys: string[] = [];
 
   for (const key of Object.keys(dataForMocks)) {
+    if (key === '_nullable') continue; // Internal marker, not a data key
+
     const fullPath = pathPrefix ? `${pathPrefix}.${key}` : key;
 
     if (mockData[key] === undefined) {
@@ -460,6 +517,193 @@ function fillMissingMockDataKeysWithDefaults(
   }
 }
 
+/**
+ * Enforce execution flow requiredValues by setting falsy paths to null.
+ *
+ * The LLM doesn't reliably generate null for `comparison: 'falsy'` requirements.
+ * For example, a flow like "diffView: falsy" should hide a modal, but the LLM
+ * might generate a truthy object, causing the modal to show in all screenshots.
+ *
+ * This function:
+ * 1. Gets requiredValues from covered flows
+ * 2. For 'falsy' comparisons: sets the value to null
+ * 3. For 'truthy' comparisons with falsy values: generates a default truthy value
+ */
+function enforceRequiredValues(
+  mockData: Record<string, unknown>,
+  coveredFlowIds: string[],
+  executionFlows: ExecutionFlow[],
+): void {
+  if (!coveredFlowIds.length || !executionFlows.length) {
+    return;
+  }
+
+  // Get all requiredValues from covered flows
+  const coveredFlows = executionFlows.filter((flow) =>
+    coveredFlowIds.includes(flow.id),
+  );
+
+  for (const flow of coveredFlows) {
+    if (!flow.requiredValues) continue;
+
+    for (const rv of flow.requiredValues) {
+      if (!rv.attributePath) continue;
+
+      // Find the value in mockData - the path could be nested
+      // e.g., attributePath: "diffView" could be at mockData['useDiffModal()'].diffView
+      const result = findAndSetValueInMockData(
+        mockData,
+        rv.attributePath,
+        rv.comparison,
+        rv.valueType,
+      );
+
+      if (result.found) {
+        awsLog(
+          `CodeYam: Enforced ${rv.comparison} for ${rv.attributePath} (set to ${result.newValue === null ? 'null' : typeof result.newValue})`,
+        );
+      }
+    }
+  }
+}
+
+/**
+ * Find a value in mockData by attributePath and enforce the comparison.
+ * The attributePath could be a simple key or a nested path.
+ *
+ * Returns { found: boolean, newValue: unknown }
+ */
+function findAndSetValueInMockData(
+  mockData: Record<string, unknown>,
+  attributePath: string,
+  comparison: string,
+  valueType?: string,
+): { found: boolean; newValue?: unknown } {
+  // Try to find the path at various nesting levels
+  // The attributePath might be "diffView" but the actual location is
+  // mockData['useDiffModal()'].diffView
+
+  // Strategy 1: Direct path (e.g., mockData[attributePath])
+  if (attributePath in mockData) {
+    const currentValue = mockData[attributePath];
+    const { shouldChange, newValue } = getEnforcedValue(
+      currentValue,
+      comparison,
+      valueType,
+    );
+    if (shouldChange) {
+      mockData[attributePath] = newValue;
+      return { found: true, newValue };
+    }
+    return { found: true, newValue: currentValue };
+  }
+
+  // Strategy 2: Search in nested objects
+  for (const [key, value] of Object.entries(mockData)) {
+    if (value && typeof value === 'object' && !Array.isArray(value)) {
+      const nestedObj = value as Record<string, unknown>;
+
+      // Check if attributePath exists in this nested object
+      if (attributePath in nestedObj) {
+        const currentValue = nestedObj[attributePath];
+        const { shouldChange, newValue } = getEnforcedValue(
+          currentValue,
+          comparison,
+          valueType,
+        );
+        if (shouldChange) {
+          nestedObj[attributePath] = newValue;
+          return { found: true, newValue };
+        }
+        return { found: true, newValue: currentValue };
+      }
+
+      // Also check dot-notation paths (e.g., "diffView.type")
+      if (attributePath.includes('.')) {
+        const parts = attributePath.split('.');
+        const firstPart = parts[0];
+        if (firstPart in nestedObj) {
+          // Recurse with the rest of the path
+          const result = findAndSetValueInMockData(
+            nestedObj,
+            attributePath,
+            comparison,
+            valueType,
+          );
+          if (result.found) return result;
+        }
+      }
+
+      // Recursively search deeper
+      const result = findAndSetValueInMockData(
+        nestedObj,
+        attributePath,
+        comparison,
+        valueType,
+      );
+      if (result.found) return result;
+    }
+  }
+
+  return { found: false };
+}
+
+/**
+ * Determine if a value should be changed to match a comparison requirement.
+ *
+ * For 'falsy' comparison: truthy values should become null
+ * For 'truthy' comparison: falsy values should become a default truthy value
+ */
+function getEnforcedValue(
+  currentValue: unknown,
+  comparison: string,
+  valueType?: string,
+): { shouldChange: boolean; newValue: unknown } {
+  const isTruthy = Boolean(currentValue);
+
+  if (comparison === 'falsy') {
+    // Value should be falsy
+    if (isTruthy) {
+      return { shouldChange: true, newValue: null };
+    }
+    return { shouldChange: false, newValue: currentValue };
+  }
+
+  if (comparison === 'truthy') {
+    // Value should be truthy
+    if (!isTruthy) {
+      // Generate a default truthy value based on valueType
+      const defaultValue = generateDefaultTruthyValue(valueType);
+      return { shouldChange: true, newValue: defaultValue };
+    }
+    return { shouldChange: false, newValue: currentValue };
+  }
+
+  // For other comparisons (equals, exists, etc.), don't auto-enforce
+  return { shouldChange: false, newValue: currentValue };
+}
+
+/**
+ * Generate a default truthy value for a given type.
+ */
+function generateDefaultTruthyValue(valueType?: string): unknown {
+  if (!valueType) return { _placeholder: true };
+
+  switch (valueType.toLowerCase()) {
+    case 'string':
+      return 'default-value';
+    case 'number':
+      return 1;
+    case 'boolean':
+      return true;
+    case 'array':
+      return [{ _placeholder: true }];
+    case 'object':
+    default:
+      return { _placeholder: true };
+  }
+}
+
 type ScenarioDataWithoutDescription = Omit<ScenarioData, 'scenarioDescription'>;
 
 interface GenerateEntityScenarioDataArgs {
@@ -653,6 +897,29 @@ export async function generateDataForScenario({
         }
       }
 
+      // Detect keys that were lost from failed or partial chunk responses
+      // and fill them with schema-based defaults so they aren't permanently lost.
+      const allChunkedKeys = chunks.flatMap((c) => Object.keys(c || {}));
+      const returnedKeys = new Set(Object.keys(chunkedMockData));
+      const missingChunkKeys = allChunkedKeys.filter(
+        (k) => !returnedKeys.has(k),
+      );
+
+      if (missingChunkKeys.length > 0) {
+        awsLog(
+          `Chunked processing: ${missingChunkKeys.length} key(s) missing from chunk results, filling with defaults: ${missingChunkKeys.join(', ')}`,
+        );
+        const dataForMocksRecord = structure.dataForMocks as Record<
+          string,
+          unknown
+        >;
+        for (const key of missingChunkKeys) {
+          chunkedMockData[key] = generateDefaultForSchemaType(
+            dataForMocksRecord[key],
+          );
+        }
+      }
+
       awsLog(
         `Chunked processing complete. Generated ${Object.keys(chunkedMockData).length} keys total`,
       );
@@ -848,6 +1115,18 @@ export async function generateDataForScenario({
     );
   }
 
+  // Strip _nullable markers from LLM-generated mock data.
+  // The _nullable marker is an internal CodeYam concept used in type definitions
+  // to indicate that a field can be null/undefined. The LLM sometimes includes
+  // these markers in its generated data, which causes runtime errors when code
+  // iterates over object keys (e.g., Object.keys(importedBy) picks up "_nullable",
+  // then Object.keys(importedBy["_nullable"]) calls Object.keys(null) and throws).
+  if (fullScenarioData.data.mockData) {
+    stripNullableMarkers(
+      fullScenarioData.data.mockData as Record<string, unknown>,
+    );
+  }
+
   // Relocate misplaced nested keys to their correct position.
   // The LLM sometimes places nested keys at root level instead of inside their
   // parent object (e.g., 'fastener' at root instead of inside 'trpc').
@@ -870,6 +1149,31 @@ export async function generateDataForScenario({
     );
   }
 
+  // Coerce objects/arrays to primitives when the schema expects a primitive type.
+  // The LLM sometimes generates an object where the schema expects "string",
+  // e.g., { body: { "env": "production" } } instead of { body: "some string" }.
+  // This causes runtime errors like "TypeError: body.match is not a function".
+  // Must run BEFORE convertCommaSeparatedStringsToArrays, which intentionally
+  // overrides schema types for array-like field names.
+  if (structure.dataForMocks && fullScenarioData.data.mockData) {
+    coerceObjectsToPrimitivesBySchema(
+      fullScenarioData.data.mockData,
+      structure.dataForMocks,
+    );
+  }
+
+  // Coerce primitives to empty arrays when the schema expects an array.
+  // The LLM sometimes generates a primitive (e.g., `webapps: true`) where the
+  // schema expects an array (e.g., `webapps: [{ name: "string" }]`).
+  // This causes runtime errors like "TypeError: config.webapps?.forEach is not a function".
+  // Must run AFTER coerceObjectsToPrimitivesBySchema and BEFORE convertCommaSeparatedStringsToArrays.
+  if (structure.dataForMocks && fullScenarioData.data.mockData) {
+    coercePrimitivesToArraysBySchema(
+      fullScenarioData.data.mockData,
+      structure.dataForMocks,
+    );
+  }
+
   // Convert comma-separated strings to arrays when appropriate.
   // The LLM sometimes generates strings like "color,size" instead of arrays
   // like ["color", "size"] when the schema type is incorrectly inferred as
@@ -897,6 +1201,18 @@ export async function generateDataForScenario({
     fixNullIdsBySchema(fullScenarioData.data.mockData, structure.dataForMocks);
   }
 
+  // Enforce execution flow requiredValues by setting falsy paths to null.
+  // The LLM doesn't reliably generate null for falsy requirements (e.g., diffView: falsy
+  // to hide a modal). This post-processing ensures that scenarios match their
+  // covered flows' requiredValues.
+  if (fullScenarioData.data.mockData && executionFlows) {
+    enforceRequiredValues(
+      fullScenarioData.data.mockData,
+      scenario.metadata?.coveredFlows || [],
+      executionFlows,
+    );
+  }
+
   if (structure.arguments && fullScenarioData.data.argumentsData) {
     for (let i = 0; i < fullScenarioData.data.argumentsData.length; i++) {
       if (structure.arguments[i]) {
@@ -1086,6 +1402,60 @@ export default async function generateEntityScenarioData({
           defaultScenarioData.data.argumentsData.map((arg) => ({ ...arg }));
       }
 
+      // Enforce requiredValues AFTER merge for non-default scenarios
+      // This ensures that if a non-default scenario doesn't cover a certain flow,
+      // it inherits the enforcement from the default scenario
+      if (scenarioData.data?.mockData && executionFlows) {
+        // Get the flows covered by this scenario
+        const scenarioCoveredFlows =
+          nonDefaultScenarios.find(
+            (s) => s.name === result.scenarioData.scenarioDescription,
+          )?.metadata?.coveredFlows || [];
+
+        // Get the paths that the scenario's flows affect
+        const scenarioAffectedPaths = new Set<string>();
+        for (const flowId of scenarioCoveredFlows) {
+          const flow = executionFlows.find((f) => f.id === flowId);
+          if (flow?.requiredValues) {
+            for (const rv of flow.requiredValues) {
+              if (rv.attributePath) {
+                // Extract base path (e.g., "diffView" from "diffView.type")
+                const basePath = rv.attributePath.split('.')[0];
+                scenarioAffectedPaths.add(basePath);
+              }
+            }
+          }
+        }
+
+        // Get the default scenario's flows
+        const defaultCoveredFlows =
+          defaultScenario.metadata?.coveredFlows || [];
+
+        // For paths NOT affected by the scenario, apply default's enforcement
+        const defaultFlowsForUnaffectedPaths = defaultCoveredFlows.filter(
+          (flowId) => {
+            const flow = executionFlows.find((f) => f.id === flowId);
+            if (!flow?.requiredValues) return false;
+
+            // Check if this flow affects a path that the scenario doesn't cover
+            return flow.requiredValues.some((rv) => {
+              if (!rv.attributePath) return false;
+              const basePath = rv.attributePath.split('.')[0];
+              return !scenarioAffectedPaths.has(basePath);
+            });
+          },
+        );
+
+        // Apply enforcement from default scenario for unaffected paths
+        if (defaultFlowsForUnaffectedPaths.length > 0) {
+          enforceRequiredValues(
+            scenarioData.data.mockData,
+            defaultFlowsForUnaffectedPaths,
+            executionFlows,
+          );
+        }
+      }
+
       return scenarioData;
     });