@codeyam/codeyam-cli 0.1.0-staging.323686 → 0.1.0-staging.4813bf3

package/analyzer-template/packages/ai/src/lib/dataStructureChunking.ts

@@ -3,9 +3,11 @@ import type { ExecutionFlow, ScenariosDataStructure } from '~codeyam/types';
 // Type for a single required value from ExecutionFlow
 type RequiredValue = NonNullable<ExecutionFlow['requiredValues']>[number];
 
-const DEFAULT_MAX_KEYS_PER_CHUNK = 8;
+const DEFAULT_MAX_CHUNK_SIZE = 10_000; // ~10K chars per chunk
 
 export interface ChunkOptions {
+  maxChunkSize?: number;
+  /** @deprecated Use maxChunkSize instead. Kept for backward compatibility in tests. */
   maxKeysPerChunk?: number;
 }
 
@@ -15,8 +17,12 @@ export interface ChunkOptions {
  * Large schemas overwhelm LLMs, causing them to make mistakes on some keys.
  * By processing smaller chunks, each key gets more focused attention.
  *
+ * Uses cumulative JSON size to decide chunk boundaries rather than a fixed
+ * key count, so a single oversized key gets its own chunk while many small
+ * keys are grouped together efficiently.
+ *
  * @param dataForMocks - The full data structure schema
- * @param options - Chunking options (maxKeysPerChunk defaults to 8)
+ * @param options - Chunking options (maxChunkSize defaults to 10_000)
  * @returns Array of smaller data structure chunks
  */
 export function chunkDataStructure(
@@ -27,27 +33,39 @@ export function chunkDataStructure(
     return [dataForMocks];
   }
 
-  const { maxKeysPerChunk = DEFAULT_MAX_KEYS_PER_CHUNK } = options;
+  const { maxChunkSize = DEFAULT_MAX_CHUNK_SIZE } = options;
   const keys = Object.keys(dataForMocks);
 
-  // Don't chunk if already small enough
-  if (keys.length <= maxKeysPerChunk) {
+  // Calculate total size to see if chunking is needed
+  const totalSize = JSON.stringify(dataForMocks).length;
+  if (totalSize <= maxChunkSize) {
     return [dataForMocks];
   }
 
+  // Greedily pack keys into chunks by cumulative size
   const chunks: Array<ScenariosDataStructure['dataForMocks']> = [];
-
-  for (let i = 0; i < keys.length; i += maxKeysPerChunk) {
-    const chunkKeys = keys.slice(i, i + maxKeysPerChunk);
-    const chunk: ScenariosDataStructure['dataForMocks'] = {};
-
-    for (const key of chunkKeys) {
-      (chunk as Record<string, unknown>)[key] = (
-        dataForMocks as Record<string, unknown>
-      )[key];
+  let currentChunk: Record<string, unknown> = {};
+  let currentSize = 0;
+
+  for (const key of keys) {
+    const keySize = JSON.stringify(
+      (dataForMocks as Record<string, unknown>)[key],
+    ).length;
+
+    // If adding this key would exceed the limit AND chunk isn't empty, start a new chunk
+    if (currentSize > 0 && currentSize + keySize > maxChunkSize) {
+      chunks.push(currentChunk as ScenariosDataStructure['dataForMocks']);
+      currentChunk = {};
+      currentSize = 0;
     }
 
-    chunks.push(chunk);
+    currentChunk[key] = (dataForMocks as Record<string, unknown>)[key];
+    currentSize += keySize;
+  }
+
+  // Push the last chunk
+  if (Object.keys(currentChunk).length > 0) {
+    chunks.push(currentChunk as ScenariosDataStructure['dataForMocks']);
   }
 
   return chunks;

package/analyzer-template/packages/ai/src/lib/generateEntityDataStructure.ts

@@ -1,9 +1,12 @@
 import { Entity } from '~codeyam/types';
 import isolateScopes from './isolateScopes';
 import analyzeScope from './analyzeScope';
-import { FileAnalyzer } from '~codeyam/analyze';
+import { FileAnalyzer, transformationTracer } from '~codeyam/analyze';
 import { AI, SerializableDataStructure } from '~codeyam/ai';
-import { resetScopeDataStructureMetrics } from './dataStructure/ScopeDataStructure';
+import {
+  resetScopeDataStructureMetrics,
+  ScopeDataStructure,
+} from './dataStructure/ScopeDataStructure';
 
 // import { awsLog } from '~codeyam/utils';
 
@@ -91,8 +94,60 @@ export default async function generateEntityDataStructure({
   }
 
   // Inspect the runtime type of dataStructure to see if it must be serialized
+  // Note: When worker threads are used, dataStructure is already serialized (no toSerializable).
+  // Workers are disabled when CODEYAM_TRACE_TRANSFORMS=1 to enable full tracing.
   if ('toSerializable' in entityScope.dataStructure) {
-    return entityScope.dataStructure.toSerializable();
+    const scopeDataStructure =
+      entityScope.dataStructure as unknown as ScopeDataStructure;
+
+    // Trace the internal state BEFORE serialization - captures what methods return
+    if (transformationTracer.isEnabled()) {
+      transformationTracer.startEntity({
+        name: entity.name,
+        entityType: entity.entityType ?? 'unknown',
+        filePath: entity.filePath,
+      });
+
+      // Capture internal state - this is what getReturnValue/getFunctionSignature return
+      transformationTracer.snapshot(entity.name, 'scopeDataStructure', {
+        signatureSchema: scopeDataStructure.getFunctionSignature({
+          fillInUnknowns: true,
+        }),
+        returnValueSchema: scopeDataStructure.getReturnValue({
+          fillInUnknowns: true,
+        }),
+      });
+    }
+
+    const serialized = scopeDataStructure.toSerializable();
+
+    // Trace AFTER serialization - shows what gets output
+    if (transformationTracer.isEnabled()) {
+      // Build schema from external function calls for comparison
+      const externalSchemas: Record<string, string> = {};
+      for (const efc of serialized.externalFunctionCalls ?? []) {
+        for (const [path, type] of Object.entries(efc.schema ?? {})) {
+          externalSchemas[path] = type;
+        }
+      }
+
+      transformationTracer.snapshot(entity.name, 'serialized', {
+        signatureSchema:
+          serialized.functionResults?.[entity.name]?.signatureWithUnknowns,
+        returnValueSchema:
+          serialized.functionResults?.[entity.name]?.returnValueWithUnknowns,
+        // Include external function schemas to detect serialization gaps
+        dependencySchemas: {
+          externalFunctions: {
+            combined: {
+              returnValueSchema: externalSchemas,
+            },
+          },
+        },
+      });
+    }
+
+    return serialized;
   }
   return entityScope.dataStructure;
 }

package/analyzer-template/packages/ai/src/lib/generateEntityScenarioData.ts

@@ -16,7 +16,9 @@ import validateJson from './validateJson';
 import { awsLog, awsLogDebugLevel } from '~codeyam/utils';
 import { AI, parseJsonSafe } from '~codeyam/ai';
 import convertNullToUndefinedBySchema from './dataStructure/helpers/convertNullToUndefinedBySchema';
+import convertTypeAnnotationsToValues from './dataStructure/helpers/convertTypeAnnotationsToValues';
 import fixNullIdsBySchema from './dataStructure/helpers/fixNullIdsBySchema';
+import coerceObjectsToPrimitivesBySchema from './dataStructure/helpers/coerceObjectsToPrimitivesBySchema';
 import { JsonTypeDefinition } from '~codeyam/types';
 import { deepMerge } from '~codeyam/generate';
 import {
@@ -292,11 +294,13 @@ function generateDefaultForSchemaType(schemaType: unknown): unknown {
     if (schemaType.includes('| null')) return null;
     return schemaType; // Return the type as a string placeholder
   }
-  if (
-    typeof schemaType === 'object' &&
-    schemaType !== null &&
-    !Array.isArray(schemaType)
-  ) {
+  if (Array.isArray(schemaType)) {
+    if (schemaType.length === 0) return [];
+    // Generate a single default element based on the first element's schema
+    const elementDefault = generateDefaultForSchemaType(schemaType[0]);
+    return elementDefault !== undefined ? [elementDefault] : [];
+  }
+  if (typeof schemaType === 'object' && schemaType !== null) {
     // Recursively generate defaults for nested objects
     const result: Record<string, unknown> = {};
     for (const [key, value] of Object.entries(schemaType)) {
@@ -424,6 +428,8 @@ function fillMissingMockDataKeysWithDefaults(
   const missingKeys: string[] = [];
 
   for (const key of Object.keys(dataForMocks)) {
+    if (key === '_nullable') continue; // Internal marker, not a data key
+
     const fullPath = pathPrefix ? `${pathPrefix}.${key}` : key;
 
     if (mockData[key] === undefined) {
@@ -459,6 +465,193 @@ function fillMissingMockDataKeysWithDefaults(
   }
 }
 
+/**
+ * Enforce execution flow requiredValues by setting falsy paths to null.
+ *
+ * The LLM doesn't reliably generate null for `comparison: 'falsy'` requirements.
+ * For example, a flow like "diffView: falsy" should hide a modal, but the LLM
+ * might generate a truthy object, causing the modal to show in all screenshots.
+ *
+ * This function:
+ * 1. Gets requiredValues from covered flows
+ * 2. For 'falsy' comparisons: sets the value to null
+ * 3. For 'truthy' comparisons with falsy values: generates a default truthy value
+ */
+function enforceRequiredValues(
+  mockData: Record<string, unknown>,
+  coveredFlowIds: string[],
+  executionFlows: ExecutionFlow[],
+): void {
+  if (!coveredFlowIds.length || !executionFlows.length) {
+    return;
+  }
+
+  // Get all requiredValues from covered flows
+  const coveredFlows = executionFlows.filter((flow) =>
+    coveredFlowIds.includes(flow.id),
+  );
+
+  for (const flow of coveredFlows) {
+    if (!flow.requiredValues) continue;
+
+    for (const rv of flow.requiredValues) {
+      if (!rv.attributePath) continue;
+
+      // Find the value in mockData - the path could be nested
+      // e.g., attributePath: "diffView" could be at mockData['useDiffModal()'].diffView
+      const result = findAndSetValueInMockData(
+        mockData,
+        rv.attributePath,
+        rv.comparison,
+        rv.valueType,
+      );
+
+      if (result.found) {
+        awsLog(
+          `CodeYam: Enforced ${rv.comparison} for ${rv.attributePath} (set to ${result.newValue === null ? 'null' : typeof result.newValue})`,
+        );
+      }
+    }
+  }
+}
+
+/**
+ * Find a value in mockData by attributePath and enforce the comparison.
+ * The attributePath could be a simple key or a nested path.
+ *
+ * Returns { found: boolean, newValue: unknown }
+ */
+function findAndSetValueInMockData(
+  mockData: Record<string, unknown>,
+  attributePath: string,
+  comparison: string,
+  valueType?: string,
+): { found: boolean; newValue?: unknown } {
+  // Try to find the path at various nesting levels
+  // The attributePath might be "diffView" but the actual location is
+  // mockData['useDiffModal()'].diffView
+
+  // Strategy 1: Direct path (e.g., mockData[attributePath])
+  if (attributePath in mockData) {
+    const currentValue = mockData[attributePath];
+    const { shouldChange, newValue } = getEnforcedValue(
+      currentValue,
+      comparison,
+      valueType,
+    );
+    if (shouldChange) {
+      mockData[attributePath] = newValue;
+      return { found: true, newValue };
+    }
+    return { found: true, newValue: currentValue };
+  }
+
+  // Strategy 2: Search in nested objects
+  for (const [key, value] of Object.entries(mockData)) {
+    if (value && typeof value === 'object' && !Array.isArray(value)) {
+      const nestedObj = value as Record<string, unknown>;
+
+      // Check if attributePath exists in this nested object
+      if (attributePath in nestedObj) {
+        const currentValue = nestedObj[attributePath];
+        const { shouldChange, newValue } = getEnforcedValue(
+          currentValue,
+          comparison,
+          valueType,
+        );
+        if (shouldChange) {
+          nestedObj[attributePath] = newValue;
+          return { found: true, newValue };
+        }
+        return { found: true, newValue: currentValue };
+      }
+
+      // Also check dot-notation paths (e.g., "diffView.type")
+      if (attributePath.includes('.')) {
+        const parts = attributePath.split('.');
+        const firstPart = parts[0];
+        if (firstPart in nestedObj) {
+          // Recurse with the rest of the path
+          const result = findAndSetValueInMockData(
+            nestedObj,
+            attributePath,
+            comparison,
+            valueType,
+          );
+          if (result.found) return result;
+        }
+      }
+
+      // Recursively search deeper
+      const result = findAndSetValueInMockData(
+        nestedObj,
+        attributePath,
+        comparison,
+        valueType,
+      );
+      if (result.found) return result;
+    }
+  }
+
+  return { found: false };
+}
+
+/**
+ * Determine if a value should be changed to match a comparison requirement.
+ *
+ * For 'falsy' comparison: truthy values should become null
+ * For 'truthy' comparison: falsy values should become a default truthy value
+ */
+function getEnforcedValue(
+  currentValue: unknown,
+  comparison: string,
+  valueType?: string,
+): { shouldChange: boolean; newValue: unknown } {
+  const isTruthy = Boolean(currentValue);
+
+  if (comparison === 'falsy') {
+    // Value should be falsy
+    if (isTruthy) {
+      return { shouldChange: true, newValue: null };
+    }
+    return { shouldChange: false, newValue: currentValue };
+  }
+
+  if (comparison === 'truthy') {
+    // Value should be truthy
+    if (!isTruthy) {
+      // Generate a default truthy value based on valueType
+      const defaultValue = generateDefaultTruthyValue(valueType);
+      return { shouldChange: true, newValue: defaultValue };
+    }
+    return { shouldChange: false, newValue: currentValue };
+  }
+
+  // For other comparisons (equals, exists, etc.), don't auto-enforce
+  return { shouldChange: false, newValue: currentValue };
+}
+
+/**
+ * Generate a default truthy value for a given type.
+ */
+function generateDefaultTruthyValue(valueType?: string): unknown {
+  if (!valueType) return { _placeholder: true };
+
+  switch (valueType.toLowerCase()) {
+    case 'string':
+      return 'default-value';
+    case 'number':
+      return 1;
+    case 'boolean':
+      return true;
+    case 'array':
+      return [{ _placeholder: true }];
+    case 'object':
+    default:
+      return { _placeholder: true };
+  }
+}
+
 type ScenarioDataWithoutDescription = Omit<ScenarioData, 'scenarioDescription'>;
 
 interface GenerateEntityScenarioDataArgs {
@@ -652,6 +845,29 @@ export async function generateDataForScenario({
         }
       }
 
+      // Detect keys that were lost from failed or partial chunk responses
+      // and fill them with schema-based defaults so they aren't permanently lost.
+      const allChunkedKeys = chunks.flatMap((c) => Object.keys(c || {}));
+      const returnedKeys = new Set(Object.keys(chunkedMockData));
+      const missingChunkKeys = allChunkedKeys.filter(
+        (k) => !returnedKeys.has(k),
+      );
+
+      if (missingChunkKeys.length > 0) {
+        awsLog(
+          `Chunked processing: ${missingChunkKeys.length} key(s) missing from chunk results, filling with defaults: ${missingChunkKeys.join(', ')}`,
+        );
+        const dataForMocksRecord = structure.dataForMocks as Record<
+          string,
+          unknown
+        >;
+        for (const key of missingChunkKeys) {
+          chunkedMockData[key] = generateDefaultForSchemaType(
+            dataForMocksRecord[key],
+          );
+        }
+      }
+
       awsLog(
         `Chunked processing complete. Generated ${Object.keys(chunkedMockData).length} keys total`,
       );
@@ -869,6 +1085,19 @@ export async function generateDataForScenario({
     );
   }
 
+  // Coerce objects/arrays to primitives when the schema expects a primitive type.
+  // The LLM sometimes generates an object where the schema expects "string",
+  // e.g., { body: { "env": "production" } } instead of { body: "some string" }.
+  // This causes runtime errors like "TypeError: body.match is not a function".
+  // Must run BEFORE convertCommaSeparatedStringsToArrays, which intentionally
+  // overrides schema types for array-like field names.
+  if (structure.dataForMocks && fullScenarioData.data.mockData) {
+    coerceObjectsToPrimitivesBySchema(
+      fullScenarioData.data.mockData,
+      structure.dataForMocks,
+    );
+  }
+
   // Convert comma-separated strings to arrays when appropriate.
   // The LLM sometimes generates strings like "color,size" instead of arrays
   // like ["color", "size"] when the schema type is incorrectly inferred as
@@ -878,6 +1107,16 @@ export async function generateDataForScenario({
     convertCommaSeparatedStringsToArrays(fullScenarioData.data.mockData);
   }
 
+  // Convert type annotation strings that appear as values to actual values.
+  // The LLM sometimes echoes the schema type annotation as the value.
+  // For example, if the schema says { filePath: "string | undefined" },
+  // the LLM might return { filePath: "string | undefined" } instead of
+  // generating an actual value. This converts those type strings to
+  // appropriate default values (e.g., "string | undefined" → undefined).
+  if (fullScenarioData.data.mockData) {
+    convertTypeAnnotationsToValues(fullScenarioData.data.mockData);
+  }
+
   // Fix null values for ID fields when the schema indicates they should be non-null.
   // The LLM sometimes generates `null` for ID fields (e.g., `"id": null`) when
   // the schema type is `"number"`. This causes runtime issues when code checks
@@ -886,6 +1125,18 @@ export async function generateDataForScenario({
     fixNullIdsBySchema(fullScenarioData.data.mockData, structure.dataForMocks);
   }
 
+  // Enforce execution flow requiredValues by setting falsy paths to null.
+  // The LLM doesn't reliably generate null for falsy requirements (e.g., diffView: falsy
+  // to hide a modal). This post-processing ensures that scenarios match their
+  // covered flows' requiredValues.
+  if (fullScenarioData.data.mockData && executionFlows) {
+    enforceRequiredValues(
+      fullScenarioData.data.mockData,
+      scenario.metadata?.coveredFlows || [],
+      executionFlows,
+    );
+  }
+
   if (structure.arguments && fullScenarioData.data.argumentsData) {
     for (let i = 0; i < fullScenarioData.data.argumentsData.length; i++) {
       if (structure.arguments[i]) {
@@ -1075,6 +1326,60 @@ export default async function generateEntityScenarioData({
           defaultScenarioData.data.argumentsData.map((arg) => ({ ...arg }));
       }
 
+      // Enforce requiredValues AFTER merge for non-default scenarios
+      // This ensures that if a non-default scenario doesn't cover a certain flow,
+      // it inherits the enforcement from the default scenario
+      if (scenarioData.data?.mockData && executionFlows) {
+        // Get the flows covered by this scenario
+        const scenarioCoveredFlows =
+          nonDefaultScenarios.find(
+            (s) => s.name === result.scenarioData.scenarioDescription,
+          )?.metadata?.coveredFlows || [];
+
+        // Get the paths that the scenario's flows affect
+        const scenarioAffectedPaths = new Set<string>();
+        for (const flowId of scenarioCoveredFlows) {
+          const flow = executionFlows.find((f) => f.id === flowId);
+          if (flow?.requiredValues) {
+            for (const rv of flow.requiredValues) {
+              if (rv.attributePath) {
+                // Extract base path (e.g., "diffView" from "diffView.type")
+                const basePath = rv.attributePath.split('.')[0];
+                scenarioAffectedPaths.add(basePath);
+              }
+            }
+          }
+        }
+
+        // Get the default scenario's flows
+        const defaultCoveredFlows =
+          defaultScenario.metadata?.coveredFlows || [];
+
+        // For paths NOT affected by the scenario, apply default's enforcement
+        const defaultFlowsForUnaffectedPaths = defaultCoveredFlows.filter(
+          (flowId) => {
+            const flow = executionFlows.find((f) => f.id === flowId);
+            if (!flow?.requiredValues) return false;
+
+            // Check if this flow affects a path that the scenario doesn't cover
+            return flow.requiredValues.some((rv) => {
+              if (!rv.attributePath) return false;
+              const basePath = rv.attributePath.split('.')[0];
+              return !scenarioAffectedPaths.has(basePath);
+            });
+          },
+        );
+
+        // Apply enforcement from default scenario for unaffected paths
+        if (defaultFlowsForUnaffectedPaths.length > 0) {
+          enforceRequiredValues(
+            scenarioData.data.mockData,
+            defaultFlowsForUnaffectedPaths,
+            executionFlows,
+          );
+        }
+      }
+
       return scenarioData;
     });
 
@@ -1104,7 +1409,8 @@ Generate COMPLETE, robust data for the entire data structure.
     : `## Non-Default Scenario
 Generate ONLY the differences from the default scenario.
 - Include only fields that need to change
-- Set to \`null\` to remove data
+- For object/scalar fields: set to \`null\` to remove/unset the data
+- For array fields: use \`[]\` for empty arrays (not \`null\`) unless the schema type explicitly includes \`| null\`
 - Omit unchanged fields—they merge from default`;
 
   // Only include the "NO ERROR DATA" instruction when the scenario doesn't require error data
@@ -1188,6 +1494,7 @@ Use simple elements only (\`<div>\`, \`<span>\`). No custom components.
 - Arrays should have many items (at least 4) unless specified otherwise
 - Each item must follow the exact structure provided
 - In general we want robust data, not minimal data unless specified otherwise
+- For empty arrays, use \`[]\` (not \`null\`) unless the schema type explicitly includes \`| null\` and the scenario requires the attribute be removed
 
 ## CRITICAL: Preserve Exact Structure
 Your response MUST mirror the EXACT nested structure provided in mockData Structure.
@@ -1267,6 +1574,7 @@ Use simple elements only (\`<div>\`, \`<span>\`). No custom components.
 ### Arrays
 - Arrays should have many items (at least 4) unless specified otherwise
 - Each item must follow the exact structure provided
+- For empty arrays, use \`[]\` (not \`null\`) unless the schema type explicitly includes \`| null\` and the scenario requires the attribute be removed
 
 ## CRITICAL: Preserve Exact Structure
 Your response MUST mirror the EXACT nested structure provided for the missing keys.
@@ -1395,4 +1703,5 @@ NEVER include "error" fields in responses. Skip them entirely.
 - No \`undefined\`—use \`null\` or omit
 - Generate data for ALL keys in the chunk (don't skip any)
 - Arrays should have many items (at least 4) unless specified otherwise
+- For empty arrays, use \`[]\` (not \`null\`) unless the schema type explicitly includes \`| null\` and the scenario requires the attribute be removed
 `;

package/analyzer-template/packages/ai/src/lib/generateEntityScenarios.ts

@@ -176,11 +176,15 @@ export default async function generateEntityScenarios({
     return { scenarios: [], llmCall };
   }
 
-  const scenarios = (
-    result[
-      `${error ? 'errorD' : 'd'}ataScenarios`
-    ] as unknown as ScenarioResult[]
-  ).map(
+  const dataScenarios = result[`${error ? 'errorD' : 'd'}ataScenarios`];
+
+  // Gracefully handle missing dataScenarios (e.g., when LLM returns unexpected format
+  // or when test fixtures don't include the expected key)
+  if (!Array.isArray(dataScenarios)) {
+    return { scenarios: [], llmCall };
+  }
+
+  const scenarios = (dataScenarios as unknown as ScenarioResult[]).map(
     (scenarioInfo) =>
       ({
         projectId: entity.projectId,