@codeyam/codeyam-cli 0.1.0-staging.6e699e5 → 0.1.0-staging.7481014

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/stripNullableMarkers.ts

@@ -0,0 +1,35 @@
+/**
+ * Recursively strips `_nullable` keys from LLM-generated mock data.
+ *
+ * The `_nullable` marker is an internal CodeYam concept used in type definitions
+ * to indicate that a field can be null/undefined. The LLM sometimes includes
+ * these markers in its generated scenario data, which causes runtime errors
+ * when code iterates over object keys (e.g., `Object.keys(importedBy)` picks
+ * up `_nullable` as a key alongside real data).
+ *
+ * This function mutates the input object in-place, consistent with other
+ * post-processing helpers like `convertNullToUndefinedBySchema`.
+ */
+export default function stripNullableMarkers(
+  data: Record<string, unknown>,
+): void {
+  if (data == null || typeof data !== 'object') return;
+
+  // Delete _nullable from this level
+  if ('_nullable' in data) {
+    delete data._nullable;
+  }
+
+  // Recurse into nested objects and arrays
+  for (const value of Object.values(data)) {
+    if (Array.isArray(value)) {
+      for (const item of value) {
+        if (item !== null && typeof item === 'object' && !Array.isArray(item)) {
+          stripNullableMarkers(item as Record<string, unknown>);
+        }
+      }
+    } else if (value !== null && typeof value === 'object') {
+      stripNullableMarkers(value as Record<string, unknown>);
+    }
+  }
+}

package/analyzer-template/packages/ai/src/lib/dataStructureChunking.ts

@@ -3,9 +3,11 @@ import type { ExecutionFlow, ScenariosDataStructure } from '~codeyam/types';
 // Type for a single required value from ExecutionFlow
 type RequiredValue = NonNullable<ExecutionFlow['requiredValues']>[number];
 
-const DEFAULT_MAX_KEYS_PER_CHUNK = 8;
+const DEFAULT_MAX_CHUNK_SIZE = 10_000; // ~10K chars per chunk
+const DEFAULT_MAX_KEYS_PER_CHUNK = 20;
 
 export interface ChunkOptions {
+  maxChunkSize?: number;
   maxKeysPerChunk?: number;
 }
 
@@ -15,8 +17,12 @@ export interface ChunkOptions {
  * Large schemas overwhelm LLMs, causing them to make mistakes on some keys.
  * By processing smaller chunks, each key gets more focused attention.
  *
+ * Uses cumulative JSON size to decide chunk boundaries rather than a fixed
+ * key count, so a single oversized key gets its own chunk while many small
+ * keys are grouped together efficiently.
+ *
  * @param dataForMocks - The full data structure schema
- * @param options - Chunking options (maxKeysPerChunk defaults to 8)
+ * @param options - Chunking options (maxChunkSize defaults to 10_000)
  * @returns Array of smaller data structure chunks
  */
 export function chunkDataStructure(
@@ -27,27 +33,48 @@ export function chunkDataStructure(
     return [dataForMocks];
   }
 
-  const { maxKeysPerChunk = DEFAULT_MAX_KEYS_PER_CHUNK } = options;
+  const {
+    maxChunkSize = DEFAULT_MAX_CHUNK_SIZE,
+    maxKeysPerChunk = DEFAULT_MAX_KEYS_PER_CHUNK,
+  } = options;
   const keys = Object.keys(dataForMocks);
 
-  // Don't chunk if already small enough
-  if (keys.length <= maxKeysPerChunk) {
+  // Calculate total size to see if chunking is needed
+  const totalSize = JSON.stringify(dataForMocks).length;
+  if (totalSize <= maxChunkSize && keys.length <= maxKeysPerChunk) {
     return [dataForMocks];
   }
 
+  // Greedily pack keys into chunks by cumulative size and key count
   const chunks: Array<ScenariosDataStructure['dataForMocks']> = [];
+  let currentChunk: Record<string, unknown> = {};
+  let currentSize = 0;
+
+  for (const key of keys) {
+    const keySize = JSON.stringify(
+      (dataForMocks as Record<string, unknown>)[key],
+    ).length;
 
-  for (let i = 0; i < keys.length; i += maxKeysPerChunk) {
-    const chunkKeys = keys.slice(i, i + maxKeysPerChunk);
-    const chunk: ScenariosDataStructure['dataForMocks'] = {};
+    const currentKeyCount = Object.keys(currentChunk).length;
 
-    for (const key of chunkKeys) {
-      (chunk as Record<string, unknown>)[key] = (
-        dataForMocks as Record<string, unknown>
-      )[key];
+    // If adding this key would exceed size or key count limit AND chunk isn't empty, start a new chunk
+    if (
+      currentSize > 0 &&
+      (currentSize + keySize > maxChunkSize ||
+        currentKeyCount >= maxKeysPerChunk)
+    ) {
+      chunks.push(currentChunk as ScenariosDataStructure['dataForMocks']);
+      currentChunk = {};
+      currentSize = 0;
     }
 
-    chunks.push(chunk);
+    currentChunk[key] = (dataForMocks as Record<string, unknown>)[key];
+    currentSize += keySize;
+  }
+
+  // Push the last chunk
+  if (Object.keys(currentChunk).length > 0) {
+    chunks.push(currentChunk as ScenariosDataStructure['dataForMocks']);
   }
 
   return chunks;

package/analyzer-template/packages/ai/src/lib/generateEntityDataStructure.ts

@@ -1,9 +1,12 @@
 import { Entity } from '~codeyam/types';
 import isolateScopes from './isolateScopes';
 import analyzeScope from './analyzeScope';
-import { FileAnalyzer } from '~codeyam/analyze';
+import { FileAnalyzer, transformationTracer } from '~codeyam/analyze';
 import { AI, SerializableDataStructure } from '~codeyam/ai';
-import { resetScopeDataStructureMetrics } from './dataStructure/ScopeDataStructure';
+import {
+  resetScopeDataStructureMetrics,
+  ScopeDataStructure,
+} from './dataStructure/ScopeDataStructure';
 
 // import { awsLog } from '~codeyam/utils';
 
@@ -91,8 +94,60 @@ export default async function generateEntityDataStructure({
   }
 
   // Inspect the runtime type of dataStructure to see if it must be serialized
+  // Note: When worker threads are used, dataStructure is already serialized (no toSerializable).
+  // Workers are disabled when CODEYAM_TRACE_TRANSFORMS=1 to enable full tracing.
   if ('toSerializable' in entityScope.dataStructure) {
-    return entityScope.dataStructure.toSerializable();
+    const scopeDataStructure =
+      entityScope.dataStructure as unknown as ScopeDataStructure;
+
+    // Trace the internal state BEFORE serialization - captures what methods return
+    if (transformationTracer.isEnabled()) {
+      transformationTracer.startEntity({
+        name: entity.name,
+        entityType: entity.entityType ?? 'unknown',
+        filePath: entity.filePath,
+      });
+
+      // Capture internal state - this is what getReturnValue/getFunctionSignature return
+      transformationTracer.snapshot(entity.name, 'scopeDataStructure', {
+        signatureSchema: scopeDataStructure.getFunctionSignature({
+          fillInUnknowns: true,
+        }),
+        returnValueSchema: scopeDataStructure.getReturnValue({
+          fillInUnknowns: true,
+        }),
+      });
+    }
+
+    const serialized = scopeDataStructure.toSerializable();
+
+    // Trace AFTER serialization - shows what gets output
+    if (transformationTracer.isEnabled()) {
+      // Build schema from external function calls for comparison
+      const externalSchemas: Record<string, string> = {};
+      for (const efc of serialized.externalFunctionCalls ?? []) {
+        for (const [path, type] of Object.entries(efc.schema ?? {})) {
+          externalSchemas[path] = type;
+        }
+      }
+
+      transformationTracer.snapshot(entity.name, 'serialized', {
+        signatureSchema:
+          serialized.functionResults?.[entity.name]?.signatureWithUnknowns,
+        returnValueSchema:
+          serialized.functionResults?.[entity.name]?.returnValueWithUnknowns,
+        // Include external function schemas to detect serialization gaps
+        dependencySchemas: {
+          externalFunctions: {
+            combined: {
+              returnValueSchema: externalSchemas,
+            },
+          },
+        },
+      });
+    }
+
+    return serialized;
   }
   return entityScope.dataStructure;
 }