@codeyam/codeyam-cli 0.1.0-staging.2a88920 → 0.1.0-staging.4684848

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/convertDotNotation.ts

@@ -14,23 +14,112 @@ const isStandaloneIndex = (s?: string) => !!s && STANDALONE_INDEX_RE.test(s);
 // The regex matches any path ending with .length that has [] somewhere before it
 const DYNAMIC_LENGTH_RE = /\[\].*\.length$/;
 
+// Cache for type string analysis to avoid repeated split/filter operations.
+// These functions are called multiple times per path segment across thousands of paths.
+const typeAnalysisCache = new Map<
+  string,
+  { isSkippable: boolean; baseType: string; isNullable: boolean }
+>();
+
+function getTypeAnalysis(t: string) {
+  const cached = typeAnalysisCache.get(t);
+  if (cached) return cached;
+  const parts = t.split('|').map((s) => s.trim());
+  const base = parts.filter((s) => s !== 'undefined' && s !== 'null');
+  const result = {
+    isSkippable:
+      base.length === 1 &&
+      (base[0] === 'object' ||
+        base[0] === 'array' ||
+        base[0] === 'function' ||
+        base[0] === 'unknown'),
+    baseType: base[0],
+    isNullable: parts.includes('undefined') || parts.includes('null'),
+  };
+  typeAnalysisCache.set(t, result);
+  return result;
+}
+
 // Treat these as structural placeholders (don't commit them as concrete leaves)
 function isSkippableLeafType(t: string) {
-  // 'unknown' by itself is a placeholder (but 'boolean | unknown' is not)
-  return (
-    t === 'object' ||
-    t === 'array' ||
-    t === 'function' ||
-    t.trim() === 'unknown'
-  );
+  return getTypeAnalysis(t).isSkippable;
+}
+
+// Extract the base structural type from a potentially nullable type string.
+// e.g., 'object | undefined' → 'object', 'array | null' → 'array'
+function getBaseSkippableType(t: string): string {
+  return getTypeAnalysis(t).baseType;
+}
+
+// Check if a type string has nullable annotations (| undefined or | null)
+function isNullableType(t: string): boolean {
+  return getTypeAnalysis(t).isNullable;
+}
+
+// Matches paths containing [][] — e.g., "items[][]" or "items[][].text"
+const DOUBLE_ARRAY_RE = /\[\]\[\]/;
+
+// Matches a [].property segment — e.g., "items[].name" (but not "items[][]")
+const ARRAY_OBJECT_PROPERTY_RE = /\[\]\.[^[\]]/;
+
+/**
+ * When scope analysis produces both `prefix[].property` (array of objects) and
+ * `prefix[][]` (array of arrays) for the same prefix, there's a conflict about
+ * what the array elements are. Object-property paths are more specific and come
+ * from actual property access; double-array paths come from mixed iteration
+ * patterns in the scope analysis. Drop the double-array paths when both exist.
+ */
+function resolveDoubleArrayConflicts(
+  schema: Record<string, string>,
+): Record<string, string> {
+  // Collect prefixes that end in [] and have object property children
+  const objectArrayPrefixes = new Set<string>();
+  for (const path of Object.keys(schema)) {
+    if (!ARRAY_OBJECT_PROPERTY_RE.test(path)) continue;
+    // Extract every "...[]" prefix that is followed by ".prop"
+    let idx = 0;
+    while ((idx = path.indexOf('[].', idx)) !== -1) {
+      objectArrayPrefixes.add(path.slice(0, idx + 2)); // includes the "[]"
+      idx += 3;
+    }
+  }
+
+  if (objectArrayPrefixes.size === 0) return schema;
+
+  // Check if any double-array paths actually conflict
+  let hasConflict = false;
+  for (const path of Object.keys(schema)) {
+    if (!DOUBLE_ARRAY_RE.test(path)) continue;
+    const idx = path.indexOf('[][]');
+    if (idx !== -1 && objectArrayPrefixes.has(path.slice(0, idx + 2))) {
+      hasConflict = true;
+      break;
+    }
+  }
+
+  if (!hasConflict) return schema;
+
+  // Filter out conflicting double-array paths
+  const resolved: Record<string, string> = {};
+  for (const [path, value] of Object.entries(schema)) {
+    if (DOUBLE_ARRAY_RE.test(path)) {
+      const idx = path.indexOf('[][]');
+      if (idx !== -1 && objectArrayPrefixes.has(path.slice(0, idx + 2))) {
+        continue; // drop this conflicting path
+      }
+    }
+    resolved[path] = value;
+  }
+  return resolved;
 }
 
 export default function convertDotNotation(
   schema: Record<string, string>,
 ): JsonTypeDefinition {
+  const resolved = resolveDoubleArrayConflicts(schema);
   const result: JsonTypeDefinition = {};
 
-  for (const [rawPath, typ] of Object.entries(schema)) {
+  for (const [rawPath, typ] of Object.entries(resolved)) {
     // Skip paths with .length after array notation from dynamic iteration
     // e.g., metadata[].length, metadata[][].length
     if (DYNAMIC_LENGTH_RE.test(rawPath)) continue;
@@ -218,6 +307,17 @@ export default function convertDotNotation(
           // If the next segment after '[]' is another '[]' or a standalone index,
           // we need an array at cursor[key][0], not an object (for patterns like "key[][]" or "key[][0]")
           if (afterArray === '[]' || isStandaloneIndex(afterArray)) {
+            // But don't overwrite an existing populated object — object properties
+            // (from paths like key[].property) take precedence over double-array hints
+            // (from paths like key[][]) which come from mixed access patterns in scope analysis
+            if (
+              cursor[key][0] &&
+              typeof cursor[key][0] === 'object' &&
+              !Array.isArray(cursor[key][0]) &&
+              Object.keys(cursor[key][0]).length > 0
+            ) {
+              break;
+            }
             if (!Array.isArray(cursor[key][0])) cursor[key][0] = [];
           } else {
             // Default case: create an object
@@ -242,15 +342,41 @@ export default function convertDotNotation(
           cursor[key] = typ;
         } else {
           // Structural/placeholder terminal
-          if (typ === 'array') {
+          const nullable = isNullableType(typ);
+          const baseType = getBaseSkippableType(typ);
+
+          if (baseType === 'array') {
             if (!Array.isArray(cursor[key])) cursor[key] = [];
+            if (nullable) {
+              (cursor[key] as any)._nullable = true;
+            }
           } else if (
-            typ === 'object' ||
-            typ === 'function' ||
-            typ.trim() === 'unknown'
+            baseType === 'object' ||
+            baseType === 'function' ||
+            baseType === 'unknown'
           ) {
-            if (cursor[key] === undefined) {
-              cursor[key] = typ;
+            if (nullable) {
+              // Nullable object: ensure it's an actual object (not a string
+              // placeholder) so _nullable can be set and child paths can
+              // populate properties on it.
+              if (
+                cursor[key] === undefined ||
+                typeof cursor[key] === 'string'
+              ) {
+                cursor[key] = {};
+              }
+              if (
+                typeof cursor[key] === 'object' &&
+                cursor[key] !== null &&
+                !Array.isArray(cursor[key])
+              ) {
+                (cursor[key] as any)._nullable = true;
+              }
+            } else {
+              // Non-nullable: preserve existing behavior (string placeholder)
+              if (cursor[key] === undefined) {
+                cursor[key] = typ;
+              }
             }
           }
         }

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/convertTypeAnnotationsToValues.ts

@@ -81,7 +81,23 @@ function isTypeAnnotation(value: string): boolean {
     'string | number | undefined',
   ];
 
-  return commonTypePatterns.includes(trimmed);
+  if (commonTypePatterns.includes(trimmed)) return true;
+
+  // Check for arrow function type annotations like "() => void", "(event: MouseEvent) => void"
+  // These contain '=>' but the body is a type keyword, not a JS expression
+  if (trimmed.includes('=>')) {
+    const arrowIndex = trimmed.indexOf('=>');
+    const body = trimmed.slice(arrowIndex + 2).trim();
+    if (
+      /^(void|string|number|boolean|any|unknown|never|null|undefined|object|Date|Promise<\w+>)$/i.test(
+        body,
+      )
+    ) {
+      return true;
+    }
+  }
+
+  return false;
 }
 
 /**
@@ -105,6 +121,9 @@ function typeAnnotationToValue(typeAnnotation: string): unknown {
     return [];
   }
 
+  // Handle arrow function type annotations like "() => void"
+  if (trimmed.includes('=>')) return () => {};
+
   // Handle primitive types
   if (trimmed === 'string') return '';
   if (trimmed === 'number') return 0;

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/fillInSchemaGapsAndUnknowns.ts

@@ -152,8 +152,11 @@ export default function fillInSchemaGapsAndUnknowns(
           schema[previousSubPath],
         )
       ) {
-        const newValue =
-          checkIfKnownType(previousSubPath, functionKeysMapping) ?? 'object';
+        // When fillInUnknowns=false, add missing gaps as 'unknown' to preserve raw schema.
+        // When fillInUnknowns=true, infer the type using checkIfKnownType.
+        const newValue = fillInUnknowns
+          ? (checkIfKnownType(previousSubPath, functionKeysMapping) ?? 'object')
+          : 'unknown';
 
         if (
           !schema[previousSubPath] ||
@@ -166,10 +169,14 @@ export default function fillInSchemaGapsAndUnknowns(
 
       // Handle the wrong function chaining function().function()
       if (isFunction && !existingSchema(previousSubPath)) {
+        // When fillInUnknowns=false, use 'unknown' for non-function paths.
+        // When fillInUnknowns=true, infer the type.
         schema[previousSubPath] = previousSubPath.endsWith(')')
           ? 'function'
-          : (checkIfKnownType(previousSubPath, functionKeysMapping) ??
-            'object');
+          : fillInUnknowns
+            ? (checkIfKnownType(previousSubPath, functionKeysMapping) ??
+              'object')
+            : 'unknown';
         changeMade = true;
       }
 
@@ -458,7 +465,10 @@ export function fillInDirectSchemaGapsAndUnknowns({
         }
       } else if (schema[key] === 'unknown') {
         const newType = guessTypeForPath(key, 'string');
-        if (newType && schema[key] !== newType) {
+        // Don't convert 'unknown' to 'object' — 'object' is just inferred from
+        // having child properties, which isn't more useful than 'unknown' and
+        // prevents enrichment from filling in the actual type later.
+        if (newType && newType !== 'object' && schema[key] !== newType) {
           schema[key] = newType;
           changeMade = true;
         }
@@ -481,12 +491,44 @@ export function fillInDirectSchemaGapsAndUnknowns({
             // This handles spurious [] paths from components like JsonNode that handle
             // both arrays and objects. When metadata is explicitly typed as 'object',
             // paths like metadata[] (from dynamic iteration) should not change it.
+            const existingType = schema[previousSubPath];
+            const baseType = existingType?.split(' | ')[0];
+
+            // Also check if this path has meaningful object property children (non-[] children).
+            // If it does, it's an object with mixed access patterns (e.g., agent.entries
+            // AND agent[].type), not a pure array. Don't infer 'array' in that case.
+            // Exclude .length since it exists on arrays too and is NOT evidence of object-ness.
+            let hasObjectPropertyChildren = false;
+            if (baseType !== 'array' && baseType !== 'object') {
+              const dotPrefix = previousSubPath + '.';
+              const lengthPath = previousSubPath + '.length';
+              for (const schemaKey in schema) {
+                if (
+                  schemaKey.startsWith(dotPrefix) &&
+                  schemaKey !== lengthPath
+                ) {
+                  hasObjectPropertyChildren = true;
+                  break;
+                }
+              }
+            }
+
             if (
-              schema[previousSubPath] !== 'array' &&
-              schema[previousSubPath] !== 'object'
+              baseType !== 'array' &&
+              baseType !== 'object' &&
+              !hasObjectPropertyChildren
             ) {
-              schema[previousSubPath] = 'array';
-              changeMade = true;
+              // Preserve nullability from existing type (e.g., 'unknown | undefined' -> 'array | undefined')
+              let newType = 'array';
+              if (existingType?.includes(' | undefined')) {
+                newType = 'array | undefined';
+              } else if (existingType?.includes(' | null')) {
+                newType = 'array | null';
+              }
+              if (schema[previousSubPath] !== newType) {
+                schema[previousSubPath] = newType;
+                changeMade = true;
+              }
             }
           }
         }
@@ -502,14 +544,36 @@ export function fillInDirectSchemaGapsAndUnknowns({
             schema[previousSubPath],
           )
         ) {
-          const newValue = getKnownType(previousSubPath) ?? 'object';
-
+          const existingType = schema[previousSubPath];
+          const baseExistingType = existingType?.split(' | ')[0];
+          // Skip if the base type is already a structured type
           if (
-            !schema[previousSubPath] ||
-            schema[previousSubPath] !== newValue
+            !['object', 'array', 'function', 'async-function'].includes(
+              baseExistingType,
+            )
           ) {
-            schema[previousSubPath] = newValue;
-            changeMade = true;
+            let newValue = getKnownType(previousSubPath) ?? 'object';
+            // Don't overwrite 'unknown' with 'object' — 'object' is just
+            // inferred from child properties and isn't more useful than 'unknown'
+            if (
+              newValue === 'object' &&
+              existingType &&
+              existingType.split(' | ')[0] === 'unknown'
+            ) {
+              // preserve existing 'unknown' (possibly with nullability)
+            } else {
+              // Preserve nullability from existing type
+              if (existingType?.includes(' | undefined')) {
+                newValue = `${newValue} | undefined`;
+              } else if (existingType?.includes(' | null')) {
+                newValue = `${newValue} | null`;
+              }
+
+              if (!existingType || existingType !== newValue) {
+                schema[previousSubPath] = newValue;
+                changeMade = true;
+              }
+            }
           }
         }
 
@@ -778,10 +842,11 @@ function checkIfKnownType(
     return 'array';
   }
 
-  // When .includes() or .indexOf() is called with a variable argument AND we have .length,
-  // prefer array. This pattern (arr.length > 0 && arr.includes(item)) is common for arrays.
-  // But str.includes('literal') is common for strings - that case is handled by hasStringLiteralArgument.
-  if (isArray && hasLengthAccess && hasIncludesOrIndexOfWithVariable) {
+  // When .includes() or .indexOf() is called with a variable argument, prefer array.
+  // Pattern: arr.includes(item) -> likely array (checking if item exists in collection)
+  // String literals are already filtered by hasStringLiteralArgument, so if we get here
+  // with a variable argument, it's strong evidence of array membership testing.
+  if (isArray && hasIncludesOrIndexOfWithVariable) {
     return 'array';
   }
 

package/analyzer-template/packages/ai/src/lib/dataStructureChunking.ts

@@ -3,9 +3,11 @@ import type { ExecutionFlow, ScenariosDataStructure } from '~codeyam/types';
 // Type for a single required value from ExecutionFlow
 type RequiredValue = NonNullable<ExecutionFlow['requiredValues']>[number];
 
-const DEFAULT_MAX_KEYS_PER_CHUNK = 8;
+const DEFAULT_MAX_CHUNK_SIZE = 10_000; // ~10K chars per chunk
 
 export interface ChunkOptions {
+  maxChunkSize?: number;
+  /** @deprecated Use maxChunkSize instead. Kept for backward compatibility in tests. */
   maxKeysPerChunk?: number;
 }
 
@@ -15,8 +17,12 @@ export interface ChunkOptions {
  * Large schemas overwhelm LLMs, causing them to make mistakes on some keys.
  * By processing smaller chunks, each key gets more focused attention.
  *
+ * Uses cumulative JSON size to decide chunk boundaries rather than a fixed
+ * key count, so a single oversized key gets its own chunk while many small
+ * keys are grouped together efficiently.
+ *
  * @param dataForMocks - The full data structure schema
- * @param options - Chunking options (maxKeysPerChunk defaults to 8)
+ * @param options - Chunking options (maxChunkSize defaults to 10_000)
  * @returns Array of smaller data structure chunks
  */
 export function chunkDataStructure(
@@ -27,27 +33,39 @@ export function chunkDataStructure(
     return [dataForMocks];
   }
 
-  const { maxKeysPerChunk = DEFAULT_MAX_KEYS_PER_CHUNK } = options;
+  const { maxChunkSize = DEFAULT_MAX_CHUNK_SIZE } = options;
   const keys = Object.keys(dataForMocks);
 
-  // Don't chunk if already small enough
-  if (keys.length <= maxKeysPerChunk) {
+  // Calculate total size to see if chunking is needed
+  const totalSize = JSON.stringify(dataForMocks).length;
+  if (totalSize <= maxChunkSize) {
     return [dataForMocks];
   }
 
+  // Greedily pack keys into chunks by cumulative size
   const chunks: Array<ScenariosDataStructure['dataForMocks']> = [];
-
-  for (let i = 0; i < keys.length; i += maxKeysPerChunk) {
-    const chunkKeys = keys.slice(i, i + maxKeysPerChunk);
-    const chunk: ScenariosDataStructure['dataForMocks'] = {};
-
-    for (const key of chunkKeys) {
-      (chunk as Record<string, unknown>)[key] = (
-        dataForMocks as Record<string, unknown>
-      )[key];
+  let currentChunk: Record<string, unknown> = {};
+  let currentSize = 0;
+
+  for (const key of keys) {
+    const keySize = JSON.stringify(
+      (dataForMocks as Record<string, unknown>)[key],
+    ).length;
+
+    // If adding this key would exceed the limit AND chunk isn't empty, start a new chunk
+    if (currentSize > 0 && currentSize + keySize > maxChunkSize) {
+      chunks.push(currentChunk as ScenariosDataStructure['dataForMocks']);
+      currentChunk = {};
+      currentSize = 0;
     }
 
-    chunks.push(chunk);
+    currentChunk[key] = (dataForMocks as Record<string, unknown>)[key];
+    currentSize += keySize;
+  }
+
+  // Push the last chunk
+  if (Object.keys(currentChunk).length > 0) {
+    chunks.push(currentChunk as ScenariosDataStructure['dataForMocks']);
   }
 
   return chunks;

package/analyzer-template/packages/ai/src/lib/generateEntityDataStructure.ts

@@ -1,9 +1,12 @@
 import { Entity } from '~codeyam/types';
 import isolateScopes from './isolateScopes';
 import analyzeScope from './analyzeScope';
-import { FileAnalyzer } from '~codeyam/analyze';
+import { FileAnalyzer, transformationTracer } from '~codeyam/analyze';
 import { AI, SerializableDataStructure } from '~codeyam/ai';
-import { resetScopeDataStructureMetrics } from './dataStructure/ScopeDataStructure';
+import {
+  resetScopeDataStructureMetrics,
+  ScopeDataStructure,
+} from './dataStructure/ScopeDataStructure';
 
 // import { awsLog } from '~codeyam/utils';
 
@@ -91,8 +94,60 @@ export default async function generateEntityDataStructure({
   }
 
   // Inspect the runtime type of dataStructure to see if it must be serialized
+  // Note: When worker threads are used, dataStructure is already serialized (no toSerializable).
+  // Workers are disabled when CODEYAM_TRACE_TRANSFORMS=1 to enable full tracing.
   if ('toSerializable' in entityScope.dataStructure) {
-    return entityScope.dataStructure.toSerializable();
+    const scopeDataStructure =
+      entityScope.dataStructure as unknown as ScopeDataStructure;
+
+    // Trace the internal state BEFORE serialization - captures what methods return
+    if (transformationTracer.isEnabled()) {
+      transformationTracer.startEntity({
+        name: entity.name,
+        entityType: entity.entityType ?? 'unknown',
+        filePath: entity.filePath,
+      });
+
+      // Capture internal state - this is what getReturnValue/getFunctionSignature return
+      transformationTracer.snapshot(entity.name, 'scopeDataStructure', {
+        signatureSchema: scopeDataStructure.getFunctionSignature({
+          fillInUnknowns: true,
+        }),
+        returnValueSchema: scopeDataStructure.getReturnValue({
+          fillInUnknowns: true,
+        }),
+      });
+    }
+
+    const serialized = scopeDataStructure.toSerializable();
+
+    // Trace AFTER serialization - shows what gets output
+    if (transformationTracer.isEnabled()) {
+      // Build schema from external function calls for comparison
+      const externalSchemas: Record<string, string> = {};
+      for (const efc of serialized.externalFunctionCalls ?? []) {
+        for (const [path, type] of Object.entries(efc.schema ?? {})) {
+          externalSchemas[path] = type;
+        }
+      }
+
+      transformationTracer.snapshot(entity.name, 'serialized', {
+        signatureSchema:
+          serialized.functionResults?.[entity.name]?.signatureWithUnknowns,
+        returnValueSchema:
+          serialized.functionResults?.[entity.name]?.returnValueWithUnknowns,
+        // Include external function schemas to detect serialization gaps
+        dependencySchemas: {
+          externalFunctions: {
+            combined: {
+              returnValueSchema: externalSchemas,
+            },
+          },
+        },
+      });
+    }
+
+    return serialized;
   }
   return entityScope.dataStructure;
 }