@codeyam/codeyam-cli 0.1.0-staging.79ef713 → 0.1.0-staging.8421896

package/analyzer-template/packages/ai/src/lib/dataStructureChunking.ts

@@ -3,9 +3,11 @@ import type { ExecutionFlow, ScenariosDataStructure } from '~codeyam/types';
 // Type for a single required value from ExecutionFlow
 type RequiredValue = NonNullable<ExecutionFlow['requiredValues']>[number];
 
-const DEFAULT_MAX_KEYS_PER_CHUNK = 8;
+const DEFAULT_MAX_CHUNK_SIZE = 10_000; // ~10K chars per chunk
 
 export interface ChunkOptions {
+  maxChunkSize?: number;
+  /** @deprecated Use maxChunkSize instead. Kept for backward compatibility in tests. */
   maxKeysPerChunk?: number;
 }
 
@@ -15,8 +17,12 @@ export interface ChunkOptions {
  * Large schemas overwhelm LLMs, causing them to make mistakes on some keys.
  * By processing smaller chunks, each key gets more focused attention.
  *
+ * Uses cumulative JSON size to decide chunk boundaries rather than a fixed
+ * key count, so a single oversized key gets its own chunk while many small
+ * keys are grouped together efficiently.
+ *
  * @param dataForMocks - The full data structure schema
- * @param options - Chunking options (maxKeysPerChunk defaults to 8)
+ * @param options - Chunking options (maxChunkSize defaults to 10_000)
  * @returns Array of smaller data structure chunks
  */
 export function chunkDataStructure(
@@ -27,27 +33,39 @@ export function chunkDataStructure(
     return [dataForMocks];
   }
 
-  const { maxKeysPerChunk = DEFAULT_MAX_KEYS_PER_CHUNK } = options;
+  const { maxChunkSize = DEFAULT_MAX_CHUNK_SIZE } = options;
   const keys = Object.keys(dataForMocks);
 
-  // Don't chunk if already small enough
-  if (keys.length <= maxKeysPerChunk) {
+  // Calculate total size to see if chunking is needed
+  const totalSize = JSON.stringify(dataForMocks).length;
+  if (totalSize <= maxChunkSize) {
     return [dataForMocks];
   }
 
+  // Greedily pack keys into chunks by cumulative size
   const chunks: Array<ScenariosDataStructure['dataForMocks']> = [];
-
-  for (let i = 0; i < keys.length; i += maxKeysPerChunk) {
-    const chunkKeys = keys.slice(i, i + maxKeysPerChunk);
-    const chunk: ScenariosDataStructure['dataForMocks'] = {};
-
-    for (const key of chunkKeys) {
-      (chunk as Record<string, unknown>)[key] = (
-        dataForMocks as Record<string, unknown>
-      )[key];
+  let currentChunk: Record<string, unknown> = {};
+  let currentSize = 0;
+
+  for (const key of keys) {
+    const keySize = JSON.stringify(
+      (dataForMocks as Record<string, unknown>)[key],
+    ).length;
+
+    // If adding this key would exceed the limit AND chunk isn't empty, start a new chunk
+    if (currentSize > 0 && currentSize + keySize > maxChunkSize) {
+      chunks.push(currentChunk as ScenariosDataStructure['dataForMocks']);
+      currentChunk = {};
+      currentSize = 0;
     }
 
-    chunks.push(chunk);
+    currentChunk[key] = (dataForMocks as Record<string, unknown>)[key];
+    currentSize += keySize;
+  }
+
+  // Push the last chunk
+  if (Object.keys(currentChunk).length > 0) {
+    chunks.push(currentChunk as ScenariosDataStructure['dataForMocks']);
   }
 
   return chunks;

package/analyzer-template/packages/ai/src/lib/generateEntityScenarioData.ts

@@ -294,11 +294,13 @@ function generateDefaultForSchemaType(schemaType: unknown): unknown {
     if (schemaType.includes('| null')) return null;
     return schemaType; // Return the type as a string placeholder
   }
-  if (
-    typeof schemaType === 'object' &&
-    schemaType !== null &&
-    !Array.isArray(schemaType)
-  ) {
+  if (Array.isArray(schemaType)) {
+    if (schemaType.length === 0) return [];
+    // Generate a single default element based on the first element's schema
+    const elementDefault = generateDefaultForSchemaType(schemaType[0]);
+    return elementDefault !== undefined ? [elementDefault] : [];
+  }
+  if (typeof schemaType === 'object' && schemaType !== null) {
     // Recursively generate defaults for nested objects
     const result: Record<string, unknown> = {};
     for (const [key, value] of Object.entries(schemaType)) {
@@ -426,6 +428,8 @@ function fillMissingMockDataKeysWithDefaults(
   const missingKeys: string[] = [];
 
   for (const key of Object.keys(dataForMocks)) {
+    if (key === '_nullable') continue; // Internal marker, not a data key
+
     const fullPath = pathPrefix ? `${pathPrefix}.${key}` : key;
 
     if (mockData[key] === undefined) {
@@ -841,6 +845,29 @@ export async function generateDataForScenario({
         }
       }
 
+      // Detect keys that were lost from failed or partial chunk responses
+      // and fill them with schema-based defaults so they aren't permanently lost.
+      const allChunkedKeys = chunks.flatMap((c) => Object.keys(c || {}));
+      const returnedKeys = new Set(Object.keys(chunkedMockData));
+      const missingChunkKeys = allChunkedKeys.filter(
+        (k) => !returnedKeys.has(k),
+      );
+
+      if (missingChunkKeys.length > 0) {
+        awsLog(
+          `Chunked processing: ${missingChunkKeys.length} key(s) missing from chunk results, filling with defaults: ${missingChunkKeys.join(', ')}`,
+        );
+        const dataForMocksRecord = structure.dataForMocks as Record<
+          string,
+          unknown
+        >;
+        for (const key of missingChunkKeys) {
+          chunkedMockData[key] = generateDefaultForSchemaType(
+            dataForMocksRecord[key],
+          );
+        }
+      }
+
       awsLog(
         `Chunked processing complete. Generated ${Object.keys(chunkedMockData).length} keys total`,
       );

package/analyzer-template/packages/ai/src/lib/generateExecutionFlows.ts

@@ -105,6 +105,19 @@ export default function generateExecutionFlows({
     fullToShortPathMap[fullPath] = shortPath;
   }
 
+  console.log(
+    `[generateExecutionFlows] ${entity.name}: attributesMap has ${Object.keys(resolvedAttributesMap).length} keys, fullToShortPathMap has ${Object.keys(fullToShortPathMap).length} entries`,
+  );
+  console.log(
+    `[generateExecutionFlows] ${entity.name}: attributesMap keys: [${Object.keys(resolvedAttributesMap).join(', ')}]`,
+  );
+  console.log(
+    `[generateExecutionFlows] ${entity.name}: fullToShortPathMap: ${JSON.stringify(fullToShortPathMap)}`,
+  );
+  console.log(
+    `[generateExecutionFlows] ${entity.name}: equivalentSignatureVariables: ${JSON.stringify(equivalentSignatureVariables)}`,
+  );
+
   // Get pre-computed conditional data from metadata
   const conditionalUsages: Record<string, ConditionalUsage[]> =
     entity.metadata?.isolatedDataStructure?.conditionalUsages ?? {};
@@ -149,6 +162,26 @@ export default function generateExecutionFlows({
     }
   }
 
+  console.log(
+    `[generateExecutionFlows] ${entity.name}: conditionalUsages keys: [${Object.keys(conditionalUsages).join(', ')}]`,
+  );
+  console.log(
+    `[generateExecutionFlows] ${entity.name}: conditionalUsages sourceDataPaths: ${JSON.stringify(
+      Object.fromEntries(
+        Object.entries(conditionalUsages).map(([k, v]) => [
+          k,
+          (v as any[]).map((u: any) => u.sourceDataPath ?? '(none)'),
+        ]),
+      ),
+    )}`,
+  );
+  console.log(
+    `[generateExecutionFlows] ${entity.name}: compoundConditionals count: ${compoundConditionals.length}`,
+  );
+  console.log(
+    `[generateExecutionFlows] ${entity.name}: sourceEquivalencies keys count: ${Object.keys(mergedDataStructure.sourceEquivalencies ?? {}).length}`,
+  );
+
   const conditionalsBasedFlows = generateExecutionFlowsFromConditionals({
     conditionalUsages,
     compoundConditionals,
@@ -503,8 +536,11 @@ function buildValueGroupMap(flows: ExecutionFlow[]): Map<string, string[]> {
   for (const [, flowValues] of pathToValues) {
     // Only create groups if there are multiple different values for the same path
     const uniqueValues = new Set(flowValues.map((fv) => fv.value));
-    if (uniqueValues.size > 1) {
-      // This attribute has multiple values, so all flows are siblings
+    if (uniqueValues.size > 1 && flowValues.length <= 10) {
+      // This attribute has multiple values, so all flows are siblings.
+      // Skip large groups (>10 flows) — these are typically shared gating conditions
+      // (like route params) that would dominate the selection by pulling in too many
+      // siblings and displacing higher-priority flows.
       const siblingIds = flowValues.map((fv) => fv.flowId);
       for (const fv of flowValues) {
         valueGroupMap.set(