@codeyam/codeyam-cli 0.1.0-staging.e38f7bd → 0.1.0-staging.fef152f

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/fixNullIdsBySchema.ts

@@ -0,0 +1,129 @@
+import { awsLog } from '~codeyam/utils';
+
+/**
+ * Fixes null values for ID fields when the schema indicates they should be non-null numbers.
+ *
+ * The LLM sometimes generates `null` for ID fields (e.g., `"id": null`) when the schema
+ * type is `"number"` or `"number | undefined"` (but NOT `"number | null"`). This causes
+ * runtime issues when code checks `if (!data?.id)` expecting a truthy value.
+ *
+ * This function traverses data and schema together:
+ * - If field name looks like an ID field AND value is null AND schema doesn't allow null,
+ *   replace with a valid default value (1 for numbers, generated UUID for strings)
+ *
+ * @param data The data object with potential null ID values
+ * @param schema The schema describing expected types
+ * @param path Current path in the data structure (for logging)
+ * @returns The data with null IDs fixed (mutates original)
+ */
+export default function fixNullIdsBySchema<T>(
+  data: T,
+  schema: unknown,
+  path: string = '',
+): T {
+  // Handle null/undefined data
+  if (data === null || data === undefined) {
+    return data;
+  }
+
+  // Only process objects
+  if (typeof data !== 'object') {
+    return data;
+  }
+
+  // Handle arrays
+  if (Array.isArray(data)) {
+    // For arrays, the schema should be an array with one element describing the item type
+    const itemSchema = Array.isArray(schema) ? schema[0] : undefined;
+    for (let i = 0; i < data.length; i++) {
+      data[i] = fixNullIdsBySchema(data[i], itemSchema, `${path}[${i}]`);
+    }
+    return data;
+  }
+
+  // Handle objects
+  const dataObj = data as Record<string, unknown>;
+  const schemaObj = schema as Record<string, unknown> | undefined;
+
+  for (const key of Object.keys(dataObj)) {
+    const value = dataObj[key];
+    const fieldSchema = schemaObj?.[key];
+    const fieldPath = path ? `${path}.${key}` : key;
+
+    if (value === null && isIdField(key)) {
+      // Check if schema indicates this should be a non-null number
+      const replacement = getIdReplacementValue(fieldSchema);
+      if (replacement !== null) {
+        dataObj[key] = replacement;
+        awsLog(
+          `CodeYam: Fixed null ID field "${fieldPath}": null -> ${replacement}`,
+        );
+      }
+    } else if (typeof value === 'object' && value !== null) {
+      // Recursively process nested objects/arrays
+      fixNullIdsBySchema(value, fieldSchema, fieldPath);
+    }
+  }
+
+  return data;
+}
+
+/**
+ * Pattern for field names that are likely ID fields.
+ * Matches: id, Id, ID, userId, user_id, quoteId, quote_id, etc.
+ */
+const ID_FIELD_PATTERNS = [
+  /^id$/i, // Exact match for "id", "Id", "ID"
+  /Id$/, // Ends with "Id" (camelCase): userId, quoteId
+  /_id$/i, // Ends with "_id" (snake_case): user_id, quote_id
+  /^.*_id$/i, // Contains "_id": some_object_id
+];
+
+/**
+ * Determines if a field name looks like an ID field.
+ */
+function isIdField(fieldName: string): boolean {
+  return ID_FIELD_PATTERNS.some((pattern) => pattern.test(fieldName));
+}
+
+/**
+ * Gets the replacement value for a null ID based on schema type.
+ * Returns null if the schema allows null values (no replacement needed).
+ *
+ * @param fieldSchema The schema type for the field
+ * @returns Replacement value (1 for numbers, generated string for strings) or null if no replacement
+ */
+function getIdReplacementValue(fieldSchema: unknown): number | string | null {
+  // If no schema for this field, be conservative - don't replace
+  if (fieldSchema === undefined || fieldSchema === null) {
+    return null;
+  }
+
+  // If schema is a string type definition
+  if (typeof fieldSchema === 'string') {
+    // If schema explicitly allows null, don't replace
+    if (fieldSchema.includes('null')) {
+      return null;
+    }
+
+    // If schema is a number type, return 1 as default
+    if (
+      fieldSchema === 'number' ||
+      fieldSchema.startsWith('number') ||
+      fieldSchema.includes('number')
+    ) {
+      return 1;
+    }
+
+    // If schema is a string type, return a generated ID
+    if (
+      fieldSchema === 'string' ||
+      fieldSchema.startsWith('string') ||
+      fieldSchema.includes('string')
+    ) {
+      return 'id-1';
+    }
+  }
+
+  return null;
+}

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/stripNullableMarkers.ts

@@ -0,0 +1,35 @@
+/**
+ * Recursively strips `_nullable` keys from LLM-generated mock data.
+ *
+ * The `_nullable` marker is an internal CodeYam concept used in type definitions
+ * to indicate that a field can be null/undefined. The LLM sometimes includes
+ * these markers in its generated scenario data, which causes runtime errors
+ * when code iterates over object keys (e.g., `Object.keys(importedBy)` picks
+ * up `_nullable` as a key alongside real data).
+ *
+ * This function mutates the input object in-place, consistent with other
+ * post-processing helpers like `convertNullToUndefinedBySchema`.
+ */
+export default function stripNullableMarkers(
+  data: Record<string, unknown>,
+): void {
+  if (data == null || typeof data !== 'object') return;
+
+  // Delete _nullable from this level
+  if ('_nullable' in data) {
+    delete data._nullable;
+  }
+
+  // Recurse into nested objects and arrays
+  for (const value of Object.values(data)) {
+    if (Array.isArray(value)) {
+      for (const item of value) {
+        if (item !== null && typeof item === 'object' && !Array.isArray(item)) {
+          stripNullableMarkers(item as Record<string, unknown>);
+        }
+      }
+    } else if (value !== null && typeof value === 'object') {
+      stripNullableMarkers(value as Record<string, unknown>);
+    }
+  }
+}

package/analyzer-template/packages/ai/src/lib/dataStructureChunking.ts

@@ -0,0 +1,183 @@
+import type { ExecutionFlow, ScenariosDataStructure } from '~codeyam/types';
+
+// Type for a single required value from ExecutionFlow
+type RequiredValue = NonNullable<ExecutionFlow['requiredValues']>[number];
+
+const DEFAULT_MAX_CHUNK_SIZE = 10_000; // ~10K chars per chunk
+const DEFAULT_MAX_KEYS_PER_CHUNK = 20;
+
+export interface ChunkOptions {
+  maxChunkSize?: number;
+  maxKeysPerChunk?: number;
+}
+
+/**
+ * Chunk a large data structure into smaller pieces for focused LLM processing.
+ *
+ * Large schemas overwhelm LLMs, causing them to make mistakes on some keys.
+ * By processing smaller chunks, each key gets more focused attention.
+ *
+ * Uses cumulative JSON size to decide chunk boundaries rather than a fixed
+ * key count, so a single oversized key gets its own chunk while many small
+ * keys are grouped together efficiently.
+ *
+ * @param dataForMocks - The full data structure schema
+ * @param options - Chunking options (maxChunkSize defaults to 10_000)
+ * @returns Array of smaller data structure chunks
+ */
+export function chunkDataStructure(
+  dataForMocks: ScenariosDataStructure['dataForMocks'],
+  options: ChunkOptions = {},
+): Array<ScenariosDataStructure['dataForMocks']> {
+  if (!dataForMocks || typeof dataForMocks !== 'object') {
+    return [dataForMocks];
+  }
+
+  const {
+    maxChunkSize = DEFAULT_MAX_CHUNK_SIZE,
+    maxKeysPerChunk = DEFAULT_MAX_KEYS_PER_CHUNK,
+  } = options;
+  const keys = Object.keys(dataForMocks);
+
+  // Calculate total size to see if chunking is needed
+  const totalSize = JSON.stringify(dataForMocks).length;
+  if (totalSize <= maxChunkSize && keys.length <= maxKeysPerChunk) {
+    return [dataForMocks];
+  }
+
+  // Greedily pack keys into chunks by cumulative size and key count
+  const chunks: Array<ScenariosDataStructure['dataForMocks']> = [];
+  let currentChunk: Record<string, unknown> = {};
+  let currentSize = 0;
+
+  for (const key of keys) {
+    const keySize = JSON.stringify(
+      (dataForMocks as Record<string, unknown>)[key],
+    ).length;
+
+    const currentKeyCount = Object.keys(currentChunk).length;
+
+    // If adding this key would exceed size or key count limit AND chunk isn't empty, start a new chunk
+    if (
+      currentSize > 0 &&
+      (currentSize + keySize > maxChunkSize ||
+        currentKeyCount >= maxKeysPerChunk)
+    ) {
+      chunks.push(currentChunk as ScenariosDataStructure['dataForMocks']);
+      currentChunk = {};
+      currentSize = 0;
+    }
+
+    currentChunk[key] = (dataForMocks as Record<string, unknown>)[key];
+    currentSize += keySize;
+  }
+
+  // Push the last chunk
+  if (Object.keys(currentChunk).length > 0) {
+    chunks.push(currentChunk as ScenariosDataStructure['dataForMocks']);
+  }
+
+  return chunks;
+}
+
+/**
+ * Get the requiredValues that are relevant to a specific chunk.
+ *
+ * When processing a chunk, we should include any requiredValues from the covered
+ * flows that reference keys in that chunk. This helps the LLM understand what
+ * values are expected for those keys.
+ *
+ * @param chunk - A chunk of the data structure
+ * @param executionFlows - All execution flows for the entity
+ * @param coveredFlowIds - The flow IDs covered by the current scenario
+ * @returns Array of requiredValues that reference keys in this chunk
+ */
+export function getRequiredValuesForChunk(
+  chunk: ScenariosDataStructure['dataForMocks'],
+  executionFlows: ExecutionFlow[],
+  coveredFlowIds: string[],
+): RequiredValue[] {
+  if (!chunk || typeof chunk !== 'object') {
+    return [];
+  }
+
+  const chunkKeys = Object.keys(chunk);
+  const relevantValues: RequiredValue[] = [];
+
+  // Get the covered flows
+  const coveredFlows = executionFlows.filter((flow) =>
+    coveredFlowIds.includes(flow.id),
+  );
+
+  for (const flow of coveredFlows) {
+    if (!flow.requiredValues) continue;
+
+    for (const rv of flow.requiredValues) {
+      if (!rv.attributePath) continue;
+
+      // Check if this requiredValue references a key in this chunk
+      for (const chunkKey of chunkKeys) {
+        if (pathMatchesKey(rv.attributePath, chunkKey)) {
+          relevantValues.push(rv);
+          break; // Don't add the same rv multiple times
+        }
+      }
+    }
+  }
+
+  return relevantValues;
+}
+
+/**
+ * Check if an attributePath references a given dataForMocks key.
+ * (Same logic as in extractCriticalDataKeys)
+ */
+function pathMatchesKey(attributePath: string, dataKey: string): boolean {
+  const normalizedPath = attributePath.replace('.functionCallReturnValue', '');
+
+  if (normalizedPath.startsWith(dataKey)) {
+    const afterKey = normalizedPath.slice(dataKey.length);
+    if (
+      afterKey === '' ||
+      afterKey.startsWith('.') ||
+      afterKey.startsWith('[')
+    ) {
+      return true;
+    }
+  }
+
+  // Try matching without type parameters
+  const keyWithoutTypes = removeTypeParameters(dataKey);
+  const pathWithoutTypes = removeTypeParameters(normalizedPath);
+
+  if (pathWithoutTypes.startsWith(keyWithoutTypes)) {
+    const afterKey = pathWithoutTypes.slice(keyWithoutTypes.length);
+    if (
+      afterKey === '' ||
+      afterKey.startsWith('.') ||
+      afterKey.startsWith('[')
+    ) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+/**
+ * Remove type parameters from a function call string.
+ */
+function removeTypeParameters(str: string): string {
+  let result = str;
+  let prevLength = 0;
+
+  while (result.length !== prevLength) {
+    prevLength = result.length;
+    result = result.replace(/<[^<>]*>/g, '');
+  }
+
+  return result;
+}
+
+// Re-export extractCriticalDataKeys for backwards compatibility
+export { extractCriticalDataKeys } from './extractCriticalDataKeys';

package/analyzer-template/packages/ai/src/lib/deepEqual.ts

@@ -0,0 +1,30 @@
+export default function deepEqual(obj1: unknown, obj2: unknown): boolean {
+  // Handle primitive types and null/undefined
+  if (obj1 === obj2) return true;
+  if (obj1 == null || obj2 == null) return false;
+  if (typeof obj1 !== 'object' || typeof obj2 !== 'object') return false;
+
+  // Handle Date objects
+  if (obj1 instanceof Date && obj2 instanceof Date) {
+    return obj1.getTime() === obj2.getTime();
+  }
+
+  // Handle Array objects
+  if (Array.isArray(obj1) && Array.isArray(obj2)) {
+    if (obj1.length !== obj2.length) return false;
+    return obj1.every((item, index) => deepEqual(item, obj2[index]));
+  }
+
+  // Handle regular objects
+  const keys1 = Object.keys(obj1);
+  const keys2 = Object.keys(obj2);
+
+  if (keys1.length !== keys2.length) return false;
+
+  return keys1.every((key) => {
+    if (!Object.prototype.hasOwnProperty.call(obj2, key)) return false;
+    const obj1AsAny = obj1 as any;
+    const obj2AsAny = obj2 as any;
+    return deepEqual(obj1AsAny[key], obj2AsAny[key]);
+  });
+}

package/analyzer-template/packages/ai/src/lib/e2eDataTracking.ts

@@ -0,0 +1,334 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import * as crypto from 'crypto';
+
+/**
+ * E2E Data Tracking Utility
+ *
+ * This module helps debug discrepancies between first and second E2E test runs.
+ * The first run makes real LLM calls and saves the results.
+ * The second run uses cached LLM responses but sometimes produces different data.
+ *
+ * Usage:
+ *   Set CODEYAM_E2E_TRACK_DATA=true to enable tracking
+ *   Data is saved to /tmp/codeyam-e2e-tracking/
+ *
+ * First run (no fixtures): Saves snapshots of key data points
+ * Second run (with fixtures): Compares current data against saved snapshots
+ */
+
+const TRACKING_DIR = '/tmp/codeyam-e2e-tracking';
+
+interface DataSnapshot {
+  timestamp: number;
+  checkpoint: string;
+  entityName?: string;
+  scenarioName?: string;
+  dataHash: string;
+  data: unknown;
+}
+
+interface ComparisonResult {
+  checkpoint: string;
+  entityName?: string;
+  scenarioName?: string;
+  matches: boolean;
+  firstRunHash?: string;
+  secondRunHash?: string;
+  differences?: string[];
+}
+
+let trackingEnabled: boolean | undefined;
+let isFirstRun: boolean | undefined;
+let comparisonResults: ComparisonResult[] = [];
+
+function isTrackingEnabled(): boolean {
+  if (trackingEnabled === undefined) {
+    trackingEnabled = process.env.CODEYAM_E2E_TRACK_DATA === 'true';
+  }
+  return trackingEnabled;
+}
+
+function determineIfFirstRun(): boolean {
+  if (isFirstRun === undefined) {
+    // First run = no LLM fixtures directory
+    isFirstRun = !process.env.CODEYAM_LLM_FIXTURES_DIR;
+  }
+  return isFirstRun;
+}
+
+function ensureTrackingDir(): void {
+  if (!fs.existsSync(TRACKING_DIR)) {
+    fs.mkdirSync(TRACKING_DIR, { recursive: true });
+  }
+}
+
+function hashData(data: unknown): string {
+  const serialized = JSON.stringify(data, null, 0);
+  return crypto.createHash('md5').update(serialized).digest('hex');
+}
+
+function getSnapshotFilename(
+  checkpoint: string,
+  entityName?: string,
+  scenarioName?: string,
+): string {
+  const parts = [checkpoint];
+  if (entityName) parts.push(entityName.replace(/[^a-zA-Z0-9]/g, '_'));
+  if (scenarioName)
+    parts.push(scenarioName.replace(/[^a-zA-Z0-9]/g, '_').substring(0, 30));
+  return parts.join('_') + '.json';
+}
+
+/**
+ * Track a data snapshot at a specific checkpoint.
+ *
+ * On first run: Saves the data to a file for later comparison.
+ * On second run: Compares with saved data and logs any differences.
+ *
+ * @param checkpoint - Unique identifier for this tracking point (e.g., 'completionCall_result', 'parsed_scenarioData')
+ * @param data - The data to track
+ * @param entityName - Optional entity name for context
+ * @param scenarioName - Optional scenario name for context
+ */
+export function trackDataSnapshot(
+  checkpoint: string,
+  data: unknown,
+  entityName?: string,
+  scenarioName?: string,
+): void {
+  if (!isTrackingEnabled()) return;
+
+  ensureTrackingDir();
+
+  const filename = getSnapshotFilename(checkpoint, entityName, scenarioName);
+  const filepath = path.join(TRACKING_DIR, filename);
+  const currentHash = hashData(data);
+
+  if (determineIfFirstRun()) {
+    // First run: Save the snapshot
+    const snapshot: DataSnapshot = {
+      timestamp: Date.now(),
+      checkpoint,
+      entityName,
+      scenarioName,
+      dataHash: currentHash,
+      data,
+    };
+    fs.writeFileSync(filepath, JSON.stringify(snapshot, null, 2));
+    console.log(
+      `[E2E Tracking] First run - saved snapshot: ${checkpoint}${entityName ? ` (${entityName})` : ''}${scenarioName ? ` [${scenarioName}]` : ''} hash=${currentHash.substring(0, 8)}`,
+    );
+  } else {
+    // Second run: Compare with saved snapshot
+    if (fs.existsSync(filepath)) {
+      const savedSnapshot: DataSnapshot = JSON.parse(
+        fs.readFileSync(filepath, 'utf-8'),
+      );
+
+      const result: ComparisonResult = {
+        checkpoint,
+        entityName,
+        scenarioName,
+        matches: currentHash === savedSnapshot.dataHash,
+        firstRunHash: savedSnapshot.dataHash,
+        secondRunHash: currentHash,
+      };
+
+      if (!result.matches) {
+        result.differences = findDifferences(savedSnapshot.data, data);
+        console.log(
+          `[E2E Tracking] MISMATCH at ${checkpoint}${entityName ? ` (${entityName})` : ''}${scenarioName ? ` [${scenarioName}]` : ''}`,
+        );
+        console.log(`  First run hash:  ${savedSnapshot.dataHash}`);
+        console.log(`  Second run hash: ${currentHash}`);
+
+        // Save both versions for comparison
+        const diffFilepath = filepath.replace('.json', '_DIFF.json');
+        fs.writeFileSync(
+          diffFilepath,
+          JSON.stringify(
+            {
+              checkpoint,
+              entityName,
+              scenarioName,
+              firstRun: savedSnapshot.data,
+              secondRun: data,
+              differences: result.differences,
+            },
+            null,
+            2,
+          ),
+        );
+        console.log(`  Diff saved to: ${diffFilepath}`);
+      } else {
+        console.log(
+          `[E2E Tracking] Match at ${checkpoint}${entityName ? ` (${entityName})` : ''} hash=${currentHash.substring(0, 8)}`,
+        );
+      }
+
+      comparisonResults.push(result);
+    } else {
+      console.log(
+        `[E2E Tracking] No first-run snapshot found for: ${checkpoint}${entityName ? ` (${entityName})` : ''}`,
+      );
+    }
+  }
+}
+
+/**
+ * Find the differences between two data structures.
+ * Returns a list of paths where values differ.
+ */
+function findDifferences(
+  obj1: unknown,
+  obj2: unknown,
+  path: string = '',
+): string[] {
+  const differences: string[] = [];
+
+  if (typeof obj1 !== typeof obj2) {
+    differences.push(
+      `${path || 'root'}: type mismatch (${typeof obj1} vs ${typeof obj2})`,
+    );
+    return differences;
+  }
+
+  if (obj1 === null || obj2 === null) {
+    if (obj1 !== obj2) {
+      differences.push(
+        `${path || 'root'}: ${JSON.stringify(obj1)} vs ${JSON.stringify(obj2)}`,
+      );
+    }
+    return differences;
+  }
+
+  if (Array.isArray(obj1) && Array.isArray(obj2)) {
+    if (obj1.length !== obj2.length) {
+      differences.push(
+        `${path || 'root'}: array length ${obj1.length} vs ${obj2.length}`,
+      );
+    }
+    const maxLen = Math.max(obj1.length, obj2.length);
+    for (let i = 0; i < maxLen; i++) {
+      differences.push(...findDifferences(obj1[i], obj2[i], `${path}[${i}]`));
+    }
+    return differences;
+  }
+
+  if (typeof obj1 === 'object' && typeof obj2 === 'object') {
+    const keys1 = Object.keys(obj1 as Record<string, unknown>);
+    const keys2 = Object.keys(obj2 as Record<string, unknown>);
+    const allKeys = Array.from(new Set([...keys1, ...keys2]));
+
+    for (const key of allKeys) {
+      const val1 = (obj1 as Record<string, unknown>)[key];
+      const val2 = (obj2 as Record<string, unknown>)[key];
+
+      if (!(key in (obj1 as Record<string, unknown>))) {
+        differences.push(
+          `${path ? path + '.' : ''}${key}: missing in first run`,
+        );
+      } else if (!(key in (obj2 as Record<string, unknown>))) {
+        differences.push(
+          `${path ? path + '.' : ''}${key}: missing in second run`,
+        );
+      } else {
+        differences.push(
+          ...findDifferences(val1, val2, `${path ? path + '.' : ''}${key}`),
+        );
+      }
+    }
+    return differences;
+  }
+
+  // Primitive comparison
+  if (obj1 !== obj2) {
+    const str1 = JSON.stringify(obj1);
+    const str2 = JSON.stringify(obj2);
+    if (str1.length < 100 && str2.length < 100) {
+      differences.push(`${path || 'root'}: ${str1} vs ${str2}`);
+    } else {
+      differences.push(
+        `${path || 'root'}: values differ (${str1.length} chars vs ${str2.length} chars)`,
+      );
+    }
+  }
+
+  return differences;
+}
+
+/**
+ * Get a summary of all comparison results.
+ * Call this at the end of the second run to see overall results.
+ */
+export function getTrackingSummary(): {
+  total: number;
+  matches: number;
+  mismatches: number;
+  results: ComparisonResult[];
+} {
+  const matches = comparisonResults.filter((r) => r.matches).length;
+  const mismatches = comparisonResults.filter((r) => !r.matches).length;
+
+  return {
+    total: comparisonResults.length,
+    matches,
+    mismatches,
+    results: comparisonResults,
+  };
+}
+
+/**
+ * Print a summary of tracking results to console.
+ */
+export function printTrackingSummary(): void {
+  if (!isTrackingEnabled()) return;
+
+  const summary = getTrackingSummary();
+  console.log('\n========== E2E Data Tracking Summary ==========');
+  console.log(`Total checkpoints: ${summary.total}`);
+  console.log(`Matches: ${summary.matches}`);
+  console.log(`Mismatches: ${summary.mismatches}`);
+
+  if (summary.mismatches > 0) {
+    console.log('\nMismatched checkpoints:');
+    for (const result of summary.results.filter((r) => !r.matches)) {
+      console.log(
+        `  - ${result.checkpoint}${result.entityName ? ` (${result.entityName})` : ''}${result.scenarioName ? ` [${result.scenarioName}]` : ''}`,
+      );
+      if (result.differences && result.differences.length <= 10) {
+        for (const diff of result.differences) {
+          console.log(`      ${diff}`);
+        }
+      } else if (result.differences) {
+        console.log(
+          `      (${result.differences.length} differences - see diff file)`,
+        );
+      }
+    }
+  }
+  console.log('================================================\n');
+}
+
+/**
+ * Clear all tracking data. Call this before starting a new test session.
+ */
+export function clearTrackingData(): void {
+  if (fs.existsSync(TRACKING_DIR)) {
+    const files = fs.readdirSync(TRACKING_DIR);
+    for (const file of files) {
+      fs.unlinkSync(path.join(TRACKING_DIR, file));
+    }
+  }
+  comparisonResults = [];
+  isFirstRun = undefined;
+}
+
+/**
+ * Reset tracking state for a new test run (keeps files, clears in-memory state).
+ */
+export function resetTrackingState(): void {
+  comparisonResults = [];
+  isFirstRun = undefined;
+}