@codeyam/codeyam-cli 0.1.0-staging.596f0eb → 0.1.0-staging.6e699e5

package/analyzer-template/packages/ai/src/lib/e2eDataTracking.ts

@@ -0,0 +1,334 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import * as crypto from 'crypto';
+
+/**
+ * E2E Data Tracking Utility
+ *
+ * This module helps debug discrepancies between first and second E2E test runs.
+ * The first run makes real LLM calls and saves the results.
+ * The second run uses cached LLM responses but sometimes produces different data.
+ *
+ * Usage:
+ *   Set CODEYAM_E2E_TRACK_DATA=true to enable tracking
+ *   Data is saved to /tmp/codeyam-e2e-tracking/
+ *
+ * First run (no fixtures): Saves snapshots of key data points
+ * Second run (with fixtures): Compares current data against saved snapshots
+ */
+
+const TRACKING_DIR = '/tmp/codeyam-e2e-tracking';
+
+interface DataSnapshot {
+  timestamp: number;
+  checkpoint: string;
+  entityName?: string;
+  scenarioName?: string;
+  dataHash: string;
+  data: unknown;
+}
+
+interface ComparisonResult {
+  checkpoint: string;
+  entityName?: string;
+  scenarioName?: string;
+  matches: boolean;
+  firstRunHash?: string;
+  secondRunHash?: string;
+  differences?: string[];
+}
+
+let trackingEnabled: boolean | undefined;
+let isFirstRun: boolean | undefined;
+let comparisonResults: ComparisonResult[] = [];
+
+function isTrackingEnabled(): boolean {
+  if (trackingEnabled === undefined) {
+    trackingEnabled = process.env.CODEYAM_E2E_TRACK_DATA === 'true';
+  }
+  return trackingEnabled;
+}
+
+function determineIfFirstRun(): boolean {
+  if (isFirstRun === undefined) {
+    // First run = no LLM fixtures directory
+    isFirstRun = !process.env.CODEYAM_LLM_FIXTURES_DIR;
+  }
+  return isFirstRun;
+}
+
+function ensureTrackingDir(): void {
+  if (!fs.existsSync(TRACKING_DIR)) {
+    fs.mkdirSync(TRACKING_DIR, { recursive: true });
+  }
+}
+
+function hashData(data: unknown): string {
+  const serialized = JSON.stringify(data, null, 0);
+  return crypto.createHash('md5').update(serialized).digest('hex');
+}
+
+function getSnapshotFilename(
+  checkpoint: string,
+  entityName?: string,
+  scenarioName?: string,
+): string {
+  const parts = [checkpoint];
+  if (entityName) parts.push(entityName.replace(/[^a-zA-Z0-9]/g, '_'));
+  if (scenarioName)
+    parts.push(scenarioName.replace(/[^a-zA-Z0-9]/g, '_').substring(0, 30));
+  return parts.join('_') + '.json';
+}
+
+/**
+ * Track a data snapshot at a specific checkpoint.
+ *
+ * On first run: Saves the data to a file for later comparison.
+ * On second run: Compares with saved data and logs any differences.
+ *
+ * @param checkpoint - Unique identifier for this tracking point (e.g., 'completionCall_result', 'parsed_scenarioData')
+ * @param data - The data to track
+ * @param entityName - Optional entity name for context
+ * @param scenarioName - Optional scenario name for context
+ */
+export function trackDataSnapshot(
+  checkpoint: string,
+  data: unknown,
+  entityName?: string,
+  scenarioName?: string,
+): void {
+  if (!isTrackingEnabled()) return;
+
+  ensureTrackingDir();
+
+  const filename = getSnapshotFilename(checkpoint, entityName, scenarioName);
+  const filepath = path.join(TRACKING_DIR, filename);
+  const currentHash = hashData(data);
+
+  if (determineIfFirstRun()) {
+    // First run: Save the snapshot
+    const snapshot: DataSnapshot = {
+      timestamp: Date.now(),
+      checkpoint,
+      entityName,
+      scenarioName,
+      dataHash: currentHash,
+      data,
+    };
+    fs.writeFileSync(filepath, JSON.stringify(snapshot, null, 2));
+    console.log(
+      `[E2E Tracking] First run - saved snapshot: ${checkpoint}${entityName ? ` (${entityName})` : ''}${scenarioName ? ` [${scenarioName}]` : ''} hash=${currentHash.substring(0, 8)}`,
+    );
+  } else {
+    // Second run: Compare with saved snapshot
+    if (fs.existsSync(filepath)) {
+      const savedSnapshot: DataSnapshot = JSON.parse(
+        fs.readFileSync(filepath, 'utf-8'),
+      );
+
+      const result: ComparisonResult = {
+        checkpoint,
+        entityName,
+        scenarioName,
+        matches: currentHash === savedSnapshot.dataHash,
+        firstRunHash: savedSnapshot.dataHash,
+        secondRunHash: currentHash,
+      };
+
+      if (!result.matches) {
+        result.differences = findDifferences(savedSnapshot.data, data);
+        console.log(
+          `[E2E Tracking] MISMATCH at ${checkpoint}${entityName ? ` (${entityName})` : ''}${scenarioName ? ` [${scenarioName}]` : ''}`,
+        );
+        console.log(`  First run hash:  ${savedSnapshot.dataHash}`);
+        console.log(`  Second run hash: ${currentHash}`);
+
+        // Save both versions for comparison
+        const diffFilepath = filepath.replace('.json', '_DIFF.json');
+        fs.writeFileSync(
+          diffFilepath,
+          JSON.stringify(
+            {
+              checkpoint,
+              entityName,
+              scenarioName,
+              firstRun: savedSnapshot.data,
+              secondRun: data,
+              differences: result.differences,
+            },
+            null,
+            2,
+          ),
+        );
+        console.log(`  Diff saved to: ${diffFilepath}`);
+      } else {
+        console.log(
+          `[E2E Tracking] Match at ${checkpoint}${entityName ? ` (${entityName})` : ''} hash=${currentHash.substring(0, 8)}`,
+        );
+      }
+
+      comparisonResults.push(result);
+    } else {
+      console.log(
+        `[E2E Tracking] No first-run snapshot found for: ${checkpoint}${entityName ? ` (${entityName})` : ''}`,
+      );
+    }
+  }
+}
+
+/**
+ * Find the differences between two data structures.
+ * Returns a list of paths where values differ.
+ */
+function findDifferences(
+  obj1: unknown,
+  obj2: unknown,
+  path: string = '',
+): string[] {
+  const differences: string[] = [];
+
+  if (typeof obj1 !== typeof obj2) {
+    differences.push(
+      `${path || 'root'}: type mismatch (${typeof obj1} vs ${typeof obj2})`,
+    );
+    return differences;
+  }
+
+  if (obj1 === null || obj2 === null) {
+    if (obj1 !== obj2) {
+      differences.push(
+        `${path || 'root'}: ${JSON.stringify(obj1)} vs ${JSON.stringify(obj2)}`,
+      );
+    }
+    return differences;
+  }
+
+  if (Array.isArray(obj1) && Array.isArray(obj2)) {
+    if (obj1.length !== obj2.length) {
+      differences.push(
+        `${path || 'root'}: array length ${obj1.length} vs ${obj2.length}`,
+      );
+    }
+    const maxLen = Math.max(obj1.length, obj2.length);
+    for (let i = 0; i < maxLen; i++) {
+      differences.push(...findDifferences(obj1[i], obj2[i], `${path}[${i}]`));
+    }
+    return differences;
+  }
+
+  if (typeof obj1 === 'object' && typeof obj2 === 'object') {
+    const keys1 = Object.keys(obj1 as Record<string, unknown>);
+    const keys2 = Object.keys(obj2 as Record<string, unknown>);
+    const allKeys = Array.from(new Set([...keys1, ...keys2]));
+
+    for (const key of allKeys) {
+      const val1 = (obj1 as Record<string, unknown>)[key];
+      const val2 = (obj2 as Record<string, unknown>)[key];
+
+      if (!(key in (obj1 as Record<string, unknown>))) {
+        differences.push(
+          `${path ? path + '.' : ''}${key}: missing in first run`,
+        );
+      } else if (!(key in (obj2 as Record<string, unknown>))) {
+        differences.push(
+          `${path ? path + '.' : ''}${key}: missing in second run`,
+        );
+      } else {
+        differences.push(
+          ...findDifferences(val1, val2, `${path ? path + '.' : ''}${key}`),
+        );
+      }
+    }
+    return differences;
+  }
+
+  // Primitive comparison
+  if (obj1 !== obj2) {
+    const str1 = JSON.stringify(obj1);
+    const str2 = JSON.stringify(obj2);
+    if (str1.length < 100 && str2.length < 100) {
+      differences.push(`${path || 'root'}: ${str1} vs ${str2}`);
+    } else {
+      differences.push(
+        `${path || 'root'}: values differ (${str1.length} chars vs ${str2.length} chars)`,
+      );
+    }
+  }
+
+  return differences;
+}
+
+/**
+ * Get a summary of all comparison results.
+ * Call this at the end of the second run to see overall results.
+ */
+export function getTrackingSummary(): {
+  total: number;
+  matches: number;
+  mismatches: number;
+  results: ComparisonResult[];
+} {
+  const matches = comparisonResults.filter((r) => r.matches).length;
+  const mismatches = comparisonResults.filter((r) => !r.matches).length;
+
+  return {
+    total: comparisonResults.length,
+    matches,
+    mismatches,
+    results: comparisonResults,
+  };
+}
+
+/**
+ * Print a summary of tracking results to console.
+ */
+export function printTrackingSummary(): void {
+  if (!isTrackingEnabled()) return;
+
+  const summary = getTrackingSummary();
+  console.log('\n========== E2E Data Tracking Summary ==========');
+  console.log(`Total checkpoints: ${summary.total}`);
+  console.log(`Matches: ${summary.matches}`);
+  console.log(`Mismatches: ${summary.mismatches}`);
+
+  if (summary.mismatches > 0) {
+    console.log('\nMismatched checkpoints:');
+    for (const result of summary.results.filter((r) => !r.matches)) {
+      console.log(
+        `  - ${result.checkpoint}${result.entityName ? ` (${result.entityName})` : ''}${result.scenarioName ? ` [${result.scenarioName}]` : ''}`,
+      );
+      if (result.differences && result.differences.length <= 10) {
+        for (const diff of result.differences) {
+          console.log(`      ${diff}`);
+        }
+      } else if (result.differences) {
+        console.log(
+          `      (${result.differences.length} differences - see diff file)`,
+        );
+      }
+    }
+  }
+  console.log('================================================\n');
+}
+
+/**
+ * Clear all tracking data. Call this before starting a new test session.
+ */
+export function clearTrackingData(): void {
+  if (fs.existsSync(TRACKING_DIR)) {
+    const files = fs.readdirSync(TRACKING_DIR);
+    for (const file of files) {
+      fs.unlinkSync(path.join(TRACKING_DIR, file));
+    }
+  }
+  comparisonResults = [];
+  isFirstRun = undefined;
+}
+
+/**
+ * Reset tracking state for a new test run (keeps files, clears in-memory state).
+ */
+export function resetTrackingState(): void {
+  comparisonResults = [];
+  isFirstRun = undefined;
+}

package/analyzer-template/packages/ai/src/lib/extractCriticalDataKeys.ts

@@ -0,0 +1,120 @@
+import type { ExecutionFlow, ScenariosDataStructure } from '~codeyam/types';
+
+/**
+ * Extract the data structure keys that are critical for satisfying execution flow requirements.
+ *
+ * When generating scenario mock data, only a subset of the full dataForMocks structure
+ * is actually needed to satisfy the scenario's covered flows' requiredValues.
+ * This function identifies those critical keys so they can be processed with more focus
+ * (e.g., in a separate LLM call or prioritized in the prompt).
+ *
+ * @param dataForMocks - The full data structure schema
+ * @param executionFlows - All execution flows for the entity
+ * @param coveredFlowIds - The flow IDs covered by the current scenario
+ * @returns Array of top-level keys from dataForMocks that are referenced by the covered flows
+ */
+export function extractCriticalDataKeys(
+  dataForMocks: ScenariosDataStructure['dataForMocks'],
+  executionFlows: ExecutionFlow[],
+  coveredFlowIds: string[],
+): string[] {
+  if (!dataForMocks || typeof dataForMocks !== 'object') {
+    return [];
+  }
+
+  const dataKeys = Object.keys(dataForMocks);
+  const criticalKeys = new Set<string>();
+
+  // Get the covered flows
+  const coveredFlows = executionFlows.filter((flow) =>
+    coveredFlowIds.includes(flow.id),
+  );
+
+  // For each covered flow, check if any requiredValue references a dataForMocks key
+  for (const flow of coveredFlows) {
+    if (!flow.requiredValues) continue;
+
+    for (const rv of flow.requiredValues) {
+      const path = rv.attributePath;
+      if (!path) continue;
+
+      // Try to match the attributePath to a dataForMocks key
+      // The path format is: "functionCall().functionCallReturnValue.property"
+      // or: "functionCall<Type>().functionCallReturnValue.property"
+      // We need to extract the function call part and match it to a key
+
+      for (const key of dataKeys) {
+        if (pathMatchesKey(path, key)) {
+          criticalKeys.add(key);
+          break; // One match per requiredValue is enough
+        }
+      }
+    }
+  }
+
+  return Array.from(criticalKeys);
+}
+
+/**
+ * Check if an attributePath references a given dataForMocks key.
+ *
+ * Handles various path formats:
+ * - "useParams().functionCallReturnValue.*" matches "useParams()"
+ * - "useLoaderData<LoaderData>().functionCallReturnValue.data" matches "useLoaderData<LoaderData>()"
+ * - "useFetcher().data.success" matches "useFetcher()"
+ */
+function pathMatchesKey(attributePath: string, dataKey: string): boolean {
+  // Normalize both for comparison
+  // Remove .functionCallReturnValue from the path
+  const normalizedPath = attributePath.replace('.functionCallReturnValue', '');
+
+  // Check if the path starts with the key (exact match for function call keys)
+  if (normalizedPath.startsWith(dataKey)) {
+    // Ensure it's not a partial match (e.g., "use" shouldn't match "useState")
+    const afterKey = normalizedPath.slice(dataKey.length);
+    // Valid continuations: end of string, '.', or '['
+    if (
+      afterKey === '' ||
+      afterKey.startsWith('.') ||
+      afterKey.startsWith('[')
+    ) {
+      return true;
+    }
+  }
+
+  // Try matching without type parameters if needed
+  // e.g., "useLoaderData<LoaderData>()" should match attributePaths with same base
+  const keyWithoutTypes = removeTypeParameters(dataKey);
+  const pathWithoutTypes = removeTypeParameters(normalizedPath);
+
+  if (pathWithoutTypes.startsWith(keyWithoutTypes)) {
+    const afterKey = pathWithoutTypes.slice(keyWithoutTypes.length);
+    if (
+      afterKey === '' ||
+      afterKey.startsWith('.') ||
+      afterKey.startsWith('[')
+    ) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+/**
+ * Remove type parameters from a function call string.
+ * e.g., "useLoaderData<LoaderData>()" -> "useLoaderData()"
+ */
+function removeTypeParameters(str: string): string {
+  // Match and remove <...> including nested angle brackets
+  let result = str;
+  let prevLength = 0;
+
+  // Keep removing until no more changes (handles nested generics)
+  while (result.length !== prevLength) {
+    prevLength = result.length;
+    result = result.replace(/<[^<>]*>/g, '');
+  }
+
+  return result;
+}

package/analyzer-template/packages/ai/src/lib/generateChangesEntityScenarioData.ts

@@ -16,6 +16,8 @@ import type {
 import { LlmCall } from '~codeyam/types';
 import validateJson from './validateJson';
 import checkAllAttributes from './checkAllAttributes';
+import convertNullToUndefinedBySchema from './dataStructure/helpers/convertNullToUndefinedBySchema';
+import deepEqual from './deepEqual';
 import { awsLog } from '~codeyam/utils';
 import { AI, parseJsonSafe } from '~codeyam/ai';
 
@@ -59,12 +61,14 @@ async function generateChangesDataForScenario({
   llmCalls: LlmCall[];
 }> {
   if (
-    structure === existingStructure &&
-    scenario.metadata.keyAttributeInstructions ===
-      existingScenario.metadata.keyAttributeInstructions
+    deepEqual(structure, existingStructure) &&
+    deepEqual(
+      scenario.metadata?.coveredFlows,
+      existingScenario?.metadata?.coveredFlows,
+    )
   ) {
     console.log(
-      "CodeYam: No changes in scenario's structure or keyAttributeInstructions - returning existing scenario data",
+      "CodeYam: No changes in scenario's structure or coveredFlows - returning existing scenario data",
       {
         analysisId: scenario.analysisId,
         scenarioId: scenario.id,
@@ -95,6 +99,7 @@ async function generateChangesDataForScenario({
       prompt = generateEntityScenarioDataGenerator(
         structure,
         scenario,
+        undefined, // executionFlows - not available in changes context
         defaultScenarioData,
         incompleteResponse,
       );
@@ -219,8 +224,8 @@ async function generateChangesDataForScenario({
     if (!Array.isArray(argumentsData)) {
       argumentsData = [];
     }
-    fullScenarioData.data.argumentsData = argumentsData.map((arg) =>
-      checkAllAttributes(arg),
+    fullScenarioData.data.argumentsData = argumentsData.map(
+      (arg) => checkAllAttributes(arg) as { [key: string]: unknown },
     );
 
     if (structure.dataForMocks && !fullScenarioData.data.mockData) {
@@ -232,7 +237,28 @@ async function generateChangesDataForScenario({
     }
     fullScenarioData.data.mockData = checkAllAttributes(
       fullScenarioData.data.mockData,
-    );
+    ) as { [key: string]: unknown };
+
+    // Convert null values to undefined based on schema type constraints.
+    // LLM uses null for "no value" (JSON doesn't support undefined), but TypeScript
+    // types like "string | undefined" don't accept null. This converts null→undefined
+    // for fields typed as "T | undefined" (but preserves null for "T | null").
+    if (structure.dataForMocks && fullScenarioData.data.mockData) {
+      convertNullToUndefinedBySchema(
+        fullScenarioData.data.mockData,
+        structure.dataForMocks,
+      );
+    }
+    if (structure.arguments && fullScenarioData.data.argumentsData) {
+      for (let i = 0; i < fullScenarioData.data.argumentsData.length; i++) {
+        if (structure.arguments[i]) {
+          convertNullToUndefinedBySchema(
+            fullScenarioData.data.argumentsData[i],
+            structure.arguments[i],
+          );
+        }
+      }
+    }
 
     return {
       llmCalls: [{ name: scenario.name, id: llmCall.id }],
@@ -346,6 +372,47 @@ NEVER include "error" fields in responses. Skip them entirely.
 - If structure has \`{ data: {...}, error: {...} }\`, only fill \`data\`
 - Leave out any attribute named "error"—do not set to null, omit entirely
 
+## CRITICAL: Preserve Exact Structure
+Your response MUST mirror the EXACT nested structure provided in mockData Structure.
+- Do NOT reorganize, split, or create duplicate keys
+- The hierarchy of nested objects must match exactly what was provided
+- Only change the leaf VALUES (replacing type descriptions like "string" with actual data like "hello")
+- Arrays should have 3-5 items to provide realistic test data variety
+
+**Example** - if given this structure:
+\`\`\`json
+{
+  "createClient()": {
+    "from(\\"users\\")": {
+      "select(\\"*\\")": { "data": [{ "id": "string", "name": "string" }] },
+      "delete()": { "eq(\\"id\\", id)": {} }
+    }
+  }
+}
+\`\`\`
+
+Return ONE \`from("users")\` key containing BOTH \`select("*")\` and \`delete()\`:
+\`\`\`json
+{
+  "createClient()": {
+    "from(\\"users\\")": {
+      "select(\\"*\\")": { "data": [{ "id": "user-1", "name": "Alice" }, { "id": "user-2", "name": "Bob" }] },
+      "delete()": { "eq(\\"id\\", \\"user-1\\")": {} }
+    }
+  }
+}
+\`\`\`
+
+**WRONG** - do NOT create duplicate keys like this:
+\`\`\`json
+{
+  "createClient()": {
+    "from(\\"users\\")": { "select(\\"*\\")": { ... } },
+    "from(\\"users\\")": { "delete()": { ... } }
+  }
+}
+\`\`\`
+
 ## Special Markers
 
 ### Dynamic Dates (\`~~codeyam-code~~\`)