@codeyam/codeyam-cli 0.1.0-staging.1669d45 → 0.1.0-staging.2a88920

package/analyzer-template/packages/ai/src/lib/dataStructureChunking.ts

@@ -0,0 +1,156 @@
+import type { ExecutionFlow, ScenariosDataStructure } from '~codeyam/types';
+
+// Type for a single required value from ExecutionFlow
+type RequiredValue = NonNullable<ExecutionFlow['requiredValues']>[number];
+
+const DEFAULT_MAX_KEYS_PER_CHUNK = 8;
+
+export interface ChunkOptions {
+  maxKeysPerChunk?: number;
+}
+
+/**
+ * Chunk a large data structure into smaller pieces for focused LLM processing.
+ *
+ * Large schemas overwhelm LLMs, causing them to make mistakes on some keys.
+ * By processing smaller chunks, each key gets more focused attention.
+ *
+ * @param dataForMocks - The full data structure schema
+ * @param options - Chunking options (maxKeysPerChunk defaults to 8)
+ * @returns Array of smaller data structure chunks
+ */
+export function chunkDataStructure(
+  dataForMocks: ScenariosDataStructure['dataForMocks'],
+  options: ChunkOptions = {},
+): Array<ScenariosDataStructure['dataForMocks']> {
+  if (!dataForMocks || typeof dataForMocks !== 'object') {
+    return [dataForMocks];
+  }
+
+  const { maxKeysPerChunk = DEFAULT_MAX_KEYS_PER_CHUNK } = options;
+  const keys = Object.keys(dataForMocks);
+
+  // Don't chunk if already small enough
+  if (keys.length <= maxKeysPerChunk) {
+    return [dataForMocks];
+  }
+
+  const chunks: Array<ScenariosDataStructure['dataForMocks']> = [];
+
+  for (let i = 0; i < keys.length; i += maxKeysPerChunk) {
+    const chunkKeys = keys.slice(i, i + maxKeysPerChunk);
+    const chunk: ScenariosDataStructure['dataForMocks'] = {};
+
+    for (const key of chunkKeys) {
+      (chunk as Record<string, unknown>)[key] = (
+        dataForMocks as Record<string, unknown>
+      )[key];
+    }
+
+    chunks.push(chunk);
+  }
+
+  return chunks;
+}
+
+/**
+ * Get the requiredValues that are relevant to a specific chunk.
+ *
+ * When processing a chunk, we should include any requiredValues from the covered
+ * flows that reference keys in that chunk. This helps the LLM understand what
+ * values are expected for those keys.
+ *
+ * @param chunk - A chunk of the data structure
+ * @param executionFlows - All execution flows for the entity
+ * @param coveredFlowIds - The flow IDs covered by the current scenario
+ * @returns Array of requiredValues that reference keys in this chunk
+ */
+export function getRequiredValuesForChunk(
+  chunk: ScenariosDataStructure['dataForMocks'],
+  executionFlows: ExecutionFlow[],
+  coveredFlowIds: string[],
+): RequiredValue[] {
+  if (!chunk || typeof chunk !== 'object') {
+    return [];
+  }
+
+  const chunkKeys = Object.keys(chunk);
+  const relevantValues: RequiredValue[] = [];
+
+  // Get the covered flows
+  const coveredFlows = executionFlows.filter((flow) =>
+    coveredFlowIds.includes(flow.id),
+  );
+
+  for (const flow of coveredFlows) {
+    if (!flow.requiredValues) continue;
+
+    for (const rv of flow.requiredValues) {
+      if (!rv.attributePath) continue;
+
+      // Check if this requiredValue references a key in this chunk
+      for (const chunkKey of chunkKeys) {
+        if (pathMatchesKey(rv.attributePath, chunkKey)) {
+          relevantValues.push(rv);
+          break; // Don't add the same rv multiple times
+        }
+      }
+    }
+  }
+
+  return relevantValues;
+}
+
+/**
+ * Check if an attributePath references a given dataForMocks key.
+ * (Same logic as in extractCriticalDataKeys)
+ */
+function pathMatchesKey(attributePath: string, dataKey: string): boolean {
+  const normalizedPath = attributePath.replace('.functionCallReturnValue', '');
+
+  if (normalizedPath.startsWith(dataKey)) {
+    const afterKey = normalizedPath.slice(dataKey.length);
+    if (
+      afterKey === '' ||
+      afterKey.startsWith('.') ||
+      afterKey.startsWith('[')
+    ) {
+      return true;
+    }
+  }
+
+  // Try matching without type parameters
+  const keyWithoutTypes = removeTypeParameters(dataKey);
+  const pathWithoutTypes = removeTypeParameters(normalizedPath);
+
+  if (pathWithoutTypes.startsWith(keyWithoutTypes)) {
+    const afterKey = pathWithoutTypes.slice(keyWithoutTypes.length);
+    if (
+      afterKey === '' ||
+      afterKey.startsWith('.') ||
+      afterKey.startsWith('[')
+    ) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+/**
+ * Remove type parameters from a function call string.
+ */
+function removeTypeParameters(str: string): string {
+  let result = str;
+  let prevLength = 0;
+
+  while (result.length !== prevLength) {
+    prevLength = result.length;
+    result = result.replace(/<[^<>]*>/g, '');
+  }
+
+  return result;
+}
+
+// Re-export extractCriticalDataKeys for backwards compatibility
+export { extractCriticalDataKeys } from './extractCriticalDataKeys';

package/analyzer-template/packages/ai/src/lib/e2eDataTracking.ts

@@ -0,0 +1,334 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import * as crypto from 'crypto';
+
+/**
+ * E2E Data Tracking Utility
+ *
+ * This module helps debug discrepancies between first and second E2E test runs.
+ * The first run makes real LLM calls and saves the results.
+ * The second run uses cached LLM responses but sometimes produces different data.
+ *
+ * Usage:
+ *   Set CODEYAM_E2E_TRACK_DATA=true to enable tracking
+ *   Data is saved to /tmp/codeyam-e2e-tracking/
+ *
+ * First run (no fixtures): Saves snapshots of key data points
+ * Second run (with fixtures): Compares current data against saved snapshots
+ */
+
+const TRACKING_DIR = '/tmp/codeyam-e2e-tracking';
+
+interface DataSnapshot {
+  timestamp: number;
+  checkpoint: string;
+  entityName?: string;
+  scenarioName?: string;
+  dataHash: string;
+  data: unknown;
+}
+
+interface ComparisonResult {
+  checkpoint: string;
+  entityName?: string;
+  scenarioName?: string;
+  matches: boolean;
+  firstRunHash?: string;
+  secondRunHash?: string;
+  differences?: string[];
+}
+
+let trackingEnabled: boolean | undefined;
+let isFirstRun: boolean | undefined;
+let comparisonResults: ComparisonResult[] = [];
+
+function isTrackingEnabled(): boolean {
+  if (trackingEnabled === undefined) {
+    trackingEnabled = process.env.CODEYAM_E2E_TRACK_DATA === 'true';
+  }
+  return trackingEnabled;
+}
+
+function determineIfFirstRun(): boolean {
+  if (isFirstRun === undefined) {
+    // First run = no LLM fixtures directory
+    isFirstRun = !process.env.CODEYAM_LLM_FIXTURES_DIR;
+  }
+  return isFirstRun;
+}
+
+function ensureTrackingDir(): void {
+  if (!fs.existsSync(TRACKING_DIR)) {
+    fs.mkdirSync(TRACKING_DIR, { recursive: true });
+  }
+}
+
+function hashData(data: unknown): string {
+  const serialized = JSON.stringify(data, null, 0);
+  return crypto.createHash('md5').update(serialized).digest('hex');
+}
+
+function getSnapshotFilename(
+  checkpoint: string,
+  entityName?: string,
+  scenarioName?: string,
+): string {
+  const parts = [checkpoint];
+  if (entityName) parts.push(entityName.replace(/[^a-zA-Z0-9]/g, '_'));
+  if (scenarioName)
+    parts.push(scenarioName.replace(/[^a-zA-Z0-9]/g, '_').substring(0, 30));
+  return parts.join('_') + '.json';
+}
+
+/**
+ * Track a data snapshot at a specific checkpoint.
+ *
+ * On first run: Saves the data to a file for later comparison.
+ * On second run: Compares with saved data and logs any differences.
+ *
+ * @param checkpoint - Unique identifier for this tracking point (e.g., 'completionCall_result', 'parsed_scenarioData')
+ * @param data - The data to track
+ * @param entityName - Optional entity name for context
+ * @param scenarioName - Optional scenario name for context
+ */
+export function trackDataSnapshot(
+  checkpoint: string,
+  data: unknown,
+  entityName?: string,
+  scenarioName?: string,
+): void {
+  if (!isTrackingEnabled()) return;
+
+  ensureTrackingDir();
+
+  const filename = getSnapshotFilename(checkpoint, entityName, scenarioName);
+  const filepath = path.join(TRACKING_DIR, filename);
+  const currentHash = hashData(data);
+
+  if (determineIfFirstRun()) {
+    // First run: Save the snapshot
+    const snapshot: DataSnapshot = {
+      timestamp: Date.now(),
+      checkpoint,
+      entityName,
+      scenarioName,
+      dataHash: currentHash,
+      data,
+    };
+    fs.writeFileSync(filepath, JSON.stringify(snapshot, null, 2));
+    console.log(
+      `[E2E Tracking] First run - saved snapshot: ${checkpoint}${entityName ? ` (${entityName})` : ''}${scenarioName ? ` [${scenarioName}]` : ''} hash=${currentHash.substring(0, 8)}`,
+    );
+  } else {
+    // Second run: Compare with saved snapshot
+    if (fs.existsSync(filepath)) {
+      const savedSnapshot: DataSnapshot = JSON.parse(
+        fs.readFileSync(filepath, 'utf-8'),
+      );
+
+      const result: ComparisonResult = {
+        checkpoint,
+        entityName,
+        scenarioName,
+        matches: currentHash === savedSnapshot.dataHash,
+        firstRunHash: savedSnapshot.dataHash,
+        secondRunHash: currentHash,
+      };
+
+      if (!result.matches) {
+        result.differences = findDifferences(savedSnapshot.data, data);
+        console.log(
+          `[E2E Tracking] MISMATCH at ${checkpoint}${entityName ? ` (${entityName})` : ''}${scenarioName ? ` [${scenarioName}]` : ''}`,
+        );
+        console.log(`  First run hash:  ${savedSnapshot.dataHash}`);
+        console.log(`  Second run hash: ${currentHash}`);
+
+        // Save both versions for comparison
+        const diffFilepath = filepath.replace('.json', '_DIFF.json');
+        fs.writeFileSync(
+          diffFilepath,
+          JSON.stringify(
+            {
+              checkpoint,
+              entityName,
+              scenarioName,
+              firstRun: savedSnapshot.data,
+              secondRun: data,
+              differences: result.differences,
+            },
+            null,
+            2,
+          ),
+        );
+        console.log(`  Diff saved to: ${diffFilepath}`);
+      } else {
+        console.log(
+          `[E2E Tracking] Match at ${checkpoint}${entityName ? ` (${entityName})` : ''} hash=${currentHash.substring(0, 8)}`,
+        );
+      }
+
+      comparisonResults.push(result);
+    } else {
+      console.log(
+        `[E2E Tracking] No first-run snapshot found for: ${checkpoint}${entityName ? ` (${entityName})` : ''}`,
+      );
+    }
+  }
+}
+
+/**
+ * Find the differences between two data structures.
+ * Returns a list of paths where values differ.
+ */
+function findDifferences(
+  obj1: unknown,
+  obj2: unknown,
+  path: string = '',
+): string[] {
+  const differences: string[] = [];
+
+  if (typeof obj1 !== typeof obj2) {
+    differences.push(
+      `${path || 'root'}: type mismatch (${typeof obj1} vs ${typeof obj2})`,
+    );
+    return differences;
+  }
+
+  if (obj1 === null || obj2 === null) {
+    if (obj1 !== obj2) {
+      differences.push(
+        `${path || 'root'}: ${JSON.stringify(obj1)} vs ${JSON.stringify(obj2)}`,
+      );
+    }
+    return differences;
+  }
+
+  if (Array.isArray(obj1) && Array.isArray(obj2)) {
+    if (obj1.length !== obj2.length) {
+      differences.push(
+        `${path || 'root'}: array length ${obj1.length} vs ${obj2.length}`,
+      );
+    }
+    const maxLen = Math.max(obj1.length, obj2.length);
+    for (let i = 0; i < maxLen; i++) {
+      differences.push(...findDifferences(obj1[i], obj2[i], `${path}[${i}]`));
+    }
+    return differences;
+  }
+
+  if (typeof obj1 === 'object' && typeof obj2 === 'object') {
+    const keys1 = Object.keys(obj1 as Record<string, unknown>);
+    const keys2 = Object.keys(obj2 as Record<string, unknown>);
+    const allKeys = Array.from(new Set([...keys1, ...keys2]));
+
+    for (const key of allKeys) {
+      const val1 = (obj1 as Record<string, unknown>)[key];
+      const val2 = (obj2 as Record<string, unknown>)[key];
+
+      if (!(key in (obj1 as Record<string, unknown>))) {
+        differences.push(
+          `${path ? path + '.' : ''}${key}: missing in first run`,
+        );
+      } else if (!(key in (obj2 as Record<string, unknown>))) {
+        differences.push(
+          `${path ? path + '.' : ''}${key}: missing in second run`,
+        );
+      } else {
+        differences.push(
+          ...findDifferences(val1, val2, `${path ? path + '.' : ''}${key}`),
+        );
+      }
+    }
+    return differences;
+  }
+
+  // Primitive comparison
+  if (obj1 !== obj2) {
+    const str1 = JSON.stringify(obj1);
+    const str2 = JSON.stringify(obj2);
+    if (str1.length < 100 && str2.length < 100) {
+      differences.push(`${path || 'root'}: ${str1} vs ${str2}`);
+    } else {
+      differences.push(
+        `${path || 'root'}: values differ (${str1.length} chars vs ${str2.length} chars)`,
+      );
+    }
+  }
+
+  return differences;
+}
+
+/**
+ * Get a summary of all comparison results.
+ * Call this at the end of the second run to see overall results.
+ */
+export function getTrackingSummary(): {
+  total: number;
+  matches: number;
+  mismatches: number;
+  results: ComparisonResult[];
+} {
+  const matches = comparisonResults.filter((r) => r.matches).length;
+  const mismatches = comparisonResults.filter((r) => !r.matches).length;
+
+  return {
+    total: comparisonResults.length,
+    matches,
+    mismatches,
+    results: comparisonResults,
+  };
+}
+
+/**
+ * Print a summary of tracking results to console.
+ */
+export function printTrackingSummary(): void {
+  if (!isTrackingEnabled()) return;
+
+  const summary = getTrackingSummary();
+  console.log('\n========== E2E Data Tracking Summary ==========');
+  console.log(`Total checkpoints: ${summary.total}`);
+  console.log(`Matches: ${summary.matches}`);
+  console.log(`Mismatches: ${summary.mismatches}`);
+
+  if (summary.mismatches > 0) {
+    console.log('\nMismatched checkpoints:');
+    for (const result of summary.results.filter((r) => !r.matches)) {
+      console.log(
+        `  - ${result.checkpoint}${result.entityName ? ` (${result.entityName})` : ''}${result.scenarioName ? ` [${result.scenarioName}]` : ''}`,
+      );
+      if (result.differences && result.differences.length <= 10) {
+        for (const diff of result.differences) {
+          console.log(`      ${diff}`);
+        }
+      } else if (result.differences) {
+        console.log(
+          `      (${result.differences.length} differences - see diff file)`,
+        );
+      }
+    }
+  }
+  console.log('================================================\n');
+}
+
+/**
+ * Clear all tracking data. Call this before starting a new test session.
+ */
+export function clearTrackingData(): void {
+  if (fs.existsSync(TRACKING_DIR)) {
+    const files = fs.readdirSync(TRACKING_DIR);
+    for (const file of files) {
+      fs.unlinkSync(path.join(TRACKING_DIR, file));
+    }
+  }
+  comparisonResults = [];
+  isFirstRun = undefined;
+}
+
+/**
+ * Reset tracking state for a new test run (keeps files, clears in-memory state).
+ */
+export function resetTrackingState(): void {
+  comparisonResults = [];
+  isFirstRun = undefined;
+}

package/analyzer-template/packages/ai/src/lib/extractCriticalDataKeys.ts

@@ -0,0 +1,120 @@
+import type { ExecutionFlow, ScenariosDataStructure } from '~codeyam/types';
+
+/**
+ * Extract the data structure keys that are critical for satisfying execution flow requirements.
+ *
+ * When generating scenario mock data, only a subset of the full dataForMocks structure
+ * is actually needed to satisfy the scenario's covered flows' requiredValues.
+ * This function identifies those critical keys so they can be processed with more focus
+ * (e.g., in a separate LLM call or prioritized in the prompt).
+ *
+ * @param dataForMocks - The full data structure schema
+ * @param executionFlows - All execution flows for the entity
+ * @param coveredFlowIds - The flow IDs covered by the current scenario
+ * @returns Array of top-level keys from dataForMocks that are referenced by the covered flows
+ */
+export function extractCriticalDataKeys(
+  dataForMocks: ScenariosDataStructure['dataForMocks'],
+  executionFlows: ExecutionFlow[],
+  coveredFlowIds: string[],
+): string[] {
+  if (!dataForMocks || typeof dataForMocks !== 'object') {
+    return [];
+  }
+
+  const dataKeys = Object.keys(dataForMocks);
+  const criticalKeys = new Set<string>();
+
+  // Get the covered flows
+  const coveredFlows = executionFlows.filter((flow) =>
+    coveredFlowIds.includes(flow.id),
+  );
+
+  // For each covered flow, check if any requiredValue references a dataForMocks key
+  for (const flow of coveredFlows) {
+    if (!flow.requiredValues) continue;
+
+    for (const rv of flow.requiredValues) {
+      const path = rv.attributePath;
+      if (!path) continue;
+
+      // Try to match the attributePath to a dataForMocks key
+      // The path format is: "functionCall().functionCallReturnValue.property"
+      // or: "functionCall<Type>().functionCallReturnValue.property"
+      // We need to extract the function call part and match it to a key
+
+      for (const key of dataKeys) {
+        if (pathMatchesKey(path, key)) {
+          criticalKeys.add(key);
+          break; // One match per requiredValue is enough
+        }
+      }
+    }
+  }
+
+  return Array.from(criticalKeys);
+}
+
+/**
+ * Check if an attributePath references a given dataForMocks key.
+ *
+ * Handles various path formats:
+ * - "useParams().functionCallReturnValue.*" matches "useParams()"
+ * - "useLoaderData<LoaderData>().functionCallReturnValue.data" matches "useLoaderData<LoaderData>()"
+ * - "useFetcher().data.success" matches "useFetcher()"
+ */
+function pathMatchesKey(attributePath: string, dataKey: string): boolean {
+  // Normalize both for comparison
+  // Remove .functionCallReturnValue from the path
+  const normalizedPath = attributePath.replace('.functionCallReturnValue', '');
+
+  // Check if the path starts with the key (exact match for function call keys)
+  if (normalizedPath.startsWith(dataKey)) {
+    // Ensure it's not a partial match (e.g., "use" shouldn't match "useState")
+    const afterKey = normalizedPath.slice(dataKey.length);
+    // Valid continuations: end of string, '.', or '['
+    if (
+      afterKey === '' ||
+      afterKey.startsWith('.') ||
+      afterKey.startsWith('[')
+    ) {
+      return true;
+    }
+  }
+
+  // Try matching without type parameters if needed
+  // e.g., "useLoaderData<LoaderData>()" should match attributePaths with same base
+  const keyWithoutTypes = removeTypeParameters(dataKey);
+  const pathWithoutTypes = removeTypeParameters(normalizedPath);
+
+  if (pathWithoutTypes.startsWith(keyWithoutTypes)) {
+    const afterKey = pathWithoutTypes.slice(keyWithoutTypes.length);
+    if (
+      afterKey === '' ||
+      afterKey.startsWith('.') ||
+      afterKey.startsWith('[')
+    ) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+/**
+ * Remove type parameters from a function call string.
+ * e.g., "useLoaderData<LoaderData>()" -> "useLoaderData()"
+ */
+function removeTypeParameters(str: string): string {
+  // Match and remove <...> including nested angle brackets
+  let result = str;
+  let prevLength = 0;
+
+  // Keep removing until no more changes (handles nested generics)
+  while (result.length !== prevLength) {
+    prevLength = result.length;
+    result = result.replace(/<[^<>]*>/g, '');
+  }
+
+  return result;
+}