@codeyam/codeyam-cli 0.1.0-staging.1669d45 → 0.1.0-staging.323686

package/packages/ai/src/lib/e2eDataTracking.js

@@ -0,0 +1,241 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import * as crypto from 'crypto';
+/**
+ * E2E Data Tracking Utility
+ *
+ * This module helps debug discrepancies between first and second E2E test runs.
+ * The first run makes real LLM calls and saves the results.
+ * The second run uses cached LLM responses but sometimes produces different data.
+ *
+ * Usage:
+ *   Set CODEYAM_E2E_TRACK_DATA=true to enable tracking
+ *   Data is saved to /tmp/codeyam-e2e-tracking/
+ *
+ * First run (no fixtures): Saves snapshots of key data points
+ * Second run (with fixtures): Compares current data against saved snapshots
+ */
+const TRACKING_DIR = '/tmp/codeyam-e2e-tracking';
+let trackingEnabled;
+let isFirstRun;
+let comparisonResults = [];
+function isTrackingEnabled() {
+    if (trackingEnabled === undefined) {
+        trackingEnabled = process.env.CODEYAM_E2E_TRACK_DATA === 'true';
+    }
+    return trackingEnabled;
+}
+function determineIfFirstRun() {
+    if (isFirstRun === undefined) {
+        // First run = no LLM fixtures directory
+        isFirstRun = !process.env.CODEYAM_LLM_FIXTURES_DIR;
+    }
+    return isFirstRun;
+}
+function ensureTrackingDir() {
+    if (!fs.existsSync(TRACKING_DIR)) {
+        fs.mkdirSync(TRACKING_DIR, { recursive: true });
+    }
+}
+function hashData(data) {
+    const serialized = JSON.stringify(data, null, 0);
+    return crypto.createHash('md5').update(serialized).digest('hex');
+}
+function getSnapshotFilename(checkpoint, entityName, scenarioName) {
+    const parts = [checkpoint];
+    if (entityName)
+        parts.push(entityName.replace(/[^a-zA-Z0-9]/g, '_'));
+    if (scenarioName)
+        parts.push(scenarioName.replace(/[^a-zA-Z0-9]/g, '_').substring(0, 30));
+    return parts.join('_') + '.json';
+}
+/**
+ * Track a data snapshot at a specific checkpoint.
+ *
+ * On first run: Saves the data to a file for later comparison.
+ * On second run: Compares with saved data and logs any differences.
+ *
+ * @param checkpoint - Unique identifier for this tracking point (e.g., 'completionCall_result', 'parsed_scenarioData')
+ * @param data - The data to track
+ * @param entityName - Optional entity name for context
+ * @param scenarioName - Optional scenario name for context
+ */
+export function trackDataSnapshot(checkpoint, data, entityName, scenarioName) {
+    if (!isTrackingEnabled())
+        return;
+    ensureTrackingDir();
+    const filename = getSnapshotFilename(checkpoint, entityName, scenarioName);
+    const filepath = path.join(TRACKING_DIR, filename);
+    const currentHash = hashData(data);
+    if (determineIfFirstRun()) {
+        // First run: Save the snapshot
+        const snapshot = {
+            timestamp: Date.now(),
+            checkpoint,
+            entityName,
+            scenarioName,
+            dataHash: currentHash,
+            data,
+        };
+        fs.writeFileSync(filepath, JSON.stringify(snapshot, null, 2));
+        console.log(`[E2E Tracking] First run - saved snapshot: ${checkpoint}${entityName ? ` (${entityName})` : ''}${scenarioName ? ` [${scenarioName}]` : ''} hash=${currentHash.substring(0, 8)}`);
+    }
+    else {
+        // Second run: Compare with saved snapshot
+        if (fs.existsSync(filepath)) {
+            const savedSnapshot = JSON.parse(fs.readFileSync(filepath, 'utf-8'));
+            const result = {
+                checkpoint,
+                entityName,
+                scenarioName,
+                matches: currentHash === savedSnapshot.dataHash,
+                firstRunHash: savedSnapshot.dataHash,
+                secondRunHash: currentHash,
+            };
+            if (!result.matches) {
+                result.differences = findDifferences(savedSnapshot.data, data);
+                console.log(`[E2E Tracking] MISMATCH at ${checkpoint}${entityName ? ` (${entityName})` : ''}${scenarioName ? ` [${scenarioName}]` : ''}`);
+                console.log(`  First run hash:  ${savedSnapshot.dataHash}`);
+                console.log(`  Second run hash: ${currentHash}`);
+                // Save both versions for comparison
+                const diffFilepath = filepath.replace('.json', '_DIFF.json');
+                fs.writeFileSync(diffFilepath, JSON.stringify({
+                    checkpoint,
+                    entityName,
+                    scenarioName,
+                    firstRun: savedSnapshot.data,
+                    secondRun: data,
+                    differences: result.differences,
+                }, null, 2));
+                console.log(`  Diff saved to: ${diffFilepath}`);
+            }
+            else {
+                console.log(`[E2E Tracking] Match at ${checkpoint}${entityName ? ` (${entityName})` : ''} hash=${currentHash.substring(0, 8)}`);
+            }
+            comparisonResults.push(result);
+        }
+        else {
+            console.log(`[E2E Tracking] No first-run snapshot found for: ${checkpoint}${entityName ? ` (${entityName})` : ''}`);
+        }
+    }
+}
+/**
+ * Find the differences between two data structures.
+ * Returns a list of paths where values differ.
+ */
+function findDifferences(obj1, obj2, path = '') {
+    const differences = [];
+    if (typeof obj1 !== typeof obj2) {
+        differences.push(`${path || 'root'}: type mismatch (${typeof obj1} vs ${typeof obj2})`);
+        return differences;
+    }
+    if (obj1 === null || obj2 === null) {
+        if (obj1 !== obj2) {
+            differences.push(`${path || 'root'}: ${JSON.stringify(obj1)} vs ${JSON.stringify(obj2)}`);
+        }
+        return differences;
+    }
+    if (Array.isArray(obj1) && Array.isArray(obj2)) {
+        if (obj1.length !== obj2.length) {
+            differences.push(`${path || 'root'}: array length ${obj1.length} vs ${obj2.length}`);
+        }
+        const maxLen = Math.max(obj1.length, obj2.length);
+        for (let i = 0; i < maxLen; i++) {
+            differences.push(...findDifferences(obj1[i], obj2[i], `${path}[${i}]`));
+        }
+        return differences;
+    }
+    if (typeof obj1 === 'object' && typeof obj2 === 'object') {
+        const keys1 = Object.keys(obj1);
+        const keys2 = Object.keys(obj2);
+        const allKeys = Array.from(new Set([...keys1, ...keys2]));
+        for (const key of allKeys) {
+            const val1 = obj1[key];
+            const val2 = obj2[key];
+            if (!(key in obj1)) {
+                differences.push(`${path ? path + '.' : ''}${key}: missing in first run`);
+            }
+            else if (!(key in obj2)) {
+                differences.push(`${path ? path + '.' : ''}${key}: missing in second run`);
+            }
+            else {
+                differences.push(...findDifferences(val1, val2, `${path ? path + '.' : ''}${key}`));
+            }
+        }
+        return differences;
+    }
+    // Primitive comparison
+    if (obj1 !== obj2) {
+        const str1 = JSON.stringify(obj1);
+        const str2 = JSON.stringify(obj2);
+        if (str1.length < 100 && str2.length < 100) {
+            differences.push(`${path || 'root'}: ${str1} vs ${str2}`);
+        }
+        else {
+            differences.push(`${path || 'root'}: values differ (${str1.length} chars vs ${str2.length} chars)`);
+        }
+    }
+    return differences;
+}
+/**
+ * Get a summary of all comparison results.
+ * Call this at the end of the second run to see overall results.
+ */
+export function getTrackingSummary() {
+    const matches = comparisonResults.filter((r) => r.matches).length;
+    const mismatches = comparisonResults.filter((r) => !r.matches).length;
+    return {
+        total: comparisonResults.length,
+        matches,
+        mismatches,
+        results: comparisonResults,
+    };
+}
+/**
+ * Print a summary of tracking results to console.
+ */
+export function printTrackingSummary() {
+    if (!isTrackingEnabled())
+        return;
+    const summary = getTrackingSummary();
+    console.log('\n========== E2E Data Tracking Summary ==========');
+    console.log(`Total checkpoints: ${summary.total}`);
+    console.log(`Matches: ${summary.matches}`);
+    console.log(`Mismatches: ${summary.mismatches}`);
+    if (summary.mismatches > 0) {
+        console.log('\nMismatched checkpoints:');
+        for (const result of summary.results.filter((r) => !r.matches)) {
+            console.log(`  - ${result.checkpoint}${result.entityName ? ` (${result.entityName})` : ''}${result.scenarioName ? ` [${result.scenarioName}]` : ''}`);
+            if (result.differences && result.differences.length <= 10) {
+                for (const diff of result.differences) {
+                    console.log(`      ${diff}`);
+                }
+            }
+            else if (result.differences) {
+                console.log(`      (${result.differences.length} differences - see diff file)`);
+            }
+        }
+    }
+    console.log('================================================\n');
+}
+/**
+ * Clear all tracking data. Call this before starting a new test session.
+ */
+export function clearTrackingData() {
+    if (fs.existsSync(TRACKING_DIR)) {
+        const files = fs.readdirSync(TRACKING_DIR);
+        for (const file of files) {
+            fs.unlinkSync(path.join(TRACKING_DIR, file));
+        }
+    }
+    comparisonResults = [];
+    isFirstRun = undefined;
+}
+/**
+ * Reset tracking state for a new test run (keeps files, clears in-memory state).
+ */
+export function resetTrackingState() {
+    comparisonResults = [];
+    isFirstRun = undefined;
+}
+//# sourceMappingURL=e2eDataTracking.js.map

package/packages/ai/src/lib/e2eDataTracking.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"e2eDataTracking.js","sourceRoot":"","sources":["../../../../../../packages/ai/src/lib/e2eDataTracking.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AAEjC;;;;;;;;;;;;;GAaG;AAEH,MAAM,YAAY,GAAG,2BAA2B,CAAC;AAqBjD,IAAI,eAAoC,CAAC;AACzC,IAAI,UAA+B,CAAC;AACpC,IAAI,iBAAiB,GAAuB,EAAE,CAAC;AAE/C,SAAS,iBAAiB;IACxB,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QAClC,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,KAAK,MAAM,CAAC;IAClE,CAAC;IACD,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,SAAS,mBAAmB;IAC1B,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,wCAAwC;QACxC,UAAU,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;IACrD,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,iBAAiB;IACxB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QACjC,EAAE,CAAC,SAAS,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAClD,CAAC;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,IAAa;IAC7B,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACjD,OAAO,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,mBAAmB,CAC1B,UAAkB,EAClB,UAAmB,EACnB,YAAqB;IAErB,MAAM,KAAK,GAAG,CAAC,UAAU,CAAC,CAAC;IAC3B,IAAI,UAAU;QAAE,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC,CAAC;IACrE,IAAI,YAAY;QACd,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IAC1E,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC;AACnC,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,iBAAiB,CAC/B,UAAkB,EAClB,IAAa,EACb,UAAmB,EACnB,YAAqB;IAErB,IAAI,CAAC,iBAAiB,EAAE;QAAE,OAAO;IAEjC,iBAAiB,EAAE,CAAC;IAEpB,MAAM,QAAQ,GAAG,mBAAmB,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC;IAC3E,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;IACnD,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAEnC,IAAI,mBAAmB,EAAE,EAAE,CAAC;QAC1B,+BAA+B;QAC/B,MAAM,QAAQ,GAAiB;YAC7B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,UAAU;YACV,UAAU;YACV,YAAY;YACZ,QAAQ,EAAE,WAAW;YACrB,IAAI;SACL,CAAC;QACF,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC9D,OAAO,CAAC,GAAG,CACT,8CAA8C,UAAU,GAAG,UAAU,CAAC,CAAC,CAAC,KAAK,UAAU,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,YAAY,CAAC,CAAC,CAAC,KAAK,YAAY,GAAG,CAAC,CAAC,CAAC,EAAE,SAAS,WAAW,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CACjL,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,0CAA0C;QAC1C,IAAI,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC5B,MAAM,aAAa,GAAiB,IAAI,CAAC,KAAK,CAC5C,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CACnC,CAAC;YAEF,MAAM,MAAM,GAAqB;gBAC/B,UAAU;gBACV,UAAU;gBACV,YAAY;gBACZ,OAAO,EAAE,WAAW,KAAK,aAAa,CAAC,QAAQ;gBAC/C,YAAY,EAAE,aAAa,CAAC,QAAQ;gBACpC,aAAa,EAAE,WAAW;aAC3B,CAAC;YAEF,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBACpB,MAAM,CAAC,WAAW,GAAG,eAAe,CAAC,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;gBAC/D,OAAO,CAAC,GAAG,CACT,8BAA8B,UAAU,GAAG,UAAU,CAAC,CAAC,CAAC,KAAK,UAAU,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,YAAY,CAAC,CAAC,CAAC,KAAK,YAAY,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC7H,CAAC;gBACF,OAAO,CAAC,GAAG,CAAC,sBAAsB,aAAa,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC5D,OAAO,CAAC,GAAG,CAAC,sBAAsB,WAAW,EAAE,CAAC,CAAC;gBAEjD,oCAAoC;gBACpC,MAAM,YAAY,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;gBAC7D,EAAE,CAAC,aAAa,CACd,YAAY,EACZ,IAAI,CAAC,SAAS,CACZ;oBACE,UAAU;oBACV,UAAU;oBACV,YAAY;oBACZ,QAAQ,EAAE,aAAa,CAAC,IAAI;oBAC5B,SAAS,EAAE,IAAI;oBACf,WAAW,EAAE,MAAM,CAAC,WAAW;iBAChC,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;gBACF,OAAO,CAAC,GAAG,CAAC,oBAAoB,YAAY,EAAE,CAAC,CAAC;YAClD,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,CACT,2BAA2B,UAAU,GAAG,UAAU,CAAC,CAAC,CAAC,KAAK,UAAU,GAAG,CAAC,CAAC,CAAC,EAAE,SAAS,WAAW,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CACnH,CAAC;YACJ,CAAC;YAED,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CACT,mDAAmD,UAAU,GAAG,UAAU,CAAC,CAAC,CAAC,KAAK,UAAU,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACvG,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,eAAe,CACtB,IAAa,EACb,IAAa,EACb,OAAe,EAAE;IAEjB,MAAM,WAAW,GAAa,EAAE,CAAC;IAEjC,IAAI,OAAO,IAAI,KAAK,OAAO,IAAI,EAAE,CAAC;QAChC,WAAW,CAAC,IAAI,CACd,GAAG,IAAI,IAAI,MAAM,oBAAoB,OAAO,IAAI,OAAO,OAAO,IAAI,GAAG,CACtE,CAAC;QACF,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QACnC,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,WAAW,CAAC,IAAI,CACd,GAAG,IAAI,IAAI,MAAM,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CACxE,CAAC;QACJ,CAAC;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/C,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;YAChC,WAAW,CAAC,IAAI,CACd,GAAG,IAAI,IAAI,MAAM,kBAAkB,IAAI,CAAC,MAAM,OAAO,IAAI,CAAC,MAAM,EAAE,CACnE,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAClD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAChC,WAAW,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QACzD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAA+B,CAAC,CAAC;QAC3D,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAA+B,CAAC,CAAC;QAC3D,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,KAAK,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAE1D,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAI,IAAgC,CAAC,GAAG,CAAC,CAAC;YACpD,MAAM,IAAI,GAAI,IAAgC,CAAC,GAAG,CAAC,CAAC;YAEpD,IAAI,CAAC,CAAC,GAAG,IAAK,IAAgC,CAAC,EAAE,CAAC;gBAChD,WAAW,CAAC,IAAI,CACd,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,GAAG,wBAAwB,CACxD,CAAC;YACJ,CAAC;iBAAM,IAAI,CAAC,CAAC,GAAG,IAAK,IAAgC,CAAC,EAAE,CAAC;gBACvD,WAAW,CAAC,IAAI,CACd,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,GAAG,yBAAyB,CACzD,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,WAAW,CAAC,IAAI,CACd,GAAG,eAAe,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,GAAG,EAAE,CAAC,CAClE,CAAC;YACJ,CAAC;QACH,CAAC;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,uBAAuB;IACvB,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YAC3C,WAAW,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,MAAM,KAAK,IAAI,OAAO,IAAI,EAAE,CAAC,CAAC;QAC5D,CAAC;aAAM,CAAC;YACN,WAAW,CAAC,IAAI,CACd,GAAG,IAAI,IAAI,MAAM,oBAAoB,IAAI,CAAC,MAAM,aAAa,IAAI,CAAC,MAAM,SAAS,CAClF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB;IAMhC,MAAM,OAAO,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IAClE,MAAM,UAAU,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IAEtE,OAAO;QACL,KAAK,EAAE,iBAAiB,CAAC,MAAM;QAC/B,OAAO;QACP,UAAU;QACV,OAAO,EAAE,iBAAiB;KAC3B,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB;IAClC,IAAI,CAAC,iBAAiB,EAAE;QAAE,OAAO;IAEjC,MAAM,OAAO,GAAG,kBAAkB,EAAE,CAAC;IACrC,OAAO,CAAC,GAAG,CAAC,mDAAmD,CAAC,CAAC;IACjE,OAAO,CAAC,GAAG,CAAC,sBAAsB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;IACnD,OAAO,CAAC,GAAG,CAAC,YAAY,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IAC3C,OAAO,CAAC,GAAG,CAAC,eAAe,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;IAEjD,IAAI,OAAO,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;QAC3B,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;QACzC,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/D,OAAO,CAAC,GAAG,CACT,OAAO,MAAM,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACzI,CAAC;YACF,IAAI,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;gBAC1D,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;oBACtC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;gBAC/B,CAAC;YACH,CAAC;iBAAM,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;gBAC9B,OAAO,CAAC,GAAG,CACT,UAAU,MAAM,CAAC,WAAW,CAAC,MAAM,+BAA+B,CACnE,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;AACpE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB;IAC/B,IAAI,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;QAC3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IACD,iBAAiB,GAAG,EAAE,CAAC;IACvB,UAAU,GAAG,SAAS,CAAC;AACzB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB;IAChC,iBAAiB,GAAG,EAAE,CAAC;IACvB,UAAU,GAAG,SAAS,CAAC;AACzB,CAAC"}

package/packages/ai/src/lib/extractCriticalDataKeys.js

@@ -0,0 +1,96 @@
+/**
+ * Extract the data structure keys that are critical for satisfying execution flow requirements.
+ *
+ * When generating scenario mock data, only a subset of the full dataForMocks structure
+ * is actually needed to satisfy the scenario's covered flows' requiredValues.
+ * This function identifies those critical keys so they can be processed with more focus
+ * (e.g., in a separate LLM call or prioritized in the prompt).
+ *
+ * @param dataForMocks - The full data structure schema
+ * @param executionFlows - All execution flows for the entity
+ * @param coveredFlowIds - The flow IDs covered by the current scenario
+ * @returns Array of top-level keys from dataForMocks that are referenced by the covered flows
+ */
+export function extractCriticalDataKeys(dataForMocks, executionFlows, coveredFlowIds) {
+    if (!dataForMocks || typeof dataForMocks !== 'object') {
+        return [];
+    }
+    const dataKeys = Object.keys(dataForMocks);
+    const criticalKeys = new Set();
+    // Get the covered flows
+    const coveredFlows = executionFlows.filter((flow) => coveredFlowIds.includes(flow.id));
+    // For each covered flow, check if any requiredValue references a dataForMocks key
+    for (const flow of coveredFlows) {
+        if (!flow.requiredValues)
+            continue;
+        for (const rv of flow.requiredValues) {
+            const path = rv.attributePath;
+            if (!path)
+                continue;
+            // Try to match the attributePath to a dataForMocks key
+            // The path format is: "functionCall().functionCallReturnValue.property"
+            // or: "functionCall<Type>().functionCallReturnValue.property"
+            // We need to extract the function call part and match it to a key
+            for (const key of dataKeys) {
+                if (pathMatchesKey(path, key)) {
+                    criticalKeys.add(key);
+                    break; // One match per requiredValue is enough
+                }
+            }
+        }
+    }
+    return Array.from(criticalKeys);
+}
+/**
+ * Check if an attributePath references a given dataForMocks key.
+ *
+ * Handles various path formats:
+ * - "useParams().functionCallReturnValue.*" matches "useParams()"
+ * - "useLoaderData<LoaderData>().functionCallReturnValue.data" matches "useLoaderData<LoaderData>()"
+ * - "useFetcher().data.success" matches "useFetcher()"
+ */
+function pathMatchesKey(attributePath, dataKey) {
+    // Normalize both for comparison
+    // Remove .functionCallReturnValue from the path
+    const normalizedPath = attributePath.replace('.functionCallReturnValue', '');
+    // Check if the path starts with the key (exact match for function call keys)
+    if (normalizedPath.startsWith(dataKey)) {
+        // Ensure it's not a partial match (e.g., "use" shouldn't match "useState")
+        const afterKey = normalizedPath.slice(dataKey.length);
+        // Valid continuations: end of string, '.', or '['
+        if (afterKey === '' ||
+            afterKey.startsWith('.') ||
+            afterKey.startsWith('[')) {
+            return true;
+        }
+    }
+    // Try matching without type parameters if needed
+    // e.g., "useLoaderData<LoaderData>()" should match attributePaths with same base
+    const keyWithoutTypes = removeTypeParameters(dataKey);
+    const pathWithoutTypes = removeTypeParameters(normalizedPath);
+    if (pathWithoutTypes.startsWith(keyWithoutTypes)) {
+        const afterKey = pathWithoutTypes.slice(keyWithoutTypes.length);
+        if (afterKey === '' ||
+            afterKey.startsWith('.') ||
+            afterKey.startsWith('[')) {
+            return true;
+        }
+    }
+    return false;
+}
+/**
+ * Remove type parameters from a function call string.
+ * e.g., "useLoaderData<LoaderData>()" -> "useLoaderData()"
+ */
+function removeTypeParameters(str) {
+    // Match and remove <...> including nested angle brackets
+    let result = str;
+    let prevLength = 0;
+    // Keep removing until no more changes (handles nested generics)
+    while (result.length !== prevLength) {
+        prevLength = result.length;
+        result = result.replace(/<[^<>]*>/g, '');
+    }
+    return result;
+}
+//# sourceMappingURL=extractCriticalDataKeys.js.map

package/packages/ai/src/lib/extractCriticalDataKeys.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extractCriticalDataKeys.js","sourceRoot":"","sources":["../../../../../../packages/ai/src/lib/extractCriticalDataKeys.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,uBAAuB,CACrC,YAAoD,EACpD,cAA+B,EAC/B,cAAwB;IAExB,IAAI,CAAC,YAAY,IAAI,OAAO,YAAY,KAAK,QAAQ,EAAE,CAAC;QACtD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC3C,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IAEvC,wBAAwB;IACxB,MAAM,YAAY,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAClD,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CACjC,CAAC;IAEF,kFAAkF;IAClF,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QAChC,IAAI,CAAC,IAAI,CAAC,cAAc;YAAE,SAAS;QAEnC,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,EAAE,CAAC,aAAa,CAAC;YAC9B,IAAI,CAAC,IAAI;gBAAE,SAAS;YAEpB,uDAAuD;YACvD,wEAAwE;YACxE,8DAA8D;YAC9D,kEAAkE;YAElE,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;gBAC3B,IAAI,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;oBAC9B,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBACtB,MAAM,CAAC,wCAAwC;gBACjD,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;AAClC,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,cAAc,CAAC,aAAqB,EAAE,OAAe;IAC5D,gCAAgC;IAChC,gDAAgD;IAChD,MAAM,cAAc,GAAG,aAAa,CAAC,OAAO,CAAC,0BAA0B,EAAE,EAAE,CAAC,CAAC;IAE7E,6EAA6E;IAC7E,IAAI,cAAc,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QACvC,2EAA2E;QAC3E,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACtD,kDAAkD;QAClD,IACE,QAAQ,KAAK,EAAE;YACf,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC;YACxB,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,EACxB,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,iDAAiD;IACjD,iFAAiF;IACjF,MAAM,eAAe,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;IACtD,MAAM,gBAAgB,GAAG,oBAAoB,CAAC,cAAc,CAAC,CAAC;IAE9D,IAAI,gBAAgB,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,CAAC;QACjD,MAAM,QAAQ,GAAG,gBAAgB,CAAC,KAAK,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;QAChE,IACE,QAAQ,KAAK,EAAE;YACf,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC;YACxB,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,EACxB,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,SAAS,oBAAoB,CAAC,GAAW;IACvC,yDAAyD;IACzD,IAAI,MAAM,GAAG,GAAG,CAAC;IACjB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,gEAAgE;IAChE,OAAO,MAAM,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QACpC,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;QAC3B,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}