codeflash 0.0.1 → 0.2.0

package/runtime/compare-results.js

@@ -0,0 +1,331 @@
+#!/usr/bin/env node
+/**
+ * Codeflash Result Comparator
+ *
+ * This script compares test results between original and optimized code runs.
+ * It reads serialized behavior data from SQLite databases and compares them
+ * using the codeflash-comparator in JavaScript land.
+ *
+ * Usage:
+ *   node codeflash-compare-results.js <original_db> <candidate_db>
+ *   node codeflash-compare-results.js --json <json_input>
+ *
+ * Output (JSON):
+ *   {
+ *     "equivalent": true/false,
+ *     "diffs": [
+ *       {
+ *         "invocation_id": "...",
+ *         "scope": "return_value|stdout|did_pass",
+ *         "original": "...",
+ *         "candidate": "..."
+ *       }
+ *     ],
+ *     "error": null | "error message"
+ *   }
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+// Import our modules
+const { deserialize } = require('./serializer');
+const { comparator } = require('./comparator');
+
+// Lazy-load better-sqlite3 to avoid process.exit during module require
+// This prevents crashes when this module is imported by test files that don't use it
+let Database = null;
+let databaseLoadError = null;
+
+function getDatabase() {
+    if (Database === null && databaseLoadError === null) {
+        try {
+            Database = require('better-sqlite3');
+        } catch (e) {
+            databaseLoadError = 'better-sqlite3 not installed. Run: npm install better-sqlite3';
+        }
+    }
+    return { Database, error: databaseLoadError };
+}
+
+/**
+ * Read test results from a SQLite database.
+ *
+ * @param {string} dbPath - Path to SQLite database
+ * @returns {Map<string, object>} Map of invocation_id -> result object
+ */
+function readTestResults(dbPath) {
+    const results = new Map();
+
+    if (!fs.existsSync(dbPath)) {
+        throw new Error(`Database not found: ${dbPath}`);
+    }
+
+    // Get Database lazily - throws if not available
+    const { Database: DB, error } = getDatabase();
+    if (error) {
+        throw new Error(error);
+    }
+
+    const db = new DB(dbPath, { readonly: true });
+
+    try {
+        const stmt = db.prepare(`
+            SELECT
+                test_module_path,
+                test_class_name,
+                test_function_name,
+                function_getting_tested,
+                loop_index,
+                iteration_id,
+                runtime,
+                return_value,
+                verification_type
+            FROM test_results
+            WHERE loop_index = 1
+        `);
+
+        for (const row of stmt.iterate()) {
+            // Build unique invocation ID (matches Python's format)
+            const invocationId = `${row.loop_index}:${row.test_module_path}:${row.test_class_name || ''}:${row.test_function_name}:${row.function_getting_tested}:${row.iteration_id}`;
+
+            // Deserialize the return value
+            let returnValue = null;
+            if (row.return_value) {
+                try {
+                    returnValue = deserialize(row.return_value);
+                } catch (e) {
+                    console.error(`Failed to deserialize result for ${invocationId}: ${e.message}`);
+                }
+            }
+
+            results.set(invocationId, {
+                testModulePath: row.test_module_path,
+                testClassName: row.test_class_name,
+                testFunctionName: row.test_function_name,
+                functionGettingTested: row.function_getting_tested,
+                loopIndex: row.loop_index,
+                iterationId: row.iteration_id,
+                runtime: row.runtime,
+                returnValue,
+                verificationType: row.verification_type,
+            });
+        }
+    } finally {
+        db.close();
+    }
+
+    return results;
+}
+
+/**
+ * Compare two sets of test results.
+ *
+ * @param {Map<string, object>} originalResults - Results from original code
+ * @param {Map<string, object>} candidateResults - Results from optimized code
+ * @returns {object} Comparison result
+ */
+function compareResults(originalResults, candidateResults) {
+    const diffs = [];
+    let allEquivalent = true;
+
+    // Get all unique invocation IDs
+    const allIds = new Set([...originalResults.keys(), ...candidateResults.keys()]);
+
+    for (const invocationId of allIds) {
+        const original = originalResults.get(invocationId);
+        const candidate = candidateResults.get(invocationId);
+
+        // If candidate has extra results not in original, that's OK
+        if (candidate && !original) {
+            continue;
+        }
+
+        // If original has results not in candidate, that's a diff
+        if (original && !candidate) {
+            allEquivalent = false;
+            diffs.push({
+                invocation_id: invocationId,
+                scope: 'missing',
+                original: summarizeValue(original.returnValue),
+                candidate: null,
+                test_info: {
+                    test_module_path: original.testModulePath,
+                    test_function_name: original.testFunctionName,
+                    function_getting_tested: original.functionGettingTested,
+                }
+            });
+            continue;
+        }
+
+        // Compare return values using the JavaScript comparator
+        // The return value format is [args, kwargs, returnValue] (behavior tuple)
+        const originalValue = original.returnValue;
+        const candidateValue = candidate.returnValue;
+
+        const isEqual = comparator(originalValue, candidateValue);
+
+        if (!isEqual) {
+            allEquivalent = false;
+            diffs.push({
+                invocation_id: invocationId,
+                scope: 'return_value',
+                original: summarizeValue(originalValue),
+                candidate: summarizeValue(candidateValue),
+                test_info: {
+                    test_module_path: original.testModulePath,
+                    test_function_name: original.testFunctionName,
+                    function_getting_tested: original.functionGettingTested,
+                }
+            });
+        }
+    }
+
+    return {
+        equivalent: allEquivalent,
+        diffs,
+        total_invocations: allIds.size,
+        original_count: originalResults.size,
+        candidate_count: candidateResults.size,
+    };
+}
+
+/**
+ * Create a summary of a value for diff reporting.
+ * Truncates long values to avoid huge output.
+ *
+ * @param {any} value - Value to summarize
+ * @returns {string} String representation
+ */
+function summarizeValue(value, maxLength = 200) {
+    try {
+        let str;
+        if (value === undefined) {
+            str = 'undefined';
+        } else if (value === null) {
+            str = 'null';
+        } else if (typeof value === 'function') {
+            str = `[Function: ${value.name || 'anonymous'}]`;
+        } else if (value instanceof Map) {
+            str = `Map(${value.size}) { ${[...value.entries()].slice(0, 3).map(([k, v]) => `${summarizeValue(k, 50)} => ${summarizeValue(v, 50)}`).join(', ')}${value.size > 3 ? ', ...' : ''} }`;
+        } else if (value instanceof Set) {
+            str = `Set(${value.size}) { ${[...value].slice(0, 3).map(v => summarizeValue(v, 50)).join(', ')}${value.size > 3 ? ', ...' : ''} }`;
+        } else if (value instanceof Date) {
+            str = value.toISOString();
+        } else if (Array.isArray(value)) {
+            if (value.length <= 5) {
+                str = JSON.stringify(value);
+            } else {
+                str = `[${value.slice(0, 3).map(v => summarizeValue(v, 50)).join(', ')}, ... (${value.length} items)]`;
+            }
+        } else if (typeof value === 'object') {
+            str = JSON.stringify(value);
+        } else {
+            str = String(value);
+        }
+
+        if (str.length > maxLength) {
+            return str.slice(0, maxLength - 3) + '...';
+        }
+        return str;
+    } catch (e) {
+        return `[Unable to stringify: ${e.message}]`;
+    }
+}
+
+/**
+ * Compare results from serialized buffers directly (for stdin input).
+ *
+ * @param {Buffer} originalBuffer - Serialized original result
+ * @param {Buffer} candidateBuffer - Serialized candidate result
+ * @returns {boolean} True if equivalent
+ */
+function compareBuffers(originalBuffer, candidateBuffer) {
+    try {
+        const original = deserialize(originalBuffer);
+        const candidate = deserialize(candidateBuffer);
+        return comparator(original, candidate);
+    } catch (e) {
+        console.error(`Comparison error: ${e.message}`);
+        return false;
+    }
+}
+
+/**
+ * Main entry point.
+ */
+function main() {
+    const args = process.argv.slice(2);
+
+    if (args.length === 0) {
+        console.error('Usage: node codeflash-compare-results.js <original_db> <candidate_db>');
+        console.error('       node codeflash-compare-results.js --stdin (reads JSON from stdin)');
+        process.exit(1);
+    }
+
+    // Handle stdin mode for programmatic use
+    if (args[0] === '--stdin') {
+        let input = '';
+        process.stdin.setEncoding('utf8');
+        process.stdin.on('data', chunk => input += chunk);
+        process.stdin.on('end', () => {
+            try {
+                const data = JSON.parse(input);
+                const originalBuffer = Buffer.from(data.original, 'base64');
+                const candidateBuffer = Buffer.from(data.candidate, 'base64');
+                const isEqual = compareBuffers(originalBuffer, candidateBuffer);
+                console.log(JSON.stringify({ equivalent: isEqual, error: null }));
+            } catch (e) {
+                console.log(JSON.stringify({ equivalent: false, error: e.message }));
+            }
+        });
+        return;
+    }
+
+    // Standard mode: compare two SQLite databases
+    if (args.length < 2) {
+        console.error('Usage: node codeflash-compare-results.js <original_db> <candidate_db>');
+        process.exit(1);
+    }
+
+    const [originalDb, candidateDb] = args;
+
+    try {
+        const originalResults = readTestResults(originalDb);
+        const candidateResults = readTestResults(candidateDb);
+
+        const comparison = compareResults(originalResults, candidateResults);
+
+        // Limit the number of diffs to avoid huge output
+        const MAX_DIFFS = 50;
+        if (comparison.diffs.length > MAX_DIFFS) {
+            const truncatedCount = comparison.diffs.length - MAX_DIFFS;
+            comparison.diffs = comparison.diffs.slice(0, MAX_DIFFS);
+            comparison.diffs_truncated = truncatedCount;
+        }
+
+        // Use compact JSON (no pretty-printing) to reduce output size
+        console.log(JSON.stringify(comparison));
+        process.exit(comparison.equivalent ? 0 : 1);
+    } catch (e) {
+        console.log(JSON.stringify({
+            equivalent: false,
+            diffs: [],
+            error: e.message
+        }));
+        process.exit(1);
+    }
+}
+
+// Export for programmatic use
+module.exports = {
+    readTestResults,
+    compareResults,
+    compareBuffers,
+    summarizeValue,
+};
+
+// Run if called directly
+if (require.main === module) {
+    main();
+}

package/runtime/index.d.ts

@@ -0,0 +1,146 @@
+/**
+ * Codeflash TypeScript Declarations
+ */
+
+/**
+ * Capture a function call for behavior verification.
+ * Records inputs, outputs, timing to SQLite database.
+ *
+ * @param funcName - Name of the function being tested
+ * @param lineId - Line number identifier in test file
+ * @param fn - The function to call
+ * @param args - Arguments to pass to the function
+ * @returns The function's return value
+ */
+export function capture<T extends (...args: any[]) => any>(
+    funcName: string,
+    lineId: string,
+    fn: T,
+    ...args: Parameters<T>
+): ReturnType<T>;
+
+/**
+ * Capture a function call for performance benchmarking.
+ * Only measures timing, prints to stdout.
+ *
+ * @param funcName - Name of the function being tested
+ * @param lineId - Line number identifier in test file
+ * @param fn - The function to call
+ * @param args - Arguments to pass to the function
+ * @returns The function's return value
+ */
+export function capturePerf<T extends (...args: any[]) => any>(
+    funcName: string,
+    lineId: string,
+    fn: T,
+    ...args: Parameters<T>
+): ReturnType<T>;
+
+/**
+ * Capture multiple invocations for benchmarking.
+ *
+ * @param funcName - Name of the function being tested
+ * @param lineId - Line number identifier
+ * @param fn - The function to call
+ * @param argsList - List of argument arrays to test
+ * @returns Array of return values
+ */
+export function captureMultiple<T extends (...args: any[]) => any>(
+    funcName: string,
+    lineId: string,
+    fn: T,
+    argsList: Parameters<T>[]
+): ReturnType<T>[];
+
+/**
+ * Write remaining results to file.
+ */
+export function writeResults(): void;
+
+/**
+ * Clear all recorded results.
+ */
+export function clearResults(): void;
+
+/**
+ * Get the current results buffer.
+ */
+export function getResults(): any[];
+
+/**
+ * Set the current test name.
+ */
+export function setTestName(name: string): void;
+
+/**
+ * Serialize a value for storage.
+ */
+export function safeSerialize(value: any): Buffer;
+
+/**
+ * Deserialize a buffer back to a value.
+ */
+export function safeDeserialize(buffer: Buffer | Uint8Array): any;
+
+/**
+ * Initialize the SQLite database.
+ */
+export function initDatabase(): void;
+
+/**
+ * Reset invocation counters.
+ */
+export function resetInvocationCounters(): void;
+
+/**
+ * Get invocation index for a testId.
+ */
+export function getInvocationIndex(testId: string): number;
+
+/**
+ * Sanitize a string for use in test IDs.
+ */
+export function sanitizeTestId(str: string): string;
+
+/**
+ * Get the serializer type being used.
+ */
+export function getSerializerType(): 'v8' | 'msgpack';
+
+/**
+ * Current loop index from environment.
+ */
+export const LOOP_INDEX: number;
+
+/**
+ * Output file path from environment.
+ */
+export const OUTPUT_FILE: string;
+
+/**
+ * Test iteration from environment.
+ */
+export const TEST_ITERATION: string;
+
+// Default export for CommonJS compatibility
+declare const codeflash: {
+    capture: typeof capture;
+    capturePerf: typeof capturePerf;
+    captureMultiple: typeof captureMultiple;
+    writeResults: typeof writeResults;
+    clearResults: typeof clearResults;
+    getResults: typeof getResults;
+    setTestName: typeof setTestName;
+    safeSerialize: typeof safeSerialize;
+    safeDeserialize: typeof safeDeserialize;
+    initDatabase: typeof initDatabase;
+    resetInvocationCounters: typeof resetInvocationCounters;
+    getInvocationIndex: typeof getInvocationIndex;
+    sanitizeTestId: typeof sanitizeTestId;
+    getSerializerType: typeof getSerializerType;
+    LOOP_INDEX: typeof LOOP_INDEX;
+    OUTPUT_FILE: typeof OUTPUT_FILE;
+    TEST_ITERATION: typeof TEST_ITERATION;
+};
+
+export default codeflash;

package/runtime/index.js

@@ -0,0 +1,86 @@
+/**
+ * codeflash
+ *
+ * Codeflash CLI runtime helpers for test instrumentation and behavior verification.
+ *
+ * Main exports:
+ * - capture: Capture function return values for behavior verification
+ * - capturePerf: Capture performance metrics (timing only)
+ * - serialize/deserialize: Value serialization for storage
+ * - comparator: Deep equality comparison
+ *
+ * Usage (CommonJS):
+ *   const { capture, capturePerf } = require('codeflash');
+ *
+ * Usage (ES Modules):
+ *   import { capture, capturePerf } from 'codeflash';
+ */
+
+'use strict';
+
+// Main capture functions (instrumentation)
+const capture = require('./capture');
+
+// Serialization utilities
+const serializer = require('./serializer');
+
+// Comparison utilities
+const comparator = require('./comparator');
+
+// Result comparison (used by CLI)
+const compareResults = require('./compare-results');
+
+// Re-export all public APIs
+module.exports = {
+    // === Main Instrumentation API ===
+    capture: capture.capture,
+    capturePerf: capture.capturePerf,
+    captureMultiple: capture.captureMultiple,
+
+    // === Test Lifecycle ===
+    writeResults: capture.writeResults,
+    clearResults: capture.clearResults,
+    getResults: capture.getResults,
+    setTestName: capture.setTestName,
+    initDatabase: capture.initDatabase,
+    resetInvocationCounters: capture.resetInvocationCounters,
+
+    // === Serialization ===
+    serialize: serializer.serialize,
+    deserialize: serializer.deserialize,
+    getSerializerType: serializer.getSerializerType,
+    safeSerialize: capture.safeSerialize,
+    safeDeserialize: capture.safeDeserialize,
+
+    // === Comparison ===
+    comparator: comparator.comparator,
+    createComparator: comparator.createComparator,
+    strictComparator: comparator.strictComparator,
+    looseComparator: comparator.looseComparator,
+    isClose: comparator.isClose,
+
+    // === Result Comparison (CLI helpers) ===
+    readTestResults: compareResults.readTestResults,
+    compareResults: compareResults.compareResults,
+    compareBuffers: compareResults.compareBuffers,
+
+    // === Utilities ===
+    getInvocationIndex: capture.getInvocationIndex,
+    sanitizeTestId: capture.sanitizeTestId,
+
+    // === Constants ===
+    LOOP_INDEX: capture.LOOP_INDEX,
+    OUTPUT_FILE: capture.OUTPUT_FILE,
+    TEST_ITERATION: capture.TEST_ITERATION,
+
+    // === Batch Looping Control (used by loop-runner) ===
+    incrementBatch: capture.incrementBatch,
+    getCurrentBatch: capture.getCurrentBatch,
+    checkSharedTimeLimit: capture.checkSharedTimeLimit,
+    PERF_BATCH_SIZE: capture.PERF_BATCH_SIZE,
+    PERF_LOOP_COUNT: capture.PERF_LOOP_COUNT,
+
+    // === Feature Detection ===
+    hasV8: serializer.hasV8,
+    hasMsgpack: serializer.hasMsgpack,
+};