codeflash 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +104 -0
- package/bin/codeflash-setup.js +13 -0
- package/bin/codeflash.js +131 -0
- package/package.json +71 -6
- package/runtime/capture.js +707 -0
- package/runtime/comparator.js +406 -0
- package/runtime/compare-results.js +329 -0
- package/runtime/index.js +79 -0
- package/runtime/serializer.js +851 -0
- package/scripts/postinstall.js +265 -0
- package/index.js +0 -7
|
@@ -0,0 +1,707 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Codeflash Jest Helper - Unified Test Instrumentation
|
|
3
|
+
*
|
|
4
|
+
* This module provides a unified approach to instrumenting JavaScript tests
|
|
5
|
+
* for both behavior verification and performance measurement.
|
|
6
|
+
*
|
|
7
|
+
* The instrumentation mirrors Python's codeflash implementation:
|
|
8
|
+
* - Static identifiers (testModule, testFunction, lineId) are passed at instrumentation time
|
|
9
|
+
* - Dynamic invocation counter increments only when same call site is seen again (e.g., in loops)
|
|
10
|
+
* - Uses hrtime for nanosecond precision timing
|
|
11
|
+
* - SQLite for consistent data format with Python implementation
|
|
12
|
+
*
|
|
13
|
+
* Usage:
|
|
14
|
+
* const { capture } = require('@codeflash/jest-runtime');
|
|
15
|
+
*
|
|
16
|
+
* // For behavior verification (writes to SQLite):
|
|
17
|
+
* const result = codeflash.capture('functionName', lineId, targetFunction, arg1, arg2);
|
|
18
|
+
*
|
|
19
|
+
* // For performance benchmarking (stdout only):
|
|
20
|
+
* const result = codeflash.capturePerf('functionName', lineId, targetFunction, arg1, arg2);
|
|
21
|
+
*
|
|
22
|
+
* Environment Variables:
|
|
23
|
+
* CODEFLASH_OUTPUT_FILE - Path to write results SQLite file
|
|
24
|
+
* CODEFLASH_LOOP_INDEX - Current benchmark loop iteration (default: 1)
|
|
25
|
+
* CODEFLASH_TEST_ITERATION - Test iteration number (default: 0)
|
|
26
|
+
* CODEFLASH_TEST_MODULE - Test module path
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
const fs = require('fs');
|
|
30
|
+
const path = require('path');
|
|
31
|
+
|
|
32
|
+
// Load the codeflash serializer for robust value serialization
|
|
33
|
+
const serializer = require('./serializer');
|
|
34
|
+
|
|
35
|
+
// Try to load better-sqlite3, fall back to JSON if not available
|
|
36
|
+
let Database;
|
|
37
|
+
let useSqlite = false;
|
|
38
|
+
try {
|
|
39
|
+
Database = require('better-sqlite3');
|
|
40
|
+
useSqlite = true;
|
|
41
|
+
} catch (e) {
|
|
42
|
+
// better-sqlite3 not available, will use JSON fallback
|
|
43
|
+
console.warn('[codeflash] better-sqlite3 not found, using JSON fallback');
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// Configuration from environment
|
|
47
|
+
const OUTPUT_FILE = process.env.CODEFLASH_OUTPUT_FILE || '/tmp/codeflash_results.sqlite';
|
|
48
|
+
const LOOP_INDEX = parseInt(process.env.CODEFLASH_LOOP_INDEX || '1', 10);
|
|
49
|
+
const TEST_ITERATION = process.env.CODEFLASH_TEST_ITERATION || '0';
|
|
50
|
+
const TEST_MODULE = process.env.CODEFLASH_TEST_MODULE || '';
|
|
51
|
+
|
|
52
|
+
// Random seed for reproducible test runs
|
|
53
|
+
// Both original and optimized runs use the same seed to get identical "random" values
|
|
54
|
+
const RANDOM_SEED = parseInt(process.env.CODEFLASH_RANDOM_SEED || '0', 10);
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Seeded random number generator using mulberry32 algorithm.
|
|
58
|
+
* This provides reproducible "random" numbers given a fixed seed.
|
|
59
|
+
*/
|
|
60
|
+
function createSeededRandom(seed) {
|
|
61
|
+
let state = seed;
|
|
62
|
+
return function() {
|
|
63
|
+
state |= 0;
|
|
64
|
+
state = state + 0x6D2B79F5 | 0;
|
|
65
|
+
let t = Math.imul(state ^ state >>> 15, 1 | state);
|
|
66
|
+
t = t + Math.imul(t ^ t >>> 7, 61 | t) ^ t;
|
|
67
|
+
return ((t ^ t >>> 14) >>> 0) / 4294967296;
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// Override non-deterministic APIs with seeded versions if seed is provided
|
|
72
|
+
// NOTE: We do NOT seed performance.now() or process.hrtime() as those are used
|
|
73
|
+
// internally by this script for timing measurements.
|
|
74
|
+
if (RANDOM_SEED !== 0) {
|
|
75
|
+
// Seed Math.random
|
|
76
|
+
const seededRandom = createSeededRandom(RANDOM_SEED);
|
|
77
|
+
Math.random = seededRandom;
|
|
78
|
+
|
|
79
|
+
// Seed Date.now() and new Date() - use fixed base timestamp that increments
|
|
80
|
+
const SEEDED_BASE_TIME = 1700000000000; // Nov 14, 2023 - fixed reference point
|
|
81
|
+
let dateOffset = 0;
|
|
82
|
+
const OriginalDate = Date;
|
|
83
|
+
const originalDateNow = Date.now;
|
|
84
|
+
|
|
85
|
+
Date.now = function() {
|
|
86
|
+
return SEEDED_BASE_TIME + (dateOffset++);
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
// Override Date constructor to use seeded time when called without arguments
|
|
90
|
+
function SeededDate(...args) {
|
|
91
|
+
if (args.length === 0) {
|
|
92
|
+
// No arguments: use seeded current time
|
|
93
|
+
return new OriginalDate(SEEDED_BASE_TIME + (dateOffset++));
|
|
94
|
+
}
|
|
95
|
+
// With arguments: use original behavior
|
|
96
|
+
return new OriginalDate(...args);
|
|
97
|
+
}
|
|
98
|
+
SeededDate.prototype = OriginalDate.prototype;
|
|
99
|
+
SeededDate.now = Date.now;
|
|
100
|
+
SeededDate.parse = OriginalDate.parse;
|
|
101
|
+
SeededDate.UTC = OriginalDate.UTC;
|
|
102
|
+
global.Date = SeededDate;
|
|
103
|
+
|
|
104
|
+
// Seed crypto.randomUUID() and crypto.getRandomValues()
|
|
105
|
+
try {
|
|
106
|
+
const crypto = require('crypto');
|
|
107
|
+
const randomForCrypto = createSeededRandom(RANDOM_SEED + 1000); // Different seed to avoid correlation
|
|
108
|
+
|
|
109
|
+
// Seed crypto.randomUUID()
|
|
110
|
+
if (crypto.randomUUID) {
|
|
111
|
+
const originalRandomUUID = crypto.randomUUID.bind(crypto);
|
|
112
|
+
crypto.randomUUID = function() {
|
|
113
|
+
// Generate a deterministic UUID v4 format
|
|
114
|
+
const hex = () => Math.floor(randomForCrypto() * 16).toString(16);
|
|
115
|
+
const bytes = Array.from({ length: 32 }, hex).join('');
|
|
116
|
+
return `${bytes.slice(0, 8)}-${bytes.slice(8, 12)}-4${bytes.slice(13, 16)}-${(8 + Math.floor(randomForCrypto() * 4)).toString(16)}${bytes.slice(17, 20)}-${bytes.slice(20, 32)}`;
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// Seed crypto.getRandomValues() - used by uuid libraries
|
|
121
|
+
const seededGetRandomValues = function(array) {
|
|
122
|
+
for (let i = 0; i < array.length; i++) {
|
|
123
|
+
if (array instanceof Uint8Array) {
|
|
124
|
+
array[i] = Math.floor(randomForCrypto() * 256);
|
|
125
|
+
} else if (array instanceof Uint16Array) {
|
|
126
|
+
array[i] = Math.floor(randomForCrypto() * 65536);
|
|
127
|
+
} else if (array instanceof Uint32Array) {
|
|
128
|
+
array[i] = Math.floor(randomForCrypto() * 4294967296);
|
|
129
|
+
} else {
|
|
130
|
+
array[i] = Math.floor(randomForCrypto() * 256);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
return array;
|
|
134
|
+
};
|
|
135
|
+
|
|
136
|
+
if (crypto.getRandomValues) {
|
|
137
|
+
crypto.getRandomValues = seededGetRandomValues;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// Also seed webcrypto if available (Node 18+)
|
|
141
|
+
// Use the same seeded function to avoid circular references
|
|
142
|
+
if (crypto.webcrypto) {
|
|
143
|
+
if (crypto.webcrypto.getRandomValues) {
|
|
144
|
+
crypto.webcrypto.getRandomValues = seededGetRandomValues;
|
|
145
|
+
}
|
|
146
|
+
if (crypto.webcrypto.randomUUID) {
|
|
147
|
+
crypto.webcrypto.randomUUID = crypto.randomUUID;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
} catch (e) {
|
|
151
|
+
// crypto module not available, skip seeding
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// Current test context (set by Jest hooks)
|
|
156
|
+
let currentTestName = null;
|
|
157
|
+
let currentTestPath = null; // Test file path from Jest
|
|
158
|
+
|
|
159
|
+
// Invocation counter map: tracks how many times each testId has been seen
|
|
160
|
+
// Key: testId (testModule:testClass:testFunction:lineId:loopIndex)
|
|
161
|
+
// Value: count (starts at 0, increments each time same key is seen)
|
|
162
|
+
const invocationCounterMap = new Map();
|
|
163
|
+
|
|
164
|
+
// Results buffer (for JSON fallback)
|
|
165
|
+
const results = [];
|
|
166
|
+
|
|
167
|
+
// SQLite database (lazy initialized)
|
|
168
|
+
let db = null;
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Get high-resolution time in nanoseconds.
|
|
172
|
+
* Prefers process.hrtime.bigint() for nanosecond precision,
|
|
173
|
+
* falls back to performance.now() * 1e6 for non-Node environments.
|
|
174
|
+
*
|
|
175
|
+
* @returns {bigint|number} - Time in nanoseconds
|
|
176
|
+
*/
|
|
177
|
+
function getTimeNs() {
|
|
178
|
+
if (typeof process !== 'undefined' && process.hrtime && process.hrtime.bigint) {
|
|
179
|
+
return process.hrtime.bigint();
|
|
180
|
+
}
|
|
181
|
+
// Fallback to performance.now() in milliseconds, converted to nanoseconds
|
|
182
|
+
const { performance } = require('perf_hooks');
|
|
183
|
+
return BigInt(Math.floor(performance.now() * 1_000_000));
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Calculate duration in nanoseconds.
|
|
188
|
+
*
|
|
189
|
+
* @param {bigint} start - Start time in nanoseconds
|
|
190
|
+
* @param {bigint} end - End time in nanoseconds
|
|
191
|
+
* @returns {number} - Duration in nanoseconds (as Number for SQLite compatibility)
|
|
192
|
+
*/
|
|
193
|
+
function getDurationNs(start, end) {
|
|
194
|
+
const duration = end - start;
|
|
195
|
+
// Convert to Number for SQLite storage (SQLite INTEGER is 64-bit)
|
|
196
|
+
return Number(duration);
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* Sanitize a string for use in test IDs.
|
|
201
|
+
* Replaces special characters that could conflict with regex extraction
|
|
202
|
+
* during stdout parsing.
|
|
203
|
+
*
|
|
204
|
+
* Characters replaced with '_': ! # : (space) ( ) [ ] { } | \ / * ? ^ $ . + -
|
|
205
|
+
*
|
|
206
|
+
* @param {string} str - String to sanitize
|
|
207
|
+
* @returns {string} - Sanitized string safe for test IDs
|
|
208
|
+
*/
|
|
209
|
+
function sanitizeTestId(str) {
|
|
210
|
+
if (!str) return str;
|
|
211
|
+
// Replace characters that could conflict with our delimiter pattern (######)
|
|
212
|
+
// or the colon-separated format, or general regex metacharacters
|
|
213
|
+
return str.replace(/[!#: ()\[\]{}|\\/*?^$.+\-]/g, '_');
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* Get or create invocation index for a testId.
|
|
218
|
+
* This mirrors Python's index tracking per wrapper function.
|
|
219
|
+
*
|
|
220
|
+
* @param {string} testId - Unique test identifier
|
|
221
|
+
* @returns {number} - Current invocation index (0-based)
|
|
222
|
+
*/
|
|
223
|
+
function getInvocationIndex(testId) {
|
|
224
|
+
const currentIndex = invocationCounterMap.get(testId);
|
|
225
|
+
if (currentIndex === undefined) {
|
|
226
|
+
invocationCounterMap.set(testId, 0);
|
|
227
|
+
return 0;
|
|
228
|
+
}
|
|
229
|
+
invocationCounterMap.set(testId, currentIndex + 1);
|
|
230
|
+
return currentIndex + 1;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* Reset invocation counter for a test.
|
|
235
|
+
* Called at the start of each test to ensure consistent indexing.
|
|
236
|
+
*/
|
|
237
|
+
function resetInvocationCounters() {
|
|
238
|
+
invocationCounterMap.clear();
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* Initialize the SQLite database.
|
|
243
|
+
*/
|
|
244
|
+
function initDatabase() {
|
|
245
|
+
if (!useSqlite || db) return;
|
|
246
|
+
|
|
247
|
+
try {
|
|
248
|
+
db = new Database(OUTPUT_FILE);
|
|
249
|
+
db.exec(`
|
|
250
|
+
CREATE TABLE IF NOT EXISTS test_results (
|
|
251
|
+
test_module_path TEXT,
|
|
252
|
+
test_class_name TEXT,
|
|
253
|
+
test_function_name TEXT,
|
|
254
|
+
function_getting_tested TEXT,
|
|
255
|
+
loop_index INTEGER,
|
|
256
|
+
iteration_id TEXT,
|
|
257
|
+
runtime INTEGER,
|
|
258
|
+
return_value BLOB,
|
|
259
|
+
verification_type TEXT
|
|
260
|
+
)
|
|
261
|
+
`);
|
|
262
|
+
} catch (e) {
|
|
263
|
+
console.error('[codeflash] Failed to initialize SQLite:', e.message);
|
|
264
|
+
useSqlite = false;
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* Safely serialize a value for storage.
|
|
270
|
+
*
|
|
271
|
+
* @param {any} value - Value to serialize
|
|
272
|
+
* @returns {Buffer} - Serialized value as Buffer
|
|
273
|
+
*/
|
|
274
|
+
function safeSerialize(value) {
|
|
275
|
+
try {
|
|
276
|
+
return serializer.serialize(value);
|
|
277
|
+
} catch (e) {
|
|
278
|
+
console.warn('[codeflash] Serialization failed:', e.message);
|
|
279
|
+
return Buffer.from(JSON.stringify({ __type: 'SerializationError', error: e.message }));
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/**
|
|
284
|
+
* Safely deserialize a buffer back to a value.
|
|
285
|
+
*
|
|
286
|
+
* @param {Buffer|Uint8Array} buffer - Serialized buffer
|
|
287
|
+
* @returns {any} - Deserialized value
|
|
288
|
+
*/
|
|
289
|
+
function safeDeserialize(buffer) {
|
|
290
|
+
try {
|
|
291
|
+
return serializer.deserialize(buffer);
|
|
292
|
+
} catch (e) {
|
|
293
|
+
console.warn('[codeflash] Deserialization failed:', e.message);
|
|
294
|
+
return { __type: 'DeserializationError', error: e.message };
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
/**
|
|
299
|
+
* Record a test result to SQLite or JSON buffer.
|
|
300
|
+
*
|
|
301
|
+
* @param {string} testModulePath - Test module path
|
|
302
|
+
* @param {string|null} testClassName - Test class name (null for Jest)
|
|
303
|
+
* @param {string} testFunctionName - Test function name
|
|
304
|
+
* @param {string} funcName - Name of the function being tested
|
|
305
|
+
* @param {string} invocationId - Unique invocation identifier (lineId_index)
|
|
306
|
+
* @param {Array} args - Arguments passed to the function
|
|
307
|
+
* @param {any} returnValue - Return value from the function
|
|
308
|
+
* @param {Error|null} error - Error thrown by the function (if any)
|
|
309
|
+
* @param {number} durationNs - Execution time in nanoseconds
|
|
310
|
+
*/
|
|
311
|
+
function recordResult(testModulePath, testClassName, testFunctionName, funcName, invocationId, args, returnValue, error, durationNs) {
|
|
312
|
+
// Serialize the return value (args, kwargs (empty for JS), return_value) like Python does
|
|
313
|
+
const serializedValue = error
|
|
314
|
+
? safeSerialize(error)
|
|
315
|
+
: safeSerialize([args, {}, returnValue]);
|
|
316
|
+
|
|
317
|
+
if (useSqlite && db) {
|
|
318
|
+
try {
|
|
319
|
+
const stmt = db.prepare(`
|
|
320
|
+
INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
321
|
+
`);
|
|
322
|
+
stmt.run(
|
|
323
|
+
testModulePath, // test_module_path
|
|
324
|
+
testClassName, // test_class_name
|
|
325
|
+
testFunctionName, // test_function_name
|
|
326
|
+
funcName, // function_getting_tested
|
|
327
|
+
LOOP_INDEX, // loop_index
|
|
328
|
+
invocationId, // iteration_id
|
|
329
|
+
durationNs, // runtime (nanoseconds) - no rounding
|
|
330
|
+
serializedValue, // return_value (serialized)
|
|
331
|
+
'function_call' // verification_type
|
|
332
|
+
);
|
|
333
|
+
} catch (e) {
|
|
334
|
+
console.error('[codeflash] Failed to write to SQLite:', e.message);
|
|
335
|
+
// Fall back to JSON
|
|
336
|
+
results.push({
|
|
337
|
+
testModulePath,
|
|
338
|
+
testClassName,
|
|
339
|
+
testFunctionName,
|
|
340
|
+
funcName,
|
|
341
|
+
loopIndex: LOOP_INDEX,
|
|
342
|
+
iterationId: invocationId,
|
|
343
|
+
durationNs,
|
|
344
|
+
returnValue: error ? null : returnValue,
|
|
345
|
+
error: error ? { name: error.name, message: error.message } : null,
|
|
346
|
+
verificationType: 'function_call'
|
|
347
|
+
});
|
|
348
|
+
}
|
|
349
|
+
} else {
|
|
350
|
+
// JSON fallback
|
|
351
|
+
results.push({
|
|
352
|
+
testModulePath,
|
|
353
|
+
testClassName,
|
|
354
|
+
testFunctionName,
|
|
355
|
+
funcName,
|
|
356
|
+
loopIndex: LOOP_INDEX,
|
|
357
|
+
iterationId: invocationId,
|
|
358
|
+
durationNs,
|
|
359
|
+
returnValue: error ? null : returnValue,
|
|
360
|
+
error: error ? { name: error.name, message: error.message } : null,
|
|
361
|
+
verificationType: 'function_call'
|
|
362
|
+
});
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
/**
|
|
367
|
+
* Capture a function call with full behavior tracking.
|
|
368
|
+
*
|
|
369
|
+
* This is the main API for instrumenting function calls for BEHAVIOR verification.
|
|
370
|
+
* It captures inputs, outputs, errors, and timing.
|
|
371
|
+
* Results are written to SQLite for comparison between original and optimized code.
|
|
372
|
+
*
|
|
373
|
+
* Static parameters (funcName, lineId) are determined at instrumentation time.
|
|
374
|
+
* The lineId enables tracking when the same call site is invoked multiple times (e.g., in loops).
|
|
375
|
+
*
|
|
376
|
+
* @param {string} funcName - Name of the function being tested (static)
|
|
377
|
+
* @param {string} lineId - Line number identifier in test file (static)
|
|
378
|
+
* @param {Function} fn - The function to call
|
|
379
|
+
* @param {...any} args - Arguments to pass to the function
|
|
380
|
+
* @returns {any} - The function's return value
|
|
381
|
+
* @throws {Error} - Re-throws any error from the function
|
|
382
|
+
*/
|
|
383
|
+
function capture(funcName, lineId, fn, ...args) {
|
|
384
|
+
// Validate that fn is actually a function
|
|
385
|
+
if (typeof fn !== 'function') {
|
|
386
|
+
const fnType = fn === null ? 'null' : (fn === undefined ? 'undefined' : typeof fn);
|
|
387
|
+
throw new TypeError(
|
|
388
|
+
`codeflash.capture: Expected function '${funcName}' but got ${fnType}. ` +
|
|
389
|
+
`This usually means the function was not imported correctly. ` +
|
|
390
|
+
`Check that the import statement matches how the module exports the function ` +
|
|
391
|
+
`(e.g., default export vs named export, CommonJS vs ES modules).`
|
|
392
|
+
);
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
// Initialize database on first capture
|
|
396
|
+
initDatabase();
|
|
397
|
+
|
|
398
|
+
// Get test context (raw values for SQLite storage)
|
|
399
|
+
// Use TEST_MODULE env var if set, otherwise derive from test file path
|
|
400
|
+
let testModulePath;
|
|
401
|
+
if (TEST_MODULE) {
|
|
402
|
+
testModulePath = TEST_MODULE;
|
|
403
|
+
} else if (currentTestPath) {
|
|
404
|
+
// Get relative path from cwd and convert to module-style path
|
|
405
|
+
const path = require('path');
|
|
406
|
+
const relativePath = path.relative(process.cwd(), currentTestPath);
|
|
407
|
+
// Convert to Python module-style path (e.g., "tests/test_foo.test.js" -> "tests.test_foo.test")
|
|
408
|
+
// This matches what Jest's junit XML produces
|
|
409
|
+
testModulePath = relativePath
|
|
410
|
+
.replace(/\\/g, '/') // Handle Windows paths
|
|
411
|
+
.replace(/\.js$/, '') // Remove .js extension
|
|
412
|
+
.replace(/\.test$/, '.test') // Keep .test suffix
|
|
413
|
+
.replace(/\//g, '.'); // Convert path separators to dots
|
|
414
|
+
} else {
|
|
415
|
+
testModulePath = currentTestName || 'unknown';
|
|
416
|
+
}
|
|
417
|
+
const testClassName = null; // Jest doesn't use classes like Python
|
|
418
|
+
const testFunctionName = currentTestName || 'unknown';
|
|
419
|
+
|
|
420
|
+
// Sanitized versions for stdout tags (avoid regex conflicts)
|
|
421
|
+
const safeModulePath = sanitizeTestId(testModulePath);
|
|
422
|
+
const safeTestFunctionName = sanitizeTestId(testFunctionName);
|
|
423
|
+
|
|
424
|
+
// Create testId for invocation tracking (matches Python format)
|
|
425
|
+
const testId = `${safeModulePath}:${testClassName}:${safeTestFunctionName}:${lineId}:${LOOP_INDEX}`;
|
|
426
|
+
|
|
427
|
+
// Get invocation index (increments if same testId seen again)
|
|
428
|
+
const invocationIndex = getInvocationIndex(testId);
|
|
429
|
+
const invocationId = `${lineId}_${invocationIndex}`;
|
|
430
|
+
|
|
431
|
+
// Format stdout tag (matches Python format, uses sanitized names)
|
|
432
|
+
const testStdoutTag = `${safeModulePath}:${testClassName ? testClassName + '.' : ''}${safeTestFunctionName}:${funcName}:${LOOP_INDEX}:${invocationId}`;
|
|
433
|
+
|
|
434
|
+
// Print start tag
|
|
435
|
+
console.log(`!$######${testStdoutTag}######$!`);
|
|
436
|
+
|
|
437
|
+
// Timing with nanosecond precision
|
|
438
|
+
const startTime = getTimeNs();
|
|
439
|
+
let returnValue;
|
|
440
|
+
let error = null;
|
|
441
|
+
|
|
442
|
+
try {
|
|
443
|
+
returnValue = fn(...args);
|
|
444
|
+
|
|
445
|
+
// Handle promises (async functions)
|
|
446
|
+
if (returnValue instanceof Promise) {
|
|
447
|
+
return returnValue.then(
|
|
448
|
+
(resolved) => {
|
|
449
|
+
const endTime = getTimeNs();
|
|
450
|
+
const durationNs = getDurationNs(startTime, endTime);
|
|
451
|
+
recordResult(testModulePath, testClassName, testFunctionName, funcName, invocationId, args, resolved, null, durationNs);
|
|
452
|
+
// Print end tag (no duration for behavior mode)
|
|
453
|
+
console.log(`!######${testStdoutTag}######!`);
|
|
454
|
+
return resolved;
|
|
455
|
+
},
|
|
456
|
+
(err) => {
|
|
457
|
+
const endTime = getTimeNs();
|
|
458
|
+
const durationNs = getDurationNs(startTime, endTime);
|
|
459
|
+
recordResult(testModulePath, testClassName, testFunctionName, funcName, invocationId, args, null, err, durationNs);
|
|
460
|
+
console.log(`!######${testStdoutTag}######!`);
|
|
461
|
+
throw err;
|
|
462
|
+
}
|
|
463
|
+
);
|
|
464
|
+
}
|
|
465
|
+
} catch (e) {
|
|
466
|
+
error = e;
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
const endTime = getTimeNs();
|
|
470
|
+
const durationNs = getDurationNs(startTime, endTime);
|
|
471
|
+
recordResult(testModulePath, testClassName, testFunctionName, funcName, invocationId, args, returnValue, error, durationNs);
|
|
472
|
+
|
|
473
|
+
// Print end tag (no duration for behavior mode, matching Python)
|
|
474
|
+
console.log(`!######${testStdoutTag}######!`);
|
|
475
|
+
|
|
476
|
+
if (error) throw error;
|
|
477
|
+
return returnValue;
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
/**
|
|
481
|
+
* Capture a function call for PERFORMANCE benchmarking only.
|
|
482
|
+
*
|
|
483
|
+
* This is a lightweight instrumentation that only measures timing.
|
|
484
|
+
* It prints start/end tags to stdout (no SQLite writes, no serialization overhead).
|
|
485
|
+
* Used when we've already verified behavior and just need accurate timing.
|
|
486
|
+
*
|
|
487
|
+
* The timing measurement is done exactly around the function call for accuracy.
|
|
488
|
+
*
|
|
489
|
+
* Output format matches Python's codeflash_performance wrapper:
|
|
490
|
+
* Start: !$######test_module:test_class.test_name:func_name:loop_index:invocation_id######$!
|
|
491
|
+
* End: !######test_module:test_class.test_name:func_name:loop_index:invocation_id:duration_ns######!
|
|
492
|
+
*
|
|
493
|
+
* @param {string} funcName - Name of the function being tested (static)
|
|
494
|
+
* @param {string} lineId - Line number identifier in test file (static)
|
|
495
|
+
* @param {Function} fn - The function to call
|
|
496
|
+
* @param {...any} args - Arguments to pass to the function
|
|
497
|
+
* @returns {any} - The function's return value
|
|
498
|
+
* @throws {Error} - Re-throws any error from the function
|
|
499
|
+
*/
|
|
500
|
+
function capturePerf(funcName, lineId, fn, ...args) {
|
|
501
|
+
// Get test context
|
|
502
|
+
// Use TEST_MODULE env var if set, otherwise derive from test file path
|
|
503
|
+
let testModulePath;
|
|
504
|
+
if (TEST_MODULE) {
|
|
505
|
+
testModulePath = TEST_MODULE;
|
|
506
|
+
} else if (currentTestPath) {
|
|
507
|
+
// Get relative path from cwd and convert to module-style path
|
|
508
|
+
const path = require('path');
|
|
509
|
+
const relativePath = path.relative(process.cwd(), currentTestPath);
|
|
510
|
+
// Convert to Python module-style path (e.g., "tests/test_foo.test.js" -> "tests.test_foo.test")
|
|
511
|
+
testModulePath = relativePath
|
|
512
|
+
.replace(/\\/g, '/')
|
|
513
|
+
.replace(/\.js$/, '')
|
|
514
|
+
.replace(/\.test$/, '.test')
|
|
515
|
+
.replace(/\//g, '.');
|
|
516
|
+
} else {
|
|
517
|
+
testModulePath = currentTestName || 'unknown';
|
|
518
|
+
}
|
|
519
|
+
const testClassName = null; // Jest doesn't use classes like Python
|
|
520
|
+
const testFunctionName = currentTestName || 'unknown';
|
|
521
|
+
|
|
522
|
+
// Sanitized versions for stdout tags (avoid regex conflicts)
|
|
523
|
+
const safeModulePath = sanitizeTestId(testModulePath);
|
|
524
|
+
const safeTestFunctionName = sanitizeTestId(testFunctionName);
|
|
525
|
+
|
|
526
|
+
// Create testId for invocation tracking (matches Python format)
|
|
527
|
+
const testId = `${safeModulePath}:${testClassName}:${safeTestFunctionName}:${lineId}:${LOOP_INDEX}`;
|
|
528
|
+
|
|
529
|
+
// Get invocation index (increments if same testId seen again)
|
|
530
|
+
const invocationIndex = getInvocationIndex(testId);
|
|
531
|
+
const invocationId = `${lineId}_${invocationIndex}`;
|
|
532
|
+
|
|
533
|
+
// Format stdout tag (matches Python format, uses sanitized names)
|
|
534
|
+
const testStdoutTag = `${safeModulePath}:${testClassName ? testClassName + '.' : ''}${safeTestFunctionName}:${funcName}:${LOOP_INDEX}:${invocationId}`;
|
|
535
|
+
|
|
536
|
+
// Print start tag
|
|
537
|
+
console.log(`!$######${testStdoutTag}######$!`);
|
|
538
|
+
|
|
539
|
+
// Timing with nanosecond precision - exactly around the function call
|
|
540
|
+
let returnValue;
|
|
541
|
+
let error = null;
|
|
542
|
+
let durationNs;
|
|
543
|
+
|
|
544
|
+
try {
|
|
545
|
+
const startTime = getTimeNs();
|
|
546
|
+
returnValue = fn(...args);
|
|
547
|
+
const endTime = getTimeNs();
|
|
548
|
+
durationNs = getDurationNs(startTime, endTime);
|
|
549
|
+
|
|
550
|
+
// Handle promises (async functions)
|
|
551
|
+
if (returnValue instanceof Promise) {
|
|
552
|
+
return returnValue.then(
|
|
553
|
+
(resolved) => {
|
|
554
|
+
// For async, we measure until resolution
|
|
555
|
+
const asyncEndTime = getTimeNs();
|
|
556
|
+
const asyncDurationNs = getDurationNs(startTime, asyncEndTime);
|
|
557
|
+
// Print end tag with timing
|
|
558
|
+
console.log(`!######${testStdoutTag}:${asyncDurationNs}######!`);
|
|
559
|
+
return resolved;
|
|
560
|
+
},
|
|
561
|
+
(err) => {
|
|
562
|
+
const asyncEndTime = getTimeNs();
|
|
563
|
+
const asyncDurationNs = getDurationNs(startTime, asyncEndTime);
|
|
564
|
+
// Print end tag with timing even on error
|
|
565
|
+
console.log(`!######${testStdoutTag}:${asyncDurationNs}######!`);
|
|
566
|
+
throw err;
|
|
567
|
+
}
|
|
568
|
+
);
|
|
569
|
+
}
|
|
570
|
+
} catch (e) {
|
|
571
|
+
const endTime = getTimeNs();
|
|
572
|
+
// For sync errors, we still need to calculate duration
|
|
573
|
+
// Use a fallback if we didn't capture startTime yet
|
|
574
|
+
durationNs = 0;
|
|
575
|
+
error = e;
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
// Print end tag with timing (no rounding)
|
|
579
|
+
console.log(`!######${testStdoutTag}:${durationNs}######!`);
|
|
580
|
+
|
|
581
|
+
if (error) throw error;
|
|
582
|
+
return returnValue;
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
/**
|
|
586
|
+
* Capture multiple invocations for benchmarking.
|
|
587
|
+
*
|
|
588
|
+
* @param {string} funcName - Name of the function being tested
|
|
589
|
+
* @param {string} lineId - Line number identifier
|
|
590
|
+
* @param {Function} fn - The function to call
|
|
591
|
+
* @param {Array<Array>} argsList - List of argument arrays to test
|
|
592
|
+
* @returns {Array} - Array of return values
|
|
593
|
+
*/
|
|
594
|
+
function captureMultiple(funcName, lineId, fn, argsList) {
|
|
595
|
+
return argsList.map(args => capture(funcName, lineId, fn, ...args));
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
/**
|
|
599
|
+
* Write remaining JSON results to file (fallback mode).
|
|
600
|
+
* Called automatically via Jest afterAll hook.
|
|
601
|
+
*/
|
|
602
|
+
function writeResults() {
|
|
603
|
+
// Close SQLite connection if open
|
|
604
|
+
if (db) {
|
|
605
|
+
try {
|
|
606
|
+
db.close();
|
|
607
|
+
} catch (e) {
|
|
608
|
+
// Ignore close errors
|
|
609
|
+
}
|
|
610
|
+
db = null;
|
|
611
|
+
return;
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
// Write JSON fallback if SQLite wasn't used
|
|
615
|
+
if (results.length === 0) return;
|
|
616
|
+
|
|
617
|
+
try {
|
|
618
|
+
// Write as JSON for fallback parsing
|
|
619
|
+
const jsonPath = OUTPUT_FILE.replace('.sqlite', '.json');
|
|
620
|
+
const output = {
|
|
621
|
+
version: '1.0.0',
|
|
622
|
+
loopIndex: LOOP_INDEX,
|
|
623
|
+
timestamp: Date.now(),
|
|
624
|
+
results
|
|
625
|
+
};
|
|
626
|
+
fs.writeFileSync(jsonPath, JSON.stringify(output, null, 2));
|
|
627
|
+
} catch (e) {
|
|
628
|
+
console.error('[codeflash] Error writing JSON results:', e.message);
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
/**
|
|
633
|
+
* Clear all recorded results.
|
|
634
|
+
* Useful for resetting between test files.
|
|
635
|
+
*/
|
|
636
|
+
function clearResults() {
|
|
637
|
+
results.length = 0;
|
|
638
|
+
resetInvocationCounters();
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
/**
|
|
642
|
+
* Get the current results buffer.
|
|
643
|
+
* Useful for debugging or custom result handling.
|
|
644
|
+
*
|
|
645
|
+
* @returns {Array} - Current results buffer
|
|
646
|
+
*/
|
|
647
|
+
function getResults() {
|
|
648
|
+
return results;
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
/**
|
|
652
|
+
* Set the current test name.
|
|
653
|
+
* Called automatically via Jest beforeEach hook.
|
|
654
|
+
*
|
|
655
|
+
* @param {string} name - Test name
|
|
656
|
+
*/
|
|
657
|
+
function setTestName(name) {
|
|
658
|
+
currentTestName = name;
|
|
659
|
+
resetInvocationCounters();
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
// Jest lifecycle hooks - these run automatically when this module is imported
|
|
663
|
+
if (typeof beforeEach !== 'undefined') {
|
|
664
|
+
beforeEach(() => {
|
|
665
|
+
// Get current test name and path from Jest's expect state
|
|
666
|
+
try {
|
|
667
|
+
const state = expect.getState();
|
|
668
|
+
currentTestName = state.currentTestName || 'unknown';
|
|
669
|
+
// testPath is the absolute path to the test file
|
|
670
|
+
currentTestPath = state.testPath || null;
|
|
671
|
+
} catch (e) {
|
|
672
|
+
currentTestName = 'unknown';
|
|
673
|
+
currentTestPath = null;
|
|
674
|
+
}
|
|
675
|
+
// Reset invocation counters for each test
|
|
676
|
+
resetInvocationCounters();
|
|
677
|
+
});
|
|
678
|
+
}
|
|
679
|
+
|
|
680
|
+
if (typeof afterAll !== 'undefined') {
|
|
681
|
+
afterAll(() => {
|
|
682
|
+
writeResults();
|
|
683
|
+
});
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
// Export public API
|
|
687
|
+
module.exports = {
|
|
688
|
+
capture, // Behavior verification (writes to SQLite)
|
|
689
|
+
capturePerf, // Performance benchmarking (prints to stdout only)
|
|
690
|
+
captureMultiple,
|
|
691
|
+
writeResults,
|
|
692
|
+
clearResults,
|
|
693
|
+
getResults,
|
|
694
|
+
setTestName,
|
|
695
|
+
safeSerialize,
|
|
696
|
+
safeDeserialize,
|
|
697
|
+
initDatabase,
|
|
698
|
+
resetInvocationCounters,
|
|
699
|
+
getInvocationIndex,
|
|
700
|
+
sanitizeTestId, // Sanitize test names for stdout tags
|
|
701
|
+
// Serializer info
|
|
702
|
+
getSerializerType: serializer.getSerializerType,
|
|
703
|
+
// Constants
|
|
704
|
+
LOOP_INDEX,
|
|
705
|
+
OUTPUT_FILE,
|
|
706
|
+
TEST_ITERATION
|
|
707
|
+
};
|