observability-toolkit 2.0.0 → 2.1.0

package/dist/lib/core/file-utils.test.js

@@ -0,0 +1,1329 @@
+import { describe, it, beforeEach, afterEach } from 'vitest';
+import assert from 'node:assert';
+import { FLOAT_TOLERANCE } from '../../test-helpers/test-data-builders.js';
+import { DEFAULT_LIMIT_10, SAMPLE_SIZE_100, COUNT_FOUR, COUNT_FIFTEEN, COUNT_FIVE, COUNT_NINE, COUNT_SIX, COUNT_THREE, COUNT_TWO, VALUE_FORTY_TWO, } from '../../test-helpers/test-constants.js';
+import { MAX_RESULTS_IN_MEMORY, RETENTION_DAYS_DEFAULT, STREAMING_THRESHOLD, } from './constants.js';
+import { TIME_MS, } from './units.js';
+import { ESTIMATED_SIZE_MAX_BYTES, ESTIMATED_SIZE_MIN_BYTES, LARGE_ARRAY_SAMPLE_SIZE, LARGE_JSONL_LINE_COUNT, LARGE_RESERVOIR_DATASET_SIZE, LARGE_RESERVOIR_P50_EXPECTED, LARGE_RESERVOIR_P95_EXPECTED, LARGE_RESERVOIR_SUM_UPPER_TERM, MIXED_ARRAY_CIRCULAR_ITEMS, MIXED_ARRAY_NORMAL_ITEMS, MOSTLY_CIRCULAR_ITEMS, MOSTLY_CIRCULAR_NORMAL_ITEMS, PERCENTILE_P50_EXPECTED, PERCENTILE_P95_EXPECTED, PERCENTILE_P99_EXPECTED, PERCENTILE_TEST_SAMPLE_SIZE, RESULT_COUNT_EIGHT, RESULT_COUNT_TEN, NUMERIC_SEQUENCE_FIVE_TWO_EIGHT, NUMERIC_SEQUENCE_NEGATIVE_TEN_TO_TEN, NUMERIC_SEQUENCE_ONE_TO_FIVE, NUMERIC_SEQUENCE_ONE_TO_TEN, NUMERIC_SEQUENCE_ONE_TO_THREE, NUMERIC_SEQUENCE_TEN_TWENTY_THIRTY, RELATIVE_ERROR_TEN_PERCENT, STREAMING_COUNT_CUSTOM_HIGH, STREAMING_COUNT_CUSTOM_LOW, STREAMING_COUNT_CUSTOM_THRESHOLD, STREAMING_COUNT_DEFAULT_THRESHOLD, WINDOW_LIMIT_FIVE, WINDOW_LIMIT_TEN, WINDOW_LIMIT_THREE, WINDOW_OFFSET_TWO, } from './file-utils.test-constants.js';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+import { gzipSync, gunzipSync } from 'zlib';
+import { expandTilde, loadJson, loadJsonSafe, getDateString, listFiles, parseDateFromFilename, readJsonlSync, readJsonlSyncWithStats, streamJsonl, filterByDateRange, paginateResults, hasReachedLimit, cleanupOldFiles, isGzipFile, createJsonlStream, BatchWriter, StreamingAggregator, enforceMemoryLimit, shouldUseStreamingAggregation, getMemoryLimits, hasCircularReference, estimateByteSize, CONSERVATIVE_FALLBACK_SIZE, } from './file-utils.js';
+describe('file-utils', () => {
+    let testDir;
+    beforeEach(() => {
+        // Create a unique temp directory for each test
+        testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'file-utils-test-'));
+    });
+    afterEach(() => {
+        // Clean up temp directory
+        if (testDir && fs.existsSync(testDir)) {
+            fs.rmSync(testDir, { recursive: true, force: true });
+        }
+    });
+    describe('expandTilde', () => {
+        it('should return path as-is if it does not start with tilde', () => {
+            const result = expandTilde('/absolute/path');
+            assert.strictEqual(result, '/absolute/path');
+        });
+        it('should expand tilde to home directory for ~ alone', () => {
+            const result = expandTilde('~');
+            assert.strictEqual(result, os.homedir());
+        });
+        it('should expand tilde to home directory for ~/path', () => {
+            const result = expandTilde('~/documents/file.txt');
+            assert.strictEqual(result, path.join(os.homedir(), 'documents/file.txt'));
+        });
+        it('should return ~user/path as-is (not supported)', () => {
+            const result = expandTilde('~someuser/path');
+            assert.strictEqual(result, '~someuser/path');
+        });
+        it('should handle multiple directory levels', () => {
+            const result = expandTilde('~/a/b/c/d');
+            assert.strictEqual(result, path.join(os.homedir(), 'a/b/c/d'));
+        });
+    });
+    describe('loadJson', () => {
+        it('should load and parse valid JSON file', () => {
+            const testData = { key: 'value', num: VALUE_FORTY_TWO };
+            const filePath = path.join(testDir, 'test.json');
+            fs.writeFileSync(filePath, JSON.stringify(testData));
+            const result = loadJson(filePath);
+            assert.deepStrictEqual(result, testData);
+        });
+        it('should return null if file does not exist', () => {
+            const result = loadJson(path.join(testDir, 'nonexistent.json'));
+            assert.strictEqual(result, null);
+        });
+        it('should return null on JSON parse error', () => {
+            const filePath = path.join(testDir, 'invalid.json');
+            fs.writeFileSync(filePath, 'invalid json {');
+            const result = loadJson(filePath);
+            assert.strictEqual(result, null);
+        });
+        it('should return fallback value if file does not exist', () => {
+            const fallback = { default: true };
+            const result = loadJson(path.join(testDir, 'nonexistent.json'), fallback);
+            assert.deepStrictEqual(result, fallback);
+        });
+        it('should return fallback value on parse error', () => {
+            const fallback = { default: true };
+            const filePath = path.join(testDir, 'invalid.json');
+            fs.writeFileSync(filePath, 'invalid json');
+            const result = loadJson(filePath, fallback);
+            assert.deepStrictEqual(result, fallback);
+        });
+        it('should handle empty JSON file', () => {
+            const filePath = path.join(testDir, 'empty.json');
+            fs.writeFileSync(filePath, '');
+            const result = loadJson(filePath);
+            assert.strictEqual(result, null);
+        });
+        it('should parse complex nested JSON', () => {
+            const testData = {
+                nested: { deep: { value: 'test' } },
+                array: [1, COUNT_TWO, COUNT_THREE],
+            };
+            const filePath = path.join(testDir, 'complex.json');
+            fs.writeFileSync(filePath, JSON.stringify(testData));
+            const result = loadJson(filePath);
+            assert.deepStrictEqual(result, testData);
+        });
+    });
+    describe('loadJsonSafe', () => {
+        it('should return loaded JSON if file exists and is valid', () => {
+            const testData = { key: 'value' };
+            const fallback = { fallback: true };
+            const filePath = path.join(testDir, 'test.json');
+            fs.writeFileSync(filePath, JSON.stringify(testData));
+            const result = loadJsonSafe(filePath, fallback);
+            assert.deepStrictEqual(result, testData);
+        });
+        it('should return fallback if file does not exist', () => {
+            const fallback = { fallback: true };
+            const result = loadJsonSafe(path.join(testDir, 'nonexistent.json'), fallback);
+            assert.deepStrictEqual(result, fallback);
+        });
+        it('should return fallback on parse error', () => {
+            const fallback = { fallback: true };
+            const filePath = path.join(testDir, 'invalid.json');
+            fs.writeFileSync(filePath, 'invalid json');
+            const result = loadJsonSafe(filePath, fallback);
+            assert.deepStrictEqual(result, fallback);
+        });
+        it('should never return null', () => {
+            const fallback = { key: 'default' };
+            const result = loadJsonSafe(path.join(testDir, 'nonexistent.json'), fallback);
+            assert.notStrictEqual(result, null);
+        });
+    });
+    describe('getDateString', () => {
+        it('should format date as YYYY-MM-DD', () => {
+            const date = new Date('2026-01-28T12:34:56.789Z');
+            const result = getDateString(date);
+            assert.strictEqual(result, '2026-01-28');
+        });
+        it('should use current date if none provided', () => {
+            const result = getDateString();
+            const now = new Date();
+            const expected = now.toISOString().split('T')[0];
+            assert.strictEqual(result, expected);
+        });
+        it('should handle leap year date', () => {
+            const date = new Date('2024-02-29T00:00:00.000Z');
+            const result = getDateString(date);
+            assert.strictEqual(result, '2024-02-29');
+        });
+        it('should handle year boundary', () => {
+            const date = new Date('2025-12-31T23:59:59.999Z');
+            const result = getDateString(date);
+            assert.strictEqual(result, '2025-12-31');
+        });
+        it('should handle January 1st', () => {
+            const date = new Date('2026-01-01T00:00:00.000Z');
+            const result = getDateString(date);
+            assert.strictEqual(result, '2026-01-01');
+        });
+    });
+    describe('listFiles', () => {
+        it('should return empty array if directory does not exist', () => {
+            const result = listFiles(path.join(testDir, 'nonexistent'), /\.json$/);
+            assert.deepStrictEqual(result, []);
+        });
+        it('should filter files by pattern', () => {
+            fs.writeFileSync(path.join(testDir, 'file1.json'), '{}');
+            fs.writeFileSync(path.join(testDir, 'file2.json'), '{}');
+            fs.writeFileSync(path.join(testDir, 'file3.txt'), 'text');
+            fs.writeFileSync(path.join(testDir, 'file4.json'), '{}');
+            const result = listFiles(testDir, /\.json$/);
+            assert.strictEqual(result.length, COUNT_THREE);
+            assert(result.every(f => f.endsWith('.json')));
+        });
+        it('should return full paths', () => {
+            fs.writeFileSync(path.join(testDir, 'file1.json'), '{}');
+            fs.writeFileSync(path.join(testDir, 'file2.json'), '{}');
+            const result = listFiles(testDir, /\.json$/);
+            assert(result.every(f => f.startsWith(testDir)));
+        });
+        it('should sort by modification time (newest first)', async () => {
+            // Create files with different modification times
+            const file1 = path.join(testDir, 'file1.json');
+            const file2 = path.join(testDir, 'file2.json');
+            const file3 = path.join(testDir, 'file3.json');
+            fs.writeFileSync(file1, '{}');
+            // Small delay to ensure different mtimes
+            await new Promise(resolve => setTimeout(resolve, STREAMING_COUNT_CUSTOM_LOW));
+            fs.writeFileSync(file2, '{}');
+            await new Promise(resolve => setTimeout(resolve, STREAMING_COUNT_CUSTOM_LOW));
+            fs.writeFileSync(file3, '{}');
+            const result = listFiles(testDir, /\.json$/);
+            assert.strictEqual(result[0], file3); // newest first
+            assert.strictEqual(result[1], file2);
+            assert.strictEqual(result[2], file1); // oldest last
+        });
+        it('should handle empty directory', () => {
+            const result = listFiles(testDir, /\.json$/);
+            assert.deepStrictEqual(result, []);
+        });
+        it('should match complex regex patterns', () => {
+            fs.writeFileSync(path.join(testDir, 'traces-2026-01-28.jsonl'), '{}');
+            fs.writeFileSync(path.join(testDir, 'metrics-2026-01-28.jsonl'), '{}');
+            fs.writeFileSync(path.join(testDir, 'logs-2026-01-28.jsonl'), '{}');
+            fs.writeFileSync(path.join(testDir, 'other.txt'), 'text');
+            const result = listFiles(testDir, /^\w+-\d{4}-\d{2}-\d{2}\.jsonl$/);
+            assert.strictEqual(result.length, COUNT_THREE);
+        });
+    });
+    describe('parseDateFromFilename', () => {
+        it('should extract date from filename with format traces-YYYY-MM-DD.jsonl', () => {
+            const result = parseDateFromFilename('traces-2026-01-28.jsonl');
+            assert.strictEqual(result, '2026-01-28');
+        });
+        it('should extract date from filename with format metrics-YYYY-MM-DD.jsonl', () => {
+            const result = parseDateFromFilename('metrics-2026-12-25.jsonl');
+            assert.strictEqual(result, '2026-12-25');
+        });
+        it('should return null if no date found', () => {
+            const result = parseDateFromFilename('nodatehere.jsonl');
+            assert.strictEqual(result, null);
+        });
+        it('should handle full path', () => {
+            const result = parseDateFromFilename('/path/to/traces-2026-01-28.jsonl');
+            assert.strictEqual(result, '2026-01-28');
+        });
+        it('should extract first matching date pattern', () => {
+            const result = parseDateFromFilename('traces-2026-01-28-backup-2026-01-29.jsonl');
+            assert.strictEqual(result, '2026-01-28');
+        });
+        it('should return null for invalid date 9999-99-99 (invalid month/day/year)', () => {
+            const result = parseDateFromFilename('file-9999-99-99.jsonl');
+            assert.strictEqual(result, null);
+        });
+        it('should return null for invalid month 13', () => {
+            const result = parseDateFromFilename('file-2024-13-01.jsonl');
+            assert.strictEqual(result, null);
+        });
+        it('should return null for invalid month 00', () => {
+            const result = parseDateFromFilename('file-2024-00-15.jsonl');
+            assert.strictEqual(result, null);
+        });
+        it('should return null for invalid day 00', () => {
+            const result = parseDateFromFilename('file-2024-05-00.jsonl');
+            assert.strictEqual(result, null);
+        });
+        it('should return null for invalid day 32', () => {
+            const result = parseDateFromFilename('file-2024-05-32.jsonl');
+            assert.strictEqual(result, null);
+        });
+        it('should return null for year before 2000', () => {
+            const result = parseDateFromFilename('file-1999-06-15.jsonl');
+            assert.strictEqual(result, null);
+        });
+        it('should return null for year after 2100', () => {
+            const result = parseDateFromFilename('file-2101-06-15.jsonl');
+            assert.strictEqual(result, null);
+        });
+        it('should accept year 2000 (boundary)', () => {
+            const result = parseDateFromFilename('file-2000-01-01.jsonl');
+            assert.strictEqual(result, '2000-01-01');
+        });
+        it('should accept year 2100 (boundary)', () => {
+            const result = parseDateFromFilename('file-2100-12-31.jsonl');
+            assert.strictEqual(result, '2100-12-31');
+        });
+        it('should return null for incomplete date patterns', () => {
+            const result = parseDateFromFilename('file-2026-01.jsonl');
+            assert.strictEqual(result, null);
+        });
+        it('should handle leap year date', () => {
+            const result = parseDateFromFilename('traces-2024-02-29.jsonl');
+            assert.strictEqual(result, '2024-02-29');
+        });
+    });
+    describe('readJsonlSync', () => {
+        it('should return empty array if file does not exist', () => {
+            const result = readJsonlSync(path.join(testDir, 'nonexistent.jsonl'));
+            assert.deepStrictEqual(result, []);
+        });
+        it('should parse JSONL file with multiple lines', () => {
+            const filePath = path.join(testDir, 'test.jsonl');
+            fs.writeFileSync(filePath, '{"id":1,"name":"first"}\n{"id":2,"name":"second"}\n{"id":3,"name":"third"}');
+            const result = readJsonlSync(filePath);
+            assert.strictEqual(result.length, COUNT_THREE);
+            assert.deepStrictEqual(result[0], { id: 1, name: 'first' });
+            assert.deepStrictEqual(result[1], { id: 2, name: 'second' });
+            assert.deepStrictEqual(result[2], { id: 3, name: 'third' });
+        });
+        it('should skip empty lines', () => {
+            const filePath = path.join(testDir, 'test.jsonl');
+            fs.writeFileSync(filePath, '{"id":1}\n\n{"id":2}\n   \n{"id":3}');
+            const result = readJsonlSync(filePath);
+            assert.strictEqual(result.length, COUNT_THREE);
+        });
+        it('should skip malformed JSON lines', () => {
+            const filePath = path.join(testDir, 'test.jsonl');
+            fs.writeFileSync(filePath, '{"id":1}\n{invalid json}\n{"id":2}');
+            const result = readJsonlSync(filePath);
+            assert.strictEqual(result.length, COUNT_TWO);
+            assert.deepStrictEqual(result[0], { id: 1 });
+            assert.deepStrictEqual(result[1], { id: 2 });
+        });
+        it('should respect limit parameter', () => {
+            const filePath = path.join(testDir, 'test.jsonl');
+            fs.writeFileSync(filePath, '{"id":1}\n{"id":2}\n{"id":3}\n{"id":4}\n{"id":5}');
+            const result = readJsonlSync(filePath, COUNT_THREE);
+            assert.strictEqual(result.length, COUNT_THREE);
+        });
+        it('should stop reading at limit even with more lines', () => {
+            const filePath = path.join(testDir, 'test.jsonl');
+            fs.writeFileSync(filePath, '{"id":1}\n{"id":2}\n{"id":3}\n{"id":4}');
+            const result = readJsonlSync(filePath, COUNT_TWO);
+            assert.strictEqual(result.length, COUNT_TWO);
+            assert.deepStrictEqual(result[1], { id: 2 });
+        });
+        it('should handle file with trailing whitespace', () => {
+            const filePath = path.join(testDir, 'test.jsonl');
+            fs.writeFileSync(filePath, '{"id":1}\n{"id":2}\n   ');
+            const result = readJsonlSync(filePath);
+            assert.strictEqual(result.length, COUNT_TWO);
+        });
+        it('should handle single line file', () => {
+            const filePath = path.join(testDir, 'test.jsonl');
+            fs.writeFileSync(filePath, '{"id":1}');
+            const result = readJsonlSync(filePath);
+            assert.strictEqual(result.length, 1);
+            assert.deepStrictEqual(result[0], { id: 1 });
+        });
+        it('should handle complex JSON objects', () => {
+            const filePath = path.join(testDir, 'test.jsonl');
+            fs.writeFileSync(filePath, '{"nested":{"deep":{"value":1}}}\n{"array":[1,2,3]}');
+            const result = readJsonlSync(filePath);
+            assert.deepStrictEqual(result[0], { nested: { deep: { value: 1 } } });
+            assert.deepStrictEqual(result[1], { array: [1, COUNT_TWO, COUNT_THREE] });
+        });
+    });
+    describe('isGzipFile', () => {
+        it('should return true for .gz extension', () => {
+            assert.strictEqual(isGzipFile('file.jsonl.gz'), true);
+            assert.strictEqual(isGzipFile('/path/to/traces-2026-01-28.jsonl.gz'), true);
+        });
+        it('should return false for non-.gz extension', () => {
+            assert.strictEqual(isGzipFile('file.jsonl'), false);
+            assert.strictEqual(isGzipFile('/path/to/traces.json'), false);
+            assert.strictEqual(isGzipFile('file.gz.bak'), false);
+        });
+    });
+    describe('createJsonlStream', () => {
+        it('should return a readable stream for plain file', async () => {
+            const filePath = path.join(testDir, 'test.jsonl');
+            fs.writeFileSync(filePath, '{"id":1}\n{"id":2}');
+            const stream = createJsonlStream(filePath);
+            assert.ok(stream);
+            assert.ok('pipe' in stream);
+            // Consume the stream to avoid async activity after test ends
+            const chunks = [];
+            for await (const chunk of stream) {
+                chunks.push(Buffer.from(chunk));
+            }
+            assert.ok(chunks.length > 0);
+        });
+        it('should return a decompressing stream for gzip file', async () => {
+            const filePath = path.join(testDir, 'test.jsonl.gz');
+            const content = '{"id":1}\n{"id":2}';
+            fs.writeFileSync(filePath, gzipSync(content));
+            const stream = createJsonlStream(filePath);
+            assert.ok(stream);
+            // Read and verify content is decompressed
+            const chunks = [];
+            for await (const chunk of stream) {
+                chunks.push(Buffer.from(chunk));
+            }
+            const result = Buffer.concat(chunks).toString('utf-8');
+            assert.strictEqual(result, content);
+        });
+    });
+    describe('streamJsonl', () => {
+        it('should yield nothing if file does not exist', async () => {
+            const results = [];
+            for await (const item of streamJsonl(path.join(testDir, 'nonexistent.jsonl'))) {
+                results.push(item);
+            }
+            assert.strictEqual(results.length, 0);
+        });
+        it('should yield parsed JSON objects from file', async () => {
+            const filePath = path.join(testDir, 'test.jsonl');
+            fs.writeFileSync(filePath, '{"id":1}\n{"id":2}\n{"id":3}');
+            const results = [];
+            for await (const item of streamJsonl(filePath)) {
+                results.push(item);
+            }
+            assert.strictEqual(results.length, COUNT_THREE);
+            assert.deepStrictEqual(results[0], { id: 1 });
+            assert.deepStrictEqual(results[1], { id: 2 });
+            assert.deepStrictEqual(results[2], { id: 3 });
+        });
+        it('should skip empty lines', async () => {
+            const filePath = path.join(testDir, 'test.jsonl');
+            fs.writeFileSync(filePath, '{"id":1}\n\n  \n{"id":2}');
+            const results = [];
+            for await (const item of streamJsonl(filePath)) {
+                results.push(item);
+            }
+            assert.strictEqual(results.length, COUNT_TWO);
+        });
+        it('should skip malformed JSON lines', async () => {
+            const filePath = path.join(testDir, 'test.jsonl');
+            fs.writeFileSync(filePath, '{"id":1}\n{invalid}\n{"id":2}');
+            const results = [];
+            for await (const item of streamJsonl(filePath)) {
+                results.push(item);
+            }
+            assert.strictEqual(results.length, COUNT_TWO);
+            assert.deepStrictEqual(results[0], { id: 1 });
+            assert.deepStrictEqual(results[1], { id: 2 });
+        });
+        it('should handle large JSONL files efficiently', async () => {
+            const filePath = path.join(testDir, 'large.jsonl');
+            const lines = Array.from({ length: LARGE_JSONL_LINE_COUNT }, (_, i) => JSON.stringify({ id: i })).join('\n');
+            fs.writeFileSync(filePath, lines);
+            let count = 0;
+            for await (const _item of streamJsonl(filePath)) {
+                count++;
+            }
+            assert.strictEqual(count, LARGE_JSONL_LINE_COUNT);
+        });
+        it('should stream gzipped JSONL file', async () => {
+            const filePath = path.join(testDir, 'test.jsonl.gz');
+            const content = '{"id":1}\n{"id":2}\n{"id":3}';
+            fs.writeFileSync(filePath, gzipSync(content));
+            const results = [];
+            for await (const item of streamJsonl(filePath)) {
+                results.push(item);
+            }
+            assert.strictEqual(results.length, COUNT_THREE);
+            assert.deepStrictEqual(results[0], { id: 1 });
+            assert.deepStrictEqual(results[1], { id: 2 });
+            assert.deepStrictEqual(results[2], { id: 3 });
+        });
+        it('should skip malformed lines in gzipped file', async () => {
+            const filePath = path.join(testDir, 'test.jsonl.gz');
+            const content = '{"id":1}\n{invalid}\n{"id":2}';
+            fs.writeFileSync(filePath, gzipSync(content));
+            const results = [];
+            for await (const item of streamJsonl(filePath)) {
+                results.push(item);
+            }
+            assert.strictEqual(results.length, COUNT_TWO);
+            assert.deepStrictEqual(results[0], { id: 1 });
+            assert.deepStrictEqual(results[1], { id: 2 });
+        });
+        it('should track parse stats when statsTracker is provided', async () => {
+            const filePath = path.join(testDir, 'test.jsonl');
+            // 2 valid, 1 invalid, 2 empty
+            fs.writeFileSync(filePath, '{"id":1}\n\n{invalid}\n  \n{"id":2}');
+            const tracker = {
+                successes: 0,
+                failures: 0,
+                empties: 0,
+                recordSuccess(_path) { this.successes++; },
+                recordFailure(_path) { this.failures++; },
+                recordEmpty(_path) { this.empties++; },
+            };
+            const results = [];
+            for await (const item of streamJsonl(filePath, { statsTracker: tracker })) {
+                results.push(item);
+            }
+            assert.strictEqual(results.length, COUNT_TWO);
+            assert.strictEqual(tracker.successes, COUNT_TWO);
+            assert.strictEqual(tracker.failures, 1);
+            assert.strictEqual(tracker.empties, COUNT_TWO);
+        });
+        it('should work without statsTracker (backward compatible)', async () => {
+            const filePath = path.join(testDir, 'test.jsonl');
+            fs.writeFileSync(filePath, '{"id":1}\n{"id":2}');
+            const results = [];
+            for await (const item of streamJsonl(filePath)) {
+                results.push(item);
+            }
+            assert.strictEqual(results.length, COUNT_TWO);
+        });
+    });
+    describe('filterByDateRange', () => {
+        const items = [
+            { id: 1, timestamp: '2026-01-10' },
+            { id: 2, timestamp: '2026-01-15' },
+            { id: 3, timestamp: '2026-01-20' },
+            { id: 4, timestamp: '2026-01-25' },
+        ];
+        it('should return all items if no date range specified', () => {
+            const result = filterByDateRange(items);
+            assert.strictEqual(result.length, WINDOW_LIMIT_FIVE - 1);
+        });
+        it('should filter items after startDate', () => {
+            const result = filterByDateRange(items, '2026-01-15');
+            assert.strictEqual(result.length, COUNT_THREE);
+            assert.deepStrictEqual(result[0], { id: 2, timestamp: '2026-01-15' });
+        });
+        it('should filter items before endDate', () => {
+            const result = filterByDateRange(items, undefined, '2026-01-20');
+            assert.strictEqual(result.length, COUNT_THREE);
+            assert.deepStrictEqual(result[2], { id: 3, timestamp: '2026-01-20' });
+        });
+        it('should filter items between startDate and endDate', () => {
+            const result = filterByDateRange(items, '2026-01-12', '2026-01-22');
+            assert.strictEqual(result.length, COUNT_TWO);
+            assert.deepStrictEqual(result[0], { id: 2, timestamp: '2026-01-15' });
+            assert.deepStrictEqual(result[1], { id: 3, timestamp: '2026-01-20' });
+        });
+        it('should include items with exact startDate', () => {
+            const result = filterByDateRange(items, '2026-01-15');
+            assert(result.some(item => item.id === COUNT_TWO));
+        });
+        it('should include items with exact endDate', () => {
+            const result = filterByDateRange(items, undefined, '2026-01-15');
+            assert(result.some(item => item.id === COUNT_TWO));
+        });
+        it('should include items without timestamp', () => {
+            const itemsWithoutTs = [
+                { id: 1, timestamp: '2026-01-10' },
+                { id: 2 },
+                { id: 3, timestamp: '2026-01-20' },
+            ];
+            const result = filterByDateRange(itemsWithoutTs, '2026-01-15');
+            assert.strictEqual(result.length, COUNT_TWO);
+            assert(result.some(item => item.id === COUNT_TWO));
+        });
+        it('should return empty array if no items in range', () => {
+            const result = filterByDateRange(items, '2026-02-01', '2026-02-28');
+            assert.deepStrictEqual(result, []);
+        });
+        it('should handle empty items array', () => {
+            const result = filterByDateRange([], '2026-01-01', '2026-12-31');
+            assert.deepStrictEqual(result, []);
+        });
+        it('should work with time component in timestamp', () => {
+            const itemsWithTime = [
+                { id: 1, timestamp: '2026-01-15T10:00:00' },
+                { id: 2, timestamp: '2026-01-15T15:00:00' },
+                { id: 3, timestamp: '2026-01-16T10:00:00' },
+            ];
+            // Note: filterByDateRange uses lexicographic comparison
+            // '2026-01-15T...' > '2026-01-15' due to 'T', so use end of day
+            const result = filterByDateRange(itemsWithTime, '2026-01-15', '2026-01-15T23:59:59');
+            assert.strictEqual(result.length, COUNT_TWO);
+        });
+        it('should handle lexicographic string comparison correctly', () => {
+            const itemsWithDates = [
+                { id: 1, timestamp: '2026-01-01' },
+                { id: 2, timestamp: '2026-02-01' },
+                { id: 3, timestamp: '2026-10-01' },
+            ];
+            const result = filterByDateRange(itemsWithDates, '2026-02-01', '2026-10-01');
+            assert.strictEqual(result.length, COUNT_TWO);
+        });
+        it('should handle generic type with optional timestamp', () => {
+            const customItems = [
+                { name: 'item1', timestamp: '2026-01-10' },
+                { name: 'item2' },
+            ];
+            const result = filterByDateRange(customItems, '2026-01-09', '2026-01-11');
+            assert.strictEqual(result.length, COUNT_TWO);
+        });
+    });
+    describe('paginateResults', () => {
+        const items = NUMERIC_SEQUENCE_ONE_TO_TEN;
+        it('should return items from offset to offset + limit', () => {
+            const result = paginateResults(items, WINDOW_OFFSET_TWO, WINDOW_LIMIT_THREE);
+            assert.deepStrictEqual(result, NUMERIC_SEQUENCE_ONE_TO_TEN.slice(WINDOW_LIMIT_THREE - 1, COUNT_SIX - 1));
+        });
+        it('should return from start when offset is 0', () => {
+            const result = paginateResults(items, 0, WINDOW_LIMIT_FIVE);
+            assert.deepStrictEqual(result, NUMERIC_SEQUENCE_ONE_TO_FIVE);
+        });
+        it('should handle offset at end of array', () => {
+            const result = paginateResults(items, RESULT_COUNT_EIGHT, WINDOW_LIMIT_FIVE);
+            assert.deepStrictEqual(result, [COUNT_NINE, DEFAULT_LIMIT_10]);
+        });
+        it('should return empty array when offset exceeds length', () => {
+            const result = paginateResults(items, COUNT_FIFTEEN, WINDOW_LIMIT_FIVE);
+            assert.deepStrictEqual(result, []);
+        });
+        it('should handle limit larger than remaining items', () => {
+            const result = paginateResults(items, COUNT_SIX + 1, WINDOW_LIMIT_TEN);
+            assert.deepStrictEqual(result, [RESULT_COUNT_EIGHT, COUNT_NINE, DEFAULT_LIMIT_10]);
+        });
+        it('should handle empty array', () => {
+            const result = paginateResults([], 0, WINDOW_LIMIT_TEN);
+            assert.deepStrictEqual(result, []);
+        });
+        it('should work with object arrays', () => {
+            const objs = NUMERIC_SEQUENCE_ONE_TO_THREE.map((id) => ({ id }));
+            const result = paginateResults(objs, 1, WINDOW_OFFSET_TWO);
+            assert.deepStrictEqual(result, objs.slice(1));
+        });
+    });
+    describe('hasReachedLimit', () => {
+        it('should return true when results equal offset + limit', () => {
+            const result = hasReachedLimit(RESULT_COUNT_TEN, WINDOW_LIMIT_FIVE, WINDOW_LIMIT_FIVE);
+            assert.strictEqual(result, true);
+        });
+        it('should return true when results exceed offset + limit', () => {
+            const result = hasReachedLimit(COUNT_FIFTEEN, WINDOW_LIMIT_FIVE, WINDOW_LIMIT_FIVE);
+            assert.strictEqual(result, true);
+        });
+        it('should return false when results are less than offset + limit', () => {
+            const result = hasReachedLimit(RESULT_COUNT_EIGHT, WINDOW_LIMIT_FIVE, WINDOW_LIMIT_FIVE);
+            assert.strictEqual(result, false);
+        });
+        it('should return false with zero results', () => {
+            const result = hasReachedLimit(0, 0, WINDOW_LIMIT_TEN);
+            assert.strictEqual(result, false);
+        });
+        it('should handle zero offset', () => {
+            const result = hasReachedLimit(RESULT_COUNT_TEN, 0, WINDOW_LIMIT_TEN);
+            assert.strictEqual(result, true);
+        });
+    });
+    describe('readJsonlSyncWithStats', () => {
+        it('should return data and stats', () => {
+            const filePath = path.join(testDir, 'test.jsonl');
+            fs.writeFileSync(filePath, '{"id":1}\n{"id":2}\n{"id":3}');
+            const result = readJsonlSyncWithStats(filePath);
+            assert.ok('data' in result);
+            assert.ok('stats' in result);
+            assert.strictEqual(result.data.length, COUNT_THREE);
+            assert.strictEqual(result.stats.totalLines, COUNT_THREE);
+            assert.strictEqual(result.stats.parsedLines, COUNT_THREE);
+            assert.strictEqual(result.stats.skippedLines, 0);
+            assert.strictEqual(result.stats.emptyLines, 0);
+        });
+        it('should count skipped lines', () => {
+            const filePath = path.join(testDir, 'test.jsonl');
+            fs.writeFileSync(filePath, '{"id":1}\n{invalid json}\n{"id":2}\n{also invalid}');
+            const result = readJsonlSyncWithStats(filePath);
+            assert.strictEqual(result.data.length, COUNT_TWO);
+            assert.strictEqual(result.stats.parsedLines, COUNT_TWO);
+            assert.strictEqual(result.stats.skippedLines, COUNT_TWO);
+        });
+        it('should count empty lines', () => {
+            const filePath = path.join(testDir, 'test.jsonl');
+            // Note: trailing empty lines are removed by trim(), so only internal empty lines are counted
+            fs.writeFileSync(filePath, '{"id":1}\n\n  \n{"id":2}');
+            const result = readJsonlSyncWithStats(filePath);
+            assert.strictEqual(result.data.length, COUNT_TWO);
+            assert.strictEqual(result.stats.emptyLines, COUNT_TWO);
+        });
+        it('should respect limit parameter', () => {
+            const filePath = path.join(testDir, 'test.jsonl');
+            fs.writeFileSync(filePath, '{"id":1}\n{"id":2}\n{"id":3}\n{"id":4}\n{"id":5}');
+            const result = readJsonlSyncWithStats(filePath, COUNT_THREE);
+            assert.strictEqual(result.data.length, COUNT_THREE);
+            assert.strictEqual(result.stats.parsedLines, COUNT_THREE);
+        });
+        it('should return empty stats for non-existent file', () => {
+            const result = readJsonlSyncWithStats(path.join(testDir, 'nonexistent.jsonl'));
+            assert.deepStrictEqual(result.data, []);
+            assert.strictEqual(result.stats.totalLines, 0);
+            assert.strictEqual(result.stats.parsedLines, 0);
+        });
+        it('should track all stats together', () => {
+            const filePath = path.join(testDir, 'test.jsonl');
+            fs.writeFileSync(filePath, '{"id":1}\n\n{bad}\n{"id":2}\n  \n{invalid}\n{"id":3}');
+            const result = readJsonlSyncWithStats(filePath);
+            assert.strictEqual(result.data.length, COUNT_THREE);
+            assert.strictEqual(result.stats.totalLines, COUNT_FIVE + COUNT_TWO);
+            assert.strictEqual(result.stats.parsedLines, COUNT_THREE);
+            assert.strictEqual(result.stats.skippedLines, COUNT_TWO);
+            assert.strictEqual(result.stats.emptyLines, COUNT_TWO);
+        });
+    });
+    describe('cleanupOldFiles', () => {
+        it('should return empty result for non-existent directory', () => {
+            const result = cleanupOldFiles(path.join(testDir, 'nonexistent'));
+            assert.strictEqual(result.deleted, 0);
+            assert.deepStrictEqual(result.errors, []);
+        });
+        it('should delete files older than retention period', () => {
+            // Create files with dates in the past
+            const oldDate = '2020-01-01';
+            const recentDate = new Date().toISOString().split('T')[0];
+            fs.writeFileSync(path.join(testDir, `traces-${oldDate}.jsonl`), '{}');
+            fs.writeFileSync(path.join(testDir, `logs-${oldDate}.jsonl`), '{}');
+            fs.writeFileSync(path.join(testDir, `traces-${recentDate}.jsonl`), '{}');
+            const result = cleanupOldFiles(testDir, RETENTION_DAYS_DEFAULT);
+            assert.strictEqual(result.deleted, COUNT_TWO);
+            assert.deepStrictEqual(result.errors, []);
+            // Verify old files are deleted
+            assert.strictEqual(fs.existsSync(path.join(testDir, `traces-${oldDate}.jsonl`)), false);
+            assert.strictEqual(fs.existsSync(path.join(testDir, `logs-${oldDate}.jsonl`)), false);
+            // Verify recent file is kept
+            assert.strictEqual(fs.existsSync(path.join(testDir, `traces-${recentDate}.jsonl`)), true);
+        });
+        it('should not delete files within retention period', () => {
+            const today = new Date();
+            const yesterday = new Date(today.getTime() - TIME_MS.DAY);
+            const yesterdayStr = yesterday.toISOString().split('T')[0];
+            fs.writeFileSync(path.join(testDir, `traces-${yesterdayStr}.jsonl`), '{}');
+            const result = cleanupOldFiles(testDir, RETENTION_DAYS_DEFAULT);
+            assert.strictEqual(result.deleted, 0);
+            assert.strictEqual(fs.existsSync(path.join(testDir, `traces-${yesterdayStr}.jsonl`)), true);
+        });
+        it('should skip files without parseable dates', () => {
+            fs.writeFileSync(path.join(testDir, 'some-random-file.jsonl'), '{}');
+            fs.writeFileSync(path.join(testDir, 'no-date.jsonl'), '{}');
+            const result = cleanupOldFiles(testDir, RETENTION_DAYS_DEFAULT);
+            assert.strictEqual(result.deleted, 0);
+            assert.strictEqual(fs.existsSync(path.join(testDir, 'some-random-file.jsonl')), true);
+            assert.strictEqual(fs.existsSync(path.join(testDir, 'no-date.jsonl')), true);
+        });
+        it('should only process .jsonl files', () => {
+            const oldDate = '2020-01-01';
+            fs.writeFileSync(path.join(testDir, `traces-${oldDate}.jsonl`), '{}');
+            fs.writeFileSync(path.join(testDir, `traces-${oldDate}.json`), '{}');
+            fs.writeFileSync(path.join(testDir, `traces-${oldDate}.txt`), 'text');
+            const result = cleanupOldFiles(testDir, RETENTION_DAYS_DEFAULT);
+            assert.strictEqual(result.deleted, 1);
+            // Only .jsonl file should be deleted
+            assert.strictEqual(fs.existsSync(path.join(testDir, `traces-${oldDate}.jsonl`)), false);
+            assert.strictEqual(fs.existsSync(path.join(testDir, `traces-${oldDate}.json`)), true);
+            assert.strictEqual(fs.existsSync(path.join(testDir, `traces-${oldDate}.txt`)), true);
+        });
+        it('should respect custom retention period', () => {
+            const today = new Date();
+            // Create file from 5 days ago
+            const fiveDaysAgo = new Date(today.getTime() - COUNT_FIVE * TIME_MS.DAY);
+            const fiveDaysAgoStr = fiveDaysAgo.toISOString().split('T')[0];
+            fs.writeFileSync(path.join(testDir, `traces-${fiveDaysAgoStr}.jsonl`), '{}');
+            // With 7 day retention, file should be kept
+            let result = cleanupOldFiles(testDir, RETENTION_DAYS_DEFAULT);
+            assert.strictEqual(result.deleted, 0);
+            assert.strictEqual(fs.existsSync(path.join(testDir, `traces-${fiveDaysAgoStr}.jsonl`)), true);
+            // With 3 day retention, file should be deleted
+            result = cleanupOldFiles(testDir, COUNT_THREE);
+            assert.strictEqual(result.deleted, 1);
+            assert.strictEqual(fs.existsSync(path.join(testDir, `traces-${fiveDaysAgoStr}.jsonl`)), false);
+        });
+        it('should handle empty directory', () => {
+            const result = cleanupOldFiles(testDir, RETENTION_DAYS_DEFAULT);
+            assert.strictEqual(result.deleted, 0);
+            assert.deepStrictEqual(result.errors, []);
+        });
+        it('should reject directories outside allowed paths', () => {
+            const outsideDir = fs.mkdtempSync(path.join(os.tmpdir(), 'outside-'));
+            try {
+                fs.writeFileSync(path.join(outsideDir, 'traces-2020-01-01.jsonl'), '{}');
+                const result = cleanupOldFiles(outsideDir, RETENTION_DAYS_DEFAULT, [testDir]);
+                assert.strictEqual(result.deleted, 0);
+                assert.strictEqual(result.errors.length, 1);
+                assert(result.errors[0].includes('not within allowed paths'));
+                // File should still exist
+                assert.strictEqual(fs.existsSync(path.join(outsideDir, 'traces-2020-01-01.jsonl')), true);
+            }
+            finally {
+                fs.rmSync(outsideDir, { recursive: true, force: true });
+            }
+        });
+        it('should allow directories within allowed paths', () => {
+            const subDir = path.join(testDir, 'subdir');
+            fs.mkdirSync(subDir);
+            fs.writeFileSync(path.join(subDir, 'traces-2020-01-01.jsonl'), '{}');
+            const result = cleanupOldFiles(subDir, RETENTION_DAYS_DEFAULT, [testDir]);
+            assert.strictEqual(result.deleted, 1);
+            assert.deepStrictEqual(result.errors, []);
+        });
+        it('should handle file types: traces, logs, metrics', () => {
+            const oldDate = '2020-01-01';
+            fs.writeFileSync(path.join(testDir, `traces-${oldDate}.jsonl`), '{}');
+            fs.writeFileSync(path.join(testDir, `logs-${oldDate}.jsonl`), '{}');
+            fs.writeFileSync(path.join(testDir, `metrics-${oldDate}.jsonl`), '{}');
+            const result = cleanupOldFiles(testDir, RETENTION_DAYS_DEFAULT);
+            assert.strictEqual(result.deleted, COUNT_THREE);
+            assert.deepStrictEqual(result.errors, []);
+        });
+        it('should use default retention of 7 days', () => {
+            const today = new Date();
+            // Create file from 8 days ago
+            const eightDaysAgo = new Date(today.getTime() - RESULT_COUNT_EIGHT * TIME_MS.DAY);
+            const eightDaysAgoStr = eightDaysAgo.toISOString().split('T')[0];
+            // Create file from COUNT_SIX days ago
+            const sixDaysAgo = new Date(today.getTime() - COUNT_SIX * TIME_MS.DAY);
+            const sixDaysAgoStr = sixDaysAgo.toISOString().split('T')[0];
+            fs.writeFileSync(path.join(testDir, `traces-${eightDaysAgoStr}.jsonl`), '{}');
+            fs.writeFileSync(path.join(testDir, `traces-${sixDaysAgoStr}.jsonl`), '{}');
+            // Use default retention (7 days)
+            const result = cleanupOldFiles(testDir);
+            assert.strictEqual(result.deleted, 1);
+            assert.strictEqual(fs.existsSync(path.join(testDir, `traces-${eightDaysAgoStr}.jsonl`)), false);
+            assert.strictEqual(fs.existsSync(path.join(testDir, `traces-${sixDaysAgoStr}.jsonl`)), true);
+        });
+        it('should delete old .jsonl.gz files', () => {
+            const oldDate = '2020-01-01';
+            const recentDate = new Date().toISOString().split('T')[0];
+            fs.writeFileSync(path.join(testDir, `traces-${oldDate}.jsonl.gz`), gzipSync('{}'));
+            fs.writeFileSync(path.join(testDir, `logs-${oldDate}.jsonl.gz`), gzipSync('{}'));
+            fs.writeFileSync(path.join(testDir, `traces-${recentDate}.jsonl.gz`), gzipSync('{}'));
+            const result = cleanupOldFiles(testDir, RETENTION_DAYS_DEFAULT);
+            assert.strictEqual(result.deleted, COUNT_TWO);
+            assert.strictEqual(fs.existsSync(path.join(testDir, `traces-${oldDate}.jsonl.gz`)), false);
+            assert.strictEqual(fs.existsSync(path.join(testDir, `logs-${oldDate}.jsonl.gz`)), false);
+            assert.strictEqual(fs.existsSync(path.join(testDir, `traces-${recentDate}.jsonl.gz`)), true);
+        });
+        it('should handle mixed .jsonl and .jsonl.gz files', () => {
+            const oldDate = '2020-01-01';
+            fs.writeFileSync(path.join(testDir, `traces-${oldDate}.jsonl`), '{}');
+            fs.writeFileSync(path.join(testDir, `logs-${oldDate}.jsonl.gz`), gzipSync('{}'));
+            const result = cleanupOldFiles(testDir, RETENTION_DAYS_DEFAULT);
+            assert.strictEqual(result.deleted, COUNT_TWO);
+            assert.strictEqual(fs.existsSync(path.join(testDir, `traces-${oldDate}.jsonl`)), false);
+            assert.strictEqual(fs.existsSync(path.join(testDir, `logs-${oldDate}.jsonl.gz`)), false);
+        });
+    });
+    describe('BatchWriter', () => {
+        it('should write records to file when batch size is reached', () => {
+            const filePath = path.join(testDir, 'batch.jsonl');
+            const writer = new BatchWriter(filePath, { batchSize: 3 });
+            writer.write({ id: 1 });
+            writer.write({ id: 2 });
+            assert.strictEqual(fs.existsSync(filePath), false); // Not flushed yet
+            writer.write({ id: 3 }); // Triggers flush
+            assert.strictEqual(fs.existsSync(filePath), true);
+            writer.close();
+            const content = fs.readFileSync(filePath, 'utf-8');
+            const lines = content.trim().split('\n');
+            assert.strictEqual(lines.length, COUNT_THREE);
+        });
+        it('should flush remaining records on close', () => {
+            const filePath = path.join(testDir, 'batch.jsonl');
+            const writer = new BatchWriter(filePath, { batchSize: DEFAULT_LIMIT_10 });
+            writer.write({ id: 1 });
+            writer.write({ id: 2 });
+            const stats = writer.close();
+            assert.strictEqual(stats.recordsWritten, COUNT_TWO);
+            assert.strictEqual(fs.existsSync(filePath), true);
+            const content = fs.readFileSync(filePath, 'utf-8');
+            const lines = content.trim().split('\n');
+            assert.strictEqual(lines.length, COUNT_TWO);
+        });
+        it('should append to file on multiple flushes', () => {
+            const filePath = path.join(testDir, 'batch.jsonl');
+            const writer = new BatchWriter(filePath, { batchSize: 2 });
+            writer.write({ id: 1 });
+            writer.write({ id: 2 }); // Flush 1
+            writer.write({ id: 3 });
+            writer.write({ id: 4 }); // Flush 2
+            writer.write({ id: 5 });
+            writer.close(); // Flush 3
+            const content = fs.readFileSync(filePath, 'utf-8');
+            const lines = content.trim().split('\n');
+            assert.strictEqual(lines.length, COUNT_FIVE);
+            // Verify content
+            assert.deepStrictEqual(JSON.parse(lines[0]), { id: 1 });
+            assert.deepStrictEqual(JSON.parse(lines[4]), { id: 5 });
+        });
+        it('should write gzip compressed output', () => {
+            const filePath = path.join(testDir, 'batch.jsonl.gz');
+            const writer = new BatchWriter(filePath, { batchSize: DEFAULT_LIMIT_10, gzip: true });
+            writer.write({ id: 1 });
+            writer.write({ id: 2 });
+            writer.close();
+            // File should be gzip compressed
+            const compressed = fs.readFileSync(filePath);
+            const decompressed = gunzipSync(compressed).toString('utf-8');
+            const lines = decompressed.trim().split('\n');
+            assert.strictEqual(lines.length, COUNT_TWO);
+        });
+        it('should handle gzip append correctly', () => {
+            const filePath = path.join(testDir, 'batch.jsonl.gz');
+            const writer = new BatchWriter(filePath, { batchSize: 2, gzip: true });
+            writer.write({ id: 1 });
+            writer.write({ id: 2 }); // Flush 1
+            writer.write({ id: 3 });
+            writer.close(); // Flush 2
+            const compressed = fs.readFileSync(filePath);
+            const decompressed = gunzipSync(compressed).toString('utf-8');
+            const lines = decompressed.trim().split('\n');
+            assert.strictEqual(lines.length, COUNT_THREE);
+        });
+        it('should create parent directories if mkdirp is true', () => {
+            const filePath = path.join(testDir, 'nested', 'dir', 'batch.jsonl');
+            const writer = new BatchWriter(filePath, { mkdirp: true });
+            writer.write({ id: 1 });
+            writer.close();
+            assert.strictEqual(fs.existsSync(filePath), true);
+        });
+        it('should use default batch size of 100', () => {
+            const filePath = path.join(testDir, 'batch.jsonl');
+            const writer = new BatchWriter(filePath);
+            const preFlushCount = SAMPLE_SIZE_100 - 1;
+            // Write 99 records - should not flush
+            for (let i = 0; i < preFlushCount; i++) {
+                writer.write({ id: i });
+            }
+            assert.strictEqual(fs.existsSync(filePath), false);
+            // 100th record triggers flush
+            writer.write({ id: preFlushCount });
+            assert.strictEqual(fs.existsSync(filePath), true);
+            writer.close();
+        });
+        it('should track statistics correctly', () => {
+            const filePath = path.join(testDir, 'batch.jsonl');
+            const writer = new BatchWriter(filePath, { batchSize: 2 });
+            writer.write({ id: 1 });
+            writer.write({ id: 2 }); // Flush 1
+            writer.write({ id: 3 });
+            const stats = writer.close(); // Flush 2
+            assert.strictEqual(stats.recordsWritten, COUNT_THREE);
+            assert.strictEqual(stats.flushCount, COUNT_TWO);
+            assert.ok(stats.bytesWritten > 0);
+        });
+        it('should report buffered count correctly', () => {
+            const filePath = path.join(testDir, 'batch.jsonl');
+            const writer = new BatchWriter(filePath, { batchSize: 5 });
+            assert.strictEqual(writer.bufferedCount, 0);
+            writer.write({ id: 1 });
+            assert.strictEqual(writer.bufferedCount, 1);
+            writer.write({ id: 2 });
+            assert.strictEqual(writer.bufferedCount, COUNT_TWO);
+            writer.close();
+            assert.strictEqual(writer.bufferedCount, 0);
+        });
+        it('should handle writeAll for multiple records', () => {
+            const filePath = path.join(testDir, 'batch.jsonl');
+            const writer = new BatchWriter(filePath, { batchSize: DEFAULT_LIMIT_10 });
+            writer.writeAll([{ id: 1 }, { id: 2 }, { id: 3 }]);
+            writer.close();
+            const content = fs.readFileSync(filePath, 'utf-8');
+            const lines = content.trim().split('\n');
+            assert.strictEqual(lines.length, COUNT_THREE);
+        });
+        it('should handle empty close gracefully', () => {
+            const filePath = path.join(testDir, 'batch.jsonl');
+            const writer = new BatchWriter(filePath);
+            const stats = writer.close();
+            assert.strictEqual(stats.recordsWritten, 0);
+            assert.strictEqual(stats.flushCount, 0);
+            assert.strictEqual(fs.existsSync(filePath), false);
+        });
+    });
+    describe('StreamingAggregator', () => {
+        it('should compute count correctly', () => {
+            const aggregator = new StreamingAggregator();
+            aggregator.addAll(NUMERIC_SEQUENCE_ONE_TO_THREE);
+            assert.strictEqual(aggregator.getResult('count'), COUNT_THREE);
+            assert.strictEqual(aggregator.getCount(), COUNT_THREE);
+        });
+        it('should compute sum correctly', () => {
+            const aggregator = new StreamingAggregator();
+            aggregator.addAll(NUMERIC_SEQUENCE_TEN_TWENTY_THIRTY);
+            assert.strictEqual(aggregator.getResult('sum'), NUMERIC_SEQUENCE_TEN_TWENTY_THIRTY.reduce((sum, value) => sum + value, 0));
+        });
+        it('should compute avg correctly', () => {
+            const aggregator = new StreamingAggregator();
+            aggregator.addAll(NUMERIC_SEQUENCE_TEN_TWENTY_THIRTY);
+            assert.strictEqual(aggregator.getResult('avg'), NUMERIC_SEQUENCE_TEN_TWENTY_THIRTY.reduce((sum, value) => sum + value, 0)
+                / COUNT_THREE);
+        });
+        it('should compute min correctly', () => {
+            const aggregator = new StreamingAggregator();
+            aggregator.addAll(NUMERIC_SEQUENCE_FIVE_TWO_EIGHT);
+            assert.strictEqual(aggregator.getResult('min'), COUNT_TWO);
+        });
+        it('should compute max correctly', () => {
+            const aggregator = new StreamingAggregator();
+            aggregator.addAll(NUMERIC_SEQUENCE_FIVE_TWO_EIGHT);
+            assert.strictEqual(aggregator.getResult('max'), RESULT_COUNT_EIGHT);
+        });
+        it('should compute p50 (median) correctly', () => {
+            const aggregator = new StreamingAggregator();
+            // Add values 1-100 for predictable p50
+            for (let i = 1; i <= PERCENTILE_TEST_SAMPLE_SIZE; i++) {
+                aggregator.add(i);
+            }
+            const p50 = aggregator.getResult('p50');
+            assert.ok(p50 !== undefined);
+            // p50 should be close to 50.5 for 1-100
+            assert.ok(Math.abs(p50 - PERCENTILE_P50_EXPECTED) < 1);
+        });
+        it('should compute p95 correctly', () => {
+            const aggregator = new StreamingAggregator();
+            for (let i = 1; i <= PERCENTILE_TEST_SAMPLE_SIZE; i++) {
+                aggregator.add(i);
+            }
+            const p95 = aggregator.getResult('p95');
+            assert.ok(p95 !== undefined);
+            // p95 should be close to 95.05 for 1-100
+            assert.ok(Math.abs(p95 - PERCENTILE_P95_EXPECTED) < 1);
+        });
+        it('should compute p99 correctly', () => {
+            const aggregator = new StreamingAggregator();
+            for (let i = 1; i <= PERCENTILE_TEST_SAMPLE_SIZE; i++) {
+                aggregator.add(i);
+            }
+            const p99 = aggregator.getResult('p99');
+            assert.ok(p99 !== undefined);
+            // p99 should be close to 99.01 for 1-100
+            assert.ok(Math.abs(p99 - PERCENTILE_P99_EXPECTED) < 1);
+        });
+        it('should return undefined for aggregations on empty aggregator', () => {
+            const aggregator = new StreamingAggregator();
+            assert.strictEqual(aggregator.getResult('count'), 0);
+            assert.strictEqual(aggregator.getResult('sum'), undefined);
+            assert.strictEqual(aggregator.getResult('avg'), undefined);
+            assert.strictEqual(aggregator.getResult('min'), undefined);
+            assert.strictEqual(aggregator.getResult('max'), undefined);
+            assert.strictEqual(aggregator.getResult('p50'), undefined);
+        });
+        it('should handle single value', () => {
+            const aggregator = new StreamingAggregator();
+            aggregator.add(VALUE_FORTY_TWO);
+            assert.strictEqual(aggregator.getResult('count'), 1);
+            assert.strictEqual(aggregator.getResult('sum'), VALUE_FORTY_TWO);
+            assert.strictEqual(aggregator.getResult('avg'), VALUE_FORTY_TWO);
+            assert.strictEqual(aggregator.getResult('min'), VALUE_FORTY_TWO);
+            assert.strictEqual(aggregator.getResult('max'), VALUE_FORTY_TWO);
+            // DDSketch uses relative error; approximate match for percentiles
+            const p50Raw = aggregator.getResult('p50');
+            assert(p50Raw !== undefined, 'p50 result should be defined');
+            assert.ok(Math.abs(p50Raw - VALUE_FORTY_TWO) < VALUE_FORTY_TWO * 0.05, `p50 ${p50Raw} should approximate ${VALUE_FORTY_TWO}`);
+        });
+        it('should handle negative values', () => {
+            const aggregator = new StreamingAggregator();
+            aggregator.addAll(NUMERIC_SEQUENCE_NEGATIVE_TEN_TO_TEN);
+            assert.strictEqual(aggregator.getResult('sum'), 0);
+            assert.strictEqual(aggregator.getResult('min'), -RESULT_COUNT_TEN);
+            assert.strictEqual(aggregator.getResult('max'), RESULT_COUNT_TEN);
+            assert.strictEqual(aggregator.getResult('avg'), 0);
+        });
+        it('should skip non-finite values', () => {
+            const aggregator = new StreamingAggregator();
+            aggregator.add(RESULT_COUNT_TEN);
+            aggregator.add(Infinity);
+            aggregator.add(-Infinity);
+            aggregator.add(NaN);
+            aggregator.add(RESULT_COUNT_TEN * COUNT_TWO);
+            assert.strictEqual(aggregator.getResult('count'), COUNT_TWO);
+            assert.strictEqual(aggregator.getResult('sum'), RESULT_COUNT_TEN * COUNT_THREE);
+            assert.strictEqual(aggregator.getResult('avg'), COUNT_FIFTEEN);
+        });
+        it('should support addAll for multiple values', () => {
+            const aggregator = new StreamingAggregator();
+            aggregator.addAll(NUMERIC_SEQUENCE_ONE_TO_FIVE);
+            assert.strictEqual(aggregator.getResult('count'), COUNT_FIVE);
+            assert.strictEqual(aggregator.getResult('sum'), COUNT_FIFTEEN);
+            assert.strictEqual(aggregator.getResult('avg'), COUNT_THREE);
+        });
+        it('should reset correctly', () => {
+            const aggregator = new StreamingAggregator();
+            aggregator.addAll(NUMERIC_SEQUENCE_ONE_TO_FIVE);
+            aggregator.reset();
+            assert.strictEqual(aggregator.getResult('count'), 0);
+            assert.strictEqual(aggregator.getResult('sum'), undefined);
+            assert.strictEqual(aggregator.getCount(), 0);
+        });
+        it('should return all stats via getStats()', () => {
+            const aggregator = new StreamingAggregator();
+            aggregator.addAll(NUMERIC_SEQUENCE_ONE_TO_FIVE);
+            const stats = aggregator.getStats();
+            assert.strictEqual(stats.count, COUNT_FIVE);
+            assert.strictEqual(stats.sum, COUNT_FIFTEEN);
+            assert.strictEqual(stats.avg, COUNT_THREE);
+            assert.strictEqual(stats.min, 1);
+            assert.strictEqual(stats.max, COUNT_FIVE);
+            assert.ok(stats.p50 !== undefined);
+        });
+        it('should handle large datasets with reservoir sampling', () => {
+            const aggregator = new StreamingAggregator();
+            // Add 50000 values - larger than reservoir size
+            for (let i = 1; i <= LARGE_RESERVOIR_DATASET_SIZE; i++) {
+                aggregator.add(i);
+            }
+            assert.strictEqual(aggregator.getResult('count'), LARGE_RESERVOIR_DATASET_SIZE);
+            assert.strictEqual(aggregator.getResult('sum'), (LARGE_RESERVOIR_DATASET_SIZE * LARGE_RESERVOIR_SUM_UPPER_TERM) / COUNT_TWO);
+            assert.strictEqual(aggregator.getResult('min'), 1);
+            assert.strictEqual(aggregator.getResult('max'), LARGE_RESERVOIR_DATASET_SIZE);
+            // Percentiles should be approximately correct (within 10% error due to sampling)
+            const p50 = aggregator.getResult('p50');
+            assert.ok(p50 !== undefined);
+            assert.ok(Math.abs(p50 - LARGE_RESERVOIR_P50_EXPECTED) / LARGE_RESERVOIR_P50_EXPECTED < RELATIVE_ERROR_TEN_PERCENT);
+            const p95 = aggregator.getResult('p95');
+            assert.ok(p95 !== undefined);
+            assert.ok(Math.abs(p95 - LARGE_RESERVOIR_P95_EXPECTED) / LARGE_RESERVOIR_P95_EXPECTED < RELATIVE_ERROR_TEN_PERCENT);
+        });
+        it('should handle decimal values', () => {
+            const aggregator = new StreamingAggregator();
+            aggregator.add(RELATIVE_ERROR_TEN_PERCENT);
+            aggregator.add(COUNT_TWO / DEFAULT_LIMIT_10);
+            aggregator.add(COUNT_THREE / DEFAULT_LIMIT_10);
+            const sum = aggregator.getResult('sum');
+            assert.ok(sum !== undefined);
+            assert.ok(Math.abs(sum - (COUNT_SIX / DEFAULT_LIMIT_10)) < FLOAT_TOLERANCE);
+        });
+    });
+    describe('enforceMemoryLimit', () => {
+        it('should return results unchanged when under limit', () => {
+            const items = NUMERIC_SEQUENCE_ONE_TO_FIVE;
+            const result = enforceMemoryLimit(items, WINDOW_LIMIT_TEN);
+            assert.deepStrictEqual(result.results, items);
+            assert.strictEqual(result.truncated, false);
+            assert.strictEqual(result.originalCount, COUNT_FIVE);
+        });
+        it('should truncate results when over limit', () => {
+            const items = NUMERIC_SEQUENCE_ONE_TO_TEN;
+            const result = enforceMemoryLimit(items, WINDOW_LIMIT_FIVE);
+            assert.deepStrictEqual(result.results, NUMERIC_SEQUENCE_ONE_TO_FIVE);
+            assert.strictEqual(result.truncated, true);
+            assert.strictEqual(result.originalCount, RESULT_COUNT_TEN);
+        });
+        it('should return exactly at limit without truncation', () => {
+            const items = NUMERIC_SEQUENCE_ONE_TO_FIVE;
+            const result = enforceMemoryLimit(items, WINDOW_LIMIT_FIVE);
+            assert.deepStrictEqual(result.results, items);
+            assert.strictEqual(result.truncated, false);
+            assert.strictEqual(result.originalCount, COUNT_FIVE);
+        });
+        it('should handle empty array', () => {
+            const result = enforceMemoryLimit([], WINDOW_LIMIT_TEN);
+            assert.deepStrictEqual(result.results, []);
+            assert.strictEqual(result.truncated, false);
+            assert.strictEqual(result.originalCount, 0);
+        });
+        it('should work with object arrays', () => {
+            const items = NUMERIC_SEQUENCE_ONE_TO_THREE.map((id) => ({ id }));
+            const result = enforceMemoryLimit(items, WINDOW_OFFSET_TWO);
+            assert.deepStrictEqual(result.results, items.slice(0, WINDOW_OFFSET_TWO));
+            assert.strictEqual(result.truncated, true);
+        });
+        it('should use default limit when not specified', () => {
+            const items = Array.from({ length: SAMPLE_SIZE_100 }, (_, i) => i);
+            const result = enforceMemoryLimit(items);
+            // Default is 10000, so 100 items should not be truncated
+            assert.strictEqual(result.truncated, false);
+            assert.strictEqual(result.results.length, SAMPLE_SIZE_100);
+        });
+        it('should log warning when truncating (verify via side effect)', () => {
+            // Store original console.warn
+            const originalWarn = console.warn;
+            let warnCalled = false;
+            let warnMessage = '';
+            console.warn = (msg) => {
+                warnCalled = true;
+                warnMessage = msg;
+            };
+            try {
+                const items = [1, COUNT_TWO, COUNT_THREE, COUNT_FOUR, COUNT_FIVE];
+                enforceMemoryLimit(items, COUNT_THREE);
+                assert.strictEqual(warnCalled, true);
+                assert.ok(warnMessage.includes('"memory"') || warnMessage.includes('memory'), 'Warning should reference memory component');
+                assert.ok(warnMessage.includes('5'));
+                assert.ok(warnMessage.includes('3'));
+            }
+            finally {
+                console.warn = originalWarn;
+            }
+        });
+    });
+    describe('shouldUseStreamingAggregation', () => {
+        it('should return false when count is below threshold', () => {
+            assert.strictEqual(shouldUseStreamingAggregation(STREAMING_COUNT_CUSTOM_THRESHOLD, STREAMING_THRESHOLD), false);
+            assert.strictEqual(shouldUseStreamingAggregation(STREAMING_THRESHOLD - 1, STREAMING_THRESHOLD), false);
+            assert.strictEqual(shouldUseStreamingAggregation(STREAMING_THRESHOLD, STREAMING_THRESHOLD), false);
+        });
+        it('should return true when count exceeds threshold', () => {
+            assert.strictEqual(shouldUseStreamingAggregation(STREAMING_THRESHOLD + 1, STREAMING_THRESHOLD), true);
+            assert.strictEqual(shouldUseStreamingAggregation(MAX_RESULTS_IN_MEMORY, STREAMING_THRESHOLD), true);
+        });
+        it('should use default threshold when not specified', () => {
+            // Default is STREAMING_THRESHOLD
+            assert.strictEqual(shouldUseStreamingAggregation(STREAMING_COUNT_DEFAULT_THRESHOLD), false);
+        });
+        it('should work with custom threshold', () => {
+            assert.strictEqual(shouldUseStreamingAggregation(STREAMING_COUNT_CUSTOM_HIGH, STREAMING_COUNT_CUSTOM_THRESHOLD), true);
+            assert.strictEqual(shouldUseStreamingAggregation(STREAMING_COUNT_CUSTOM_LOW, STREAMING_COUNT_CUSTOM_THRESHOLD), false);
+        });
+    });
+    describe('getMemoryLimits', () => {
+        it('should return memory limit constants', () => {
+            const limits = getMemoryLimits();
+            assert.ok('maxResults' in limits);
+            assert.ok('streamingThreshold' in limits);
+            assert.strictEqual(typeof limits.maxResults, 'number');
+            assert.strictEqual(typeof limits.streamingThreshold, 'number');
+            assert.ok(limits.maxResults > 0);
+            assert.ok(limits.streamingThreshold > 0);
+        });
+    });
+    describe('hasCircularReference', () => {
+        it('should return false for primitive values', () => {
+            assert.strictEqual(hasCircularReference(null), false);
+            assert.strictEqual(hasCircularReference(undefined), false);
+            assert.strictEqual(hasCircularReference(VALUE_FORTY_TWO), false);
+            assert.strictEqual(hasCircularReference('string'), false);
+            assert.strictEqual(hasCircularReference(true), false);
+        });
+        it('should return false for simple objects', () => {
+            assert.strictEqual(hasCircularReference({ a: 1, b: 2 }), false);
+            assert.strictEqual(hasCircularReference({ nested: { deep: { value: 1 } } }), false);
+        });
+        it('should return false for simple arrays', () => {
+            assert.strictEqual(hasCircularReference([1, COUNT_TWO, COUNT_THREE]), false);
+            assert.strictEqual(hasCircularReference([{ id: 1 }, { id: 2 }]), false);
+        });
+        it('should detect self-referencing object', () => {
+            const obj = { a: 1 };
+            obj.self = obj;
+            assert.strictEqual(hasCircularReference(obj), true);
+        });
+        it('should detect self-referencing array', () => {
+            const arr = [1, COUNT_TWO, COUNT_THREE];
+            arr.push(arr);
+            assert.strictEqual(hasCircularReference(arr), true);
+        });
+        it('should detect nested circular reference', () => {
+            const obj = { a: { b: { c: {} } } };
+            obj.a.b = obj;
+            assert.strictEqual(hasCircularReference(obj), true);
+        });
+        it('should detect circular reference in array of objects', () => {
+            const obj1 = { id: 1 };
+            const obj2 = { id: 2, ref: obj1 };
+            obj1.ref = obj2;
+            assert.strictEqual(hasCircularReference([obj1, obj2]), true);
+        });
+        it('should handle deeply nested non-circular structures', () => {
+            const deep = { a: { b: { c: { d: { e: { f: 1 } } } } } };
+            assert.strictEqual(hasCircularReference(deep), false);
+        });
+    });
+    describe('estimateByteSize', () => {
+        it('should return 0 for empty array', () => {
+            assert.strictEqual(estimateByteSize([]), 0);
+        });
+        it('should return correct size for small arrays', () => {
+            const items = [{ id: 1 }, { id: 2 }];
+            const expected = JSON.stringify(items).length;
+            assert.strictEqual(estimateByteSize(items), expected);
+        });
+        it('should return conservative estimate for circular references in small array', () => {
+            const obj = { id: 1 };
+            obj.self = obj;
+            const result = estimateByteSize([obj]);
+            assert.strictEqual(result, CONSERVATIVE_FALLBACK_SIZE);
+        });
+        it('should return conservative estimate for circular references in large array', () => {
+            // Create array larger than sample size (100)
+            const circularObj = { id: 1 };
+            circularObj.self = circularObj;
+            const items = [];
+            for (let i = 0; i < LARGE_ARRAY_SAMPLE_SIZE; i++) {
+                items.push(circularObj);
+            }
+            const result = estimateByteSize(items);
+            assert.strictEqual(result, CONSERVATIVE_FALLBACK_SIZE);
+        });
+        it('should handle mixed circular and non-circular items in large array', () => {
+            // Create array with some circular and some non-circular items
+            const circularObj = { id: 1 };
+            circularObj.self = circularObj;
+            const items = [];
+            // Add 150 normal items (75% of 200)
+            for (let i = 0; i < MIXED_ARRAY_NORMAL_ITEMS; i++) {
+                items.push({ id: i, data: 'normal' });
+            }
+            // Add 50 circular items (25% of 200)
+            for (let i = 0; i < MIXED_ARRAY_CIRCULAR_ITEMS; i++) {
+                items.push(circularObj);
+            }
+            // Since less than 50% failed, should extrapolate from successful samples
+            const result = estimateByteSize(items);
+            assert.ok(result > 0);
+            // Should not return conservative fallback since <50% failed
+            assert.notStrictEqual(result, CONSERVATIVE_FALLBACK_SIZE);
+        });
+        it('should return conservative estimate when >50% of samples fail', () => {
+            // Create array where most items are circular
+            const circularObj = { id: 1 };
+            circularObj.self = circularObj;
+            const items = [];
+            // Add 40 normal items (20% of 200)
+            for (let i = 0; i < MOSTLY_CIRCULAR_NORMAL_ITEMS; i++) {
+                items.push({ id: i });
+            }
+            // Add 160 circular items (80% of 200)
+            for (let i = 0; i < MOSTLY_CIRCULAR_ITEMS; i++) {
+                items.push(circularObj);
+            }
+            const result = estimateByteSize(items);
+            // Should return conservative fallback since >50% failed
+            assert.strictEqual(result, CONSERVATIVE_FALLBACK_SIZE);
+        });
+        it('should extrapolate size for large non-circular arrays', () => {
+            // Create array larger than sample size
+            const items = Array.from({ length: LARGE_ARRAY_SAMPLE_SIZE }, (_, i) => ({ id: i, name: `item-${i}` }));
+            const result = estimateByteSize(items);
+            // Result should be positive and reasonable
+            assert.ok(result > 0);
+            // Rough estimate: each item is ~25 bytes, so 200 items ~5000 bytes
+            assert.ok(result > ESTIMATED_SIZE_MIN_BYTES);
+            assert.ok(result < ESTIMATED_SIZE_MAX_BYTES);
+        });
+    });
+    describe('enforceMemoryLimit with circular references', () => {
+        it('should trigger size-based truncation for circular references', () => {
+            // Create items with circular references
+            const obj = { id: 1, data: 'test' };
+            obj.self = obj;
+            // The circular reference should cause estimateByteSize to return 1MB
+            // which exceeds the 50MB limit would not trigger, but the detection
+            // prevents bypass (the conservative estimate ensures the check works)
+            const items = [obj];
+            const result = enforceMemoryLimit(items, MAX_RESULTS_IN_MEMORY);
+            // Since a single item with circular ref returns 1MB estimate,
+            // and 1MB < 50MB limit, it won't truncate
+            // But the important thing is it doesn't crash and returns valid result
+            assert.ok(Array.isArray(result.results));
+            assert.strictEqual(result.originalCount, 1);
+        });
+        it('should handle array with nested circular references without crashing', () => {
+            const obj1 = { id: 1 };
+            const obj2 = { id: 2 };
+            obj1.ref = obj2;
+            obj2.ref = obj1;
+            const items = [obj1, obj2, { normal: 'object' }];
+            const result = enforceMemoryLimit(items, MAX_RESULTS_IN_MEMORY);
+            assert.ok(Array.isArray(result.results));
+            assert.strictEqual(result.originalCount, COUNT_THREE);
+        });
+        it('should not crash when JSON.stringify would throw', () => {
+            // Create an object that would cause JSON.stringify to throw
+            const obj = {};
+            obj.circular = obj;
+            const items = Array.from({ length: 50 }, () => obj);
+            const result = enforceMemoryLimit(items, MAX_RESULTS_IN_MEMORY);
+            // Should not throw and should return valid result
+            assert.ok(Array.isArray(result.results));
+            assert.strictEqual(result.originalCount, STREAMING_COUNT_CUSTOM_LOW);
+        });
+    });
+});
+//# sourceMappingURL=file-utils.test.js.map