observability-toolkit 1.8.0 → 1.8.4

package/dist/lib/file-utils.test.js

@@ -4,7 +4,7 @@ import * as fs from 'fs';
 import * as path from 'path';
 import * as os from 'os';
 import { gzipSync, gunzipSync } from 'zlib';
-import { expandTilde, loadJson, loadJsonSafe, getDateString, listFiles, parseDateFromFilename, readJsonlSync, readJsonlSyncWithStats, streamJsonl, filterByDateRange, paginateResults, hasReachedLimit, cleanupOldFiles, isGzipFile, createJsonlStream, BatchWriter, } from './file-utils.js';
+import { expandTilde, loadJson, loadJsonSafe, getDateString, listFiles, parseDateFromFilename, readJsonlSync, readJsonlSyncWithStats, streamJsonl, filterByDateRange, paginateResults, hasReachedLimit, cleanupOldFiles, isGzipFile, createJsonlStream, BatchWriter, StreamingAggregator, enforceMemoryLimit, shouldUseStreamingAggregation, getMemoryLimits, hasCircularReference, estimateByteSize, CONSERVATIVE_FALLBACK_SIZE, } from './file-utils.js';
 describe('file-utils', () => {
     let testDir;
     beforeEach(() => {
@@ -211,9 +211,41 @@ describe('file-utils', () => {
             const result = parseDateFromFilename('traces-2026-01-28-backup-2026-01-29.jsonl');
             assert.strictEqual(result, '2026-01-28');
         });
-        it('should handle invalid date numbers', () => {
+        it('should return null for invalid date 9999-99-99 (invalid month/day/year)', () => {
             const result = parseDateFromFilename('file-9999-99-99.jsonl');
-            assert.strictEqual(result, '9999-99-99');
+            assert.strictEqual(result, null);
+        });
+        it('should return null for invalid month 13', () => {
+            const result = parseDateFromFilename('file-2024-13-01.jsonl');
+            assert.strictEqual(result, null);
+        });
+        it('should return null for invalid month 00', () => {
+            const result = parseDateFromFilename('file-2024-00-15.jsonl');
+            assert.strictEqual(result, null);
+        });
+        it('should return null for invalid day 00', () => {
+            const result = parseDateFromFilename('file-2024-05-00.jsonl');
+            assert.strictEqual(result, null);
+        });
+        it('should return null for invalid day 32', () => {
+            const result = parseDateFromFilename('file-2024-05-32.jsonl');
+            assert.strictEqual(result, null);
+        });
+        it('should return null for year before 2000', () => {
+            const result = parseDateFromFilename('file-1999-06-15.jsonl');
+            assert.strictEqual(result, null);
+        });
+        it('should return null for year after 2100', () => {
+            const result = parseDateFromFilename('file-2101-06-15.jsonl');
+            assert.strictEqual(result, null);
+        });
+        it('should accept year 2000 (boundary)', () => {
+            const result = parseDateFromFilename('file-2000-01-01.jsonl');
+            assert.strictEqual(result, '2000-01-01');
+        });
+        it('should accept year 2100 (boundary)', () => {
+            const result = parseDateFromFilename('file-2100-12-31.jsonl');
+            assert.strictEqual(result, '2100-12-31');
         });
         it('should return null for incomplete date patterns', () => {
             const result = parseDateFromFilename('file-2026-01.jsonl');
@@ -858,5 +890,414 @@ describe('file-utils', () => {
             assert.strictEqual(fs.existsSync(filePath), false);
         });
     });
+    describe('StreamingAggregator', () => {
+        it('should compute count correctly', () => {
+            const aggregator = new StreamingAggregator();
+            aggregator.add(1);
+            aggregator.add(2);
+            aggregator.add(3);
+            assert.strictEqual(aggregator.getResult('count'), 3);
+            assert.strictEqual(aggregator.getCount(), 3);
+        });
+        it('should compute sum correctly', () => {
+            const aggregator = new StreamingAggregator();
+            aggregator.add(10);
+            aggregator.add(20);
+            aggregator.add(30);
+            assert.strictEqual(aggregator.getResult('sum'), 60);
+        });
+        it('should compute avg correctly', () => {
+            const aggregator = new StreamingAggregator();
+            aggregator.add(10);
+            aggregator.add(20);
+            aggregator.add(30);
+            assert.strictEqual(aggregator.getResult('avg'), 20);
+        });
+        it('should compute min correctly', () => {
+            const aggregator = new StreamingAggregator();
+            aggregator.add(5);
+            aggregator.add(2);
+            aggregator.add(8);
+            assert.strictEqual(aggregator.getResult('min'), 2);
+        });
+        it('should compute max correctly', () => {
+            const aggregator = new StreamingAggregator();
+            aggregator.add(5);
+            aggregator.add(2);
+            aggregator.add(8);
+            assert.strictEqual(aggregator.getResult('max'), 8);
+        });
+        it('should compute p50 (median) correctly', () => {
+            const aggregator = new StreamingAggregator();
+            // Add values 1-100 for predictable p50
+            for (let i = 1; i <= 100; i++) {
+                aggregator.add(i);
+            }
+            const p50 = aggregator.getResult('p50');
+            assert.ok(p50 !== undefined);
+            // p50 should be close to 50.5 for 1-100
+            assert.ok(Math.abs(p50 - 50.5) < 1);
+        });
+        it('should compute p95 correctly', () => {
+            const aggregator = new StreamingAggregator();
+            for (let i = 1; i <= 100; i++) {
+                aggregator.add(i);
+            }
+            const p95 = aggregator.getResult('p95');
+            assert.ok(p95 !== undefined);
+            // p95 should be close to 95.05 for 1-100
+            assert.ok(Math.abs(p95 - 95.05) < 1);
+        });
+        it('should compute p99 correctly', () => {
+            const aggregator = new StreamingAggregator();
+            for (let i = 1; i <= 100; i++) {
+                aggregator.add(i);
+            }
+            const p99 = aggregator.getResult('p99');
+            assert.ok(p99 !== undefined);
+            // p99 should be close to 99.01 for 1-100
+            assert.ok(Math.abs(p99 - 99.01) < 1);
+        });
+        it('should return undefined for aggregations on empty aggregator', () => {
+            const aggregator = new StreamingAggregator();
+            assert.strictEqual(aggregator.getResult('count'), 0);
+            assert.strictEqual(aggregator.getResult('sum'), undefined);
+            assert.strictEqual(aggregator.getResult('avg'), undefined);
+            assert.strictEqual(aggregator.getResult('min'), undefined);
+            assert.strictEqual(aggregator.getResult('max'), undefined);
+            assert.strictEqual(aggregator.getResult('p50'), undefined);
+        });
+        it('should handle single value', () => {
+            const aggregator = new StreamingAggregator();
+            aggregator.add(42);
+            assert.strictEqual(aggregator.getResult('count'), 1);
+            assert.strictEqual(aggregator.getResult('sum'), 42);
+            assert.strictEqual(aggregator.getResult('avg'), 42);
+            assert.strictEqual(aggregator.getResult('min'), 42);
+            assert.strictEqual(aggregator.getResult('max'), 42);
+            assert.strictEqual(aggregator.getResult('p50'), 42);
+        });
+        it('should handle negative values', () => {
+            const aggregator = new StreamingAggregator();
+            aggregator.add(-10);
+            aggregator.add(-5);
+            aggregator.add(0);
+            aggregator.add(5);
+            aggregator.add(10);
+            assert.strictEqual(aggregator.getResult('sum'), 0);
+            assert.strictEqual(aggregator.getResult('min'), -10);
+            assert.strictEqual(aggregator.getResult('max'), 10);
+            assert.strictEqual(aggregator.getResult('avg'), 0);
+        });
+        it('should skip non-finite values', () => {
+            const aggregator = new StreamingAggregator();
+            aggregator.add(10);
+            aggregator.add(Infinity);
+            aggregator.add(-Infinity);
+            aggregator.add(NaN);
+            aggregator.add(20);
+            assert.strictEqual(aggregator.getResult('count'), 2);
+            assert.strictEqual(aggregator.getResult('sum'), 30);
+            assert.strictEqual(aggregator.getResult('avg'), 15);
+        });
+        it('should support addAll for multiple values', () => {
+            const aggregator = new StreamingAggregator();
+            aggregator.addAll([1, 2, 3, 4, 5]);
+            assert.strictEqual(aggregator.getResult('count'), 5);
+            assert.strictEqual(aggregator.getResult('sum'), 15);
+            assert.strictEqual(aggregator.getResult('avg'), 3);
+        });
+        it('should reset correctly', () => {
+            const aggregator = new StreamingAggregator();
+            aggregator.addAll([1, 2, 3, 4, 5]);
+            aggregator.reset();
+            assert.strictEqual(aggregator.getResult('count'), 0);
+            assert.strictEqual(aggregator.getResult('sum'), undefined);
+            assert.strictEqual(aggregator.getCount(), 0);
+        });
+        it('should return all stats via getStats()', () => {
+            const aggregator = new StreamingAggregator();
+            aggregator.addAll([1, 2, 3, 4, 5]);
+            const stats = aggregator.getStats();
+            assert.strictEqual(stats.count, 5);
+            assert.strictEqual(stats.sum, 15);
+            assert.strictEqual(stats.avg, 3);
+            assert.strictEqual(stats.min, 1);
+            assert.strictEqual(stats.max, 5);
+            assert.ok(stats.p50 !== undefined);
+        });
+        it('should handle large datasets with reservoir sampling', () => {
+            const aggregator = new StreamingAggregator();
+            // Add 50000 values - larger than reservoir size
+            for (let i = 1; i <= 50000; i++) {
+                aggregator.add(i);
+            }
+            assert.strictEqual(aggregator.getResult('count'), 50000);
+            assert.strictEqual(aggregator.getResult('sum'), (50000 * 50001) / 2);
+            assert.strictEqual(aggregator.getResult('min'), 1);
+            assert.strictEqual(aggregator.getResult('max'), 50000);
+            // Percentiles should be approximately correct (within 10% error due to sampling)
+            const p50 = aggregator.getResult('p50');
+            assert.ok(p50 !== undefined);
+            assert.ok(Math.abs(p50 - 25000) / 25000 < 0.10);
+            const p95 = aggregator.getResult('p95');
+            assert.ok(p95 !== undefined);
+            assert.ok(Math.abs(p95 - 47500) / 47500 < 0.10);
+        });
+        it('should handle decimal values', () => {
+            const aggregator = new StreamingAggregator();
+            aggregator.add(0.1);
+            aggregator.add(0.2);
+            aggregator.add(0.3);
+            const sum = aggregator.getResult('sum');
+            assert.ok(sum !== undefined);
+            assert.ok(Math.abs(sum - 0.6) < 0.0001);
+        });
+    });
+    describe('enforceMemoryLimit', () => {
+        it('should return results unchanged when under limit', () => {
+            const items = [1, 2, 3, 4, 5];
+            const result = enforceMemoryLimit(items, 10);
+            assert.deepStrictEqual(result.results, items);
+            assert.strictEqual(result.truncated, false);
+            assert.strictEqual(result.originalCount, 5);
+        });
+        it('should truncate results when over limit', () => {
+            const items = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
+            const result = enforceMemoryLimit(items, 5);
+            assert.deepStrictEqual(result.results, [1, 2, 3, 4, 5]);
+            assert.strictEqual(result.truncated, true);
+            assert.strictEqual(result.originalCount, 10);
+        });
+        it('should return exactly at limit without truncation', () => {
+            const items = [1, 2, 3, 4, 5];
+            const result = enforceMemoryLimit(items, 5);
+            assert.deepStrictEqual(result.results, items);
+            assert.strictEqual(result.truncated, false);
+            assert.strictEqual(result.originalCount, 5);
+        });
+        it('should handle empty array', () => {
+            const result = enforceMemoryLimit([], 10);
+            assert.deepStrictEqual(result.results, []);
+            assert.strictEqual(result.truncated, false);
+            assert.strictEqual(result.originalCount, 0);
+        });
+        it('should work with object arrays', () => {
+            const items = [{ id: 1 }, { id: 2 }, { id: 3 }];
+            const result = enforceMemoryLimit(items, 2);
+            assert.deepStrictEqual(result.results, [{ id: 1 }, { id: 2 }]);
+            assert.strictEqual(result.truncated, true);
+        });
+        it('should use default limit when not specified', () => {
+            const items = Array.from({ length: 100 }, (_, i) => i);
+            const result = enforceMemoryLimit(items);
+            // Default is 10000, so 100 items should not be truncated
+            assert.strictEqual(result.truncated, false);
+            assert.strictEqual(result.results.length, 100);
+        });
+        it('should log warning when truncating (verify via side effect)', () => {
+            // Store original console.warn
+            const originalWarn = console.warn;
+            let warnCalled = false;
+            let warnMessage = '';
+            console.warn = (msg) => {
+                warnCalled = true;
+                warnMessage = msg;
+            };
+            try {
+                const items = [1, 2, 3, 4, 5];
+                enforceMemoryLimit(items, 3);
+                assert.strictEqual(warnCalled, true);
+                assert.ok(warnMessage.includes('[MEMORY]'));
+                assert.ok(warnMessage.includes('5'));
+                assert.ok(warnMessage.includes('3'));
+            }
+            finally {
+                console.warn = originalWarn;
+            }
+        });
+    });
+    describe('shouldUseStreamingAggregation', () => {
+        it('should return false when count is below threshold', () => {
+            assert.strictEqual(shouldUseStreamingAggregation(100, 5000), false);
+            assert.strictEqual(shouldUseStreamingAggregation(4999, 5000), false);
+            assert.strictEqual(shouldUseStreamingAggregation(5000, 5000), false);
+        });
+        it('should return true when count exceeds threshold', () => {
+            assert.strictEqual(shouldUseStreamingAggregation(5001, 5000), true);
+            assert.strictEqual(shouldUseStreamingAggregation(10000, 5000), true);
+        });
+        it('should use default threshold when not specified', () => {
+            // Default is 5000
+            assert.strictEqual(shouldUseStreamingAggregation(4000), false);
+        });
+        it('should work with custom threshold', () => {
+            assert.strictEqual(shouldUseStreamingAggregation(500, 100), true);
+            assert.strictEqual(shouldUseStreamingAggregation(50, 100), false);
+        });
+    });
+    describe('getMemoryLimits', () => {
+        it('should return memory limit constants', () => {
+            const limits = getMemoryLimits();
+            assert.ok('maxResults' in limits);
+            assert.ok('streamingThreshold' in limits);
+            assert.strictEqual(typeof limits.maxResults, 'number');
+            assert.strictEqual(typeof limits.streamingThreshold, 'number');
+            assert.ok(limits.maxResults > 0);
+            assert.ok(limits.streamingThreshold > 0);
+        });
+    });
+    describe('hasCircularReference', () => {
+        it('should return false for primitive values', () => {
+            assert.strictEqual(hasCircularReference(null), false);
+            assert.strictEqual(hasCircularReference(undefined), false);
+            assert.strictEqual(hasCircularReference(42), false);
+            assert.strictEqual(hasCircularReference('string'), false);
+            assert.strictEqual(hasCircularReference(true), false);
+        });
+        it('should return false for simple objects', () => {
+            assert.strictEqual(hasCircularReference({ a: 1, b: 2 }), false);
+            assert.strictEqual(hasCircularReference({ nested: { deep: { value: 1 } } }), false);
+        });
+        it('should return false for simple arrays', () => {
+            assert.strictEqual(hasCircularReference([1, 2, 3]), false);
+            assert.strictEqual(hasCircularReference([{ id: 1 }, { id: 2 }]), false);
+        });
+        it('should detect self-referencing object', () => {
+            const obj = { a: 1 };
+            obj.self = obj;
+            assert.strictEqual(hasCircularReference(obj), true);
+        });
+        it('should detect self-referencing array', () => {
+            const arr = [1, 2, 3];
+            arr.push(arr);
+            assert.strictEqual(hasCircularReference(arr), true);
+        });
+        it('should detect nested circular reference', () => {
+            const obj = { a: { b: { c: {} } } };
+            obj.a.b = obj;
+            assert.strictEqual(hasCircularReference(obj), true);
+        });
+        it('should detect circular reference in array of objects', () => {
+            const obj1 = { id: 1 };
+            const obj2 = { id: 2, ref: obj1 };
+            obj1.ref = obj2;
+            assert.strictEqual(hasCircularReference([obj1, obj2]), true);
+        });
+        it('should handle deeply nested non-circular structures', () => {
+            const deep = { a: { b: { c: { d: { e: { f: 1 } } } } } };
+            assert.strictEqual(hasCircularReference(deep), false);
+        });
+    });
+    describe('estimateByteSize', () => {
+        it('should return 0 for empty array', () => {
+            assert.strictEqual(estimateByteSize([]), 0);
+        });
+        it('should return correct size for small arrays', () => {
+            const items = [{ id: 1 }, { id: 2 }];
+            const expected = JSON.stringify(items).length;
+            assert.strictEqual(estimateByteSize(items), expected);
+        });
+        it('should return conservative estimate for circular references in small array', () => {
+            const obj = { id: 1 };
+            obj.self = obj;
+            const result = estimateByteSize([obj]);
+            assert.strictEqual(result, CONSERVATIVE_FALLBACK_SIZE);
+        });
+        it('should return conservative estimate for circular references in large array', () => {
+            // Create array larger than sample size (100)
+            const circularObj = { id: 1 };
+            circularObj.self = circularObj;
+            const items = [];
+            for (let i = 0; i < 200; i++) {
+                items.push(circularObj);
+            }
+            const result = estimateByteSize(items);
+            assert.strictEqual(result, CONSERVATIVE_FALLBACK_SIZE);
+        });
+        it('should handle mixed circular and non-circular items in large array', () => {
+            // Create array with some circular and some non-circular items
+            const circularObj = { id: 1 };
+            circularObj.self = circularObj;
+            const items = [];
+            // Add 150 normal items (75% of 200)
+            for (let i = 0; i < 150; i++) {
+                items.push({ id: i, data: 'normal' });
+            }
+            // Add 50 circular items (25% of 200)
+            for (let i = 0; i < 50; i++) {
+                items.push(circularObj);
+            }
+            // Since less than 50% failed, should extrapolate from successful samples
+            const result = estimateByteSize(items);
+            assert.ok(result > 0);
+            // Should not return conservative fallback since <50% failed
+            assert.notStrictEqual(result, CONSERVATIVE_FALLBACK_SIZE);
+        });
+        it('should return conservative estimate when >50% of samples fail', () => {
+            // Create array where most items are circular
+            const circularObj = { id: 1 };
+            circularObj.self = circularObj;
+            const items = [];
+            // Add 40 normal items (20% of 200)
+            for (let i = 0; i < 40; i++) {
+                items.push({ id: i });
+            }
+            // Add 160 circular items (80% of 200)
+            for (let i = 0; i < 160; i++) {
+                items.push(circularObj);
+            }
+            const result = estimateByteSize(items);
+            // Should return conservative fallback since >50% failed
+            assert.strictEqual(result, CONSERVATIVE_FALLBACK_SIZE);
+        });
+        it('should extrapolate size for large non-circular arrays', () => {
+            // Create array larger than sample size
+            const items = Array.from({ length: 200 }, (_, i) => ({ id: i, name: `item-${i}` }));
+            const result = estimateByteSize(items);
+            // Result should be positive and reasonable
+            assert.ok(result > 0);
+            // Rough estimate: each item is ~25 bytes, so 200 items ~5000 bytes
+            assert.ok(result > 1000);
+            assert.ok(result < 100000);
+        });
+    });
+    describe('enforceMemoryLimit with circular references', () => {
+        it('should trigger size-based truncation for circular references', () => {
+            // Create items with circular references
+            const obj = { id: 1, data: 'test' };
+            obj.self = obj;
+            // The circular reference should cause estimateByteSize to return 1MB
+            // which exceeds the 50MB limit would not trigger, but the detection
+            // prevents bypass (the conservative estimate ensures the check works)
+            const items = [obj];
+            const result = enforceMemoryLimit(items, 10000);
+            // Since a single item with circular ref returns 1MB estimate,
+            // and 1MB < 50MB limit, it won't truncate
+            // But the important thing is it doesn't crash and returns valid result
+            assert.ok(Array.isArray(result.results));
+            assert.strictEqual(result.originalCount, 1);
+        });
+        it('should handle array with nested circular references without crashing', () => {
+            const obj1 = { id: 1 };
+            const obj2 = { id: 2 };
+            obj1.ref = obj2;
+            obj2.ref = obj1;
+            const items = [obj1, obj2, { normal: 'object' }];
+            const result = enforceMemoryLimit(items, 10000);
+            assert.ok(Array.isArray(result.results));
+            assert.strictEqual(result.originalCount, 3);
+        });
+        it('should not crash when JSON.stringify would throw', () => {
+            // Create an object that would cause JSON.stringify to throw
+            const obj = {};
+            obj.circular = obj;
+            const items = Array.from({ length: 50 }, () => obj);
+            const result = enforceMemoryLimit(items, 10000);
+            // Should not throw and should return valid result
+            assert.ok(Array.isArray(result.results));
+            assert.strictEqual(result.originalCount, 50);
+        });
+    });
 });
 //# sourceMappingURL=file-utils.test.js.map