observability-toolkit 2.0.0 → 2.1.1

package/dist/lib/file-utils.js

@@ -1,816 +0,0 @@
-/**
- * File system utilities for observability-toolkit
- */
-import { existsSync, readFileSync, readdirSync, statSync, createReadStream, realpathSync, unlinkSync, writeFileSync, appendFileSync, mkdirSync } from 'fs';
-import { join, basename, resolve, normalize, dirname } from 'path';
-import { homedir } from 'os';
-import { createInterface } from 'readline';
-import { createGunzip, gzipSync, gunzipSync } from 'zlib';
-// Import memory limits - uses default values during module initialization
-let MAX_RESULTS_IN_MEMORY = 10000;
-let STREAMING_THRESHOLD = 5000;
-// Lazy load constants to avoid circular dependency
-async function loadConstants() {
-    try {
-        const constants = await import('./constants.js');
-        MAX_RESULTS_IN_MEMORY = constants.MAX_RESULTS_IN_MEMORY ?? 10000;
-        STREAMING_THRESHOLD = constants.STREAMING_THRESHOLD ?? 5000;
-    }
-    catch {
-        // Use defaults if import fails (e.g., during testing)
-    }
-}
-// Initialize constants on module load
-loadConstants();
-/**
- * Expand tilde (~) to home directory
- * Handles ~/path and ~user/path (falls back to ~ for named users)
- */
-export function expandTilde(filepath) {
-    if (!filepath.startsWith('~'))
-        return filepath;
-    if (filepath === '~' || filepath.startsWith('~/')) {
-        return join(homedir(), filepath.slice(1));
-    }
-    // ~user/path - not supported, return as-is
-    return filepath;
-}
-/**
- * Load and parse JSON file with fallback
- */
-export function loadJson(filePath, fallback) {
-    try {
-        if (existsSync(filePath)) {
-            return JSON.parse(readFileSync(filePath, 'utf-8'));
-        }
-    }
-    catch {
-        // Ignore parse/read errors
-    }
-    return fallback ?? null;
-}
-/**
- * Load JSON file, returning fallback on error (never null)
- */
-export function loadJsonSafe(filePath, fallback) {
-    return loadJson(filePath, fallback) ?? fallback;
-}
-/**
- * Get date string in YYYY-MM-DD format
- */
-export function getDateString(date = new Date()) {
-    return date.toISOString().split('T')[0];
-}
-/**
- * Validate that a path is within allowed base directories
- * Prevents path traversal attacks by resolving symlinks and checking containment
- *
- * Security measures:
- * - Uses realpathSync() to resolve symlinks (prevents symlink attacks)
- * - Requires non-empty allowedBasePaths
- * - Uses platform-specific path separator
- * - Logs security failures for audit trail
- */
-export function isPathWithinAllowed(targetPath, allowedBasePaths) {
-    // Require explicit allowed paths - fail closed
-    if (!allowedBasePaths || allowedBasePaths.length === 0) {
-        console.error('[SECURITY] isPathWithinAllowed called without allowedBasePaths');
-        return false;
-    }
-    try {
-        // Resolve the target path, following symlinks if the path exists
-        let resolvedTarget;
-        if (existsSync(targetPath)) {
-            // Path exists - resolve symlinks to get the real path
-            resolvedTarget = realpathSync(targetPath);
-        }
-        else {
-            // Path doesn't exist - resolve parent directory and append filename
-            // This handles cases where we're checking a path before creating it
-            const parentDir = dirname(targetPath);
-            const filename = basename(targetPath);
-            if (existsSync(parentDir)) {
-                resolvedTarget = join(realpathSync(parentDir), filename);
-            }
-            else {
-                // Neither path nor parent exists - normalize without symlink resolution
-                // This is still safe as the path can't be accessed anyway
-                resolvedTarget = normalize(resolve(targetPath));
-            }
-        }
-        // Use platform-specific separator for cross-platform compatibility
-        const sep = process.platform === 'win32' ? '\\' : '/';
-        for (const basePath of allowedBasePaths) {
-            // Resolve base path symlinks too (base paths should exist)
-            let resolvedBase;
-            try {
-                resolvedBase = existsSync(basePath) ? realpathSync(basePath) : normalize(resolve(basePath));
-            }
-            catch {
-                // Skip invalid base paths
-                continue;
-            }
-            // Check if target equals base or is within base directory
-            if (resolvedTarget === resolvedBase || resolvedTarget.startsWith(resolvedBase + sep)) {
-                return true;
-            }
-        }
-        // Log rejection for security audit (but not the full path to avoid log injection)
-        console.warn(`[SECURITY] Path validation rejected: path not within allowed directories`);
-        return false;
-    }
-    catch (error) {
-        // Log security failures for audit trail
-        const message = error instanceof Error ? error.message : String(error);
-        console.error(`[SECURITY] Path validation error: ${message}`);
-        return false;
-    }
-}
-/**
- * List files in directory matching a pattern
- * @param dir - Directory to list files from
- * @param pattern - RegExp to filter filenames
- * @param allowedBasePaths - Optional list of allowed base directories (for security validation)
- */
-export function listFiles(dir, pattern, allowedBasePaths) {
-    if (!existsSync(dir))
-        return [];
-    // Security: validate directory is within allowed paths if specified
-    if (allowedBasePaths && allowedBasePaths.length > 0) {
-        if (!isPathWithinAllowed(dir, allowedBasePaths)) {
-            return [];
-        }
-    }
-    try {
-        return readdirSync(dir)
-            .filter(f => pattern.test(f))
-            .map(f => join(dir, f))
-            .sort((a, b) => {
-            // Sort by modification time, newest first
-            try {
-                return statSync(b).mtime.getTime() - statSync(a).mtime.getTime();
-            }
-            catch {
-                return 0;
-            }
-        });
-    }
-    catch {
-        return [];
-    }
-}
-/**
- * Parse date from filename like traces-2026-01-28.jsonl
- * Validates date components to prevent path traversal via malicious date strings.
- * Returns null for invalid dates (month > 12, day > 31, unreasonable years).
- */
-export function parseDateFromFilename(filename) {
-    const match = basename(filename).match(/(\d{4})-(\d{2})-(\d{2})/);
-    if (!match)
-        return null;
-    const [, year, month, day] = match;
-    const y = parseInt(year, 10);
-    const m = parseInt(month, 10);
-    const d = parseInt(day, 10);
-    // Validate date components
-    if (m < 1 || m > 12)
-        return null;
-    if (d < 1 || d > 31)
-        return null;
-    if (y < 2000 || y > 2100)
-        return null; // Reasonable range for telemetry files
-    return `${year}-${month}-${day}`;
-}
-/**
- * Read JSONL file and parse each line
- */
-export function readJsonlSync(filePath, limit) {
-    if (!existsSync(filePath))
-        return [];
-    try {
-        const content = readFileSync(filePath, 'utf-8');
-        const lines = content.trim().split('\n');
-        const results = [];
-        for (const line of lines) {
-            if (limit && results.length >= limit)
-                break;
-            if (!line.trim())
-                continue;
-            try {
-                results.push(JSON.parse(line));
-            }
-            catch {
-                // Skip malformed lines
-            }
-        }
-        return results;
-    }
-    catch {
-        return [];
-    }
-}
-/**
- * Read JSONL file with verbose statistics about parse failures
- * Use this for debugging to see how many lines were skipped
- */
-export function readJsonlSyncWithStats(filePath, limit) {
-    const emptyResult = {
-        data: [],
-        stats: { totalLines: 0, parsedLines: 0, skippedLines: 0, emptyLines: 0 },
-    };
-    if (!existsSync(filePath))
-        return emptyResult;
-    try {
-        const content = readFileSync(filePath, 'utf-8');
-        const lines = content.trim().split('\n');
-        const results = [];
-        let skippedLines = 0;
-        let emptyLines = 0;
-        for (const line of lines) {
-            if (limit && results.length >= limit)
-                break;
-            if (!line.trim()) {
-                emptyLines++;
-                continue;
-            }
-            try {
-                results.push(JSON.parse(line));
-            }
-            catch {
-                skippedLines++;
-            }
-        }
-        return {
-            data: results,
-            stats: {
-                totalLines: lines.length,
-                parsedLines: results.length,
-                skippedLines,
-                emptyLines,
-            },
-        };
-    }
-    catch {
-        return emptyResult;
-    }
-}
-/**
- * Check if file is gzip compressed based on extension
- */
-export function isGzipFile(filePath) {
-    return filePath.endsWith('.gz');
-}
-/**
- * Create a readable stream for JSONL files, handling gzip decompression transparently
- */
-export function createJsonlStream(filePath) {
-    const fileStream = createReadStream(filePath);
-    if (isGzipFile(filePath)) {
-        return fileStream.pipe(createGunzip());
-    }
-    return fileStream;
-}
-/**
- * Stream JSONL file line by line (for large files)
- * Handles both plain .jsonl and gzip compressed .jsonl.gz files
- *
- * @param filePath - Path to JSONL file (plain or gzip compressed)
- * @param options - Optional configuration including stats tracker
- */
-export async function* streamJsonl(filePath, options) {
-    if (!existsSync(filePath))
-        return;
-    const stream = createJsonlStream(filePath);
-    const rl = createInterface({
-        input: stream,
-        crlfDelay: Infinity,
-    });
-    const tracker = options?.statsTracker;
-    for await (const line of rl) {
-        if (!line.trim()) {
-            tracker?.recordEmpty(filePath);
-            continue;
-        }
-        try {
-            yield JSON.parse(line);
-            tracker?.recordSuccess(filePath);
-        }
-        catch {
-            // Skip malformed lines
-            tracker?.recordFailure(filePath);
-        }
-    }
-}
-/**
- * Filter items by date range
- */
-export function filterByDateRange(items, startDate, endDate) {
-    return items.filter(item => {
-        if (!item.timestamp)
-            return true;
-        const ts = item.timestamp;
-        if (startDate && ts < startDate)
-            return false;
-        if (endDate && ts > endDate)
-            return false;
-        return true;
-    });
-}
-/**
- * Paginate results array with offset and limit
- */
-export function paginateResults(results, offset, limit) {
-    return results.slice(offset, offset + limit);
-}
-/**
- * Check if we've collected enough results for early termination
- */
-export function hasReachedLimit(resultsLength, offset, limit) {
-    return resultsLength >= offset + limit;
-}
-/**
- * Clean up old telemetry files based on retention policy
- * @param dir - Directory containing telemetry files
- * @param retentionDays - Number of days to retain files (default: 7)
- * @param allowedBasePaths - Optional list of allowed base directories (for security validation)
- * @returns Object with count of deleted files and any errors encountered
- */
-export function cleanupOldFiles(dir, retentionDays = 7, allowedBasePaths) {
-    const result = { deleted: 0, errors: [] };
-    if (!existsSync(dir)) {
-        return result;
-    }
-    // Security: validate directory is within allowed paths if specified
-    if (allowedBasePaths && allowedBasePaths.length > 0) {
-        if (!isPathWithinAllowed(dir, allowedBasePaths)) {
-            result.errors.push(`Directory ${dir} is not within allowed paths`);
-            return result;
-        }
-    }
-    // Calculate cutoff date
-    const now = new Date();
-    const cutoffDate = new Date(now.getTime() - retentionDays * 24 * 60 * 60 * 1000);
-    const cutoffDateStr = getDateString(cutoffDate);
-    try {
-        // List all .jsonl and .jsonl.gz files
-        const files = readdirSync(dir).filter(f => f.endsWith('.jsonl') || f.endsWith('.jsonl.gz'));
-        for (const file of files) {
-            const filePath = join(dir, file);
-            const fileDate = parseDateFromFilename(file);
-            // Skip files without parseable dates (don't delete unknown files)
-            if (!fileDate) {
-                continue;
-            }
-            // Delete files older than retention period
-            if (fileDate < cutoffDateStr) {
-                try {
-                    unlinkSync(filePath);
-                    result.deleted++;
-                }
-                catch (err) {
-                    const message = err instanceof Error ? err.message : String(err);
-                    result.errors.push(`Failed to delete ${filePath}: ${message}`);
-                }
-            }
-        }
-    }
-    catch (err) {
-        const message = err instanceof Error ? err.message : String(err);
-        result.errors.push(`Failed to read directory ${dir}: ${message}`);
-    }
-    return result;
-}
-/**
- * Batched JSONL writer that reduces I/O by buffering writes.
- * Supports optional gzip compression.
- *
- * @example
- * ```ts
- * const writer = new BatchWriter('output.jsonl', { batchSize: 50 });
- * for (const record of records) {
- *   writer.write(record);
- * }
- * writer.close(); // Flush remaining buffered records
- * ```
- */
-export class BatchWriter {
-    buffer = [];
-    filePath;
-    batchSize;
-    useGzip;
-    mkdirp;
-    initialized = false;
-    stats = { recordsWritten: 0, bytesWritten: 0, flushCount: 0 };
-    constructor(filePath, options = {}) {
-        this.filePath = filePath;
-        this.batchSize = options.batchSize ?? 100;
-        this.useGzip = options.gzip ?? false;
-        this.mkdirp = options.mkdirp ?? true;
-    }
-    /**
-     * Add a record to the buffer. Flushes automatically when batch size is reached.
-     */
-    write(record) {
-        this.buffer.push(JSON.stringify(record));
-        if (this.buffer.length >= this.batchSize) {
-            this.flush();
-        }
-    }
-    /**
-     * Write multiple records at once
-     */
-    writeAll(records) {
-        for (const record of records) {
-            this.write(record);
-        }
-    }
-    /**
-     * Flush buffered records to disk
-     */
-    flush() {
-        if (this.buffer.length === 0)
-            return;
-        const content = this.buffer.join('\n') + '\n';
-        this.buffer = [];
-        // Ensure directory exists on first write
-        if (!this.initialized) {
-            if (this.mkdirp) {
-                const dir = dirname(this.filePath);
-                if (!existsSync(dir)) {
-                    mkdirSync(dir, { recursive: true });
-                }
-            }
-            // Write (create/overwrite) on first flush
-            if (this.useGzip) {
-                writeFileSync(this.filePath, gzipSync(content));
-            }
-            else {
-                writeFileSync(this.filePath, content, 'utf-8');
-            }
-            this.initialized = true;
-        }
-        else {
-            // Append on subsequent flushes
-            if (this.useGzip) {
-                // For gzip, we need to read, decompress, append, recompress
-                // This is less efficient but maintains valid gzip format
-                const existing = existsSync(this.filePath)
-                    ? gunzipSync(readFileSync(this.filePath)).toString('utf-8')
-                    : '';
-                writeFileSync(this.filePath, gzipSync(existing + content));
-            }
-            else {
-                appendFileSync(this.filePath, content, 'utf-8');
-            }
-        }
-        this.stats.bytesWritten += Buffer.byteLength(content, 'utf-8');
-        this.stats.recordsWritten += content.split('\n').length - 1;
-        this.stats.flushCount++;
-    }
-    /**
-     * Flush remaining records and finalize the file.
-     * Always call this when done writing.
-     */
-    close() {
-        this.flush();
-        return { ...this.stats };
-    }
-    /**
-     * Get current statistics
-     */
-    getStats() {
-        return {
-            ...this.stats,
-            recordsWritten: this.stats.recordsWritten + this.buffer.length,
-        };
-    }
-    /**
-     * Get number of records currently buffered (not yet written)
-     */
-    get bufferedCount() {
-        return this.buffer.length;
-    }
-}
-/**
- * Streaming aggregator that computes aggregations incrementally without holding all values.
- * Uses reservoir sampling for percentile approximations.
- *
- * @example
- * ```ts
- * const aggregator = new StreamingAggregator();
- * for (const value of largeDataset) {
- *   aggregator.add(value);
- * }
- * const avg = aggregator.getResult('avg');
- * const p95 = aggregator.getResult('p95');
- * ```
- */
-export class StreamingAggregator {
-    count = 0;
-    sum = 0;
-    min = Infinity;
-    max = -Infinity;
-    // Reservoir sampling for percentile approximation
-    // Keep a fixed-size sorted sample for accurate percentile estimates
-    static RESERVOIR_SIZE = 1000;
-    reservoir = [];
-    seenCount = 0;
-    /**
-     * Add a value to the aggregator
-     */
-    add(value) {
-        if (!Number.isFinite(value))
-            return;
-        this.count++;
-        this.sum += value;
-        this.min = Math.min(this.min, value);
-        this.max = Math.max(this.max, value);
-        this.seenCount++;
-        // Reservoir sampling (Algorithm R)
-        if (this.reservoir.length < StreamingAggregator.RESERVOIR_SIZE) {
-            this.reservoir.push(value);
-        }
-        else {
-            // Randomly replace elements with decreasing probability
-            const j = Math.floor(Math.random() * this.seenCount);
-            if (j < StreamingAggregator.RESERVOIR_SIZE) {
-                this.reservoir[j] = value;
-            }
-        }
-    }
-    /**
-     * Add multiple values at once
-     */
-    addAll(values) {
-        for (const value of values) {
-            this.add(value);
-        }
-    }
-    /**
-     * Get the result for a specific aggregation type
-     */
-    getResult(type) {
-        if (this.count === 0 && type !== 'count') {
-            return undefined;
-        }
-        switch (type) {
-            case 'count':
-                return this.count;
-            case 'sum':
-                return this.sum;
-            case 'avg':
-                return this.sum / this.count;
-            case 'min':
-                return this.min === Infinity ? undefined : this.min;
-            case 'max':
-                return this.max === -Infinity ? undefined : this.max;
-            case 'p50':
-                return this.getPercentile(50);
-            case 'p95':
-                return this.getPercentile(95);
-            case 'p99':
-                return this.getPercentile(99);
-        }
-    }
-    /**
-     * Get all computed statistics
-     */
-    getStats() {
-        return {
-            count: this.count,
-            sum: this.sum,
-            avg: this.count > 0 ? this.sum / this.count : undefined,
-            min: this.min === Infinity ? undefined : this.min,
-            max: this.max === -Infinity ? undefined : this.max,
-            p50: this.getPercentile(50),
-            p95: this.getPercentile(95),
-            p99: this.getPercentile(99),
-        };
-    }
-    /**
-     * Reset the aggregator to initial state
-     */
-    reset() {
-        this.count = 0;
-        this.sum = 0;
-        this.min = Infinity;
-        this.max = -Infinity;
-        this.reservoir = [];
-        this.seenCount = 0;
-    }
-    /**
-     * Get the current count of values added
-     */
-    getCount() {
-        return this.count;
-    }
-    /**
-     * Calculate percentile using R-7 linear interpolation on the reservoir sample
-     */
-    getPercentile(percentile) {
-        if (this.reservoir.length === 0)
-            return undefined;
-        // Sort reservoir for percentile calculation
-        const sorted = [...this.reservoir].sort((a, b) => a - b);
-        // R-7 method: linear interpolation
-        const rank = (percentile / 100) * (sorted.length - 1);
-        const lowerIndex = Math.floor(rank);
-        const upperIndex = Math.ceil(rank);
-        const weight = rank - lowerIndex;
-        if (lowerIndex === upperIndex) {
-            return sorted[lowerIndex];
-        }
-        return sorted[lowerIndex] * (1 - weight) + sorted[upperIndex] * weight;
-    }
-}
-// Maximum approximate byte size for results (50MB)
-const MAX_RESULTS_BYTE_SIZE = 50 * 1024 * 1024;
-// Conservative fallback size (1MB) for objects that cannot be serialized
-// This prevents memory limit bypass when JSON.stringify() fails (e.g., circular references)
-export const CONSERVATIVE_FALLBACK_SIZE = 1000000;
-/**
- * Maximum recursion depth for circular reference detection.
- * Objects nested deeper than this are assumed safe to avoid performance issues.
- */
-const MAX_CIRCULAR_CHECK_DEPTH = 100;
-/**
- * Check if a value contains circular references using a WeakSet
- * @param obj - The object to check
- * @param seen - WeakSet to track seen objects
- * @param depth - Current recursion depth (internal use)
- * @returns true if circular reference detected
- */
-export function hasCircularReference(obj, seen = new WeakSet(), depth = 0) {
-    // Early bailout for primitives
-    if (obj === null || typeof obj !== 'object') {
-        return false;
-    }
-    // Depth limit to prevent performance issues with deeply nested objects
-    if (depth > MAX_CIRCULAR_CHECK_DEPTH) {
-        return false;
-    }
-    if (seen.has(obj)) {
-        return true;
-    }
-    seen.add(obj);
-    if (Array.isArray(obj)) {
-        for (const item of obj) {
-            if (hasCircularReference(item, seen, depth + 1)) {
-                return true;
-            }
-        }
-    }
-    else {
-        for (const key of Object.keys(obj)) {
-            if (hasCircularReference(obj[key], seen, depth + 1)) {
-                return true;
-            }
-        }
-    }
-    return false;
-}
-/**
- * Estimate approximate byte size of an object (rough approximation)
- * Uses a sample-based approach for large arrays to avoid performance issues
- *
- * Security: Returns a conservative estimate (1MB) when serialization fails
- * to prevent memory limit bypass via circular references or other edge cases.
- */
-export function estimateByteSize(results, sampleSize = 100) {
-    if (results.length === 0)
-        return 0;
-    // For small arrays, measure directly
-    if (results.length <= sampleSize) {
-        try {
-            // Check for circular references first to avoid JSON.stringify throwing
-            if (hasCircularReference(results)) {
-                console.warn('[MEMORY] Circular reference detected in results, using conservative size estimate');
-                return CONSERVATIVE_FALLBACK_SIZE;
-            }
-            return JSON.stringify(results).length;
-        }
-        catch {
-            // If serialization fails for any reason, return conservative estimate
-            // This prevents memory limit bypass on error
-            console.warn('[MEMORY] Failed to serialize results, using conservative size estimate');
-            return CONSERVATIVE_FALLBACK_SIZE;
-        }
-    }
-    // For large arrays, sample and extrapolate
-    const step = Math.floor(results.length / sampleSize);
-    let sampleBytes = 0;
-    let sampledCount = 0;
-    let failedCount = 0;
-    for (let i = 0; i < results.length && sampledCount < sampleSize; i += step) {
-        try {
-            // Check individual items for circular references
-            if (hasCircularReference(results[i])) {
-                failedCount++;
-                sampledCount++;
-                continue;
-            }
-            sampleBytes += JSON.stringify(results[i]).length;
-            sampledCount++;
-        }
-        catch {
-            failedCount++;
-            sampledCount++;
-        }
-    }
-    // If too many samples failed (>50%), return conservative estimate
-    if (failedCount > sampledCount / 2) {
-        console.warn('[MEMORY] Too many serialization failures in sample, using conservative size estimate');
-        return CONSERVATIVE_FALLBACK_SIZE;
-    }
-    // If all samples failed, return conservative estimate
-    if (sampledCount === failedCount) {
-        console.warn('[MEMORY] All samples failed to serialize, using conservative size estimate');
-        return CONSERVATIVE_FALLBACK_SIZE;
-    }
-    const successfulSamples = sampledCount - failedCount;
-    const avgBytesPerItem = sampleBytes / successfulSamples;
-    return Math.ceil(avgBytesPerItem * results.length);
-}
-/**
- * Enforce memory limit on results array.
- * Truncates results if they exceed count limit OR estimated byte size limit.
- * Uses sampling for large arrays to avoid performance overhead.
- *
- * @param results - Array of results to limit
- * @param limit - Maximum number of results (defaults to MAX_RESULTS_IN_MEMORY)
- * @returns Object with results, truncated flag, and original count
- *
- * @example
- * ```ts
- * const { results, truncated } = enforceMemoryLimit(largeArray);
- * if (truncated) {
- *   console.warn('Results were truncated');
- * }
- * ```
- */
-export function enforceMemoryLimit(results, limit = MAX_RESULTS_IN_MEMORY) {
-    const originalCount = results.length;
-    // Check count limit first
-    if (results.length > limit) {
-        console.warn(`[MEMORY] Results truncated from ${results.length} to ${limit} items (MAX_RESULTS_IN_MEMORY exceeded)`);
-        return {
-            results: results.slice(0, limit),
-            truncated: true,
-            originalCount,
-            truncationReason: 'count',
-        };
-    }
-    // Estimate byte size to prevent memory exhaustion with large objects
-    const estimatedBytes = estimateByteSize(results);
-    if (estimatedBytes > MAX_RESULTS_BYTE_SIZE) {
-        // Binary search to find safe size
-        let safeCount = Math.floor(results.length / 2);
-        let low = 0;
-        let high = results.length;
-        while (low < high - 1) {
-            const mid = Math.floor((low + high) / 2);
-            const midSize = estimateByteSize(results.slice(0, mid));
-            if (midSize <= MAX_RESULTS_BYTE_SIZE) {
-                low = mid;
-                safeCount = mid;
-            }
-            else {
-                high = mid;
-            }
-        }
-        // Ensure at least some results are returned
-        safeCount = Math.max(1, Math.min(safeCount, results.length));
-        console.warn(`[MEMORY] Results truncated from ${results.length} to ${safeCount} items (estimated ${Math.round(estimatedBytes / 1024 / 1024)}MB exceeded ${MAX_RESULTS_BYTE_SIZE / 1024 / 1024}MB limit)`);
-        return {
-            results: results.slice(0, safeCount),
-            truncated: true,
-            originalCount,
-            truncationReason: 'size',
-        };
-    }
-    return {
-        results,
-        truncated: false,
-        originalCount,
-    };
-}
-/**
- * Check if streaming aggregation should be used based on result count
- *
- * @param count - Number of results
- * @param threshold - Threshold to trigger streaming (defaults to STREAMING_THRESHOLD)
- * @returns true if streaming aggregation should be used
- */
-export function shouldUseStreamingAggregation(count, threshold = STREAMING_THRESHOLD) {
-    return count > threshold;
-}
-/**
- * Get current memory limit constants (useful for testing)
- */
-export function getMemoryLimits() {
-    return {
-        maxResults: MAX_RESULTS_IN_MEMORY,
-        streamingThreshold: STREAMING_THRESHOLD,
-    };
-}
-//# sourceMappingURL=file-utils.js.map