observability-toolkit 2.0.0 → 2.1.0

package/dist/lib/core/file-utils.js

@@ -0,0 +1,890 @@
+/**
+ * File system utilities for observability-toolkit
+ */
+import { DDSketch } from '@datadog/sketches-js';
+import { BYTES, MONTHS_PER_YEAR, MAX_DAY_OF_MONTH, MIN_VALID_YEAR, MAX_VALID_YEAR, ONE_MILLION, PERCENTILE, TIME_MS } from './units.js';
+import { MAX_RESULTS_IN_MEMORY, STREAMING_THRESHOLD, RETENTION_DAYS_DEFAULT, } from './constants.js';
+import { existsSync, readFileSync, readdirSync, statSync, createReadStream, realpathSync, unlinkSync, writeFileSync, appendFileSync, mkdirSync } from 'fs';
+import { join, basename, resolve, normalize, dirname } from 'path';
+import { homedir } from 'os';
+import { createInterface } from 'readline';
+import { createGunzip, gzipSync, gunzipSync } from 'zlib';
+import { createLogger } from './logger.js';
+const securityLog = createLogger('security');
+const memoryLog = createLogger('memory');
+const BATCH_SIZE = Math.sqrt(MAX_RESULTS_IN_MEMORY);
+const MAX_RESULTS_SIZE_MB = STREAMING_THRESHOLD / BATCH_SIZE;
+const MAX_RESULTS_BYTE_SIZE = MAX_RESULTS_SIZE_MB * BYTES.MB;
+const MAX_GZIP_FILE_SIZE_BYTES = MAX_RESULTS_SIZE_MB * ONE_MILLION;
+const BINARY_SEARCH_DIVISOR = 2;
+const BINARY_SEARCH_DELTA = 1;
+/**
+ * Expand tilde (~) to home directory
+ * Handles ~/path and ~user/path (falls back to ~ for named users)
+ */
+export function expandTilde(filepath) {
+    if (!filepath.startsWith('~'))
+        return filepath;
+    if (filepath === '~' || filepath.startsWith('~/')) {
+        return join(homedir(), filepath.slice(1));
+    }
+    // ~user/path - not supported, return as-is
+    return filepath;
+}
+/**
+ * Load and parse JSON file with fallback
+ */
+export function loadJson(filePath, fallback) {
+    try {
+        if (existsSync(filePath)) {
+            return JSON.parse(readFileSync(filePath, 'utf-8'));
+        }
+    }
+    catch {
+        // Ignore parse/read errors
+    }
+    return fallback ?? null;
+}
+/**
+ * Load JSON file, returning fallback on error (never null)
+ */
+export function loadJsonSafe(filePath, fallback) {
+    return loadJson(filePath, fallback) ?? fallback;
+}
+/**
+ * Load and validate JSON file against a Zod schema.
+ * Throws if validation fails or file does not exist.
+ *
+ * @param filePath - Path to JSON file
+ * @param schema - Zod schema to validate against
+ * @throws {Error} If validation fails
+ */
+export function loadJsonWithValidation(filePath, schema) {
+    try {
+        if (!existsSync(filePath)) {
+            throw new Error(`File not found: ${filePath}`);
+        }
+        const parsed = JSON.parse(readFileSync(filePath, 'utf-8'));
+        const result = schema.safeParse(parsed);
+        if (!result.success) {
+            throw new Error(`Validation failed: ${result.error.message}`);
+        }
+        return result.data;
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        throw new Error(`Failed to load and validate JSON from ${filePath}: ${message}`, { cause: error });
+    }
+}
+/**
+ * Load and validate JSON file, returning fallback on error.
+ * Unlike loadJsonWithValidation, this does not throw.
+ *
+ * @param filePath - Path to JSON file
+ * @param schema - Zod schema to validate against
+ * @param fallback - Default value if file not found or validation fails
+ */
+export function loadJsonWithValidationSafe(filePath, schema, fallback) {
+    try {
+        return loadJsonWithValidation(filePath, schema);
+    }
+    catch {
+        return fallback;
+    }
+}
+/**
+ * Get date string in YYYY-MM-DD format
+ */
+export function getDateString(date = new Date()) {
+    return date.toISOString().split('T')[0];
+}
+/**
+ * Validate that a path is within allowed base directories
+ * Prevents path traversal attacks by resolving symlinks and checking containment
+ *
+ * Security measures:
+ * - Uses realpathSync() to resolve symlinks (prevents symlink attacks)
+ * - Requires non-empty allowedBasePaths
+ * - Uses platform-specific path separator
+ * - Logs security failures for audit trail
+ */
+export function isPathWithinAllowed(targetPath, allowedBasePaths) {
+    // Require explicit allowed paths - fail closed
+    if (!allowedBasePaths || allowedBasePaths.length === 0) {
+        securityLog.error('isPathWithinAllowed called without allowedBasePaths');
+        return false;
+    }
+    try {
+        // Resolve the target path, following symlinks if the path exists
+        let resolvedTarget;
+        if (existsSync(targetPath)) {
+            // Path exists - resolve symlinks to get the real path
+            resolvedTarget = realpathSync(targetPath);
+        }
+        else {
+            // Path doesn't exist - resolve parent directory and append filename
+            // This handles cases where we're checking a path before creating it
+            const parentDir = dirname(targetPath);
+            const filename = basename(targetPath);
+            if (existsSync(parentDir)) {
+                resolvedTarget = join(realpathSync(parentDir), filename);
+            }
+            else {
+                // Neither path nor parent exists - normalize without symlink resolution
+                // This is still safe as the path can't be accessed anyway
+                resolvedTarget = normalize(resolve(targetPath));
+            }
+        }
+        // Use platform-specific separator for cross-platform compatibility
+        const sep = process.platform === 'win32' ? '\\' : '/';
+        for (const basePath of allowedBasePaths) {
+            // Resolve base path symlinks too (base paths should exist)
+            let resolvedBase;
+            try {
+                resolvedBase = existsSync(basePath) ? realpathSync(basePath) : normalize(resolve(basePath));
+            }
+            catch {
+                // Skip invalid base paths
+                continue;
+            }
+            // Check if target equals base or is within base directory
+            if (resolvedTarget === resolvedBase || resolvedTarget.startsWith(resolvedBase + sep)) {
+                return true;
+            }
+        }
+        // Log rejection for security audit (but not the full path to avoid log injection)
+        securityLog.warn('Path validation rejected: path not within allowed directories');
+        return false;
+    }
+    catch (error) {
+        // Log security failures for audit trail
+        const message = error instanceof Error ? error.message : String(error);
+        securityLog.error(`Path validation error: ${message}`);
+        return false;
+    }
+}
+/**
+ * List files in directory matching a pattern
+ * @param dir - Directory to list files from
+ * @param pattern - RegExp to filter filenames
+ * @param allowedBasePaths - Optional list of allowed base directories (for security validation)
+ */
+export function listFiles(dir, pattern, allowedBasePaths) {
+    if (!existsSync(dir))
+        return [];
+    // Security: validate directory is within allowed paths if specified
+    if (allowedBasePaths && allowedBasePaths.length > 0) {
+        if (!isPathWithinAllowed(dir, allowedBasePaths)) {
+            return [];
+        }
+    }
+    try {
+        return readdirSync(dir)
+            .filter(f => pattern.test(f))
+            .map(f => join(dir, f))
+            .sort((a, b) => {
+            // Sort by modification time, newest first
+            try {
+                return statSync(b).mtime.getTime() - statSync(a).mtime.getTime();
+            }
+            catch {
+                return 0;
+            }
+        });
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Parse date from filename like traces-2026-01-28.jsonl
+ * Validates date components to prevent path traversal via malicious date strings.
+ * Returns null for invalid dates (month > 12, day > 31, unreasonable years).
+ */
+export function parseDateFromFilename(filename) {
+    const match = basename(filename).match(/(\d{4})-(\d{2})-(\d{2})/);
+    if (!match)
+        return null;
+    const [, year, month, day] = match;
+    const y = parseInt(year, 10);
+    const m = parseInt(month, 10);
+    const d = parseInt(day, 10);
+    // Validate date components
+    if (m < 1 || m > MONTHS_PER_YEAR)
+        return null;
+    if (d < 1 || d > MAX_DAY_OF_MONTH)
+        return null;
+    if (y < MIN_VALID_YEAR || y > MAX_VALID_YEAR)
+        return null; // Reasonable range for telemetry files
+    return `${year}-${month}-${day}`;
+}
+/**
+ * Read JSONL file and parse each line.
+ * Delegates to readJsonlSyncWithStats to avoid duplicating the parse loop (CS-D1).
+ */
+export function readJsonlSync(filePath, limit) {
+    return readJsonlSyncWithStats(filePath, limit).data;
+}
+/**
+ * Read JSONL file with Zod schema validation.
+ * Each line is validated against the schema; invalid lines are skipped silently.
+ *
+ * @param filePath - Path to JSONL file
+ * @param schema - Zod schema for validation
+ * @param limit - Maximum number of valid records to return
+ * @returns Array of validated records
+ */
+export function readJsonlWithValidation(filePath, schema, limit) {
+    const emptyResult = [];
+    if (!existsSync(filePath))
+        return emptyResult;
+    try {
+        const content = readFileSync(filePath, 'utf-8');
+        const lines = content.trim().split('\n');
+        const results = [];
+        for (const line of lines) {
+            if (limit && results.length >= limit)
+                break;
+            if (!line.trim())
+                continue;
+            try {
+                const parsed = JSON.parse(line);
+                const validationResult = schema.safeParse(parsed);
+                if (validationResult.success) {
+                    results.push(validationResult.data);
+                }
+            }
+            catch {
+                // Skip malformed lines
+            }
+        }
+        return results;
+    }
+    catch {
+        return emptyResult;
+    }
+}
+/**
+ * Read JSONL file with verbose statistics about parse failures
+ * Use this for debugging to see how many lines were skipped
+ */
+export function readJsonlSyncWithStats(filePath, limit) {
+    const emptyResult = {
+        data: [],
+        stats: { totalLines: 0, parsedLines: 0, skippedLines: 0, emptyLines: 0 },
+    };
+    if (!existsSync(filePath))
+        return emptyResult;
+    try {
+        const content = readFileSync(filePath, 'utf-8');
+        const lines = content.trim().split('\n');
+        const results = [];
+        let skippedLines = 0;
+        let emptyLines = 0;
+        for (const line of lines) {
+            if (limit && results.length >= limit)
+                break;
+            if (!line.trim()) {
+                emptyLines++;
+                continue;
+            }
+            try {
+                results.push(JSON.parse(line));
+            }
+            catch {
+                skippedLines++;
+            }
+        }
+        return {
+            data: results,
+            stats: {
+                totalLines: lines.length,
+                parsedLines: results.length,
+                skippedLines,
+                emptyLines,
+            },
+        };
+    }
+    catch {
+        return emptyResult;
+    }
+}
+/**
+ * Check if file is gzip compressed based on extension
+ */
+export function isGzipFile(filePath) {
+    return filePath.endsWith('.gz');
+}
+/**
+ * Create a readable stream for JSONL files, handling gzip decompression transparently
+ */
+export function createJsonlStream(filePath) {
+    const fileStream = createReadStream(filePath);
+    if (isGzipFile(filePath)) {
+        return fileStream.pipe(createGunzip());
+    }
+    return fileStream;
+}
+/**
+ * Stream JSONL file line by line (for large files)
+ * Handles both plain .jsonl and gzip compressed .jsonl.gz files
+ *
+ * @param filePath - Path to JSONL file (plain or gzip compressed)
+ * @param options - Optional configuration including stats tracker
+ */
+export async function* streamJsonl(filePath, options) {
+    if (!existsSync(filePath))
+        return;
+    const stream = createJsonlStream(filePath);
+    const rl = createInterface({
+        input: stream,
+        crlfDelay: Infinity,
+    });
+    const tracker = options?.statsTracker;
+    for await (const line of rl) {
+        if (!line.trim()) {
+            tracker?.recordEmpty(filePath);
+            continue;
+        }
+        try {
+            yield JSON.parse(line);
+            tracker?.recordSuccess(filePath);
+        }
+        catch {
+            // Skip malformed lines
+            tracker?.recordFailure(filePath);
+        }
+    }
+}
+/**
+ * Stream JSONL file with Zod schema validation (for large files).
+ * Each line is validated against the schema; invalid lines are skipped.
+ * Handles both plain .jsonl and gzip compressed .jsonl.gz files.
+ *
+ * @param filePath - Path to JSONL file (plain or gzip compressed)
+ * @param schema - Zod schema for validation
+ * @param options - Optional configuration including stats tracker
+ */
+export async function* streamJsonlWithValidation(filePath, schema, options) {
+    if (!existsSync(filePath))
+        return;
+    const stream = createJsonlStream(filePath);
+    const rl = createInterface({
+        input: stream,
+        crlfDelay: Infinity,
+    });
+    const tracker = options?.statsTracker;
+    for await (const line of rl) {
+        if (!line.trim()) {
+            tracker?.recordEmpty(filePath);
+            continue;
+        }
+        try {
+            const parsed = JSON.parse(line);
+            const validationResult = schema.safeParse(parsed);
+            if (validationResult.success) {
+                yield validationResult.data;
+                tracker?.recordSuccess(filePath);
+            }
+            else {
+                tracker?.recordFailure(filePath);
+            }
+        }
+        catch {
+            tracker?.recordFailure(filePath);
+        }
+    }
+}
+/**
+ * Filter items by date range
+ */
+export function filterByDateRange(items, startDate, endDate) {
+    return items.filter(item => {
+        if (!item.timestamp)
+            return true;
+        const ts = item.timestamp;
+        if (startDate && ts < startDate)
+            return false;
+        if (endDate && ts > endDate)
+            return false;
+        return true;
+    });
+}
+/**
+ * Paginate results array with offset and limit
+ */
+export function paginateResults(results, offset, limit) {
+    return results.slice(offset, offset + limit);
+}
+/**
+ * Check if we've collected enough results for early termination
+ */
+export function hasReachedLimit(resultsLength, offset, limit) {
+    return resultsLength >= offset + limit;
+}
+/**
+ * Clean up old telemetry files based on retention policy
+ * @param dir - Directory containing telemetry files
+ * @param retentionDays - Number of days to retain files (default: 7)
+ * @param allowedBasePaths - Optional list of allowed base directories (for security validation)
+ * @returns Object with count of deleted files and any errors encountered
+ */
+export function cleanupOldFiles(dir, retentionDays = RETENTION_DAYS_DEFAULT, allowedBasePaths) {
+    const result = { deleted: 0, errors: [] };
+    if (!existsSync(dir)) {
+        return result;
+    }
+    // Security: validate directory is within allowed paths if specified
+    if (allowedBasePaths && allowedBasePaths.length > 0) {
+        if (!isPathWithinAllowed(dir, allowedBasePaths)) {
+            result.errors.push(`Directory ${dir} is not within allowed paths`);
+            return result;
+        }
+    }
+    // Calculate cutoff date
+    const now = new Date();
+    const cutoffDate = new Date(now.getTime() - retentionDays * TIME_MS.DAY);
+    const cutoffDateStr = getDateString(cutoffDate);
+    try {
+        // List all .jsonl and .jsonl.gz files
+        const files = readdirSync(dir).filter(f => f.endsWith('.jsonl') || f.endsWith('.jsonl.gz'));
+        for (const file of files) {
+            const filePath = join(dir, file);
+            const fileDate = parseDateFromFilename(file);
+            // Skip files without parseable dates (don't delete unknown files)
+            if (!fileDate) {
+                continue;
+            }
+            // Delete files older than retention period
+            if (fileDate < cutoffDateStr) {
+                try {
+                    unlinkSync(filePath);
+                    result.deleted++;
+                }
+                catch (err) {
+                    const message = err instanceof Error ? err.message : String(err);
+                    result.errors.push(`Failed to delete ${filePath}: ${message}`);
+                }
+            }
+        }
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        result.errors.push(`Failed to read directory ${dir}: ${message}`);
+    }
+    return result;
+}
+/**
+ * Batched JSONL writer that reduces I/O by buffering writes.
+ * Supports optional gzip compression.
+ *
+ * @example
+ * ```ts
+ * const writer = new BatchWriter('output.jsonl', { batchSize: 50 });
+ * for (const record of records) {
+ *   writer.write(record);
+ * }
+ * writer.close(); // Flush remaining buffered records
+ * ```
+ */
+export class BatchWriter {
+    buffer = [];
+    filePath;
+    batchSize;
+    useGzip;
+    mkdirp;
+    initialized = false;
+    stats = { recordsWritten: 0, bytesWritten: 0, flushCount: 0 };
+    constructor(filePath, options = {}) {
+        this.filePath = filePath;
+        this.batchSize = options.batchSize ?? BATCH_SIZE;
+        this.useGzip = options.gzip ?? false;
+        this.mkdirp = options.mkdirp ?? true;
+    }
+    /**
+     * Add a record to the buffer. Flushes automatically when batch size is reached.
+     */
+    write(record) {
+        this.buffer.push(JSON.stringify(record));
+        if (this.buffer.length >= this.batchSize) {
+            this.flush();
+        }
+    }
+    /**
+     * Write multiple records at once
+     */
+    writeAll(records) {
+        for (const record of records) {
+            this.write(record);
+        }
+    }
+    /**
+     * Flush buffered records to disk
+     */
+    flush() {
+        if (this.buffer.length === 0)
+            return;
+        const content = this.buffer.join('\n') + '\n';
+        this.buffer = [];
+        // Ensure directory exists on first write
+        if (!this.initialized) {
+            if (this.mkdirp) {
+                const dir = dirname(this.filePath);
+                if (!existsSync(dir)) {
+                    mkdirSync(dir, { recursive: true });
+                }
+            }
+            // Write (create/overwrite) on first flush
+            if (this.useGzip) {
+                writeFileSync(this.filePath, gzipSync(content));
+            }
+            else {
+                writeFileSync(this.filePath, content, 'utf-8');
+            }
+            this.initialized = true;
+        }
+        else {
+            // Append on subsequent flushes
+            if (this.useGzip) {
+                // For gzip, we need to read, decompress, append, recompress — O(n) per flush
+                const existing = existsSync(this.filePath)
+                    ? gunzipSync(readFileSync(this.filePath)).toString('utf-8')
+                    : '';
+                const combined = existing + content;
+                if (combined.length > MAX_GZIP_FILE_SIZE_BYTES) {
+                    throw new Error(`Gzip file would exceed 50MB limit (${Math.round(combined.length / ONE_MILLION)}MB). ` +
+                        'Use file rotation or disable gzip compression.');
+                }
+                writeFileSync(this.filePath, gzipSync(combined));
+            }
+            else {
+                appendFileSync(this.filePath, content, 'utf-8');
+            }
+        }
+        this.stats.bytesWritten += Buffer.byteLength(content, 'utf-8');
+        this.stats.recordsWritten += content.split('\n').length - 1;
+        this.stats.flushCount++;
+    }
+    /**
+     * Flush remaining records and finalize the file.
+     * Always call this when done writing.
+     */
+    close() {
+        this.flush();
+        return { ...this.stats };
+    }
+    /**
+     * Get current statistics
+     */
+    getStats() {
+        return {
+            ...this.stats,
+            recordsWritten: this.stats.recordsWritten + this.buffer.length,
+        };
+    }
+    /**
+     * Get number of records currently buffered (not yet written)
+     */
+    get bufferedCount() {
+        return this.buffer.length;
+    }
+}
+/**
+ * Streaming aggregator that computes aggregations incrementally without holding all values.
+ * Uses DDSketch for memory-bounded percentile estimation.
+ *
+ * @example
+ * ```ts
+ * const aggregator = new StreamingAggregator();
+ * for (const value of largeDataset) {
+ *   aggregator.add(value);
+ * }
+ * const avg = aggregator.getResult('avg');
+ * const p95 = aggregator.getResult('p95');
+ * ```
+ */
+export class StreamingAggregator {
+    count = 0;
+    sum = 0;
+    min = Infinity;
+    max = -Infinity;
+    sketch = new DDSketch();
+    /** DDSketch.getValueAtQuantile() takes 0-1; PERCENTILE enum values are 0-100 */
+    static PERCENTILE_SCALE = 100;
+    /**
+     * Add a value to the aggregator
+     */
+    add(value) {
+        if (!Number.isFinite(value))
+            return;
+        this.count++;
+        this.sum += value;
+        this.min = Math.min(this.min, value);
+        this.max = Math.max(this.max, value);
+        this.sketch.accept(value);
+    }
+    /**
+     * Add multiple values at once
+     */
+    addAll(values) {
+        for (const value of values) {
+            this.add(value);
+        }
+    }
+    /**
+     * Get the result for a specific aggregation type
+     */
+    getResult(type) {
+        if (this.count === 0 && type !== 'count') {
+            return undefined;
+        }
+        switch (type) {
+            case 'count':
+                return this.count;
+            case 'sum':
+                return this.sum;
+            case 'avg':
+                return this.sum / this.count;
+            case 'min':
+                return this.min === Infinity ? undefined : this.min;
+            case 'max':
+                return this.max === -Infinity ? undefined : this.max;
+            case 'p50':
+                return this.getPercentile(PERCENTILE.P50);
+            case 'p95':
+                return this.getPercentile(PERCENTILE.P95);
+            case 'p99':
+                return this.getPercentile(PERCENTILE.P99);
+        }
+    }
+    /**
+     * Get all computed statistics
+     */
+    getStats() {
+        return {
+            count: this.count,
+            sum: this.sum,
+            avg: this.count > 0 ? this.sum / this.count : undefined,
+            min: this.min === Infinity ? undefined : this.min,
+            max: this.max === -Infinity ? undefined : this.max,
+            p50: this.getPercentile(PERCENTILE.P50),
+            p95: this.getPercentile(PERCENTILE.P95),
+            p99: this.getPercentile(PERCENTILE.P99),
+        };
+    }
+    /**
+     * Reset the aggregator to initial state
+     */
+    reset() {
+        this.count = 0;
+        this.sum = 0;
+        this.min = Infinity;
+        this.max = -Infinity;
+        this.sketch = new DDSketch();
+    }
+    /**
+     * Get the current count of values added
+     */
+    getCount() {
+        return this.count;
+    }
+    /**
+     * Estimate percentile via DDSketch
+     */
+    getPercentile(percentile) {
+        if (this.count === 0)
+            return undefined;
+        return this.sketch.getValueAtQuantile(percentile / StreamingAggregator.PERCENTILE_SCALE);
+    }
+}
+// Conservative fallback size (1MB) for objects that cannot be serialized
+// This prevents memory limit bypass when JSON.stringify() fails (e.g., circular references)
+export const CONSERVATIVE_FALLBACK_SIZE = 1e6;
+/**
+ * Maximum recursion depth for circular reference detection.
+ * Objects nested deeper than this are assumed safe to avoid performance issues.
+ */
+const MAX_CIRCULAR_CHECK_DEPTH = 100;
+/**
+ * Check if a value contains circular references using a WeakSet
+ * @param obj - The object to check
+ * @param seen - WeakSet to track seen objects
+ * @param depth - Current recursion depth (internal use)
+ * @returns true if circular reference detected
+ */
+export function hasCircularReference(obj, seen = new WeakSet(), depth = 0) {
+    // Early bailout for primitives
+    if (obj === null || typeof obj !== 'object') {
+        return false;
+    }
+    // Depth limit to prevent performance issues with deeply nested objects
+    if (depth > MAX_CIRCULAR_CHECK_DEPTH) {
+        return false;
+    }
+    if (seen.has(obj)) {
+        return true;
+    }
+    seen.add(obj);
+    if (Array.isArray(obj)) {
+        for (const item of obj) {
+            if (hasCircularReference(item, seen, depth + 1)) {
+                return true;
+            }
+        }
+    }
+    else {
+        for (const key of Object.keys(obj)) {
+            if (hasCircularReference(obj[key], seen, depth + 1)) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+/**
+ * Estimate approximate byte size of an object (rough approximation)
+ * Uses a sample-based approach for large arrays to avoid performance issues
+ *
+ * Security: Returns a conservative estimate (1MB) when serialization fails
+ * to prevent memory limit bypass via circular references or other edge cases.
+ */
+export function estimateByteSize(results, sampleSize = BATCH_SIZE) {
+    if (results.length === 0)
+        return 0;
+    // For small arrays, measure directly
+    if (results.length <= sampleSize) {
+        try {
+            // Check for circular references first to avoid JSON.stringify throwing
+            if (hasCircularReference(results)) {
+                memoryLog.warn('Circular reference detected in results, using conservative size estimate');
+                return CONSERVATIVE_FALLBACK_SIZE;
+            }
+            return JSON.stringify(results).length;
+        }
+        catch {
+            // If serialization fails for any reason, return conservative estimate
+            // This prevents memory limit bypass on error
+            memoryLog.warn('Failed to serialize results, using conservative size estimate');
+            return CONSERVATIVE_FALLBACK_SIZE;
+        }
+    }
+    // For large arrays, sample and extrapolate
+    const step = Math.floor(results.length / sampleSize);
+    let sampleBytes = 0;
+    let sampledCount = 0;
+    let failedCount = 0;
+    for (let i = 0; i < results.length && sampledCount < sampleSize; i += step) {
+        try {
+            // Only check objects for circular references — primitives can't have them,
+            // and skipping the check avoids allocating a new WeakSet per sample item.
+            const item = results[i];
+            if (item !== null && typeof item === 'object' && hasCircularReference(item)) {
+                failedCount++;
+                sampledCount++;
+                continue;
+            }
+            sampleBytes += JSON.stringify(item).length;
+            sampledCount++;
+        }
+        catch {
+            failedCount++;
+            sampledCount++;
+        }
+    }
+    // If too many samples failed (>50%), return conservative estimate
+    if (failedCount > sampledCount / BINARY_SEARCH_DIVISOR) {
+        memoryLog.warn('Too many serialization failures in sample, using conservative size estimate');
+        return CONSERVATIVE_FALLBACK_SIZE;
+    }
+    // If all samples failed, return conservative estimate
+    if (sampledCount === failedCount) {
+        memoryLog.warn('All samples failed to serialize, using conservative size estimate');
+        return CONSERVATIVE_FALLBACK_SIZE;
+    }
+    const successfulSamples = sampledCount - failedCount;
+    const avgBytesPerItem = sampleBytes / successfulSamples;
+    return Math.ceil(avgBytesPerItem * results.length);
+}
+/**
+ * Enforce memory limit on results array.
+ * Truncates results if they exceed count limit OR estimated byte size limit.
+ * Uses sampling for large arrays to avoid performance overhead.
+ *
+ * @param results - Array of results to limit
+ * @param limit - Maximum number of results (defaults to MAX_RESULTS_IN_MEMORY)
+ * @returns Object with results, truncated flag, and original count
+ *
+ * @example
+ * ```ts
+ * const { results, truncated } = enforceMemoryLimit(largeArray);
+ * if (truncated) {
+ *   console.warn('Results were truncated');
+ * }
+ * ```
+ */
+export function enforceMemoryLimit(results, limit = MAX_RESULTS_IN_MEMORY) {
+    const originalCount = results.length;
+    // Check count limit first
+    if (results.length > limit) {
+        memoryLog.warn(`Results truncated from ${results.length} to ${limit} items (MAX_RESULTS_IN_MEMORY exceeded)`, { from: results.length, to: limit, reason: 'count' });
+        return {
+            results: results.slice(0, limit),
+            truncated: true,
+            originalCount,
+            truncationReason: 'count',
+        };
+    }
+    // Estimate byte size to prevent memory exhaustion with large objects
+    const estimatedBytes = estimateByteSize(results);
+    if (estimatedBytes > MAX_RESULTS_BYTE_SIZE) {
+        // Binary search to find safe size
+        let safeCount = Math.floor(results.length / BINARY_SEARCH_DIVISOR);
+        let low = 0;
+        let high = results.length;
+        while (low < high - BINARY_SEARCH_DELTA) {
+            const mid = Math.floor((low + high) / BINARY_SEARCH_DIVISOR);
+            const midSize = estimateByteSize(results.slice(0, mid));
+            if (midSize <= MAX_RESULTS_BYTE_SIZE) {
+                low = mid;
+                safeCount = mid;
+            }
+            else {
+                high = mid;
+            }
+        }
+        // Ensure at least some results are returned
+        safeCount = Math.max(1, Math.min(safeCount, results.length));
+        memoryLog.warn(`Results truncated from ${results.length} to ${safeCount} items (estimated ${Math.round(estimatedBytes / BYTES.MB)}MB exceeded ${MAX_RESULTS_BYTE_SIZE / BYTES.MB}MB limit)`, { from: results.length, to: safeCount, estimatedMB: Math.round(estimatedBytes / BYTES.MB), reason: 'bytes' });
+        return {
+            results: results.slice(0, safeCount),
+            truncated: true,
+            originalCount,
+            truncationReason: 'size',
+        };
+    }
+    return {
+        results,
+        truncated: false,
+        originalCount,
+    };
+}
+/**
+ * Check if streaming aggregation should be used based on result count
+ *
+ * @param count - Number of results
+ * @param threshold - Threshold to trigger streaming (defaults to STREAMING_THRESHOLD)
+ * @returns true if streaming aggregation should be used
+ */
+export function shouldUseStreamingAggregation(count, threshold = STREAMING_THRESHOLD) {
+    return count > threshold;
+}
+/**
+ * Get current memory limit constants (useful for testing)
+ */
+export function getMemoryLimits() {
+    return {
+        maxResults: MAX_RESULTS_IN_MEMORY,
+        streamingThreshold: STREAMING_THRESHOLD,
+    };
+}
+//# sourceMappingURL=file-utils.js.map