observability-toolkit 2.0.0 → 2.1.0

package/dist/lib/file-utils.d.ts

@@ -1,320 +0,0 @@
-/**
- * File system utilities for observability-toolkit
- */
-/**
- * Expand tilde (~) to home directory
- * Handles ~/path and ~user/path (falls back to ~ for named users)
- */
-export declare function expandTilde(filepath: string): string;
-/**
- * Load and parse JSON file with fallback
- */
-export declare function loadJson<T>(filePath: string, fallback?: T): T | null;
-/**
- * Load JSON file, returning fallback on error (never null)
- */
-export declare function loadJsonSafe<T>(filePath: string, fallback: T): T;
-/**
- * Get date string in YYYY-MM-DD format
- */
-export declare function getDateString(date?: Date): string;
-/**
- * Validate that a path is within allowed base directories
- * Prevents path traversal attacks by resolving symlinks and checking containment
- *
- * Security measures:
- * - Uses realpathSync() to resolve symlinks (prevents symlink attacks)
- * - Requires non-empty allowedBasePaths
- * - Uses platform-specific path separator
- * - Logs security failures for audit trail
- */
-export declare function isPathWithinAllowed(targetPath: string, allowedBasePaths: string[]): boolean;
-/**
- * List files in directory matching a pattern
- * @param dir - Directory to list files from
- * @param pattern - RegExp to filter filenames
- * @param allowedBasePaths - Optional list of allowed base directories (for security validation)
- */
-export declare function listFiles(dir: string, pattern: RegExp, allowedBasePaths?: string[]): string[];
-/**
- * Parse date from filename like traces-2026-01-28.jsonl
- * Validates date components to prevent path traversal via malicious date strings.
- * Returns null for invalid dates (month > 12, day > 31, unreasonable years).
- */
-export declare function parseDateFromFilename(filename: string): string | null;
-/**
- * Statistics from JSONL parsing
- */
-export interface JsonlParseStats {
-    totalLines: number;
-    parsedLines: number;
-    skippedLines: number;
-    emptyLines: number;
-}
-/**
- * Result from readJsonlSyncWithStats
- */
-export interface ReadJsonlResult<T> {
-    data: T[];
-    stats: JsonlParseStats;
-}
-/**
- * Read JSONL file and parse each line
- */
-export declare function readJsonlSync<T>(filePath: string, limit?: number): T[];
-/**
- * Read JSONL file with verbose statistics about parse failures
- * Use this for debugging to see how many lines were skipped
- */
-export declare function readJsonlSyncWithStats<T>(filePath: string, limit?: number): ReadJsonlResult<T>;
-/**
- * Check if file is gzip compressed based on extension
- */
-export declare function isGzipFile(filePath: string): boolean;
-/**
- * Create a readable stream for JSONL files, handling gzip decompression transparently
- */
-export declare function createJsonlStream(filePath: string): NodeJS.ReadableStream;
-/**
- * Options for streamJsonl
- */
-export interface StreamJsonlOptions {
-    /** Optional stats tracker for recording parse success/failure */
-    statsTracker?: {
-        recordSuccess(filePath: string): void;
-        recordFailure(filePath: string): void;
-        recordEmpty(filePath: string): void;
-    };
-}
-/**
- * Stream JSONL file line by line (for large files)
- * Handles both plain .jsonl and gzip compressed .jsonl.gz files
- *
- * @param filePath - Path to JSONL file (plain or gzip compressed)
- * @param options - Optional configuration including stats tracker
- */
-export declare function streamJsonl<T>(filePath: string, options?: StreamJsonlOptions): AsyncGenerator<T>;
-/**
- * Filter items by date range
- */
-export declare function filterByDateRange<T extends {
-    timestamp?: string;
-}>(items: T[], startDate?: string, endDate?: string): T[];
-/**
- * Paginate results array with offset and limit
- */
-export declare function paginateResults<T>(results: T[], offset: number, limit: number): T[];
-/**
- * Check if we've collected enough results for early termination
- */
-export declare function hasReachedLimit(resultsLength: number, offset: number, limit: number): boolean;
-/**
- * Result from cleanupOldFiles operation
- */
-export interface CleanupResult {
-    deleted: number;
-    errors: string[];
-}
-/**
- * Clean up old telemetry files based on retention policy
- * @param dir - Directory containing telemetry files
- * @param retentionDays - Number of days to retain files (default: 7)
- * @param allowedBasePaths - Optional list of allowed base directories (for security validation)
- * @returns Object with count of deleted files and any errors encountered
- */
-export declare function cleanupOldFiles(dir: string, retentionDays?: number, allowedBasePaths?: string[]): CleanupResult;
-/**
- * Options for BatchWriter
- */
-export interface BatchWriterOptions {
-    /** Number of records to buffer before writing (default: 100) */
-    batchSize?: number;
-    /** Compress output with gzip (default: false) */
-    gzip?: boolean;
-    /** Create parent directories if they don't exist (default: true) */
-    mkdirp?: boolean;
-}
-/**
- * Statistics from BatchWriter
- */
-export interface BatchWriterStats {
-    recordsWritten: number;
-    bytesWritten: number;
-    flushCount: number;
-}
-/**
- * Batched JSONL writer that reduces I/O by buffering writes.
- * Supports optional gzip compression.
- *
- * @example
- * ```ts
- * const writer = new BatchWriter('output.jsonl', { batchSize: 50 });
- * for (const record of records) {
- *   writer.write(record);
- * }
- * writer.close(); // Flush remaining buffered records
- * ```
- */
-export declare class BatchWriter<T = unknown> {
-    private buffer;
-    private readonly filePath;
-    private readonly batchSize;
-    private readonly useGzip;
-    private readonly mkdirp;
-    private initialized;
-    private stats;
-    constructor(filePath: string, options?: BatchWriterOptions);
-    /**
-     * Add a record to the buffer. Flushes automatically when batch size is reached.
-     */
-    write(record: T): void;
-    /**
-     * Write multiple records at once
-     */
-    writeAll(records: T[]): void;
-    /**
-     * Flush buffered records to disk
-     */
-    flush(): void;
-    /**
-     * Flush remaining records and finalize the file.
-     * Always call this when done writing.
-     */
-    close(): BatchWriterStats;
-    /**
-     * Get current statistics
-     */
-    getStats(): BatchWriterStats;
-    /**
-     * Get number of records currently buffered (not yet written)
-     */
-    get bufferedCount(): number;
-}
-/**
- * Aggregation types supported by StreamingAggregator
- */
-export type AggregationType = 'sum' | 'avg' | 'min' | 'max' | 'count' | 'p50' | 'p95' | 'p99';
-/**
- * Streaming aggregator that computes aggregations incrementally without holding all values.
- * Uses reservoir sampling for percentile approximations.
- *
- * @example
- * ```ts
- * const aggregator = new StreamingAggregator();
- * for (const value of largeDataset) {
- *   aggregator.add(value);
- * }
- * const avg = aggregator.getResult('avg');
- * const p95 = aggregator.getResult('p95');
- * ```
- */
-export declare class StreamingAggregator {
-    private count;
-    private sum;
-    private min;
-    private max;
-    private static readonly RESERVOIR_SIZE;
-    private reservoir;
-    private seenCount;
-    /**
-     * Add a value to the aggregator
-     */
-    add(value: number): void;
-    /**
-     * Add multiple values at once
-     */
-    addAll(values: number[]): void;
-    /**
-     * Get the result for a specific aggregation type
-     */
-    getResult(type: AggregationType): number | undefined;
-    /**
-     * Get all computed statistics
-     */
-    getStats(): {
-        count: number;
-        sum: number;
-        avg: number | undefined;
-        min: number | undefined;
-        max: number | undefined;
-        p50: number | undefined;
-        p95: number | undefined;
-        p99: number | undefined;
-    };
-    /**
-     * Reset the aggregator to initial state
-     */
-    reset(): void;
-    /**
-     * Get the current count of values added
-     */
-    getCount(): number;
-    /**
-     * Calculate percentile using R-7 linear interpolation on the reservoir sample
-     */
-    private getPercentile;
-}
-/**
- * Result from enforceMemoryLimit when truncation occurs
- */
-export interface MemoryLimitResult<T> {
-    /** The (possibly truncated) results */
-    results: T[];
-    /** Whether results were truncated */
-    truncated: boolean;
-    /** Original count before truncation */
-    originalCount: number;
-    /** Reason for truncation if truncated */
-    truncationReason?: 'count' | 'size';
-}
-export declare const CONSERVATIVE_FALLBACK_SIZE = 1000000;
-/**
- * Check if a value contains circular references using a WeakSet
- * @param obj - The object to check
- * @param seen - WeakSet to track seen objects
- * @param depth - Current recursion depth (internal use)
- * @returns true if circular reference detected
- */
-export declare function hasCircularReference(obj: unknown, seen?: WeakSet<object>, depth?: number): boolean;
-/**
- * Estimate approximate byte size of an object (rough approximation)
- * Uses a sample-based approach for large arrays to avoid performance issues
- *
- * Security: Returns a conservative estimate (1MB) when serialization fails
- * to prevent memory limit bypass via circular references or other edge cases.
- */
-export declare function estimateByteSize<T>(results: T[], sampleSize?: number): number;
-/**
- * Enforce memory limit on results array.
- * Truncates results if they exceed count limit OR estimated byte size limit.
- * Uses sampling for large arrays to avoid performance overhead.
- *
- * @param results - Array of results to limit
- * @param limit - Maximum number of results (defaults to MAX_RESULTS_IN_MEMORY)
- * @returns Object with results, truncated flag, and original count
- *
- * @example
- * ```ts
- * const { results, truncated } = enforceMemoryLimit(largeArray);
- * if (truncated) {
- *   console.warn('Results were truncated');
- * }
- * ```
- */
-export declare function enforceMemoryLimit<T>(results: T[], limit?: number): MemoryLimitResult<T>;
-/**
- * Check if streaming aggregation should be used based on result count
- *
- * @param count - Number of results
- * @param threshold - Threshold to trigger streaming (defaults to STREAMING_THRESHOLD)
- * @returns true if streaming aggregation should be used
- */
-export declare function shouldUseStreamingAggregation(count: number, threshold?: number): boolean;
-/**
- * Get current memory limit constants (useful for testing)
- */
-export declare function getMemoryLimits(): {
-    maxResults: number;
-    streamingThreshold: number;
-};
-//# sourceMappingURL=file-utils.d.ts.map

package/dist/lib/file-utils.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"file-utils.d.ts","sourceRoot":"","sources":["../../src/lib/file-utils.ts"],"names":[],"mappings":"AAAA;;GAEG;AA0BH;;;GAGG;AACH,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CASpD;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,IAAI,CASpE;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,CAAC,CAEhE;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,IAAI,GAAE,IAAiB,GAAG,MAAM,CAE7D;AAED;;;;;;;;;GASG;AACH,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,EAAE,GAAG,OAAO,CAuD3F;AAED;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,gBAAgB,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAyB7F;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAerE;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe,CAAC,CAAC;IAChC,IAAI,EAAE,CAAC,EAAE,CAAC;IACV,KAAK,EAAE,eAAe,CAAC;CACxB;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE,CAuBtE;AAED;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,eAAe,CAAC,CAAC,CAAC,CAyC9F;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAEpD;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC,cAAc,CAMzE;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,iEAAiE;IACjE,YAAY,CAAC,EAAE;QACb,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QACtC,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QACtC,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;KACrC,CAAC;CACH;AAED;;;;;;GAMG;AACH,wBAAuB,WAAW,CAAC,CAAC,EAClC,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,kBAAkB,GAC3B,cAAc,CAAC,CAAC,CAAC,CAwBnB;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,CAAC,SAAS;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,EAChE,KAAK,EAAE,CAAC,EAAE,EACV,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,MAAM,GACf,CAAC,EAAE,CAQL;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,CAAC,EAAE,CAEnF;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAE7F;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED;;;;;;GAMG;AACH,wBAAgB,eAAe,CAC7B,GAAG,EAAE,MAAM,EACX,aAAa,GAAE,MAAU,EACzB,gBAAgB,CAAC,EAAE,MAAM,EAAE,GAC1B,aAAa,CAkDf;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iDAAiD;IACjD,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,oEAAoE;IACpE,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;;;;;GAYG;AACH,qBAAa,WAAW,CAAC,CAAC,GAAG,OAAO;IAClC,OAAO,CAAC,MAAM,CAAgB;IAC9B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,KAAK,CAA2E;gBAE5E,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,kBAAuB;IAO9D;;OAEG;IACH,KAAK,CAAC,MAAM,EAAE,CAAC,GAAG,IAAI;IAOtB;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,CAAC,EAAE,GAAG,IAAI;IAM5B;;OAEG;IACH,KAAK,IAAI,IAAI;IAwCb;;;OAGG;IACH,KAAK,IAAI,gBAAgB;IAKzB;;OAEG;IACH,QAAQ,IAAI,gBAAgB;IAO5B;;OAEG;IACH,IAAI,aAAa,IAAI,MAAM,CAE1B;CACF;AAED;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,OAAO,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,CAAC;AAE9F;;;;;;;;;;;;;GAaG;AACH,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,KAAK,CAAK;IAClB,OAAO,CAAC,GAAG,CAAK;IAChB,OAAO,CAAC,GAAG,CAAY;IACvB,OAAO,CAAC,GAAG,CAAa;IAIxB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAQ;IAC9C,OAAO,CAAC,SAAS,CAAgB;IACjC,OAAO,CAAC,SAAS,CAAK;IAEtB;;OAEG;IACH,GAAG,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAqBxB;;OAEG;IACH,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI;IAM9B;;OAEG;IACH,SAAS,CAAC,IAAI,EAAE,eAAe,GAAG,MAAM,GAAG,SAAS;IAyBpD;;OAEG;IACH,QAAQ,IAAI;QACV,KAAK,EAAE,MAAM,CAAC;QACd,GAAG,EAAE,MAAM,CAAC;QACZ,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;QACxB,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;QACxB,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;QACxB,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;QACxB,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;QACxB,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;KACzB;IAaD;;OAEG;IACH,KAAK,IAAI,IAAI;IASb;;OAEG;IACH,QAAQ,IAAI,MAAM;IAIlB;;OAEG;IACH,OAAO,CAAC,aAAa;CAkBtB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB,CAAC,CAAC;IAClC,uCAAuC;IACvC,OAAO,EAAE,CAAC,EAAE,CAAC;IACb,qCAAqC;IACrC,SAAS,EAAE,OAAO,CAAC;IACnB,uCAAuC;IACvC,aAAa,EAAE,MAAM,CAAC;IACtB,yCAAyC;IACzC,gBAAgB,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;CACrC;AAOD,eAAO,MAAM,0BAA0B,UAAU,CAAC;AAQlD;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAClC,GAAG,EAAE,OAAO,EACZ,IAAI,GAAE,OAAO,CAAC,MAAM,CAAiB,EACrC,KAAK,GAAE,MAAU,GAChB,OAAO,CAgCT;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,EAAE,UAAU,SAAM,GAAG,MAAM,CAyD1E;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,kBAAkB,CAAC,CAAC,EAClC,OAAO,EAAE,CAAC,EAAE,EACZ,KAAK,GAAE,MAA8B,GACpC,iBAAiB,CAAC,CAAC,CAAC,CAsDtB;AAED;;;;;;GAMG;AACH,wBAAgB,6BAA6B,CAC3C,KAAK,EAAE,MAAM,EACb,SAAS,GAAE,MAA4B,GACtC,OAAO,CAET;AAED;;GAEG;AACH,wBAAgB,eAAe,IAAI;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,kBAAkB,EAAE,MAAM,CAAA;CAAE,CAKpF"}