observability-toolkit 1.8.5 → 2.1.0

package/dist/lib/{file-utils.d.ts → core/file-utils.d.ts}

@@ -1,6 +1,7 @@
 /**
  * File system utilities for observability-toolkit
  */
+import type { z } from 'zod';
 /**
  * Expand tilde (~) to home directory
  * Handles ~/path and ~user/path (falls back to ~ for named users)
@@ -14,6 +15,24 @@ export declare function loadJson<T>(filePath: string, fallback?: T): T | null;
  * Load JSON file, returning fallback on error (never null)
  */
 export declare function loadJsonSafe<T>(filePath: string, fallback: T): T;
+/**
+ * Load and validate JSON file against a Zod schema.
+ * Throws if validation fails or file does not exist.
+ *
+ * @param filePath - Path to JSON file
+ * @param schema - Zod schema to validate against
+ * @throws {Error} If validation fails
+ */
+export declare function loadJsonWithValidation<T>(filePath: string, schema: z.ZodType<T>): T;
+/**
+ * Load and validate JSON file, returning fallback on error.
+ * Unlike loadJsonWithValidation, this does not throw.
+ *
+ * @param filePath - Path to JSON file
+ * @param schema - Zod schema to validate against
+ * @param fallback - Default value if file not found or validation fails
+ */
+export declare function loadJsonWithValidationSafe<T>(filePath: string, schema: z.ZodType<T>, fallback: T): T;
 /**
  * Get date string in YYYY-MM-DD format
  */
@@ -59,9 +78,20 @@ export interface ReadJsonlResult<T> {
     stats: JsonlParseStats;
 }
 /**
- * Read JSONL file and parse each line
+ * Read JSONL file and parse each line.
+ * Delegates to readJsonlSyncWithStats to avoid duplicating the parse loop (CS-D1).
  */
 export declare function readJsonlSync<T>(filePath: string, limit?: number): T[];
+/**
+ * Read JSONL file with Zod schema validation.
+ * Each line is validated against the schema; invalid lines are skipped silently.
+ *
+ * @param filePath - Path to JSONL file
+ * @param schema - Zod schema for validation
+ * @param limit - Maximum number of valid records to return
+ * @returns Array of validated records
+ */
+export declare function readJsonlWithValidation<T>(filePath: string, schema: z.ZodType<T>, limit?: number): T[];
 /**
  * Read JSONL file with verbose statistics about parse failures
  * Use this for debugging to see how many lines were skipped
@@ -75,11 +105,35 @@ export declare function isGzipFile(filePath: string): boolean;
  * Create a readable stream for JSONL files, handling gzip decompression transparently
  */
 export declare function createJsonlStream(filePath: string): NodeJS.ReadableStream;
+/**
+ * Options for streamJsonl
+ */
+export interface StreamJsonlOptions {
+    /** Optional stats tracker for recording parse success/failure */
+    statsTracker?: {
+        recordSuccess(filePath: string): void;
+        recordFailure(filePath: string): void;
+        recordEmpty(filePath: string): void;
+    };
+}
 /**
  * Stream JSONL file line by line (for large files)
  * Handles both plain .jsonl and gzip compressed .jsonl.gz files
+ *
+ * @param filePath - Path to JSONL file (plain or gzip compressed)
+ * @param options - Optional configuration including stats tracker
+ */
+export declare function streamJsonl<T>(filePath: string, options?: StreamJsonlOptions): AsyncGenerator<T>;
+/**
+ * Stream JSONL file with Zod schema validation (for large files).
+ * Each line is validated against the schema; invalid lines are skipped.
+ * Handles both plain .jsonl and gzip compressed .jsonl.gz files.
+ *
+ * @param filePath - Path to JSONL file (plain or gzip compressed)
+ * @param schema - Zod schema for validation
+ * @param options - Optional configuration including stats tracker
  */
-export declare function streamJsonl<T>(filePath: string): AsyncGenerator<T>;
+export declare function streamJsonlWithValidation<T>(filePath: string, schema: z.ZodType<T>, options?: StreamJsonlOptions): AsyncGenerator<T>;
 /**
  * Filter items by date range
  */
@@ -182,7 +236,7 @@ export declare class BatchWriter<T = unknown> {
 export type AggregationType = 'sum' | 'avg' | 'min' | 'max' | 'count' | 'p50' | 'p95' | 'p99';
 /**
  * Streaming aggregator that computes aggregations incrementally without holding all values.
- * Uses reservoir sampling for percentile approximations.
+ * Uses DDSketch for memory-bounded percentile estimation.
  *
  * @example
  * ```ts
@@ -199,9 +253,9 @@ export declare class StreamingAggregator {
     private sum;
     private min;
     private max;
-    private static readonly RESERVOIR_SIZE;
-    private reservoir;
-    private seenCount;
+    private sketch;
+    /** DDSketch.getValueAtQuantile() takes 0-1; PERCENTILE enum values are 0-100 */
+    private static readonly PERCENTILE_SCALE;
     /**
      * Add a value to the aggregator
      */
@@ -236,7 +290,7 @@ export declare class StreamingAggregator {
      */
     getCount(): number;
     /**
-     * Calculate percentile using R-7 linear interpolation on the reservoir sample
+     * Estimate percentile via DDSketch
      */
     private getPercentile;
 }
@@ -258,9 +312,10 @@ export declare const CONSERVATIVE_FALLBACK_SIZE = 1000000;
  * Check if a value contains circular references using a WeakSet
  * @param obj - The object to check
  * @param seen - WeakSet to track seen objects
+ * @param depth - Current recursion depth (internal use)
  * @returns true if circular reference detected
  */
-export declare function hasCircularReference(obj: unknown, seen?: WeakSet<object>): boolean;
+export declare function hasCircularReference(obj: unknown, seen?: WeakSet<object>, depth?: number): boolean;
 /**
  * Estimate approximate byte size of an object (rough approximation)
  * Uses a sample-based approach for large arrays to avoid performance issues

package/dist/lib/core/file-utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-utils.d.ts","sourceRoot":"","sources":["../../../src/lib/core/file-utils.ts"],"names":[],"mappings":"AAAA;;GAEG;AAeH,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAW7B;;;GAGG;AACH,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CASpD;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,IAAI,CASpE;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,CAAC,CAEhE;AAED;;;;;;;GAOG;AACH,wBAAgB,sBAAsB,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAenF;AAED;;;;;;;GAOG;AACH,wBAAgB,0BAA0B,CAAC,CAAC,EAC1C,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EACpB,QAAQ,EAAE,CAAC,GACV,CAAC,CAMH;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,IAAI,GAAE,IAAiB,GAAG,MAAM,CAE7D;AAED;;;;;;;;;GASG;AACH,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,EAAE,GAAG,OAAO,CAuD3F;AAED;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,gBAAgB,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAyB7F;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAerE;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe,CAAC,CAAC;IAChC,IAAI,EAAE,CAAC,EAAE,CAAC;IACV,KAAK,EAAE,eAAe,CAAC;CACxB;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE,CAEtE;AAED;;;;;;;;GAQG;AACH,wBAAgB,uBAAuB,CAAC,CAAC,EACvC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EACpB,KAAK,CAAC,EAAE,MAAM,GACb,CAAC,EAAE,CA6BL;AAED;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,eAAe,CAAC,CAAC,CAAC,CAyC9F;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAEpD;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC,cAAc,CAMzE;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,iEAAiE;IACjE,YAAY,CAAC,EAAE;QACb,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QACtC,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QACtC,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;KACrC,CAAC;CACH;AAED;;;;;;GAMG;AACH,wBAAuB,WAAW,CAAC,CAAC,EAClC,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,kBAAkB,GAC3B,cAAc,CAAC,CAAC,CAAC,CAwBnB;AAED;;;;;;;;GAQG;AACH,wBAAuB,yBAAyB,CAAC,CAAC,EAChD,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EACpB,OAAO,CAAC,EAAE,kBAAkB,GAC3B,cAAc,CAAC,CAAC,CAAC,CA6BnB;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,CAAC,SAAS;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,EAChE,KAAK,EAAE,CAAC,EAAE,EACV,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,MAAM,GACf,CAAC,EAAE,CAQL;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,CAAC,EAAE,CAEnF;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAE7F;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED;;;;;;GAMG;AACH,wBAAgB,eAAe,CAC7B,GAAG,EAAE,MAAM,EACX,aAAa,GAAE,MAA+B,EAC9C,gBAAgB,CAAC,EAAE,MAAM,EAAE,GAC1B,aAAa,CAkDf;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iDAAiD;IACjD,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,oEAAoE;IACpE,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;;;;;GAYG;AACH,qBAAa,WAAW,CAAC,CAAC,GAAG,OAAO;IAClC,OAAO,CAAC,MAAM,CAAgB;IAC9B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,KAAK,CAA2E;gBAE5E,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,kBAAuB;IAO9D;;OAEG;IACH,KAAK,CAAC,MAAM,EAAE,CAAC,GAAG,IAAI;IAOtB;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,CAAC,EAAE,GAAG,IAAI;IAM5B;;OAEG;IACH,KAAK,IAAI,IAAI;IA8Cb;;;OAGG;IACH,KAAK,IAAI,gBAAgB;IAKzB;;OAEG;IACH,QAAQ,IAAI,gBAAgB;IAO5B;;OAEG;IACH,IAAI,aAAa,IAAI,MAAM,CAE1B;CACF;AAED;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,OAAO,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,CAAC;AAE9F;;;;;;;;;;;;;GAaG;AACH,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,KAAK,CAAK;IAClB,OAAO,CAAC,GAAG,CAAK;IAChB,OAAO,CAAC,GAAG,CAAY;IACvB,OAAO,CAAC,GAAG,CAAa;IACxB,OAAO,CAAC,MAAM,CAAkB;IAEhC,gFAAgF;IAChF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAO;IAE/C;;OAEG;IACH,GAAG,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAUxB;;OAEG;IACH,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI;IAM9B;;OAEG;IACH,SAAS,CAAC,IAAI,EAAE,eAAe,GAAG,MAAM,GAAG,SAAS;IAyBpD;;OAEG;IACH,QAAQ,IAAI;QACV,KAAK,EAAE,MAAM,CAAC;QACd,GAAG,EAAE,MAAM,CAAC;QACZ,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;QACxB,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;QACxB,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;QACxB,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;QACxB,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;QACxB,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;KACzB;IAaD;;OAEG;IACH,KAAK,IAAI,IAAI;IAQb;;OAEG;IACH,QAAQ,IAAI,MAAM;IAIlB;;OAEG;IACH,OAAO,CAAC,aAAa;CAItB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB,CAAC,CAAC;IAClC,uCAAuC;IACvC,OAAO,EAAE,CAAC,EAAE,CAAC;IACb,qCAAqC;IACrC,SAAS,EAAE,OAAO,CAAC;IACnB,uCAAuC;IACvC,aAAa,EAAE,MAAM,CAAC;IACtB,yCAAyC;IACzC,gBAAgB,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;CACrC;AAID,eAAO,MAAM,0BAA0B,UAAM,CAAC;AAQ9C;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAClC,GAAG,EAAE,OAAO,EACZ,IAAI,GAAE,OAAO,CAAC,MAAM,CAAiB,EACrC,KAAK,GAAE,MAAU,GAChB,OAAO,CAgCT;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,EAAE,UAAU,SAAa,GAAG,MAAM,CA2DjF;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,kBAAkB,CAAC,CAAC,EAClC,OAAO,EAAE,CAAC,EAAE,EACZ,KAAK,GAAE,MAA8B,GACpC,iBAAiB,CAAC,CAAC,CAAC,CAwDtB;AAED;;;;;;GAMG;AACH,wBAAgB,6BAA6B,CAC3C,KAAK,EAAE,MAAM,EACb,SAAS,GAAE,MAA4B,GACtC,OAAO,CAET;AAED;;GAEG;AACH,wBAAgB,eAAe,IAAI;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,kBAAkB,EAAE,MAAM,CAAA;CAAE,CAKpF"}