@devilsdev/rag-pipeline-utils 2.6.1 → 2.7.0

package/dist/types/core/observability/metrics.d.ts

@@ -0,0 +1,585 @@
+/** Monotonic counter that only goes up until reset. */
+export class Counter {
+    /**
+     * @param {string} name - Metric name
+     * @param {string} [description=''] - Human-readable description
+     * @param {Object<string, string>} [labels={}] - Static labels attached to the metric
+     */
+    constructor(name: string, description?: string, labels?: {
+        [x: string]: string;
+    });
+    _type: string;
+    name: string;
+    description: string;
+    labels: {
+        [x: string]: string;
+    };
+    value: number;
+    /**
+     * Add to the counter.
+     * @param {number} [n=1] - Amount to add
+     * @returns {void}
+     * @throws {Error} If n is negative
+     */
+    inc(n?: number): void;
+    /**
+     * Set the value back to 0.
+     * @returns {void}
+     */
+    reset(): void;
+    /**
+     * Snapshot of the metric with the current time as timestamp.
+     * @returns {{name: string, description: string, type: string, value: number, labels: Object<string, string>, timestamp: string}}
+     */
+    toJSON(): {
+        name: string;
+        description: string;
+        type: string;
+        value: number;
+        labels: {
+            [x: string]: string;
+        };
+        timestamp: string;
+    };
+}
+/**
+ * Histogram with cumulative bucket counts. It also keeps every observed value
+ * in memory to compute percentiles and standard deviation.
+ */
+export class Histogram {
+    /**
+     * @param {string} name - Metric name
+     * @param {string} [description=''] - Human-readable description
+     * @param {Object<string, string>} [labels={}] - Static labels attached to the metric
+     * @param {number[]} [buckets] - Bucket upper bounds, sorted ascending on construction. A +Inf bucket is added.
+     */
+    constructor(name: string, description?: string, labels?: {
+        [x: string]: string;
+    }, buckets?: number[]);
+    _type: string;
+    name: string;
+    description: string;
+    labels: {
+        [x: string]: string;
+    };
+    buckets: number[];
+    bucketCounts: any[];
+    sum: number;
+    count: number;
+    min: number;
+    max: number;
+    _obs: any[];
+    /**
+     * Record a value. Non-numbers and NaN are ignored.
+     * @param {number} v - Observed value
+     * @returns {void}
+     */
+    observe(v: number): void;
+    /**
+     * Percentiles of the observed values by the nearest-rank method. Returns 0
+     * for each percentile when nothing has been observed.
+     * @param {number[]} [percentiles=[50, 95, 99]] - Percentiles to compute, 0-100
+     * @returns {Object<number, number>} Value for each requested percentile
+     */
+    getPercentiles(percentiles?: number[]): {
+        [x: number]: number;
+    };
+    /**
+     * Mean, population standard deviation, min, max, count and sum of the
+     * observed values. With no observations, min is Infinity and max is -Infinity.
+     * @returns {{mean: number, stdDev: number, min: number, max: number, count: number, sum: number}}
+     */
+    getStatistics(): {
+        mean: number;
+        stdDev: number;
+        min: number;
+        max: number;
+        count: number;
+        sum: number;
+    };
+    /**
+     * Clear all bucket counts, statistics and stored observations.
+     * @returns {void}
+     */
+    reset(): void;
+    /**
+     * Cumulative count per bucket bound (plus '+Inf') together with getStatistics().
+     * @returns {{buckets: Object<string, number>, mean: number, stdDev: number, min: number, max: number, count: number, sum: number}}
+     */
+    export(): {
+        buckets: {
+            [x: string]: number;
+        };
+        mean: number;
+        stdDev: number;
+        min: number;
+        max: number;
+        count: number;
+        sum: number;
+    };
+    /**
+     * Snapshot of the histogram with the current time as timestamp. Does not
+     * include min, max or percentiles.
+     * @returns {{name: string, description: string, type: string, count: number, sum: number, buckets: number[], bucketCounts: number[], labels: Object<string, string>, timestamp: string}}
+     */
+    toJSON(): {
+        name: string;
+        description: string;
+        type: string;
+        count: number;
+        sum: number;
+        buckets: number[];
+        bucketCounts: number[];
+        labels: {
+            [x: string]: string;
+        };
+        timestamp: string;
+    };
+}
+/** Gauge holding a value that can be set, raised or lowered. */
+export class Gauge {
+    /**
+     * @param {string} name - Metric name
+     * @param {string} [description=''] - Human-readable description
+     * @param {Object<string, string>} [labels={}] - Static labels attached to the metric
+     */
+    constructor(name: string, description?: string, labels?: {
+        [x: string]: string;
+    });
+    _type: string;
+    name: string;
+    description: string;
+    labels: {
+        [x: string]: string;
+    };
+    value: number;
+    /**
+     * Set the value.
+     * @param {number} v - New value
+     * @returns {void}
+     */
+    set(v: number): void;
+    /**
+     * Add to the value.
+     * @param {number} [n=1] - Amount to add
+     * @returns {void}
+     */
+    inc(n?: number): void;
+    /**
+     * Subtract from the value.
+     * @param {number} [n=1] - Amount to subtract
+     * @returns {void}
+     */
+    dec(n?: number): void;
+    /**
+     * Set the value back to 0.
+     * @returns {void}
+     */
+    reset(): void;
+    /**
+     * Snapshot of the metric with the current time as timestamp.
+     * @returns {{name: string, description: string, type: string, value: number, labels: Object<string, string>, timestamp: string}}
+     */
+    toJSON(): {
+        name: string;
+        description: string;
+        type: string;
+        value: number;
+        labels: {
+            [x: string]: string;
+        };
+        timestamp: string;
+    };
+}
+/**
+ * In-memory totals for pipeline operations: embedding, retrieval and LLM
+ * calls, errors, operation counts, concurrency, backpressure and heap usage.
+ */
+export class PipelineMetrics {
+    counters: {
+        operationsTotal: Counter;
+        operationsSuccessful: Counter;
+        operationsFailed: Counter;
+        operationsActive: Counter;
+    };
+    errorsByType: {
+        plugin: number;
+        stage: number;
+    };
+    errorsByPlugin: {};
+    backpressureSamples: any[];
+    backpressureApplied: number;
+    backpressureReleased: number;
+    memory: {
+        heapUsed: number;
+        heapTotal: number;
+        usagePercentage: number;
+    };
+    embedding: {
+        operations: number;
+        totalDuration: number;
+        totalTokens: number;
+        totalBatches: number;
+        durations: any[];
+    };
+    retrieval: {
+        operations: number;
+        totalDuration: number;
+        totalResults: number;
+        durations: any[];
+    };
+    llm: {
+        operations: number;
+        totalDuration: number;
+        totalInputTokens: number;
+        totalOutputTokens: number;
+        streamingOps: number;
+        durations: any[];
+        inputTokens: any[];
+        outputTokens: any[];
+    };
+    concurrency: {
+        maxConcurrency: number;
+        concurrencyValues: any[];
+    };
+    /**
+     * Record one embedding call.
+     * @param {string} provider - Provider name (unused)
+     * @param {number} duration - Duration in milliseconds
+     * @param {number} tokens - Tokens embedded
+     * @param {number} batches - Batches sent
+     * @returns {void}
+     */
+    recordEmbedding(provider: string, duration: number, tokens: number, batches: number): void;
+    /**
+     * Record one retrieval call.
+     * @param {string} provider - Provider name (unused)
+     * @param {number} duration - Duration in milliseconds
+     * @param {number} results - Number of results returned
+     * @returns {void}
+     */
+    recordRetrieval(provider: string, duration: number, results: number): void;
+    /**
+     * Record one LLM call.
+     * @param {string} provider - Provider name (unused)
+     * @param {number} duration - Duration in milliseconds
+     * @param {number} inputTokens - Prompt tokens
+     * @param {number} outputTokens - Completion tokens
+     * @param {boolean} [streaming] - Whether the call streamed its output
+     * @returns {void}
+     */
+    recordLLM(provider: string, duration: number, inputTokens: number, outputTokens: number, streaming?: boolean): void;
+    /**
+     * Store heapUsed and heapTotal from process.memoryUsage() and the used
+     * percentage, rounded.
+     * @returns {void}
+     */
+    recordMemoryUsage(): void;
+    /**
+     * Count an error by type and by component. Only the types 'plugin' and
+     * 'stage' are counted by type; other types are counted only by component.
+     * @param {string} type - Error type, 'plugin' or 'stage'
+     * @param {string} [component] - Plugin or component name
+     * @param {string} [provider] - Provider name (unused)
+     * @param {Error} [error] - The error (unused)
+     * @returns {void}
+     */
+    recordError(type: string, component?: string, provider?: string, error?: Error): void;
+    /**
+     * Count an operation as started and active.
+     * @param {string} [type] - Operation type (unused)
+     * @param {string} [component] - Component name (unused)
+     * @param {string} [provider] - Provider name (unused)
+     * @returns {void}
+     */
+    recordOperationStart(type?: string, component?: string, provider?: string): void;
+    /**
+     * Mark an active operation as finished. A status of 'error' or 'failed'
+     * counts as failed; any other status counts as successful.
+     * @param {string} [type] - Operation type (unused)
+     * @param {string} [component] - Component name (unused)
+     * @param {string} [provider] - Provider name (unused)
+     * @param {number} [duration] - Duration in milliseconds (unused)
+     * @param {string} [status] - Outcome of the operation
+     * @returns {void}
+     */
+    recordOperationEnd(type?: string, component?: string, provider?: string, duration?: number, status?: string): void;
+    /**
+     * Record a concurrency sample. Keeps the highest max seen and every average.
+     * @param {number} maxConcurrency - Peak concurrency in the sample
+     * @param {number} avgConcurrency - Average concurrency in the sample
+     * @returns {void}
+     */
+    recordConcurrency(maxConcurrency: number, avgConcurrency: number): void;
+    /**
+     * Count a backpressure event and keep the buffer size as a sample.
+     * @param {string} action - 'applied' or 'released'; other values are not counted
+     * @param {number} [bufferSize] - Buffer size at the time of the event
+     * @param {number} [threshold] - Backpressure threshold (unused)
+     * @returns {void}
+     */
+    recordBackpressure(action: string, bufferSize?: number, threshold?: number): void;
+    /**
+     * Totals and averages for every recorded category. Averages are 0 when
+     * nothing was recorded.
+     * @returns Summary grouped by embedding, retrieval, llm, memory, errors, operations, concurrency and backpressure
+     */
+    getSummary(): {
+        embedding: {
+            totalOperations: number;
+            totalDuration: number;
+            totalTokens: number;
+            totalBatches: number;
+            avgDuration: {
+                mean: number;
+            };
+        };
+        retrieval: {
+            totalOperations: number;
+            totalResults: number;
+            avgDuration: {
+                mean: number;
+            };
+        };
+        llm: {
+            totalOperations: number;
+            streamingOperations: number;
+            avgDuration: {
+                mean: number;
+            };
+            avgInputTokens: {
+                mean: number;
+            };
+            avgOutputTokens: {
+                mean: number;
+            };
+        };
+        memory: {
+            heapUsed: number;
+            heapTotal: number;
+            usagePercentage: number;
+        };
+        errors: {
+            total: number;
+            byType: {
+                plugin: number;
+                stage: number;
+            };
+            byPlugin: {};
+        };
+        operations: {
+            total: number;
+            successful: number;
+            failed: number;
+        };
+        concurrency: {
+            maxConcurrency: number;
+            avgConcurrency: {
+                mean: number;
+            };
+        };
+        backpressure: {
+            totalEvents: number;
+            appliedEvents: number;
+            releasedEvents: number;
+            avgBufferSize: {
+                mean: number;
+            };
+        };
+    };
+    /**
+     * getSummary() plus the raw recorded values, with a timestamp.
+     * @returns Object with timestamp, summary and rawMetrics
+     */
+    exportMetrics(): {
+        timestamp: string;
+        summary: {
+            embedding: {
+                totalOperations: number;
+                totalDuration: number;
+                totalTokens: number;
+                totalBatches: number;
+                avgDuration: {
+                    mean: number;
+                };
+            };
+            retrieval: {
+                totalOperations: number;
+                totalResults: number;
+                avgDuration: {
+                    mean: number;
+                };
+            };
+            llm: {
+                totalOperations: number;
+                streamingOperations: number;
+                avgDuration: {
+                    mean: number;
+                };
+                avgInputTokens: {
+                    mean: number;
+                };
+                avgOutputTokens: {
+                    mean: number;
+                };
+            };
+            memory: {
+                heapUsed: number;
+                heapTotal: number;
+                usagePercentage: number;
+            };
+            errors: {
+                total: number;
+                byType: {
+                    plugin: number;
+                    stage: number;
+                };
+                byPlugin: {};
+            };
+            operations: {
+                total: number;
+                successful: number;
+                failed: number;
+            };
+            concurrency: {
+                maxConcurrency: number;
+                avgConcurrency: {
+                    mean: number;
+                };
+            };
+            backpressure: {
+                totalEvents: number;
+                appliedEvents: number;
+                releasedEvents: number;
+                avgBufferSize: {
+                    mean: number;
+                };
+            };
+        };
+        rawMetrics: {
+            counters: {
+                operationsTotal: Counter;
+                operationsSuccessful: Counter;
+                operationsFailed: Counter;
+                operationsActive: Counter;
+            };
+            embedding: {
+                operations: number;
+                totalDuration: number;
+                totalTokens: number;
+                totalBatches: number;
+                durations: any[];
+            };
+            retrieval: {
+                operations: number;
+                totalDuration: number;
+                totalResults: number;
+                durations: any[];
+            };
+            llm: {
+                operations: number;
+                totalDuration: number;
+                totalInputTokens: number;
+                totalOutputTokens: number;
+                streamingOps: number;
+                durations: any[];
+                inputTokens: any[];
+                outputTokens: any[];
+            };
+            memory: {
+                heapUsed: number;
+                heapTotal: number;
+                usagePercentage: number;
+            };
+            errors: {
+                byType: {
+                    plugin: number;
+                    stage: number;
+                };
+                byPlugin: {};
+            };
+            concurrency: {
+                maxConcurrency: number;
+                concurrencyValues: any[];
+            };
+            backpressure: {
+                samples: any[];
+                applied: number;
+                released: number;
+            };
+        };
+    };
+    /**
+     * Reset every counter, total and sample to its initial value.
+     * @returns {void}
+     */
+    clearMetrics(): void;
+}
+/** Registry of Counter, Histogram and Gauge instances keyed by name. */
+export class MetricsRegistry {
+    _metrics: Map<any, any>;
+    /**
+     * Add a metric under its name.
+     * @param {Counter|Histogram|Gauge} metric - Metric to register
+     * @returns {void}
+     * @throws {Error} If a metric with the same name is already registered
+     */
+    register(metric: Counter | Histogram | Gauge): void;
+    /**
+     * Look up a metric by name.
+     * @param {string} name - Metric name
+     * @returns {Counter|Histogram|Gauge|undefined} The metric, or undefined if not registered
+     */
+    getMetric(name: string): Counter | Histogram | Gauge | undefined;
+    /**
+     * All registered metrics in registration order.
+     * @returns {Array<Counter|Histogram|Gauge>}
+     */
+    getAllMetrics(): Array<Counter | Histogram | Gauge>;
+    /**
+     * Remove every registered metric.
+     * @returns {void}
+     */
+    clear(): void;
+    /**
+     * Return the metric registered under name, or create and register a Counter.
+     * An existing metric is returned as is, even if it is not a Counter.
+     * @param {string} name - Metric name
+     * @param {string} [description] - Description for a new counter
+     * @param {Object<string, string>} [labels={}] - Labels for a new counter
+     * @returns {Counter}
+     */
+    getOrCreateCounter(name: string, description?: string, labels?: {
+        [x: string]: string;
+    }): Counter;
+    /**
+     * Return the metric registered under name, or create and register a
+     * Histogram. An existing metric is returned as is, even if it is not a Histogram.
+     * @param {string} name - Metric name
+     * @param {string} [description] - Description for a new histogram
+     * @param {Object<string, string>} [labels={}] - Labels for a new histogram
+     * @param {number[]} [buckets] - Bucket bounds for a new histogram
+     * @returns {Histogram}
+     */
+    getOrCreateHistogram(name: string, description?: string, labels?: {
+        [x: string]: string;
+    }, buckets?: number[]): Histogram;
+    /**
+     * Return the metric registered under name, or create and register a Gauge.
+     * An existing metric is returned as is, even if it is not a Gauge.
+     * @param {string} name - Metric name
+     * @param {string} [description] - Description for a new gauge
+     * @param {Object<string, string>} [labels={}] - Labels for a new gauge
+     * @returns {Gauge}
+     */
+    getOrCreateGauge(name: string, description?: string, labels?: {
+        [x: string]: string;
+    }): Gauge;
+    /**
+     * Serialize all metrics (their toJSON output) with a timestamp.
+     * @returns {string} JSON string with timestamp and metrics fields
+     */
+    exportMetrics(): string;
+}
+/** Shared PipelineMetrics instance. */
+export const pipelineMetrics: PipelineMetrics;

package/dist/types/core/performance/parallel-processor.d.ts

@@ -0,0 +1,92 @@
+/**
+ * Parallel processor for embedding operations with configurable concurrency
+ */
+export class ParallelEmbedder {
+    /**
+     * @param {object} embedder - Embedder with embed() and/or embedBatch()
+     * @param {object} [_options]
+     * @param {number} [_options.batchSize=10] - Texts per batch
+     * @param {number} [_options.maxConcurrency=3] - Batches in flight at once
+     * @param {number} [_options.retryAttempts=2] - Retries per failed batch
+     * @param {number} [_options.retryDelay=1000] - Base retry delay in ms
+     * @param {'auto'|'batch'|'single'} [_options.mode='auto'] - How a batch is
+     *   embedded: 'batch' calls embed(texts) and expects one vector per text,
+     *   'single' calls embed(text) for each text. 'auto' uses embedBatch(texts)
+     *   when the embedder has it, 'single' for the built-in connectors (which
+     *   embed one string at a time) and 'batch' otherwise.
+     */
+    constructor(embedder: object, _options?: {
+        batchSize?: number;
+        maxConcurrency?: number;
+        retryAttempts?: number;
+        retryDelay?: number;
+        mode?: "auto" | "batch" | "single";
+    });
+    embedder: any;
+    batchSize: number;
+    maxConcurrency: number;
+    retryAttempts: number;
+    retryDelay: number;
+    mode: "auto" | "batch" | "single";
+    /**
+     * Embed one batch according to the mode.
+     * @param {string[]} batch
+     * @returns {Promise<number[][]>}
+     * @private
+     */
+    private _embed;
+    /**
+     * Process chunks in parallel batches with concurrency control
+     * @param {string[]} chunks - Text chunks to embed
+     * @returns {Promise<number[][]>} Array of embedding vectors
+     */
+    embedBatch(chunks: string[]): Promise<number[][]>;
+    /**
+     * Process a single batch with retry logic
+     * @param {string[]} batch - Batch of chunks to process
+     * @param {number} batchIndex - Index of the batch for logging
+     * @returns {Promise<number[][]>} Embedding vectors for the batch
+     */
+    processBatchWithRetry(batch: string[], batchIndex: number): Promise<number[][]>;
+    /**
+     * Create batches from chunks array
+     * @param {string[]} items - Items to batch
+     * @param {number} size - Batch size
+     * @returns {string[][]} Array of batches
+     */
+    createBatches(items: string[], size: number): string[][];
+    /**
+     * Delay utility for retry logic
+     * @param {number} ms - Milliseconds to delay
+     * @returns {Promise<void>}
+     */
+    delay(ms: number): Promise<void>;
+}
+/**
+ * Parallel processor for retrieval operations
+ */
+export class ParallelRetriever {
+    constructor(retriever: any, _options?: {});
+    retriever: any;
+    maxConcurrency: any;
+    chunkQueries: any;
+    /**
+     * Process multiple queries in parallel
+     * @param {number[][]} queryVectors - Array of query vectors
+     * @returns {Promise<any[]>} Retrieved results
+     */
+    retrieveBatch(queryVectors: number[][]): Promise<any[]>;
+}
+/**
+ * Semaphore for controlling concurrent operations
+ */
+export class Semaphore {
+    constructor(maxConcurrency: any);
+    maxConcurrency: any;
+    currentCount: number;
+    waitQueue: any[];
+    /** Resolves when a slot is free and takes it. */
+    acquire(): Promise<any>;
+    /** Frees a slot, starting the next waiter if there is one. */
+    release(): void;
+}

package/dist/types/core/plugin-registry.d.ts

@@ -1,4 +1,9 @@
 export = registry;
+/**
+ * The shared plugin registry: maps (type, name) to plugins, checking each
+ * against its contract. Signed manifests are required when NODE_ENV is
+ * production.
+ */
 declare const registry: PluginRegistry;
 declare class PluginRegistry {
     constructor(options?: {});

package/dist/types/dag/core/execution-scheduler.d.ts

@@ -1,4 +1,5 @@
 export = ExecutionScheduler;
+/** @typedef {import('../dag-node')} DAGNode */
 /**
  * ExecutionScheduler class for managing DAG node execution
  *
@@ -186,6 +187,7 @@ declare class ExecutionScheduler {
     }): boolean;
 }
 declare namespace ExecutionScheduler {
-    export { ExecutionScheduler as default };
+    export { ExecutionScheduler as default, DAGNode };
 }
 import ErrorContext = require("./error-context.js");
+type DAGNode = import("../dag-node");