@devilsdev/rag-pipeline-utils 2.6.1 → 2.7.0

package/dist/types/loader/html-loader.d.ts

@@ -0,0 +1,21 @@
+/**
+ * HTMLLoader reads HTML files and extracts visible text.
+ * Implements the Loader interface.
+ */
+export class HTMLLoader {
+    /**
+     * Load and extract text from an HTML file
+     * @param {string} filePath - Path to .html file
+     * @returns {Promise<Array<{ chunk(): string[] }>>}
+     */
+    load(filePath: string): Promise<Array<{
+        chunk(): string[];
+    }>>;
+    /**
+     * Chunk HTML text content using the ChunkingEngine
+     * @param {string} input
+     * @param {number} maxLen
+     * @returns {string[]}
+     */
+    _chunkText(input: string, maxLen?: number): string[];
+}

package/dist/types/loader/markdown-loader.d.ts

@@ -0,0 +1,21 @@
+/**
+ * MarkdownLoader reads and parses .md files into plain text chunks.
+ * Implements the Loader interface.
+ */
+export class MarkdownLoader {
+    /**
+     * Load and parse Markdown file
+     * @param {string} _filePath - Path to .md file
+     * @returns {Promise<Array<{ chunk(): string[] }>>}
+     */
+    load(_filePath: string): Promise<Array<{
+        chunk(): string[];
+    }>>;
+    /**
+     * Chunk text using the ChunkingEngine
+     * @param {string} input
+     * @param {number} maxLen
+     * @returns {string[]}
+     */
+    _chunkText(input: string, maxLen?: number): string[];
+}

package/dist/types/retrieval/bm25-search.d.ts

@@ -1,3 +1,7 @@
+/**
+ * Keyword search with BM25 ranking. Index documents with index(), then
+ * search(query) returns them scored by term frequency and rarity.
+ */
 export class BM25Search extends EventEmitter<[never]> {
     constructor(options?: {});
     config: {

package/dist/types/retrieval/file-vector-store.d.ts

@@ -47,7 +47,10 @@ export class FileVectorStore {
     save(): void;
     /** @returns {object|null} Embedder metadata recorded with the vectors */
     getEmbedderInfo(): object | null;
-    /** @param {{name: string, dimensions: number, state?: object}} info */
+    /**
+     * Records which embedder wrote the index, saved with it.
+     * @param {{name: string, dimensions: number, state?: object}} info
+     */
     setEmbedderInfo(info: {
         name: string;
         dimensions: number;

package/dist/types/retrieval/hybrid-retriever.d.ts

@@ -1,3 +1,7 @@
+/**
+ * Runs several retrievers in parallel (for example vector and BM25) and
+ * fuses their results with weighted reciprocal rank fusion.
+ */
 export class HybridRetriever extends EventEmitter<[never]> {
     /**
      * @param {Array<{retriever: object, weight: number, name: string}>} retrievers

package/dist/types/retrieval/pgvector-retriever.d.ts

@@ -0,0 +1,123 @@
+/**
+ * Retriever on PostgreSQL with the pgvector extension. Creates the
+ * extension, table and index on first use; similarity is cosine.
+ */
+export class PgVectorRetriever {
+    /**
+     * @param {Object} options
+     * @param {{ query: Function, connect?: Function }} [options.pool] - A pg Pool or Client you manage
+     * @param {string} [options.connectionString] - Used to create a Pool when no `pool` is given
+     * @param {string} [options.table='rag_documents'] - Table name, optionally `schema.table`
+     * @param {number} [options.dimensions] - Vector size for a new table; defaults to the first stored vector's
+     * @param {boolean} [options.createTable=true] - Create the extension, table and index on first store
+     * @param {'hnsw'|'none'} [options.index='hnsw'] - Index created with the table
+     * @param {number} [options.efSearch] - HNSW candidate list size per query (pgvector's default is 40). Higher finds more of the true nearest neighbours, more slowly; a query always uses at least its topK
+     * @param {string} [options.cacheKey] - Cache identity; change it when the table changes (see getCacheKey)
+     */
+    constructor(options?: {
+        pool?: {
+            query: Function;
+            connect?: Function;
+        };
+        connectionString?: string;
+        table?: string;
+        dimensions?: number;
+        createTable?: boolean;
+        index?: "hnsw" | "none";
+        efSearch?: number;
+        cacheKey?: string;
+    });
+    efSearch: number;
+    table: string;
+    tableName: string;
+    dimensions: number;
+    createTable: boolean;
+    index: "none" | "hnsw";
+    cacheKey: string;
+    _pool: {
+        query: Function;
+        connect?: Function;
+    };
+    _connectionString: string;
+    _ownsPool: boolean;
+    _ready: any;
+    /** The pg Pool or Client in use; creates the Pool on first use. */
+    _db(): {
+        query: Function;
+        connect?: Function;
+    };
+    _query(text: any, params: any, signal: any): Promise<any>;
+    /** Runs one query with `SET LOCAL hnsw.ef_search`, in its own transaction. */
+    _queryWithEfSearch(text: any, params: any, ef: any, signal: any): Promise<any>;
+    /** Creates the extension, table and index once, sized to `dimensions`. */
+    _ensureTable(dimensions: any): any;
+    /**
+     * Cache identity for createRagPipeline({ cache }). Other processes can
+     * write to the table without this one knowing, so there is none unless
+     * you set `cacheKey`: then cached retrieval results are reused until you
+     * change the key or their time to live ends.
+     * @param {'retriever'} role
+     * @returns {string|null}
+     */
+    getCacheKey(role: "retriever"): string | null;
+    /**
+     * Inserts or replaces documents, in one transaction: a failed batch
+     * changes nothing.
+     * @param {Array<{ id: string, vector: number[], content?: string, metadata?: Object }>} docs
+     * @returns {Promise<void>}
+     */
+    store(docs: Array<{
+        id: string;
+        vector: number[];
+        content?: string;
+        metadata?: any;
+    }>): Promise<void>;
+    /**
+     * The documents most similar to a query vector.
+     * @param {number[]|{ queryVector: number[], topK?: number, signal?: AbortSignal }} request
+     * @param {number} [k=10] - Results when `request` is a vector
+     * @returns {Promise<Array<{ id: string, score: number, content: string, metadata: Object }>>}
+     */
+    retrieve(request: number[] | {
+        queryVector: number[];
+        topK?: number;
+        signal?: AbortSignal;
+    }, k?: number): Promise<Array<{
+        id: string;
+        score: number;
+        content: string;
+        metadata: any;
+    }>>;
+    /**
+     * Deletes the chunks of one source (`metadata.source`), for re-ingesting.
+     * @param {string} source
+     * @returns {Promise<number>} Rows deleted
+     */
+    removeBySource(source: string): Promise<number>;
+    /**
+     * Deletes documents by id.
+     * @param {string[]} ids
+     * @returns {Promise<number>} Rows deleted
+     */
+    delete(ids: string[]): Promise<number>;
+    /** Deletes every document; the table stays. */
+    clear(): Promise<void>;
+    /** @returns {Promise<number>} */
+    size(): Promise<number>;
+    /**
+     * Every document, without vectors, in id order.
+     * @param {{ limit?: number }} [options]
+     * @returns {Promise<Array<{ id: string, content: string, metadata: Object }>>}
+     */
+    list({ limit }?: {
+        limit?: number;
+    }): Promise<Array<{
+        id: string;
+        content: string;
+        metadata: any;
+    }>>;
+    /** Closes the Pool this retriever created; a pool you passed in stays open. */
+    close(): Promise<void>;
+}
+/** Quotes a table name, optionally schema-qualified, after validating it. */
+export function quoteTable(name: any): string;

package/dist/types/retrieval/qdrant-retriever.d.ts

@@ -0,0 +1,106 @@
+/**
+ * Retriever on Qdrant over its REST API, with no client package. Creates
+ * the collection on first store; similarity is cosine.
+ */
+export class QdrantRetriever {
+    /**
+     * @param {Object} [options]
+     * @param {string} [options.url='http://localhost:6333'] - Qdrant's REST endpoint
+     * @param {string} [options.apiKey] - Sent as the `api-key` header
+     * @param {string} [options.collection='rag_documents']
+     * @param {number} [options.dimensions] - Vector size for a new collection; defaults to the first stored vector's
+     * @param {boolean} [options.createCollection=true] - Create the collection on first store
+     * @param {string} [options.cacheKey] - Cache identity; change it when the collection changes
+     * @param {Function} [options.fetch] - fetch implementation (defaults to the global)
+     */
+    constructor(options?: {
+        url?: string;
+        apiKey?: string;
+        collection?: string;
+        dimensions?: number;
+        createCollection?: boolean;
+        cacheKey?: string;
+        fetch?: Function;
+    });
+    url: string;
+    collection: string;
+    dimensions: number;
+    createCollection: boolean;
+    cacheKey: string;
+    _apiKey: string;
+    _fetch: Function;
+    _ready: any;
+    /**
+     * Cache identity for createRagPipeline({ cache }): none unless you set
+     * `cacheKey`, since other clients can change the collection.
+     * @param {'retriever'} role
+     * @returns {string|null}
+     */
+    getCacheKey(role: "retriever"): string | null;
+    /**
+     * One REST call to the collection; resolves Qdrant's `result`.
+     * @private
+     * @param {string} method
+     * @param {string} path - After /collections/<name>
+     * @param {Object} [body]
+     * @param {{ signal?: AbortSignal, allow404?: boolean }} [options] - allow404 resolves null for a missing collection
+     * @returns {Promise<any>}
+     */
+    private _request;
+    _ensureCollection(dimensions: any): any;
+    /**
+     * Inserts or replaces documents.
+     * @param {Array<{ id: string, vector: number[], content?: string, metadata?: Object }>} docs
+     * @returns {Promise<void>}
+     */
+    store(docs: Array<{
+        id: string;
+        vector: number[];
+        content?: string;
+        metadata?: any;
+    }>): Promise<void>;
+    /**
+     * The documents most similar to a query vector.
+     * @param {number[]|{ queryVector: number[], topK?: number, signal?: AbortSignal }} request
+     * @param {number} [k=10] - Results when `request` is a vector
+     * @returns {Promise<Array<{ id: string, score: number, content: string, metadata: Object }>>}
+     */
+    retrieve(request: number[] | {
+        queryVector: number[];
+        topK?: number;
+        signal?: AbortSignal;
+    }, k?: number): Promise<Array<{
+        id: string;
+        score: number;
+        content: string;
+        metadata: any;
+    }>>;
+    /**
+     * Deletes the chunks of one source (`metadata.source`), for re-ingesting.
+     * @param {string} source
+     */
+    removeBySource(source: string): Promise<void>;
+    /**
+     * Deletes documents by id.
+     * @param {string[]} ids
+     */
+    delete(ids: string[]): Promise<void>;
+    /** Deletes every document this class stored; the collection stays. */
+    clear(): Promise<void>;
+    /** @returns {Promise<number>} */
+    size(): Promise<number>;
+    /**
+     * Every document, without vectors.
+     * @param {{ limit?: number }} [options]
+     * @returns {Promise<Array<{ id: string, content: string, metadata: Object }>>}
+     */
+    list({ limit }?: {
+        limit?: number;
+    }): Promise<Array<{
+        id: string;
+        content: string;
+        metadata: any;
+    }>>;
+}
+/** A stable UUID (version 8, RFC 9562) for a document id. */
+export function pointId(id: any): string;

package/dist/types/security/validator.d.ts

@@ -1,4 +1,14 @@
-export function validateInput(input: any, opts?: {}): {
+/**
+ * Checks input against a maximum length and a fixed list of disallowed
+ * tokens (code fences, `${`, `</script`, `--`, `;`, `||`, `&&`).
+ * @param {*} input - Converted to a string
+ * @param {Object} [opts]
+ * @param {number} [opts.maxLength=20000]
+ * @returns {{valid: boolean, reasons: string[], sanitized: string}} `sanitized` is the input cut to `maxLength`
+ */
+export function validateInput(input: any, opts?: {
+    maxLength?: number;
+}): {
     valid: boolean;
     reasons: string[];
     sanitized: string;

package/dist/types/testing.d.ts

@@ -66,7 +66,9 @@ export class EchoLLM {
         answer?: string | ((arg0: string, arg1: Array<any>) => string);
         fallback?: string;
     });
+    /** The fixed answer, or a function of (query, docs) that returns it. */
     answer: string | ((arg0: string, arg1: Array<any>) => string);
+    /** Returned when there are no documents and no `answer`. */
     fallback: string;
     /** @type {Array<{query: string, docs: Array<Object>, options: Object}>} */
     calls: Array<{

package/dist/types/utils/batch-processor.d.ts

@@ -0,0 +1,205 @@
+/**
+ * Batch Processor Class
+ */
+export class BatchProcessor extends EventEmitter<[never]> {
+    constructor(options?: {});
+    config: {
+        maxTokensPerBatch: number;
+        maxItemsPerBatch: number;
+        minItemsPerBatch: number;
+        maxMemoryMB: number;
+        streamingThreshold: number;
+        targetBatchUtilization: number;
+        adaptiveSizing: boolean;
+        maxRetries: number;
+        retryDelay: number;
+        retryBackoff: number;
+        progressInterval: number;
+        enableMetrics: boolean;
+    };
+    cancelled: boolean;
+    processing: boolean;
+    abortController: AbortController;
+    metrics: {
+        totalItems: number;
+        processedItems: number;
+        totalBatches: number;
+        successfulBatches: number;
+        failedBatches: number;
+        totalTokens: number;
+        totalTime: number;
+        avgBatchSize: number;
+        avgTokensPerBatch: number;
+        apiCallsSaved: number;
+        memoryUsed: number;
+        peakMemory: number;
+    };
+    adaptiveState: {
+        recentBatchSizes: any[];
+        recentProcessingTimes: any[];
+        optimalBatchSize: any;
+    };
+    /**
+     * Process items in optimized batches
+     *
+     * @param {Array<string|Object>} items - Items to process; objects are read from `text` or `content`
+     * @param {function(string[], {batchIndex: number, estimatedTokens: number, signal: AbortSignal}): Promise<Array>} processFn - Processes one batch of texts, returning one result per text
+     * @param {Object} [options] - Processing options
+     * @param {function(Object): void} [options.onProgress]
+     * @param {function(Object): void} [options.onBatchComplete]
+     * @param {AbortController} [options.abortController] - Abort to stop between batches
+     * @param {boolean} [options.returnOriginalOrder=true]
+     * @returns {Promise<Array>} Processed results
+     */
+    processBatches(items: Array<string | any>, processFn: (arg0: string[], arg1: {
+        batchIndex: number;
+        estimatedTokens: number;
+        signal: AbortSignal;
+    }) => Promise<any[]>, options?: {
+        onProgress?: (arg0: any) => void;
+        onBatchComplete?: (arg0: any) => void;
+        abortController?: AbortController;
+        returnOriginalOrder?: boolean;
+    }): Promise<any[]>;
+    /**
+     * Create optimized batches from items
+     * @private
+     */
+    private _createBatches;
+    /**
+     * Process batch with retry logic
+     * @private
+     */
+    private _processBatchWithRetry;
+    /**
+     * Estimate token count for text
+     * Uses approximation: ~4 characters per token for English text
+     * @private
+     */
+    private _estimateTokens;
+    /**
+     * Estimate batch count
+     * @private
+     */
+    private _estimateBatchCount;
+    /**
+     * Estimate remaining time
+     * @private
+     */
+    private _estimateTimeRemaining;
+    /**
+     * Update memory estimate
+     * @private
+     */
+    private _updateMemoryEstimate;
+    /**
+     * Update adaptive sizing state
+     * @private
+     */
+    private _updateAdaptiveState;
+    /**
+     * Re-optimize batches based on adaptive learning
+     * @private
+     */
+    private _reoptimizeBatches;
+    /**
+     * Restore original order of results
+     * @private
+     */
+    private _restoreOriginalOrder;
+    /**
+     * Cancel ongoing processing
+     */
+    cancel(): void;
+    /**
+     * Get current metrics
+     */
+    getMetrics(): {
+        processing: boolean;
+        cancelled: boolean;
+        efficiency: {
+            batchUtilization: number;
+            tokenUtilization: number;
+            apiCallReduction: number;
+            throughput: number;
+        };
+        totalItems: number;
+        processedItems: number;
+        totalBatches: number;
+        successfulBatches: number;
+        failedBatches: number;
+        totalTokens: number;
+        totalTime: number;
+        avgBatchSize: number;
+        avgTokensPerBatch: number;
+        apiCallsSaved: number;
+        memoryUsed: number;
+        peakMemory: number;
+    };
+    /**
+     * Get processing status
+     */
+    getStatus(): {
+        processing: boolean;
+        cancelled: boolean;
+        progress: number;
+        processedItems: number;
+        totalItems: number;
+        successfulBatches: number;
+        failedBatches: number;
+    };
+    /**
+     * Reset metrics
+     */
+    resetMetrics(): void;
+}
+/**
+ * Create embedder with batch processing
+ *
+ * Wraps an embedder to use intelligent batch processing
+ */
+export function createBatchedEmbedder(embedder: any, options?: {}): any;
+export namespace DEFAULT_CONFIG {
+    let maxTokensPerBatch: number;
+    let maxItemsPerBatch: number;
+    let minItemsPerBatch: number;
+    let maxMemoryMB: number;
+    let streamingThreshold: number;
+    let targetBatchUtilization: number;
+    let adaptiveSizing: boolean;
+    let maxRetries: number;
+    let retryDelay: number;
+    let retryBackoff: number;
+    let progressInterval: number;
+    let enableMetrics: boolean;
+}
+/**
+ * Model-specific token limits
+ */
+export const MODEL_LIMITS: {
+    'text-embedding-ada-002': {
+        maxTokens: number;
+        maxItems: number;
+    };
+    'text-embedding-3-small': {
+        maxTokens: number;
+        maxItems: number;
+    };
+    'text-embedding-3-large': {
+        maxTokens: number;
+        maxItems: number;
+    };
+    'voyage-large-2': {
+        maxTokens: number;
+        maxItems: number;
+    };
+    'cohere-embed-v3': {
+        maxTokens: number;
+        maxItems: number;
+    };
+    default: {
+        maxTokens: number;
+        maxItems: number;
+    };
+};
+import { EventEmitter } from "events";

package/dist/types/utils/connection-pool.d.ts

@@ -0,0 +1,158 @@
+/**
+ * Connection Pool Manager Class
+ */
+export class ConnectionPoolManager extends EventEmitter<[never]> {
+    constructor(options?: {});
+    config: {
+        maxSockets: number;
+        maxFreeSockets: number;
+        maxSocketsPerHost: number;
+        timeout: number;
+        freeSocketTimeout: number;
+        keepAliveTimeout: number;
+        keepAlive: boolean;
+        keepAliveMsecs: number;
+        scheduling: string;
+        trackMetrics: boolean;
+        metricsInterval: number;
+    };
+    httpAgent: http.Agent;
+    httpsAgent: https.Agent;
+    pools: Map<any, any>;
+    activeConnections: number;
+    totalConnections: number;
+    metrics: {
+        totalRequests: number;
+        cachedConnections: number;
+        newConnections: number;
+        connectionReuse: number;
+        timeouts: number;
+        errors: number;
+        avgResponseTime: number;
+        totalResponseTime: number;
+        poolExhausted: number;
+        connectionsCreated: number;
+        connectionsDestroyed: number;
+        currentPoolSize: number;
+        peakPoolSize: number;
+    };
+    requestTimings: any[];
+    maxTimingsSamples: number;
+    /**
+     * Initialize HTTP and HTTPS agents with connection pooling
+     * @private
+     */
+    private _initializeAgents;
+    /**
+     * Instrument agent for metrics collection
+     * @private
+     */
+    private _instrumentAgent;
+    /**
+     * Get HTTP agent
+     *
+     * @returns {http.Agent} HTTP agent with connection pooling
+     */
+    getHttpAgent(): http.Agent;
+    /**
+     * Get HTTPS agent
+     *
+     * @returns {https.Agent} HTTPS agent with connection pooling
+     */
+    getHttpsAgent(): https.Agent;
+    /**
+     * Get agent for URL
+     *
+     * @param {string} url - URL to get agent for
+     * @returns {http.Agent|https.Agent} Appropriate agent
+     */
+    getAgentForUrl(url: string): http.Agent | https.Agent;
+    /**
+     * Track request timing
+     *
+     * @param {number} duration - Request duration in milliseconds
+     */
+    trackRequest(duration: number): void;
+    /**
+     * Track request error
+     *
+     * @param {Error} error - Request error
+     * @param {string} type - Error type (timeout, network, etc.)
+     */
+    trackError(error: Error, type?: string): void;
+    /**
+     * Track pool exhaustion
+     */
+    trackPoolExhaustion(): void;
+    /**
+     * Get current pool metrics
+     *
+     * @returns {Object} Current metrics
+     */
+    getMetrics(): any;
+    /**
+     * Get agent statistics
+     * @private
+     */
+    private _getAgentStats;
+    /**
+     * Get pool status
+     *
+     * @returns {Object} Pool status information
+     */
+    getStatus(): any;
+    /**
+     * Reset metrics
+     */
+    resetMetrics(): void;
+    /**
+     * Start periodic metrics collection
+     * @private
+     */
+    private _startMetricsCollection;
+    metricsInterval: NodeJS.Timeout;
+    /**
+     * Destroy all connections and cleanup
+     */
+    destroy(): void;
+    /**
+     * Get connection pool report
+     *
+     * @returns {string} Human-readable pool report
+     */
+    getReport(): string;
+}
+/**
+ * Get or create global connection pool
+ *
+ * @param {Object} options - Pool configuration options
+ * @returns {ConnectionPoolManager} Global pool instance
+ */
+export function getGlobalPool(options?: any): ConnectionPoolManager;
+/**
+ * Reset global connection pool
+ */
+export function resetGlobalPool(): void;
+/**
+ * Create fetch wrapper with connection pooling
+ *
+ * @param {ConnectionPoolManager} pool - Connection pool manager
+ * @returns {Function} Fetch function with pooling
+ */
+export function createPooledFetch(pool: ConnectionPoolManager): Function;
+export namespace DEFAULT_CONFIG {
+    let maxSockets: number;
+    let maxFreeSockets: number;
+    let maxSocketsPerHost: number;
+    let timeout: number;
+    let freeSocketTimeout: number;
+    let keepAliveTimeout: number;
+    let keepAlive: boolean;
+    let keepAliveMsecs: number;
+    let scheduling: string;
+    let trackMetrics: boolean;
+    let metricsInterval: number;
+}
+import { EventEmitter } from "events";
+import http = require("http");
+import https = require("https");