@devilsdev/rag-pipeline-utils 2.4.5 → 2.6.0

package/dist/types/guardrails/injection-detector.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Detect prompt-injection attempts in text.
+ *
+ * Built-in rules run on the folded text. Custom patterns run on the
+ * normalised text (case preserved) and on the folded text.
+ *
+ * @param {string} text
+ * @param {object} [opts]
+ * @param {Array<RegExp|string>} [opts.patterns=[]] - Extra patterns
+ * @param {boolean} [opts.useDefaults=true] - Include the built-in rules
+ * @returns {{ detected: boolean, matches: Array<{ id: string, pattern: string, match: string }>, normalized: string }}
+ */
+export function detectInjection(text: string, opts?: {
+    patterns?: Array<RegExp | string>;
+    useDefaults?: boolean;
+}): {
+    detected: boolean;
+    matches: Array<{
+        id: string;
+        pattern: string;
+        match: string;
+    }>;
+    normalized: string;
+};
+/**
+ * Normalise text for matching, without leetspeak/homoglyph folding.
+ * Case is preserved so case-sensitive custom patterns still work.
+ * @param {string} text
+ * @returns {string}
+ */
+export function normalizeText(text: string): string;
+/**
+ * Fully folded, lower-case form used by the built-in patterns.
+ * @param {string} text
+ * @returns {string}
+ */
+export function foldText(text: string): string;
+/**
+ * Built-in injection patterns. They run against the folded, lower-case text,
+ * so they are written in lower case and must stay narrow enough not to flag
+ * ordinary questions (e.g. "How does Redis act as a cache?").
+ * @type {Array<{ id: string, pattern: RegExp }>}
+ */
+export const DEFAULT_RULES: Array<{
+    id: string;
+    pattern: RegExp;
+}>;

package/dist/types/guardrails/post-generation-guard.d.ts

@@ -0,0 +1,61 @@
+/**
+ * Post-generation guard that checks LLM output for PII leakage,
+ * groundedness against source documents, and length constraints.
+ * Sanitizes output by redacting detected PII.
+ *
+ * @extends EventEmitter
+ */
+export class PostGenerationGuard extends EventEmitter<[never]> {
+    /**
+     * @param {object} [options] - Override default configuration
+     * @param {boolean} [options.enablePIIDetection=true] - Whether to detect PII in output
+     * @param {boolean} [options.enableGroundednessCheck=true] - Whether to check groundedness
+     * @param {number} [options.groundednessThreshold=0.3] - Minimum groundedness score
+     * @param {number} [options.maxResponseLength=50000] - Maximum allowed response length
+     * @param {number} [options.minResponseLength=0] - Minimum response length
+     * @param {object} [options.piiPatterns] - Custom PII regex patterns (merged with defaults)
+     */
+    constructor(options?: {
+        enablePIIDetection?: boolean;
+        enableGroundednessCheck?: boolean;
+        groundednessThreshold?: number;
+        maxResponseLength?: number;
+        minResponseLength?: number;
+        piiPatterns?: object;
+    });
+    config: any;
+    /**
+     * Check generated output for safety issues, PII, and groundedness.
+     *
+     * @param {string} output - The LLM-generated output text
+     * @param {Array<object>} [retrievedDocs=[]] - Source documents for groundedness checking
+     * @returns {{
+     *   safe: boolean,
+     *   issues: string[],
+     *   sanitizedOutput: string,
+     *   groundednessScore?: number
+     * }}
+     * @fires PostGenerationGuard#checked
+     */
+    check(output: string, retrievedDocs?: Array<object>): {
+        safe: boolean;
+        issues: string[];
+        sanitizedOutput: string;
+        groundednessScore?: number;
+    };
+    /**
+     * Replace PII matches in arbitrary text with [REDACTED], using the same
+     * patterns as check(). Used to scrub text derived from the answer, such as
+     * citation sentences.
+     *
+     * @param {string} text
+     * @returns {string}
+     */
+    redact(text: string): string;
+}
+/**
+ * Default configuration for the PostGenerationGuard.
+ * @type {object}
+ */
+export const DEFAULT_CONFIG: object;
+import { EventEmitter } from "events";

package/dist/types/guardrails/pre-retrieval-guard.d.ts

@@ -0,0 +1,64 @@
+/**
+ * Pre-retrieval guard that validates and sanitizes user queries before
+ * they reach the retrieval layer. Checks for injection attacks, topic
+ * violations, and input length constraints.
+ *
+ * @extends EventEmitter
+ */
+export class PreRetrievalGuard extends EventEmitter<[never]> {
+    /**
+     * @param {object} [options] - Override default configuration
+     * @param {number} [options.maxQueryLength=10000] - Maximum allowed query length
+     * @param {boolean} [options.enableInjectionDetection=true] - Whether to check for prompt injection
+     * @param {boolean} [options.enableTopicFiltering=false] - Whether to enforce topic constraints
+     * @param {string[]} [options.blockedTopics=[]] - Topics to reject
+     * @param {string[]} [options.allowedTopics=[]] - If non-empty, only these topics are permitted
+     * @param {RegExp[]} [options.injectionPatterns] - Additional injection patterns (merged with defaults)
+     */
+    constructor(options?: {
+        maxQueryLength?: number;
+        enableInjectionDetection?: boolean;
+        enableTopicFiltering?: boolean;
+        blockedTopics?: string[];
+        allowedTopics?: string[];
+        injectionPatterns?: RegExp[];
+    });
+    config: any;
+    /** @private Extra patterns supplied by the caller */
+    private _customPatterns;
+    /**
+     * Check a query for safety issues, injection attempts, and topic violations.
+     *
+     * @param {string} query - The user query to validate
+     * @returns {{
+     *   safe: boolean,
+     *   issues: string[],
+     *   sanitizedQuery: string,
+     *   injection: { detected: boolean, matches: Array<{id: string, pattern: string, match: string}> }
+     * }}
+     * `sanitizedQuery` is the query truncated to maxQueryLength; it is never
+     * rewritten. Injection findings are reported in `issues` and `injection`
+     * and make `safe` false; blocking is up to the caller (GuardrailsPipeline
+     * blocks in strict mode).
+     * @fires PreRetrievalGuard#checked
+     */
+    check(query: string): {
+        safe: boolean;
+        issues: string[];
+        sanitizedQuery: string;
+        injection: {
+            detected: boolean;
+            matches: Array<{
+                id: string;
+                pattern: string;
+                match: string;
+            }>;
+        };
+    };
+}
+/**
+ * Default configuration for the PreRetrievalGuard.
+ * @type {object}
+ */
+export const DEFAULT_CONFIG: object;
+import { EventEmitter } from "events";

package/dist/types/guardrails/retrieval-guard.d.ts

@@ -0,0 +1,57 @@
+/**
+ * Retrieval guard that filters search results based on relevance score,
+ * document freshness, and access control lists. Applied between the
+ * retrieval and generation stages of the pipeline.
+ *
+ * @extends EventEmitter
+ */
+export class RetrievalGuard extends EventEmitter<[never]> {
+    /**
+     * @param {object} [options] - Override default configuration
+     * @param {number} [options.minRelevanceScore=0] - Minimum relevance score threshold
+     * @param {number} [options.maxAgeDays=0] - Maximum document age in days (0 = disabled)
+     * @param {boolean} [options.enforceACL=false] - Whether to enforce access control
+     * @param {boolean} [options.detectInjection=false] - Scan document text for prompt injection
+     * @param {'remove'|'flag'} [options.injectionAction='remove'] - Remove flagged documents or keep and mark them
+     * @param {RegExp[]} [options.injectionPatterns=[]] - Extra injection patterns for the scan
+     */
+    constructor(options?: {
+        minRelevanceScore?: number;
+        maxAgeDays?: number;
+        enforceACL?: boolean;
+        detectInjection?: boolean;
+        injectionAction?: "remove" | "flag";
+        injectionPatterns?: RegExp[];
+    });
+    config: any;
+    /**
+     * Filter retrieval results based on configured guards.
+     *
+     * @param {Array<object>} results - Array of retrieval results, each with .score and optional .metadata
+     * @param {object} [context={}] - Request context
+     * @param {string[]} [context.userPermissions] - User's permission roles for ACL filtering
+     * @returns {{
+     *   results: Array<object>,
+     *   removed: Array<object>,
+     *   metadata: { reasons: Object<string, number> }
+     * }}
+     * @fires RetrievalGuard#filtered
+     */
+    filter(results: Array<object>, context?: {
+        userPermissions?: string[];
+    }): {
+        results: Array<object>;
+        removed: Array<object>;
+        metadata: {
+            reasons: {
+                [x: string]: number;
+            };
+        };
+    };
+}
+/**
+ * Default configuration for the RetrievalGuard.
+ * @type {object}
+ */
+export const DEFAULT_CONFIG: object;
+import { EventEmitter } from "events";

package/dist/types/mcp/mcp-server.d.ts

@@ -0,0 +1,85 @@
+/**
+ * MCP server that registers RAG pipelines as callable MCP tools.
+ *
+ * @extends EventEmitter
+ * @fires MCPServer#tool_use
+ *
+ * @example
+ * const server = MCPServer.fromPipeline(myPipeline, { name: 'my_tool' });
+ * const tools = server.getToolDefinitions();
+ * const result = await server.handleToolUse('my_tool', { query: 'What is X?' });
+ */
+export class MCPServer extends EventEmitter<[never]> {
+    /**
+     * Convenience factory: create an MCPServer with a single pipeline registered.
+     *
+     * @param {Object} pipeline - Pipeline instance with a `run` method
+     * @param {Object} [options={}] - Server and registration options
+     * @param {string} [options.name='rag_query'] - Tool name for the pipeline
+     * @param {Object} [options.config] - Pipeline-specific configuration
+     * @returns {MCPServer} Configured server instance
+     */
+    static fromPipeline(pipeline: any, options?: {
+        name?: string;
+        config?: any;
+    }): MCPServer;
+    /**
+     * @param {Object} [options={}] - Server configuration
+     * @param {boolean} [options.includeMetadata] - Include metadata in responses
+     * @param {number} [options.maxResponseLength] - Maximum response text length
+     * @param {Object} [options.toolOptions] - Options forwarded to MCPToolBuilder
+     */
+    constructor(options?: {
+        includeMetadata?: boolean;
+        maxResponseLength?: number;
+        toolOptions?: any;
+    });
+    config: any;
+    toolBuilder: MCPToolBuilder;
+    /** @type {Map<string, {pipeline: Object, config: Object, toolDefinition: Object}>} */
+    pipelines: Map<string, {
+        pipeline: any;
+        config: any;
+        toolDefinition: any;
+    }>;
+    /**
+     * Register a pipeline that can be called as an MCP tool.
+     *
+     * @param {string} name - Tool name used to invoke this pipeline
+     * @param {Object} pipeline - Pipeline instance with a `run` method
+     * @param {Object} [config={}] - Pipeline-specific configuration
+     * @returns {void}
+     */
+    registerPipeline(name: string, pipeline: any, config?: any): void;
+    /**
+     * Return array of all registered tool definitions.
+     * Each is a valid MCP tool schema.
+     *
+     * @returns {Array<Object>} Array of MCP tool definition objects
+     */
+    getToolDefinitions(): Array<any>;
+    /**
+     * Handle an MCP tool_use request by running the matching pipeline.
+     *
+     * @param {string} toolName - Name of the tool to invoke
+     * @param {Object} toolInput - Tool input parameters
+     * @param {string} toolInput.query - The search query (required)
+     * @param {number} [toolInput.topK] - Number of results to retrieve
+     * @param {boolean} [toolInput.stream] - Enable streaming response
+     * @param {boolean} [toolInput.citations] - Include source citations
+     * @param {boolean} [toolInput.evaluate] - Include quality evaluation metrics
+     * @returns {Promise<Object>} MCP-formatted response with type and text fields
+     * @throws {Error} If tool name is not registered or query is missing
+     */
+    handleToolUse(toolName: string, toolInput: {
+        query: string;
+        topK?: number;
+        stream?: boolean;
+        citations?: boolean;
+        evaluate?: boolean;
+    }): Promise<any>;
+}
+/** @type {Object} Default configuration for the MCP server */
+export const DEFAULT_CONFIG: any;
+import { EventEmitter } from "events";
+import { MCPToolBuilder } from "./mcp-tool-builder";

package/dist/types/mcp/mcp-tool-builder.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Builds MCP-compatible tool definitions from RAG pipeline configuration.
+ *
+ * @example
+ * const builder = new MCPToolBuilder({ toolName: 'my_rag_tool' });
+ * const definition = builder.buildToolDefinition({ capabilities: { stream: true } });
+ */
+export class MCPToolBuilder {
+    /**
+     * @param {Object} [options={}] - Builder configuration
+     * @param {string} [options.toolName] - Name for the generated MCP tool
+     * @param {string} [options.toolDescription] - Description for the generated MCP tool
+     */
+    constructor(options?: {
+        toolName?: string;
+        toolDescription?: string;
+    });
+    config: any;
+    /**
+     * Generate an MCP-compatible tool definition from pipeline configuration.
+     *
+     * @param {Object} [pipelineConfig={}] - Pipeline-specific configuration
+     * @param {Object} [pipelineConfig.capabilities] - Pipeline capabilities to reflect in schema
+     * @param {boolean} [pipelineConfig.capabilities.stream] - Whether streaming is supported
+     * @param {boolean} [pipelineConfig.capabilities.citations] - Whether citations are supported
+     * @param {boolean} [pipelineConfig.capabilities.evaluate] - Whether evaluation is supported
+     * @param {number} [pipelineConfig.capabilities.maxTopK] - Maximum allowed topK value
+     * @returns {Object} MCP tool definition with name, description, and input_schema
+     */
+    buildToolDefinition(pipelineConfig?: {
+        capabilities?: {
+            stream?: boolean;
+            citations?: boolean;
+            evaluate?: boolean;
+            maxTopK?: number;
+        };
+    }): any;
+    /**
+     * Build tool definitions for multiple named pipelines.
+     *
+     * @param {Array<Object>} pipelines - Array of pipeline descriptors
+     * @param {string} pipelines[].name - Tool name for this pipeline
+     * @param {string} [pipelines[].description] - Tool description
+     * @param {Object} [pipelines[].pipeline] - Pipeline instance (unused during definition building)
+     * @param {Object} [pipelines[].config] - Pipeline configuration passed to buildToolDefinition
+     * @returns {Array<Object>} Array of MCP tool definitions
+     */
+    buildMultipleTools(pipelines: Array<any>): Array<any>;
+}
+/**
+ * @module mcp/mcp-tool-builder
+ * @description Builds MCP tool definitions from pipeline configuration.
+ */
+/** @type {Object} Default configuration for the MCP tool builder */
+export const DEFAULT_CONFIG: any;

package/dist/types/reranker/cascade-reranker.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Chains multiple rerankers in sequence, passing the output of each
+ * stage as input to the next. Implements the reranker contract:
+ * rerank(query, documents, options).
+ *
+ * @extends EventEmitter
+ */
+export class CascadeReranker extends EventEmitter<[never]> {
+    /**
+     * @param {object[]} [rerankers=[]] - Array of objects with .rerank() method
+     * @param {object} [options]
+     */
+    constructor(rerankers?: object[], options?: object);
+    rerankers: any[];
+    config: any;
+    /**
+     * Run documents through each reranker in sequence.
+     *
+     * @param {string} query - The search query
+     * @param {Array<object|string>} documents - Documents to rerank
+     * @param {object} [options]
+     * @param {number} [options.topK] - Maximum number of results to return
+     * @param {AbortSignal} [options.signal] - Forwarded to every stage; checked between stages
+     * @returns {Promise<object[]>} Reranked documents from the final stage
+     * @throws {Error} If no rerankers are configured
+     */
+    rerank(query: string, documents: Array<object | string>, options?: {
+        topK?: number;
+        signal?: AbortSignal;
+    }): Promise<object[]>;
+    /**
+     * Add a reranker to the end of the cascade.
+     *
+     * @param {object} reranker - Must have a .rerank() method
+     */
+    addReranker(reranker: object): void;
+}
+/** @type {object} */
+export const DEFAULT_CONFIG: object;
+import { EventEmitter } from "events";

package/dist/types/reranker/embedding-reranker.d.ts

@@ -0,0 +1,77 @@
+/**
+ * Reranks documents by computing cosine similarity between query and
+ * document embeddings produced by an external embedder.
+ * Implements the reranker contract: rerank(query, documents, options).
+ *
+ * Document embeddings are cached in a bounded LRU keyed by a SHA-256 of the
+ * document text, and embedding calls run with bounded concurrency.
+ *
+ * @extends EventEmitter
+ */
+export class EmbeddingReranker extends EventEmitter<[never]> {
+    /**
+     * @param {object} embedder - Must expose .embedQuery(text) or .embed(text)
+     * @param {object} [options]
+     * @param {number} [options.cacheSize=1000] - Max cached document embeddings (0 disables caching)
+     * @param {number} [options.concurrency=4] - Max concurrent embedding calls
+     */
+    constructor(embedder: object, options?: {
+        cacheSize?: number;
+        concurrency?: number;
+    });
+    embedder: any;
+    config: {
+        cacheSize: number;
+        concurrency: number;
+    };
+    /** @type {Map<string, Promise<number[]>>} LRU: oldest entry first */
+    _cache: Map<string, Promise<number[]>>;
+    cacheStats: {
+        hits: number;
+        misses: number;
+    };
+    /**
+     * Call the embedder, forwarding the signal only when one is given.
+     * @private
+     */
+    private _embed;
+    /**
+     * Embed a document, using the LRU cache. In-flight promises are cached so
+     * duplicate documents in one batch are embedded once; failed embeddings
+     * are evicted so they can be retried.
+     * @private
+     */
+    private _embedDocument;
+    /**
+     * Clear the document embedding cache.
+     */
+    clearCache(): void;
+    /**
+     * Rerank documents by cosine similarity to the query embedding.
+     *
+     * @param {string} query - The search query
+     * @param {Array<object|string>} documents - Documents to rerank
+     * @param {object} [options]
+     * @param {number} [options.topK] - Maximum number of results to return
+     * @param {AbortSignal} [options.signal] - Cancels pending embedding calls
+     * @returns {Promise<object[]>} Reranked documents with .relevanceScore
+     */
+    rerank(query: string, documents: Array<object | string>, options?: {
+        topK?: number;
+        signal?: AbortSignal;
+    }): Promise<object[]>;
+    /**
+     * Compute cosine similarity between two vectors.
+     *
+     * @param {number[]} a - First vector
+     * @param {number[]} b - Second vector
+     * @returns {number} Cosine similarity in [-1, 1], or 0 for zero vectors
+     */
+    _cosineSimilarity(a: number[], b: number[]): number;
+}
+/** @type {{ cacheSize: number, concurrency: number }} */
+export const DEFAULT_CONFIG: {
+    cacheSize: number;
+    concurrency: number;
+};
+import { EventEmitter } from "events";

package/dist/types/reranker/scoring-reranker.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Reranks documents using BM25-like scoring with IDF weighting.
+ * Implements the reranker contract: rerank(query, documents, options).
+ *
+ * @extends EventEmitter
+ */
+export class ScoringReranker extends EventEmitter<[never]> {
+    /**
+     * @param {object} [options]
+     * @param {number} [options.k1=1.2] - Term frequency saturation parameter
+     * @param {number} [options.b=0.75] - Length normalisation parameter
+     */
+    constructor(options?: {
+        k1?: number;
+        b?: number;
+    });
+    config: {
+        k1: number;
+        b: number;
+    };
+    /**
+     * Rerank documents against a query using BM25 scoring.
+     *
+     * @param {string} query - The search query
+     * @param {Array<object|string>} documents - Documents to rerank
+     * @param {object} [options]
+     * @param {number} [options.topK] - Maximum number of results to return
+     * @param {AbortSignal} [options.signal] - Rejects with AbortError when already aborted
+     * @returns {Promise<object[]>} Reranked documents with .relevanceScore
+     */
+    rerank(query: string, documents: Array<object | string>, options?: {
+        topK?: number;
+        signal?: AbortSignal;
+    }): Promise<object[]>;
+}
+/** @type {{ k1: number, b: number }} */
+export const DEFAULT_CONFIG: {
+    k1: number;
+    b: number;
+};
+import { EventEmitter } from "events";

package/dist/types/retrieval/bm25-search.d.ts

@@ -0,0 +1,43 @@
+export class BM25Search extends EventEmitter<[never]> {
+    constructor(options?: {});
+    config: {
+        k1: number;
+        b: number;
+    };
+    invertedIndex: Map<any, any>;
+    documents: any[];
+    /** @type {number[]} token count per document, computed at index time */
+    docLengths: number[];
+    avgDocLength: number;
+    docCount: number;
+    /**
+     * Index documents for BM25 search
+     * @param {Array} documents - [{id, content|text, metadata?}]
+     */
+    index(documents: any[]): void;
+    /**
+     * Search using BM25 scoring
+     * @param {string} query
+     * @param {object} options - {k: 10}
+     * @returns {Array} [{id, score, content, metadata}] sorted by BM25 score
+     */
+    search(query: string, options?: object): any[];
+    /**
+     * Clear the index
+     */
+    clear(): void;
+    /**
+     * Get index stats
+     * @returns {{docCount: number, uniqueTerms: number, avgDocLength: number}}
+     */
+    getStats(): {
+        docCount: number;
+        uniqueTerms: number;
+        avgDocLength: number;
+    };
+}
+export namespace DEFAULT_CONFIG {
+    let k1: number;
+    let b: number;
+}
+import { EventEmitter } from "events";

package/dist/types/retrieval/file-vector-store.d.ts

@@ -0,0 +1,99 @@
+/**
+ * Vector store persisted to a single JSON file, so separate processes (e.g.
+ * `rag-pipeline ingest` then `rag-pipeline query`) share one index. Intended
+ * for local use and small corpora: every query is a linear scan.
+ *
+ * The index records which embedder produced the vectors (name, dimensions and
+ * optional serialized state) and refuses queries whose vector dimension does
+ * not match, instead of silently comparing incompatible vectors.
+ */
+export class FileVectorStore {
+    /**
+     * @param {object} options
+     * @param {string} options.path - Index file location
+     */
+    constructor(options?: {
+        path: string;
+    });
+    path: string;
+    _index: any;
+    _loadedDigest: string;
+    _cacheId: string;
+    _revision: number;
+    /**
+     * Cache identity for createRagPipeline({ cache }): a digest of the index
+     * as loaded, so processes reading the same unchanged index share cached
+     * results; after this instance changes the index the key also carries the
+     * instance and a revision.
+     * @param {'retriever'} role
+     * @returns {string|null}
+     */
+    getCacheKey(role: "retriever"): string | null;
+    _emptyIndex(): {
+        version: number;
+        embedder: any;
+        documents: {};
+    };
+    /**
+     * Read the index from disk (cached after the first call).
+     * @returns {{version: number, embedder: object|null, documents: object}}
+     */
+    load(): {
+        version: number;
+        embedder: object | null;
+        documents: object;
+    };
+    /** Write the index atomically: temp file in the same directory, then rename. */
+    save(): void;
+    /** @returns {object|null} Embedder metadata recorded with the vectors */
+    getEmbedderInfo(): object | null;
+    /** @param {{name: string, dimensions: number, state?: object}} info */
+    setEmbedderInfo(info: {
+        name: string;
+        dimensions: number;
+        state?: object;
+    }): void;
+    /**
+     * Add or replace documents. Ids are caller-chosen; re-storing an id
+     * replaces the previous entry.
+     *
+     * @param {Array<{id: string, vector: number[], content: string, metadata?: object}>} docs
+     */
+    store(docs: Array<{
+        id: string;
+        vector: number[];
+        content: string;
+        metadata?: object;
+    }>): Promise<void>;
+    /**
+     * Remove every document whose metadata.source equals `source`.
+     * @param {string} source
+     * @returns {number} Number of documents removed
+     */
+    removeBySource(source: string): number;
+    /** Remove every document and the recorded embedder info. */
+    clear(): void;
+    /** @returns {Array<{id: string, content: string, metadata: object}>} */
+    list(): Array<{
+        id: string;
+        content: string;
+        metadata: object;
+    }>;
+    /** @returns {number} */
+    size(): number;
+    /**
+     * Cosine-similarity search.
+     *
+     * @param {{queryVector: number[], topK?: number}} request
+     * @returns {Promise<Array<{id: string, score: number, content: string, metadata: object}>>}
+     */
+    retrieve(request?: {
+        queryVector: number[];
+        topK?: number;
+    }): Promise<Array<{
+        id: string;
+        score: number;
+        content: string;
+        metadata: object;
+    }>>;
+}