@devilsdev/rag-pipeline-utils 2.4.5 → 2.5.0

package/dist/types/connectors/cohere-connector.d.ts

@@ -0,0 +1,89 @@
+/**
+ * Embedder + reranker connector for the Cohere API.
+ * Provides text embeddings (with separate document/query input types) and
+ * document reranking capabilities.
+ *
+ * @extends BaseConnector
+ */
+export class CohereConnector extends BaseConnector {
+    /**
+     * @param {Object} [options]
+     * @param {string} [options.apiKey] - Cohere API key
+     * @param {string} [options.baseURL='https://api.cohere.ai/v1'] - API base URL
+     * @param {string} [options.embeddingModel='embed-english-v3.0'] - Embedding model
+     * @param {string} [options.rerankModel='rerank-english-v3.0'] - Reranking model
+     * @param {string} [options.inputType='search_document'] - Input type for document embeddings
+     * @param {string} [options.queryInputType='search_query'] - Input type for query embeddings
+     */
+    constructor(options?: {
+        apiKey?: string;
+        baseURL?: string;
+        embeddingModel?: string;
+        rerankModel?: string;
+        inputType?: string;
+        queryInputType?: string;
+    });
+    config: any;
+    client: axios.AxiosInstance;
+    /**
+     * @throws {Error} code MISSING_API_KEY when no key is configured
+     * @private
+     */
+    private _requireApiKey;
+    /**
+     * POST with the connector's retry policy and cancellation.
+     * @param {string} path
+     * @param {object} body
+     * @param {AbortSignal} [signal]
+     * @returns {Promise<object>} Response data
+     * @private
+     */
+    private _post;
+    /**
+     * Generate an embedding vector for document text.
+     * Uses the document input type by default.
+     *
+     * @param {string} text - Input text to embed
+     * @param {Object} [options={}]
+     * @param {AbortSignal} [options.signal] - Cancels the request
+     * @returns {Promise<number[]>} Embedding vector
+     */
+    embed(text: string, options?: {
+        signal?: AbortSignal;
+    }): Promise<number[]>;
+    /**
+     * Generate an embedding vector for a search query.
+     * Uses the query input type for better retrieval accuracy.
+     *
+     * @param {string} query - Query text to embed
+     * @param {Object} [options={}]
+     * @param {AbortSignal} [options.signal] - Cancels the request
+     * @returns {Promise<number[]>} Embedding vector
+     */
+    embedQuery(query: string, options?: {
+        signal?: AbortSignal;
+    }): Promise<number[]>;
+    /**
+     * Rerank documents by relevance to a query using Cohere's reranker.
+     *
+     * @param {string} query - The search query
+     * @param {Array<Object|string>} documents - Documents to rerank
+     * @param {Object} [options={}]
+     * @param {number} [options.topK] - Number of top results to return (defaults to all)
+     * @param {AbortSignal} [options.signal] - Cancels the request
+     * @returns {Promise<Array<Object>>} Reranked documents with relevance scores
+     */
+    rerank(query: string, documents: Array<any | string>, options?: {
+        topK?: number;
+        signal?: AbortSignal;
+    }): Promise<Array<any>>;
+}
+/**
+ * Default configuration for the Cohere connector.
+ * `apiKey` is a getter so the environment is read when a connector is
+ * constructed, not when this module is first required.
+ * @type {Object}
+ */
+export const DEFAULT_CONFIG: any;
+import { BaseConnector } from "./base-connector";
+import axios = require("axios");

package/dist/types/connectors/connector-utils.d.ts

@@ -0,0 +1,110 @@
+/**
+ * Create an AbortError for the given signal.
+ * Reuses signal.reason when it already is an AbortError.
+ *
+ * @param {AbortSignal} [signal]
+ * @returns {Error} Error with name 'AbortError' and code 'ABORT_ERR'
+ */
+export function createAbortError(signal?: AbortSignal): Error;
+/**
+ * @param {*} err
+ * @returns {boolean} True when err represents a cancellation.
+ */
+export function isAbortError(err: any): boolean;
+/**
+ * Throw an AbortError when the signal is already aborted.
+ * @param {AbortSignal} [signal]
+ */
+export function throwIfAborted(signal?: AbortSignal): void;
+/**
+ * Race a promise against an AbortSignal so callers reject promptly even when
+ * the underlying SDK or HTTP client ignores the signal. The abort listener is
+ * removed once the promise settles.
+ *
+ * @template T
+ * @param {Promise<T>|(() => Promise<T>)} work - Promise or function returning one
+ * @param {AbortSignal} [signal]
+ * @returns {Promise<T>}
+ */
+export function raceAbort<T>(work: Promise<T> | (() => Promise<T>), signal?: AbortSignal): Promise<T>;
+/**
+ * Sleep that rejects with AbortError as soon as the signal aborts.
+ * @param {number} ms
+ * @param {AbortSignal} [signal]
+ * @returns {Promise<void>}
+ */
+export function abortableSleep(ms: number, signal?: AbortSignal): Promise<void>;
+/**
+ * Extract the HTTP status from an openai SDK error or an axios error.
+ * @param {*} err
+ * @returns {number|undefined}
+ */
+export function getErrorStatus(err: any): number | undefined;
+/**
+ * Parse Retry-After / retry-after-ms from an error's response headers.
+ *
+ * @param {*} err
+ * @param {number} [now=Date.now()]
+ * @returns {number|undefined} Delay in milliseconds, or undefined when absent
+ */
+export function getRetryAfterMs(err: any, now?: number): number | undefined;
+/**
+ * Default retry classification: network failures and HTTP
+ * 408/409/425/429/5xx are retried; aborts, other 4xx (400, 401, 403, 404,
+ * 422, ...) and errors without a status or network code are not.
+ *
+ * @param {*} err
+ * @returns {boolean}
+ */
+export function isRetryableError(err: any): boolean;
+/**
+ * Compute the delay before the next retry attempt.
+ * Honours Retry-After (capped at maxDelay); otherwise uses exponential backoff
+ * with full jitter: random() * min(maxDelay, baseDelay * 2^attempt).
+ *
+ * @param {*} err - The error that triggered the retry
+ * @param {number} attempt - Zero-based retry attempt
+ * @param {object} [opts]
+ * @param {number} [opts.baseDelay=1000]
+ * @param {number} [opts.maxDelay=30000]
+ * @param {() => number} [opts.random=Math.random]
+ * @returns {number} Delay in milliseconds
+ */
+export function computeRetryDelay(err: any, attempt: number, opts?: {
+    baseDelay?: number;
+    maxDelay?: number;
+    random?: () => number;
+}): number;
+/**
+ * Resolve an API key at construction time. An explicit option (even '')
+ * wins over the environment variable.
+ *
+ * @param {object} options - Connector options
+ * @param {string} envVar - Environment variable name
+ * @returns {string}
+ */
+export function resolveApiKey(options: object, envVar: string): string;
+/**
+ * Error thrown when a provider call is attempted without an API key.
+ * @param {string} provider - Display name, e.g. 'OpenAI'
+ * @param {string} envVar - Environment variable name
+ * @returns {Error}
+ */
+export function missingApiKeyError(provider: string, envVar: string): Error;
+/**
+ * Build request options carrying the signal only when one was supplied, so
+ * calls without a signal look exactly as before.
+ * @param {AbortSignal} [signal]
+ * @returns {object}
+ */
+export function signalOptions(signal?: AbortSignal): object;
+/**
+ * Shared helpers for the built-in connectors: AbortSignal handling, retry
+ * classification and backoff, and API key resolution.
+ *
+ * Everything here must run on Node 18 (no AbortSignal.any, no crypto.hash).
+ */
+/** HTTP status codes that are worth retrying. */
+export const RETRYABLE_STATUS: Set<number>;
+/** Error codes produced by Node, axios and undici for transient network failures. */
+export const RETRYABLE_CODES: Set<string>;

package/dist/types/connectors/local-embedder.d.ts

@@ -0,0 +1,86 @@
+/**
+ * TF-IDF based embedder for fully offline use.
+ * No external API required -- builds a vocabulary from training documents
+ * and produces sparse TF-IDF vectors.
+ * @extends BaseConnector
+ */
+export class LocalEmbedder extends BaseConnector {
+    /**
+     * @param {Object} options
+     * @param {number} [options.maxVocabularySize=10000] - Maximum vocabulary size
+     * @param {number} [options.minDocFrequency=1] - Minimum document frequency for a token to be included
+     */
+    constructor(options?: {
+        maxVocabularySize?: number;
+        minDocFrequency?: number;
+    });
+    config: any;
+    /** @type {Map<string, number>} token -> vocabulary index */
+    vocabulary: Map<string, number>;
+    /** @type {Map<string, number>} token -> IDF weight */
+    idf: Map<string, number>;
+    /** @type {number} */
+    docCount: number;
+    /**
+     * Build vocabulary and IDF weights from a corpus of documents.
+     * @param {Array<string|{ content?: string, text?: string }>} documents - Training documents
+     */
+    train(documents: Array<string | {
+        content?: string;
+        text?: string;
+    }>): void;
+    /**
+     * Compute a TF-IDF vector for the given text.
+     * The vector length equals the vocabulary size. Synchronous; when
+     * options.signal is already aborted it throws an AbortError (which an
+     * awaiting caller sees as a rejection).
+     * @param {string} text - Text to embed
+     * @param {Object} [options={}]
+     * @param {AbortSignal} [options.signal] - Cancellation signal
+     * @returns {number[]} L2-normalized TF-IDF vector
+     */
+    embed(text: string, options?: {
+        signal?: AbortSignal;
+    }): number[];
+    /**
+     * Compute a TF-IDF vector for a query string.
+     * Alias for {@link embed}.
+     * @param {string} query - Query text to embed
+     * @param {Object} [options={}] - Same as embed()
+     * @returns {number[]} L2-normalized TF-IDF vector
+     */
+    embedQuery(query: string, options?: any): number[];
+    /**
+     * Serializable vocabulary and IDF weights, so a trained embedder can be
+     * restored in another process (e.g. by FileVectorStore-backed CLI queries).
+     *
+     * @returns {{docCount: number, vocabulary: Array<[string, number]>, idf: Array<[string, number]>}}
+     */
+    getState(): {
+        docCount: number;
+        vocabulary: Array<[string, number]>;
+        idf: Array<[string, number]>;
+    };
+    /**
+     * Restore state produced by getState().
+     * @param {{docCount: number, vocabulary: Array<[string, number]>, idf: Array<[string, number]>}} state
+     */
+    setState(state: {
+        docCount: number;
+        vocabulary: Array<[string, number]>;
+        idf: Array<[string, number]>;
+    }): void;
+    /** @returns {number} Vector length produced by embed() once trained */
+    get dimensions(): number;
+    /**
+     * Get the current vocabulary size.
+     * @returns {number}
+     */
+    getVocabularySize(): number;
+}
+/**
+ * Default configuration for the local TF-IDF embedder.
+ * @type {Object}
+ */
+export const DEFAULT_CONFIG: any;
+import { BaseConnector } from "./base-connector";

package/dist/types/connectors/memory-retriever.d.ts

@@ -0,0 +1,91 @@
+/**
+ * In-memory vector store with cosine similarity retrieval.
+ * Stores document vectors and retrieves the most similar ones to a query vector.
+ * All vectors in one store must have the same dimension.
+ * @extends BaseConnector
+ */
+export class MemoryRetriever extends BaseConnector {
+    /**
+     * @param {Object} options
+     * @param {number} [options.maxDocuments=100000] - Maximum number of documents to store
+     * @param {'error'|'fifo'} [options.eviction='error'] - What to do when a store()
+     *   would exceed maxDocuments: throw (default) or evict the oldest documents
+     */
+    constructor(options?: {
+        maxDocuments?: number;
+        eviction?: "error" | "fifo";
+    });
+    config: any;
+    /** @type {Map<string, { vector: number[], metadata?: Object, content?: string }>} */
+    documents: Map<string, {
+        vector: number[];
+        metadata?: any;
+        content?: string;
+    }>;
+    /** @type {number|null} Vector dimension, fixed by the first stored vector */
+    dimensions: number | null;
+    /**
+     * Store one or more vector documents. The batch is validated before
+     * anything is inserted, so a rejected call leaves the store unchanged.
+     *
+     * @param {Array<{ id: string, vector: number[], metadata?: Object, content?: string }>} vectors
+     * @returns {Promise<void>}
+     * @throws {Error} When a vector's dimension differs from the store's, or
+     *   (eviction 'error') when the store would exceed maxDocuments
+     */
+    store(vectors: Array<{
+        id: string;
+        vector: number[];
+        metadata?: any;
+        content?: string;
+    }>): Promise<void>;
+    /**
+     * Retrieve the top-k most similar documents to a query vector.
+     * @param {number[]|{ query?: string, queryVector: number[], topK?: number, signal?: AbortSignal }} queryOrObj
+     *   Either a raw vector array or an object with queryVector, optional topK
+     *   and optional AbortSignal.
+     * @param {number} [k=10] - Number of results to return (used when queryOrObj is an array)
+     * @returns {Promise<Array<{ id: string, score: number, metadata: Object, content: string }>>}
+     * @throws {Error} When the query vector's dimension differs from the stored vectors
+     */
+    retrieve(queryOrObj: number[] | {
+        query?: string;
+        queryVector: number[];
+        topK?: number;
+        signal?: AbortSignal;
+    }, k?: number): Promise<Array<{
+        id: string;
+        score: number;
+        metadata: any;
+        content: string;
+    }>>;
+    /**
+     * Compute cosine similarity between two vectors of equal length.
+     * @param {number[]} a - First vector
+     * @param {number[]} b - Second vector
+     * @returns {number} Cosine similarity in the range [-1, 1], or 0 for zero vectors
+     * @throws {Error} When the vectors have different lengths
+     * @private
+     */
+    private _cosineSimilarity;
+    /**
+     * Remove all stored documents and reset the vector dimension.
+     * @returns {Promise<void>}
+     */
+    clear(): Promise<void>;
+    /**
+     * Get statistics about the store.
+     * @returns {{ documentCount: number, maxDocuments: number, dimensions: number|null }}
+     */
+    getStats(): {
+        documentCount: number;
+        maxDocuments: number;
+        dimensions: number | null;
+    };
+}
+/**
+ * Default configuration for the in-memory retriever.
+ * @type {Object}
+ */
+export const DEFAULT_CONFIG: any;
+import { BaseConnector } from "./base-connector";

package/dist/types/connectors/ollama-connector.d.ts

@@ -0,0 +1,110 @@
+/**
+ * Connector for Ollama local LLM server.
+ * Supports text generation (sync and streaming) and embeddings.
+ * Ollama needs no API key.
+ * @extends BaseConnector
+ */
+export class OllamaConnector extends BaseConnector {
+    /**
+     * @param {Object} options
+     * @param {string} [options.baseURL='http://localhost:11434'] - Ollama server URL
+     * @param {string} [options.model='llama3'] - Default generation model
+     * @param {string} [options.embeddingModel='nomic-embed-text'] - Default embedding model
+     * @param {number} [options.timeout=60000] - Request timeout in ms
+     * @param {number} [options.temperature=0.7] - Generation temperature
+     * @param {number} [options.maxRetries=3] - Retries for transient failures
+     * @param {number} [options.retryDelay=1000] - Base backoff delay in ms
+     * @param {string} [options.systemPrompt] - System prompt used when RAG context is supplied
+     */
+    constructor(options?: {
+        baseURL?: string;
+        model?: string;
+        embeddingModel?: string;
+        timeout?: number;
+        temperature?: number;
+        maxRetries?: number;
+        retryDelay?: number;
+        systemPrompt?: string;
+    });
+    config: any;
+    client: axios.AxiosInstance;
+    /**
+     * POST with the connector's retry policy and cancellation.
+     * @private
+     */
+    private _post;
+    /**
+     * Build the /api/generate body. Retrieved context goes into the prompt as
+     * delimited, untrusted reference data; instructions go in `system`.
+     * @private
+     */
+    private _buildBody;
+    /**
+     * Generate a response for the given query with optional context documents.
+     * @param {string} query - The user query
+     * @param {Array<Object|string>} [context=[]] - Context documents for RAG
+     * @param {Object} [options={}] - Override options
+     * @param {string} [options.model] - Override the generation model
+     * @param {number} [options.temperature] - Override temperature
+     * @param {AbortSignal} [options.signal] - Cancels the request
+     * @returns {Promise<string>} Generated response text
+     */
+    generate(query: string, context?: Array<any | string>, options?: {
+        model?: string;
+        temperature?: number;
+        signal?: AbortSignal;
+    }): Promise<string>;
+    /**
+     * Stream a response token-by-token for the given query.
+     * @param {string} query - The user query
+     * @param {Array<Object|string>} [context=[]] - Context documents for RAG
+     * @param {Object} [options={}] - Override options
+     * @param {string} [options.model] - Override the generation model
+     * @param {number} [options.temperature] - Override temperature
+     * @param {AbortSignal} [options.signal] - Cancels the stream
+     * @yields {{ token: string, done: boolean }}
+     */
+    generateStream(query: string, context?: Array<any | string>, options?: {
+        model?: string;
+        temperature?: number;
+        signal?: AbortSignal;
+    }): AsyncGenerator<{
+        token: any;
+        done: boolean;
+    }, void, unknown>;
+    /**
+     * Generate an embedding vector for the given text.
+     * @param {string} text - Text to embed
+     * @param {Object} [options={}]
+     * @param {AbortSignal} [options.signal] - Cancels the request
+     * @returns {Promise<number[]>} Embedding vector
+     */
+    embed(text: string, options?: {
+        signal?: AbortSignal;
+    }): Promise<number[]>;
+    /**
+     * Generate an embedding vector for a query string.
+     * Alias for {@link embed}.
+     * @param {string} query - Query text to embed
+     * @param {Object} [options={}] - Same as embed()
+     * @returns {Promise<number[]>} Embedding vector
+     */
+    embedQuery(query: string, options?: any): Promise<number[]>;
+    /**
+     * Build a prompt string incorporating context documents.
+     * Kept for backward compatibility; context is delimited and labelled as
+     * untrusted reference data.
+     * @param {string} query - The user query
+     * @param {Array<Object|string>} context - Context documents
+     * @returns {string} Formatted prompt
+     * @private
+     */
+    private _buildPrompt;
+}
+/**
+ * Default configuration for the Ollama connector.
+ * @type {Object}
+ */
+export const DEFAULT_CONFIG: any;
+import { BaseConnector } from "./base-connector";
+import axios = require("axios");

package/dist/types/connectors/openai-connector.d.ts

@@ -0,0 +1,114 @@
+/**
+ * Full LLM + embedder connector for the OpenAI API.
+ * Supports chat completions (with streaming) and text embeddings.
+ *
+ * @extends BaseConnector
+ */
+export class OpenAIConnector extends BaseConnector {
+    /**
+     * @param {Object} [options]
+     * @param {string} [options.model='gpt-4o-mini'] - Chat model to use
+     * @param {string} [options.embeddingModel='text-embedding-3-small'] - Embedding model
+     * @param {number} [options.temperature=0.7] - Sampling temperature
+     * @param {number} [options.maxTokens=4096] - Max tokens in completion
+     * @param {string} [options.apiKey] - OpenAI API key (defaults to OPENAI_API_KEY at construction)
+     * @param {number} [options.maxRetries=3] - Retries for transient failures (the SDK's own retries are disabled)
+     * @param {number} [options.retryDelay=1000] - Base backoff delay in ms
+     * @param {string} [options.systemPrompt] - System prompt used when RAG context is supplied
+     */
+    constructor(options?: {
+        model?: string;
+        embeddingModel?: string;
+        temperature?: number;
+        maxTokens?: number;
+        apiKey?: string;
+        maxRetries?: number;
+        retryDelay?: number;
+        systemPrompt?: string;
+    });
+    config: any;
+    _client: any;
+    /**
+     * Lazily initialise and return the OpenAI client.
+     * SDK retries are disabled (maxRetries: 0); withRetry() owns the policy.
+     * @returns {Object} OpenAI client instance
+     * @private
+     */
+    private _getClient;
+    /**
+     * @throws {Error} code MISSING_API_KEY when no key is configured
+     * @private
+     */
+    private _requireApiKey;
+    /**
+     * Run a request with the connector's retry policy and cancellation.
+     * @private
+     */
+    private _request;
+    /**
+     * Generate a chat completion using the provided query and optional RAG context.
+     *
+     * @param {string} query - The user query / prompt
+     * @param {Array<Object|string>} [context=[]] - Retrieved documents for RAG
+     * @param {Object} [options={}] - Override model, temperature, maxTokens per-call
+     * @param {AbortSignal} [options.signal] - Cancels the request
+     * @returns {Promise<string>} The generated text
+     */
+    generate(query: string, context?: Array<any | string>, options?: {
+        signal?: AbortSignal;
+    }): Promise<string>;
+    /**
+     * Stream a chat completion token-by-token.
+     *
+     * @param {string} query - The user query / prompt
+     * @param {Array<Object|string>} [context=[]] - Retrieved documents for RAG
+     * @param {Object} [options={}] - Override model, temperature, maxTokens per-call
+     * @param {AbortSignal} [options.signal] - Cancels the stream
+     * @yields {{ token: string, done: boolean }}
+     */
+    generateStream(query: string, context?: Array<any | string>, options?: {
+        signal?: AbortSignal;
+    }): AsyncGenerator<{
+        token: any;
+        done: boolean;
+    }, void, unknown>;
+    /**
+     * Generate an embedding vector for the given text.
+     *
+     * @param {string} text - Input text to embed
+     * @param {Object} [options={}]
+     * @param {AbortSignal} [options.signal] - Cancels the request
+     * @returns {Promise<number[]>} Embedding vector
+     */
+    embed(text: string, options?: {
+        signal?: AbortSignal;
+    }): Promise<number[]>;
+    /**
+     * Generate an embedding vector for a search query.
+     * Alias for {@link OpenAIConnector#embed}.
+     *
+     * @param {string} query - Query text to embed
+     * @param {Object} [options={}] - Same as embed()
+     * @returns {Promise<number[]>} Embedding vector
+     */
+    embedQuery(query: string, options?: any): Promise<number[]>;
+    /**
+     * Build the messages array for a chat completion request. Retrieved
+     * context goes into the user message as delimited, untrusted reference
+     * data; the system message only carries instructions.
+     *
+     * @param {string} query - User query
+     * @param {Array<Object|string>} context - RAG context documents
+     * @returns {Array<{ role: string, content: string }>}
+     * @private
+     */
+    private _buildMessages;
+}
+/**
+ * Default configuration for the OpenAI connector.
+ * `apiKey` is a getter so the environment is read when a connector is
+ * constructed, not when this module is first required.
+ * @type {Object}
+ */
+export const DEFAULT_CONFIG: any;
+import { BaseConnector } from "./base-connector";

package/dist/types/connectors/rag-prompt.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Prompt construction shared by the LLM connectors.
+ *
+ * Retrieved documents are untrusted: they may contain text written to look
+ * like instructions (indirect prompt injection). They are therefore placed in
+ * the USER message inside clearly labelled delimiters, never in the system
+ * role, and the system prompt tells the model to treat them as data only.
+ */
+/** System prompt used when retrieved context is supplied. */
+export const RAG_SYSTEM_PROMPT: string;
+/**
+ * Build the system prompt and user message for a RAG request.
+ * Without context, the user message is the query unchanged and there is no
+ * system prompt (same as before).
+ *
+ * @param {string} query
+ * @param {Array<object|string>} [context=[]]
+ * @param {object} [opts]
+ * @param {string} [opts.systemPrompt] - Override the default RAG system prompt
+ * @returns {{ system: string|null, user: string }}
+ */
+export function buildRagPrompt(query: string, context?: Array<object | string>, opts?: {
+    systemPrompt?: string;
+}): {
+    system: string | null;
+    user: string;
+};
+/**
+ * Format retrieved documents as a delimited, labelled block.
+ * @param {Array<object|string>} context
+ * @returns {string}
+ */
+export function formatContext(context: Array<object | string>): string;
+/**
+ * Neutralise anything in document text that could close or open our
+ * delimiters, so a document cannot break out of its block.
+ * @param {string} text
+ * @returns {string}
+ */
+export function escapeDelimiters(text: string): string;