@devilsdev/rag-pipeline-utils 2.4.5 → 2.6.0

package/dist/types/connectors/memory-retriever.d.ts

@@ -0,0 +1,99 @@
+/**
+ * In-memory vector store with cosine similarity retrieval.
+ * Stores document vectors and retrieves the most similar ones to a query vector.
+ * All vectors in one store must have the same dimension.
+ * @extends BaseConnector
+ */
+export class MemoryRetriever extends BaseConnector {
+    /**
+     * @param {Object} options
+     * @param {number} [options.maxDocuments=100000] - Maximum number of documents to store
+     * @param {'error'|'fifo'} [options.eviction='error'] - What to do when a store()
+     *   would exceed maxDocuments: throw (default) or evict the oldest documents
+     */
+    constructor(options?: {
+        maxDocuments?: number;
+        eviction?: "error" | "fifo";
+    });
+    config: any;
+    /** @type {Map<string, { vector: number[], metadata?: Object, content?: string }>} */
+    documents: Map<string, {
+        vector: number[];
+        metadata?: any;
+        content?: string;
+    }>;
+    /** @type {number|null} Vector dimension, fixed by the first stored vector */
+    dimensions: number | null;
+    _cacheId: string;
+    _revision: number;
+    /**
+     * Cache identity for createRagPipeline({ cache }).
+     * @param {'retriever'} role
+     * @returns {string|null}
+     */
+    getCacheKey(role: "retriever"): string | null;
+    /**
+     * Store one or more vector documents. The batch is validated before
+     * anything is inserted, so a rejected call leaves the store unchanged.
+     *
+     * @param {Array<{ id: string, vector: number[], metadata?: Object, content?: string }>} vectors
+     * @returns {Promise<void>}
+     * @throws {Error} When a vector's dimension differs from the store's, or
+     *   (eviction 'error') when the store would exceed maxDocuments
+     */
+    store(vectors: Array<{
+        id: string;
+        vector: number[];
+        metadata?: any;
+        content?: string;
+    }>): Promise<void>;
+    /**
+     * Retrieve the top-k most similar documents to a query vector.
+     * @param {number[]|{ query?: string, queryVector: number[], topK?: number, signal?: AbortSignal }} queryOrObj
+     *   Either a raw vector array or an object with queryVector, optional topK
+     *   and optional AbortSignal.
+     * @param {number} [k=10] - Number of results to return (used when queryOrObj is an array)
+     * @returns {Promise<Array<{ id: string, score: number, metadata: Object, content: string }>>}
+     * @throws {Error} When the query vector's dimension differs from the stored vectors
+     */
+    retrieve(queryOrObj: number[] | {
+        query?: string;
+        queryVector: number[];
+        topK?: number;
+        signal?: AbortSignal;
+    }, k?: number): Promise<Array<{
+        id: string;
+        score: number;
+        metadata: any;
+        content: string;
+    }>>;
+    /**
+     * Compute cosine similarity between two vectors of equal length.
+     * @param {number[]} a - First vector
+     * @param {number[]} b - Second vector
+     * @returns {number} Cosine similarity in the range [-1, 1], or 0 for zero vectors
+     * @throws {Error} When the vectors have different lengths
+     * @private
+     */
+    private _cosineSimilarity;
+    /**
+     * Remove all stored documents and reset the vector dimension.
+     * @returns {Promise<void>}
+     */
+    clear(): Promise<void>;
+    /**
+     * Get statistics about the store.
+     * @returns {{ documentCount: number, maxDocuments: number, dimensions: number|null }}
+     */
+    getStats(): {
+        documentCount: number;
+        maxDocuments: number;
+        dimensions: number | null;
+    };
+}
+/**
+ * Default configuration for the in-memory retriever.
+ * @type {Object}
+ */
+export const DEFAULT_CONFIG: any;
+import { BaseConnector } from "./base-connector";

package/dist/types/connectors/ollama-connector.d.ts

@@ -0,0 +1,115 @@
+/**
+ * Connector for Ollama local LLM server.
+ * Supports text generation (sync and streaming) and embeddings.
+ * Ollama needs no API key.
+ * @extends BaseConnector
+ */
+export class OllamaConnector extends BaseConnector {
+    /**
+     * @param {Object} options
+     * @param {string} [options.baseURL='http://localhost:11434'] - Ollama server URL
+     * @param {string} [options.model='llama3'] - Default generation model
+     * @param {string} [options.embeddingModel='nomic-embed-text'] - Default embedding model
+     * @param {number} [options.timeout=60000] - Request timeout in ms
+     * @param {number} [options.temperature=0.7] - Generation temperature
+     * @param {number} [options.maxRetries=3] - Retries for transient failures
+     * @param {number} [options.retryDelay=1000] - Base backoff delay in ms
+     * @param {string} [options.systemPrompt] - System prompt used when RAG context is supplied
+     */
+    constructor(options?: {
+        baseURL?: string;
+        model?: string;
+        embeddingModel?: string;
+        timeout?: number;
+        temperature?: number;
+        maxRetries?: number;
+        retryDelay?: number;
+        systemPrompt?: string;
+    });
+    /**
+     * Cache identity for createRagPipeline({ cache }).
+     * @param {'embedder'|'llm'} role
+     * @returns {string|null}
+     */
+    getCacheKey(role: "embedder" | "llm"): string | null;
+    config: any;
+    client: import("axios").AxiosInstance;
+    /**
+     * POST with the connector's retry policy and cancellation.
+     * @private
+     */
+    private _post;
+    /**
+     * Build the /api/generate body. Retrieved context goes into the prompt as
+     * delimited, untrusted reference data; instructions go in `system`.
+     * @private
+     */
+    private _buildBody;
+    /**
+     * Generate a response for the given query with optional context documents.
+     * @param {string} query - The user query
+     * @param {Array<Object|string>} [context=[]] - Context documents for RAG
+     * @param {Object} [options={}] - Override options
+     * @param {string} [options.model] - Override the generation model
+     * @param {number} [options.temperature] - Override temperature
+     * @param {AbortSignal} [options.signal] - Cancels the request
+     * @returns {Promise<string>} Generated response text
+     */
+    generate(query: string, context?: Array<any | string>, options?: {
+        model?: string;
+        temperature?: number;
+        signal?: AbortSignal;
+    }): Promise<string>;
+    /**
+     * Stream a response token-by-token for the given query.
+     * @param {string} query - The user query
+     * @param {Array<Object|string>} [context=[]] - Context documents for RAG
+     * @param {Object} [options={}] - Override options
+     * @param {string} [options.model] - Override the generation model
+     * @param {number} [options.temperature] - Override temperature
+     * @param {AbortSignal} [options.signal] - Cancels the stream
+     * @yields {{ token: string, done: boolean }}
+     */
+    generateStream(query: string, context?: Array<any | string>, options?: {
+        model?: string;
+        temperature?: number;
+        signal?: AbortSignal;
+    }): AsyncGenerator<{
+        token: any;
+        done: boolean;
+    }, void, unknown>;
+    /**
+     * Generate an embedding vector for the given text.
+     * @param {string} text - Text to embed
+     * @param {Object} [options={}]
+     * @param {AbortSignal} [options.signal] - Cancels the request
+     * @returns {Promise<number[]>} Embedding vector
+     */
+    embed(text: string, options?: {
+        signal?: AbortSignal;
+    }): Promise<number[]>;
+    /**
+     * Generate an embedding vector for a query string.
+     * Alias for {@link embed}.
+     * @param {string} query - Query text to embed
+     * @param {Object} [options={}] - Same as embed()
+     * @returns {Promise<number[]>} Embedding vector
+     */
+    embedQuery(query: string, options?: any): Promise<number[]>;
+    /**
+     * Build a prompt string incorporating context documents.
+     * Kept for backward compatibility; context is delimited and labelled as
+     * untrusted reference data.
+     * @param {string} query - The user query
+     * @param {Array<Object|string>} context - Context documents
+     * @returns {string} Formatted prompt
+     * @private
+     */
+    private _buildPrompt;
+}
+/**
+ * Default configuration for the Ollama connector.
+ * @type {Object}
+ */
+export const DEFAULT_CONFIG: any;
+import { BaseConnector } from "./base-connector";

package/dist/types/connectors/openai-connector.d.ts

@@ -0,0 +1,129 @@
+/**
+ * Full LLM + embedder connector for the OpenAI API.
+ * Supports chat completions (with streaming) and text embeddings.
+ *
+ * @extends BaseConnector
+ */
+export class OpenAIConnector extends BaseConnector {
+    /**
+     * @param {Object} [options]
+     * @param {string} [options.model='gpt-4o-mini'] - Chat model to use
+     * @param {string} [options.embeddingModel='text-embedding-3-small'] - Embedding model
+     * @param {number} [options.temperature=0.7] - Sampling temperature
+     * @param {number} [options.maxTokens=4096] - Max tokens in completion
+     * @param {string} [options.apiKey] - API key. Defaults to OPENAI_API_KEY at
+     *   construction, except when `baseURL` is set (see below)
+     * @param {string} [options.baseURL] - Base URL of an OpenAI-compatible API,
+     *   e.g. Groq, Together, the Hugging Face router, vLLM, LM Studio or
+     *   Ollama's `/v1` endpoint. With a custom base URL the key is never read
+     *   from OPENAI_API_KEY (that would send your OpenAI key to another
+     *   provider): pass `apiKey` explicitly, or omit it for servers that need
+     *   none
+     * @param {number} [options.maxRetries=3] - Retries for transient failures (the SDK's own retries are disabled)
+     * @param {number} [options.retryDelay=1000] - Base backoff delay in ms
+     * @param {string} [options.systemPrompt] - System prompt used when RAG context is supplied
+     */
+    constructor(options?: {
+        model?: string;
+        embeddingModel?: string;
+        temperature?: number;
+        maxTokens?: number;
+        apiKey?: string;
+        baseURL?: string;
+        maxRetries?: number;
+        retryDelay?: number;
+        systemPrompt?: string;
+    });
+    /**
+     * Cache identity for createRagPipeline({ cache }): the settings that
+     * change this connector's output in the given role.
+     * @param {'embedder'|'llm'} role
+     * @returns {string|null}
+     */
+    getCacheKey(role: "embedder" | "llm"): string | null;
+    config: any;
+    _client: any;
+    /**
+     * Lazily initialise and return the OpenAI client.
+     * SDK retries are disabled (maxRetries: 0); withRetry() owns the policy.
+     * @returns {Object} OpenAI client instance
+     * @private
+     */
+    private _getClient;
+    /**
+     * @throws {Error} code MISSING_API_KEY when no key is configured
+     * @private
+     */
+    private _requireApiKey;
+    /**
+     * Run a request with the connector's retry policy and cancellation.
+     * @private
+     */
+    private _request;
+    /**
+     * Generate a chat completion using the provided query and optional RAG context.
+     *
+     * @param {string} query - The user query / prompt
+     * @param {Array<Object|string>} [context=[]] - Retrieved documents for RAG
+     * @param {Object} [options={}] - Override model, temperature, maxTokens per-call
+     * @param {AbortSignal} [options.signal] - Cancels the request
+     * @returns {Promise<string>} The generated text
+     */
+    generate(query: string, context?: Array<any | string>, options?: {
+        signal?: AbortSignal;
+    }): Promise<string>;
+    /**
+     * Stream a chat completion token-by-token.
+     *
+     * @param {string} query - The user query / prompt
+     * @param {Array<Object|string>} [context=[]] - Retrieved documents for RAG
+     * @param {Object} [options={}] - Override model, temperature, maxTokens per-call
+     * @param {AbortSignal} [options.signal] - Cancels the stream
+     * @yields {{ token: string, done: boolean }}
+     */
+    generateStream(query: string, context?: Array<any | string>, options?: {
+        signal?: AbortSignal;
+    }): AsyncGenerator<{
+        token: any;
+        done: boolean;
+    }, void, unknown>;
+    /**
+     * Generate an embedding vector for the given text.
+     *
+     * @param {string} text - Input text to embed
+     * @param {Object} [options={}]
+     * @param {AbortSignal} [options.signal] - Cancels the request
+     * @returns {Promise<number[]>} Embedding vector
+     */
+    embed(text: string, options?: {
+        signal?: AbortSignal;
+    }): Promise<number[]>;
+    /**
+     * Generate an embedding vector for a search query.
+     * Alias for {@link OpenAIConnector#embed}.
+     *
+     * @param {string} query - Query text to embed
+     * @param {Object} [options={}] - Same as embed()
+     * @returns {Promise<number[]>} Embedding vector
+     */
+    embedQuery(query: string, options?: any): Promise<number[]>;
+    /**
+     * Build the messages array for a chat completion request. Retrieved
+     * context goes into the user message as delimited, untrusted reference
+     * data; the system message only carries instructions.
+     *
+     * @param {string} query - User query
+     * @param {Array<Object|string>} context - RAG context documents
+     * @returns {Array<{ role: string, content: string }>}
+     * @private
+     */
+    private _buildMessages;
+}
+/**
+ * Default configuration for the OpenAI connector.
+ * `apiKey` is a getter so the environment is read when a connector is
+ * constructed, not when this module is first required.
+ * @type {Object}
+ */
+export const DEFAULT_CONFIG: any;
+import { BaseConnector } from "./base-connector";

package/dist/types/connectors/rag-prompt.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Prompt construction shared by the LLM connectors.
+ *
+ * Retrieved documents are untrusted: they may contain text written to look
+ * like instructions (indirect prompt injection). They are therefore placed in
+ * the USER message inside clearly labelled delimiters, never in the system
+ * role, and the system prompt tells the model to treat them as data only.
+ */
+/** System prompt used when retrieved context is supplied. */
+export const RAG_SYSTEM_PROMPT: string;
+/**
+ * Build the system prompt and user message for a RAG request.
+ * Without context, the user message is the query unchanged and there is no
+ * system prompt (same as before).
+ *
+ * @param {string} query
+ * @param {Array<object|string>} [context=[]]
+ * @param {object} [opts]
+ * @param {string} [opts.systemPrompt] - Override the default RAG system prompt
+ * @returns {{ system: string|null, user: string }}
+ */
+export function buildRagPrompt(query: string, context?: Array<object | string>, opts?: {
+    systemPrompt?: string;
+}): {
+    system: string | null;
+    user: string;
+};
+/**
+ * Format retrieved documents as a delimited, labelled block.
+ * @param {Array<object|string>} context
+ * @returns {string}
+ */
+export function formatContext(context: Array<object | string>): string;
+/**
+ * Neutralise anything in document text that could close or open our
+ * delimiters, so a document cannot break out of its block.
+ * @param {string} text
+ * @returns {string}
+ */
+export function escapeDelimiters(text: string): string;

package/dist/types/core/create-pipeline.d.ts

@@ -0,0 +1,8 @@
+export function createRagPipeline({ registry, loader, embedder, retriever, llm, reranker, cache, }?: {}): {
+    run: ({ query, queryVector, options }?: {
+        options?: {};
+    }) => Promise<any>;
+    cleanup(): void;
+    /** Cache hits, misses and skips per layer; null when caching is off. */
+    cacheStats(): {};
+};

package/dist/types/core/plugin-registry.d.ts

@@ -0,0 +1,136 @@
+export = registry;
+declare const registry: PluginRegistry;
+declare class PluginRegistry {
+    constructor(options?: {});
+    _plugins: Map<any, any>;
+    _validTypes: Set<string>;
+    _contracts: Map<any, any>;
+    _contractWarningsShown: Set<any>;
+    _disableContractWarnings: any;
+    _validateContractSchema: any;
+    _signatureVerifier: import("../security/plugin-signature-verifier.js").PluginSignatureVerifier;
+    _requireSignedPlugins: boolean;
+    _contractsLoaded: boolean;
+    /**
+     * Load the contract schema and the stage contracts once.
+     * @private
+     */
+    private _ensureContracts;
+    _ajv: any;
+    _contractSchema: any;
+    _contractSchemaValidator: any;
+    /**
+     * Load contract schema for validation
+     *
+     * @private
+     * @returns {Object|null} Contract schema or null if not found
+     *
+     * @since 2.2.4
+     */
+    private _loadContractSchema;
+    /**
+     * Validate contract against schema
+     *
+     * @private
+     * @param {Object} contract - Contract to validate
+     * @param {string} type - Contract type
+     * @returns {Object} Validation result { valid: boolean, errors: Array }
+     *
+     * @since 2.2.4
+     */
+    private _validateContractAgainstSchema;
+    /**
+     * Format schema validation errors for display
+     *
+     * @private
+     * @param {Array} errors - Ajv validation errors
+     * @returns {string} Formatted error message
+     *
+     * @since 2.2.4
+     */
+    private _formatSchemaErrors;
+    /**
+     * Check if a contract exists for the given plugin type
+     *
+     * @param {string} type - Plugin type (loader, embedder, etc.)
+     * @returns {boolean} True if contract exists, false otherwise
+     *
+     * @example
+     * const hasContract = registry.checkContractExists('loader');
+     * if (!hasContract) {
+     *   console.warn('No contract for loader plugins');
+     * }
+     *
+     * @since 2.2.4
+     */
+    checkContractExists(type: string): boolean;
+    /**
+     * Log one-time warning for missing contract
+     *
+     * Logs a helpful warning message about the benefits of contracts
+     * and how to add them. Only logs once per contract type and only
+     * in non-production environments.
+     *
+     * @private
+     * @param {string} type - Plugin type
+     * @param {string} context - Context where warning is generated (e.g., 'load', 'register')
+     *
+     * @since 2.2.4
+     */
+    private _warnMissingContract;
+    _loadContracts(): void;
+    _validatePluginContract(category: any, impl: any): void;
+    _getRequiredMethodsFromContract(contract: any): any[];
+    /**
+     * Register a plugin.
+     *
+     * @param {string} category - Plugin type
+     * @param {string} name - Plugin name
+     * @param {object} impl - Plugin implementation
+     * @param {object|null} [manifest] - Signed manifest; when given, its
+     *   signature is verified before registration
+     * @param {object} [options]
+     * @param {object} [options.content] - Plugin content checked against the
+     *   manifest's contentDigest: `{ files, baseDir, contents }`
+     * @returns {Promise<PluginRegistry>}
+     */
+    register(category: string, name: string, impl: object, manifest?: object | null, options?: {
+        content?: object;
+    }): Promise<PluginRegistry>;
+    /**
+     * Verify plugin signature
+     * @param {object} manifest - Plugin manifest
+     * @param {string} pluginName - Plugin name
+     */
+    _verifyPluginSignature(manifest: object, pluginName: string, content?: any): Promise<void>;
+    /**
+     * Whether plugin signature verification is enabled for this registry.
+     * @returns {boolean}
+     */
+    isSignatureVerificationEnabled(): boolean;
+    /**
+     * Whether plugins without a signed manifest are rejected.
+     * @returns {boolean}
+     */
+    requiresSignedPlugins(): boolean;
+    /**
+     * Verify a plugin's signed manifest and code before it is loaded (use this
+     * before importing plugin files, since importing executes them).
+     *
+     * @param {string} pluginName - Plugin name (for audit entries)
+     * @param {object} manifest - Signed manifest
+     * @param {object} content - `{ files, baseDir, contents }`
+     * @returns {Promise<void>} Rejects if verification fails
+     */
+    verifyPluginContent(pluginName: string, manifest: object, content: object): Promise<void>;
+    /**
+     * Emit audit entry
+     * @param {string} action - Audit action
+     * @param {object} details - Audit details
+     */
+    _emitAuditEntry(action: string, details: object): void;
+    get(category: any, name: any): any;
+    resolve(category: any, name: any): any;
+    list(category: any): any[];
+    clear(): void;
+}

package/dist/types/core/stage-contracts.d.ts

@@ -0,0 +1,36 @@
+/**
+ * What each pipeline stage must provide, as createRagPipeline and the CLI
+ * use it. A plugin needs at least one of `anyOf`; a missing `recommended`
+ * method only matters for the command named in its note.
+ *
+ * Shared by validatePlugin() and the `rag-pipeline init` wizard. The
+ * stricter rules pluginRegistry.register() applies (metadata, and the
+ * methods in contracts/*.json) are checked by validatePlugin(..., {
+ * registry: true }).
+ */
+export const STAGE_CONTRACTS: Readonly<{
+    loader: {
+        anyOf: string[];
+        recommended: {};
+    };
+    embedder: {
+        anyOf: string[];
+        recommended: {
+            embed: string;
+        };
+    };
+    retriever: {
+        anyOf: string[];
+        recommended: {
+            store: string;
+        };
+    };
+    reranker: {
+        anyOf: string[];
+        recommended: {};
+    };
+    llm: {
+        anyOf: string[];
+        recommended: {};
+    };
+}>;

package/dist/types/core/validate-plugin.d.ts

@@ -0,0 +1,45 @@
+export type PluginValidationResult = {
+    /**
+     * - True when there are no errors
+     */
+    valid: boolean;
+    /**
+     * - Problems that stop the plugin from working
+     */
+    errors: string[];
+    /**
+     * - Methods some commands need, e.g. `store`
+     * for `rag-pipeline ingest`
+     */
+    warnings: string[];
+};
+/**
+ * @typedef {Object} PluginValidationResult
+ * @property {boolean} valid - True when there are no errors
+ * @property {string[]} errors - Problems that stop the plugin from working
+ * @property {string[]} warnings - Methods some commands need, e.g. `store`
+ *   for `rag-pipeline ingest`
+ */
+/**
+ * Checks that an object can be used as a pipeline stage, without throwing,
+ * so plugin authors can assert on it in their own tests.
+ *
+ * By default it checks what createRagPipeline() calls. With
+ * `{ registry: true }` it also applies the rules pluginRegistry.register()
+ * enforces: `metadata` with a string `name` and `version` and a `type`
+ * equal to the stage, plus the methods in the stage's contract.
+ *
+ * @example
+ * const { validatePlugin } = require('@devilsdev/rag-pipeline-utils');
+ * const result = validatePlugin('llm', myLlm, { registry: true });
+ * expect(result.errors).toEqual([]);
+ *
+ * @param {'loader'|'embedder'|'retriever'|'reranker'|'llm'} stage
+ * @param {object} plugin - Plugin instance or object
+ * @param {Object} [options]
+ * @param {boolean} [options.registry=false] - Also check registry rules
+ * @returns {PluginValidationResult}
+ */
+export function validatePlugin(stage: "loader" | "embedder" | "retriever" | "reranker" | "llm", plugin: object, options?: {
+    registry?: boolean;
+}): PluginValidationResult;