@devilsdev/rag-pipeline-utils 2.4.5 → 2.6.0

package/dist/types/streaming/sse-adapter.d.ts

@@ -0,0 +1,59 @@
+/**
+ * Adapts an AsyncGenerator to the Server-Sent Events protocol.
+ *
+ * Writes well-formed SSE frames to an HTTP response, manages heartbeat
+ * keep-alive comments, and handles client disconnects gracefully.
+ *
+ * @extends EventEmitter
+ *
+ * @example
+ *   const adapter = new SSEAdapter({ heartbeatIntervalMs: 10000 });
+ *   adapter.on('token', (t) => console.log('sent', t));
+ *   await adapter.stream(asyncTokenGenerator, res);
+ */
+export class SSEAdapter extends EventEmitter<[never]> {
+    /**
+     * @param {Object} [options] - Configuration overrides
+     * @param {string} [options.eventName='token'] - SSE event name for token frames
+     * @param {string} [options.doneEvent='[DONE]'] - Sentinel value for the final data frame
+     * @param {number} [options.heartbeatIntervalMs=15000] - Heartbeat interval in milliseconds
+     */
+    constructor(options?: {
+        eventName?: string;
+        doneEvent?: string;
+        heartbeatIntervalMs?: number;
+    });
+    config: any;
+    /**
+     * Stream tokens from an AsyncGenerator to an HTTP response as SSE.
+     *
+     * Sets the required SSE headers, starts a heartbeat timer, and iterates
+     * over the generator. Each yielded value should be an object with at least
+     * a `token` property and an optional `done` boolean.
+     *
+     * @param {AsyncGenerator<{token: string, done?: boolean}>} asyncGenerator - Token source
+     * @param {import('http').ServerResponse} httpResponse - Writable HTTP response
+     * @returns {Promise<void>} Resolves when the stream ends or the client disconnects
+     */
+    stream(asyncGenerator: AsyncGenerator<{
+        token: string;
+        done?: boolean;
+    }>, httpResponse: import("http").ServerResponse): Promise<void>;
+    /**
+     * Format a single SSE event string.
+     *
+     * Multi-line data is split so each line is prefixed with `data: ` as
+     * required by the SSE specification.
+     *
+     * @param {string} eventName - The SSE event type
+     * @param {*} data - Payload (will be JSON-stringified)
+     * @returns {string} Fully formatted SSE frame including trailing blank line
+     */
+    formatEvent(eventName: string, data: any): string;
+}
+/**
+ * Default configuration for SSE streaming
+ * @type {Object}
+ */
+export const DEFAULT_CONFIG: any;
+import { EventEmitter } from "events";

package/dist/types/streaming/stream-router.d.ts

@@ -0,0 +1,88 @@
+/**
+ * Routes pipeline streaming output to SSE or WebSocket transports.
+ *
+ * Provides both an Express/Node-HTTP-compatible request handler via
+ * {@link StreamRouter#createHandler} and lower-level convenience methods
+ * ({@link StreamRouter#streamSSE}, {@link StreamRouter#streamWS}) for
+ * manual integration.
+ *
+ * @extends EventEmitter
+ *
+ * @example
+ *   const router = new StreamRouter({ enableCORS: true });
+ *   const handler = router.createHandler(pipeline);
+ *   app.post('/api/query', handler);
+ */
+export class StreamRouter extends EventEmitter<[never]> {
+    /**
+     * @param {Object} [options] - Configuration overrides
+     * @param {string} [options.defaultTransport='sse'] - Fallback transport type
+     * @param {boolean} [options.enableCORS=true] - Enable CORS headers
+     * @param {string} [options.corsOrigin='*'] - CORS origin value
+     * @param {Object} [options.sseOptions] - Options forwarded to SSEAdapter
+     * @param {Object} [options.wsOptions] - Options forwarded to WebSocketAdapter
+     */
+    constructor(options?: {
+        defaultTransport?: string;
+        enableCORS?: boolean;
+        corsOrigin?: string;
+        sseOptions?: any;
+        wsOptions?: any;
+    });
+    config: any;
+    sseAdapter: SSEAdapter;
+    wsAdapter: WebSocketAdapter;
+    /**
+     * Create an HTTP request handler that runs the pipeline in streaming mode
+     * and routes output to the appropriate transport.
+     *
+     * The returned function is compatible with Node's `http.createServer`,
+     * Express, and similar frameworks.
+     *
+     * Transport detection:
+     * - `Accept: text/event-stream` header selects SSE
+     * - `Upgrade: websocket` header selects WebSocket (note: actual upgrade
+     *   negotiation is typically handled at a higher level)
+     * - Otherwise falls back to {@link DEFAULT_CONFIG.defaultTransport}
+     *
+     * @param {Object} pipeline - A pipeline object with a `.run()` method
+     * @returns {Function} Async request handler `(req, res) => Promise<void>`
+     */
+    createHandler(pipeline: any): Function;
+    /**
+     * Run the pipeline in streaming mode and pipe tokens to an HTTP response
+     * via Server-Sent Events.
+     *
+     * If the pipeline returns a non-iterable result, it is sent as a single
+     * SSE data frame followed by the done sentinel.
+     *
+     * @param {Object} pipeline - Pipeline with a `.run()` method
+     * @param {string} query - User query string
+     * @param {import('http').ServerResponse} httpResponse - Writable HTTP response
+     * @param {Object} [options={}] - Additional pipeline run options
+     * @returns {Promise<void>}
+     */
+    streamSSE(pipeline: any, query: string, httpResponse: import("http").ServerResponse, options?: any): Promise<void>;
+    /**
+     * Run the pipeline in streaming mode and pipe tokens over a WebSocket
+     * connection.
+     *
+     * If the pipeline returns a non-iterable result, it is sent as a single
+     * `result` message followed by a `done` message.
+     *
+     * @param {Object} pipeline - Pipeline with a `.run()` method
+     * @param {string} query - User query string
+     * @param {Object} wsConnection - WebSocket-like connection with `.send()`
+     * @param {Object} [options={}] - Additional pipeline run options
+     * @returns {Promise<void>}
+     */
+    streamWS(pipeline: any, query: string, wsConnection: any, options?: any): Promise<void>;
+}
+/**
+ * Default configuration for the stream router
+ * @type {Object}
+ */
+export const DEFAULT_CONFIG: any;
+import { EventEmitter } from "events";
+import { SSEAdapter } from "./sse-adapter";
+import { WebSocketAdapter } from "./websocket-adapter";

package/dist/types/streaming/streaming-embedder.d.ts

@@ -0,0 +1,99 @@
+/**
+ * EventEmitter-based wrapper around {@link createEmbeddingStream}.
+ *
+ * @extends EventEmitter
+ *
+ * @fires StreamingEmbedder#start
+ * @fires StreamingEmbedder#embedding
+ * @fires StreamingEmbedder#batch
+ * @fires StreamingEmbedder#backpressure
+ * @fires StreamingEmbedder#done
+ * @fires StreamingEmbedder#error
+ *
+ * @example
+ *   const se = new StreamingEmbedder(myEmbedder, { batchSize: 25 });
+ *   se.on('embedding', (e) => console.log(e.id, e.progress));
+ *   se.on('done', (stats) => console.log('finished', stats));
+ *   const all = await se.embedAll(documents);
+ */
+export class StreamingEmbedder extends EventEmitter<[never]> {
+    /**
+     * @param {Object} embedder - Embedder instance with `.embed()` or `.embedQuery()`
+     * @param {Object} [options] - Override any key from {@link DEFAULT_CONFIG}
+     */
+    constructor(embedder: any, options?: any);
+    /** @type {Object} */
+    embedder: any;
+    /** @type {Object} */
+    config: any;
+    /** @type {{ processed: number, failed: number, totalTime: number }} */
+    stats: {
+        processed: number;
+        failed: number;
+        totalTime: number;
+    };
+    /**
+     * Async generator that yields embeddings while emitting lifecycle events.
+     *
+     * @param {Array<{id: string, content: string}>} documents - Documents to embed
+     * @yields {{ id: string, vector: number[], index: number, progress: number }}
+     *
+     * @fires StreamingEmbedder#start
+     * @fires StreamingEmbedder#embedding
+     * @fires StreamingEmbedder#batch
+     * @fires StreamingEmbedder#backpressure
+     * @fires StreamingEmbedder#done
+     * @fires StreamingEmbedder#error
+     */
+    embedStream(documents: Array<{
+        id: string;
+        content: string;
+    }>): AsyncGenerator<{
+        backpressure?: boolean;
+        id: string;
+        vector: any;
+        index: number;
+        progress: number;
+        error?: undefined;
+    } | {
+        id: string;
+        error: any;
+        index: number;
+        progress: number;
+    }, void, unknown>;
+    /**
+     * Convenience method that collects all embeddings into an array.
+     *
+     * @param {Array<{id: string, content: string}>} documents - Documents to embed
+     * @returns {Promise<Array<{ id: string, vector: number[], index: number, progress: number }>>}
+     */
+    embedAll(documents: Array<{
+        id: string;
+        content: string;
+    }>): Promise<Array<{
+        id: string;
+        vector: number[];
+        index: number;
+        progress: number;
+    }>>;
+    /**
+     * Returns a snapshot of the cumulative statistics.
+     *
+     * @returns {{ processed: number, failed: number, totalTime: number }}
+     */
+    getStats(): {
+        processed: number;
+        failed: number;
+        totalTime: number;
+    };
+    /**
+     * Resets the cumulative statistics to zero.
+     */
+    resetStats(): void;
+}
+/**
+ * Default configuration for the StreamingEmbedder
+ * @type {Object}
+ */
+export const DEFAULT_CONFIG: any;
+import { EventEmitter } from "events";

package/dist/types/streaming/websocket-adapter.d.ts

@@ -0,0 +1,71 @@
+/**
+ * Adapts an AsyncGenerator to a WebSocket connection.
+ *
+ * Sends JSON-framed messages for each token, a completion message when
+ * the generator is exhausted, and an error message on failure. Handles
+ * premature connection closure gracefully.
+ *
+ * @extends EventEmitter
+ *
+ * @example
+ *   const adapter = new WebSocketAdapter();
+ *   adapter.on('done', () => console.log('stream complete'));
+ *   await adapter.stream(asyncTokenGenerator, ws);
+ */
+export class WebSocketAdapter extends EventEmitter<[never]> {
+    /**
+     * @param {Object} [options] - Configuration overrides
+     * @param {string} [options.messageType='token'] - Type field for token messages
+     * @param {string} [options.doneType='done'] - Type field for completion message
+     * @param {string} [options.errorType='error'] - Type field for error messages
+     */
+    constructor(options?: {
+        messageType?: string;
+        doneType?: string;
+        errorType?: string;
+    });
+    config: any;
+    /**
+     * Stream tokens from an AsyncGenerator over a WebSocket connection.
+     *
+     * Each yielded value should be an object with at least a `token` property
+     * and an optional `done` boolean. Messages are sent as JSON strings via
+     * `wsConnection.send()`.
+     *
+     * @param {AsyncGenerator<{token: string, done?: boolean}>} asyncGenerator - Token source
+     * @param {Object} wsConnection - WebSocket-like object with a `send(data)` method
+     * @param {Function} wsConnection.send - Sends a string message over the socket
+     * @param {number} [wsConnection.readyState] - Optional W3C readyState property
+     * @returns {Promise<void>} Resolves when the stream ends or the connection closes
+     * @throws {TypeError} If wsConnection lacks a send method
+     */
+    stream(asyncGenerator: AsyncGenerator<{
+        token: string;
+        done?: boolean;
+    }>, wsConnection: {
+        send: Function;
+        readyState?: number;
+    }): Promise<void>;
+    /**
+     * Check whether the WebSocket connection is still open.
+     *
+     * @param {Object} wsConnection
+     * @returns {boolean}
+     * @private
+     */
+    private _isOpen;
+    /**
+     * Send a JSON-serialised message over the WebSocket.
+     *
+     * @param {Object} wsConnection
+     * @param {Object} payload
+     * @private
+     */
+    private _send;
+}
+/**
+ * Default configuration for WebSocket streaming
+ * @type {Object}
+ */
+export const DEFAULT_CONFIG: any;
+import { EventEmitter } from "events";

package/dist/types/templates/scaffold.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Creates complete project directories from registered templates.
+ *
+ * @extends EventEmitter
+ * @fires ProjectScaffolder#created
+ *
+ * @example
+ * const scaffolder = new ProjectScaffolder();
+ * const result = scaffolder.create('document-qa', '/path/to/my-project');
+ * console.log(result.files); // list of created files
+ */
+export class ProjectScaffolder extends EventEmitter<[never]> {
+    /**
+     * @param {Object} [options={}] - Scaffolder configuration
+     * @param {boolean} [options.installDependencies=false] - Whether to auto-run npm install
+     * @param {boolean} [options.overwrite=false] - Allow overwriting existing directories
+     */
+    constructor(options?: {
+        installDependencies?: boolean;
+        overwrite?: boolean;
+    });
+    config: any;
+    registry: TemplateRegistry;
+    /**
+     * Create a new project from a template.
+     *
+     * @param {string} templateKey - Key of the template to use
+     * @param {string} projectDir - Target directory for the new project
+     * @param {Object} [options={}] - Per-call overrides
+     * @param {boolean} [options.overwrite] - Override the instance-level overwrite setting
+     * @returns {{projectDir: string, files: string[], template: string}} Creation result
+     * @throws {Error} If template is not found or directory already exists
+     */
+    create(templateKey: string, projectDir: string, options?: {
+        overwrite?: boolean;
+    }): {
+        projectDir: string;
+        files: string[];
+        template: string;
+    };
+    /**
+     * List all available templates.
+     *
+     * @returns {Array<{key: string, name: string, description: string}>} Available templates
+     */
+    listTemplates(): Array<{
+        key: string;
+        name: string;
+        description: string;
+    }>;
+}
+/** @type {Object} Default configuration for the project scaffolder */
+export const DEFAULT_CONFIG: any;
+import { EventEmitter } from "events";
+import { TemplateRegistry } from "./template-registry";

package/dist/types/templates/template-registry.d.ts

@@ -0,0 +1,63 @@
+/**
+ * Registry of built-in and custom project templates.
+ *
+ * @example
+ * const registry = new TemplateRegistry();
+ * const templates = registry.list();
+ * const template = registry.get('document-qa');
+ */
+export class TemplateRegistry {
+    /** @type {Map<string, Object>} */
+    templates: Map<string, any>;
+    /**
+     * List all available templates.
+     *
+     * @returns {Array<{key: string, name: string, description: string}>} Summary of each template
+     */
+    list(): Array<{
+        key: string;
+        name: string;
+        description: string;
+    }>;
+    /**
+     * Get a full template definition by key.
+     *
+     * @param {string} templateKey - Template identifier
+     * @returns {Object|null} Full template object, or null if not found
+     */
+    get(templateKey: string): any | null;
+    /**
+     * Register a custom template.
+     *
+     * @param {string} key - Template identifier
+     * @param {Object} template - Template definition
+     * @param {string} template.name - Display name
+     * @param {string} template.description - Short description
+     * @param {Object} template.dependencies - npm dependencies
+     * @param {Object} template.config - Pipeline configuration
+     * @param {Object} template.files - Map of filename to file content
+     * @returns {void}
+     */
+    register(key: string, template: {
+        name: string;
+        description: string;
+        dependencies: any;
+        config: any;
+        files: any;
+    }): void;
+    /**
+     * Check if a template exists.
+     *
+     * @param {string} key - Template identifier
+     * @returns {boolean} Whether the template is registered
+     */
+    has(key: string): boolean;
+}
+/**
+ * @module templates/template-registry
+ * @description Registry of built-in project templates for RAG pipeline scaffolding.
+ */
+/** @type {Object<string, Object>} Built-in template definitions */
+export const TEMPLATES: {
+    [x: string]: any;
+};

package/dist/types/testing.d.ts

@@ -0,0 +1,168 @@
+/**
+ * Deterministic embedder: hashed bag of words, L2-normalised. Texts that
+ * share words get similar vectors, so retrieval ranks sensibly in tests.
+ */
+export class HashEmbedder {
+    /**
+     * @param {Object} [options]
+     * @param {number} [options.dimensions=256] - Vector length
+     */
+    constructor({ dimensions }?: {
+        dimensions?: number;
+    });
+    dimensions: number;
+    metadata: {
+        name: string;
+        version: string;
+        type: string;
+    };
+    /**
+     * Cache identity for createRagPipeline({ cache }).
+     * @param {string} role
+     * @returns {string|null}
+     */
+    getCacheKey(role: string): string | null;
+    /** @private */
+    private _vector;
+    /**
+     * @param {string} text
+     * @param {{ signal?: AbortSignal }} [options]
+     * @returns {Promise<number[]>}
+     */
+    embed(text: string, { signal }?: {
+        signal?: AbortSignal;
+    }): Promise<number[]>;
+    /**
+     * Same as embed().
+     * @param {string} text
+     * @param {{ signal?: AbortSignal }} [options]
+     * @returns {Promise<number[]>}
+     */
+    embedQuery(text: string, options?: {
+        signal?: AbortSignal;
+    }): Promise<number[]>;
+    /**
+     * @param {string[]} texts
+     * @param {{ signal?: AbortSignal }} [options]
+     * @returns {Promise<number[][]>}
+     */
+    embedBatch(texts: string[], { signal }?: {
+        signal?: AbortSignal;
+    }): Promise<number[][]>;
+}
+/**
+ * LLM stand-in. Answers with `answer` (a string, or a function of the query
+ * and documents), or else with the top document's content. Every call is
+ * recorded in `calls`, so tests can assert on what the pipeline sent.
+ */
+export class EchoLLM {
+    /**
+     * @param {Object} [options]
+     * @param {string|function(string, Array<Object>): string} [options.answer]
+     * @param {string} [options.fallback='No information found.'] - Answer when
+     *   there are no documents and no `answer`
+     */
+    constructor({ answer, fallback }?: {
+        answer?: string | ((arg0: string, arg1: Array<any>) => string);
+        fallback?: string;
+    });
+    answer: string | ((arg0: string, arg1: Array<any>) => string);
+    fallback: string;
+    /** @type {Array<{query: string, docs: Array<Object>, options: Object}>} */
+    calls: Array<{
+        query: string;
+        docs: Array<any>;
+        options: any;
+    }>;
+    metadata: {
+        name: string;
+        version: string;
+        type: string;
+    };
+    /**
+     * Cache identity for createRagPipeline({ cache }); none when `answer` is a
+     * function, since its output cannot be identified.
+     * @param {string} role
+     * @returns {string|null}
+     */
+    getCacheKey(role: string): string | null;
+    /** @private */
+    private _answer;
+    /**
+     * @param {string} query
+     * @param {Array<Object>} [docs]
+     * @param {{ signal?: AbortSignal }} [options]
+     * @returns {Promise<string>}
+     */
+    generate(query: string, docs?: Array<any>, options?: {
+        signal?: AbortSignal;
+    }): Promise<string>;
+    /**
+     * Streams the same answer word by word, then a done chunk.
+     * @param {string} query
+     * @param {Array<Object>} [docs]
+     * @param {{ signal?: AbortSignal }} [options]
+     * @yields {{ token: string, done: boolean }}
+     */
+    generateStream(query: string, docs?: Array<any>, options?: {
+        signal?: AbortSignal;
+    }): AsyncGenerator<{
+        token: string;
+        done: boolean;
+    }, void, unknown>;
+}
+/** Short documents for tests and examples. */
+export const fixtures: Readonly<{
+    documents: readonly (Readonly<{
+        id: "vacation";
+        content: "Full-time employees receive twenty days of paid vacation per year. Unused days carry over to March.";
+        metadata: Readonly<{
+            source: "vacation.md";
+        }>;
+    }> | Readonly<{
+        id: "remote-work";
+        content: "Employees may work remotely up to three days per week with manager approval.";
+        metadata: Readonly<{
+            source: "remote-work.md";
+        }>;
+    }> | Readonly<{
+        id: "expenses";
+        content: "Submit receipts within thirty days. Travel above 500 EUR needs pre-approval from finance.";
+        metadata: Readonly<{
+            source: "expenses.md";
+        }>;
+    }>)[];
+}>;
+/**
+ * A pipeline with HashEmbedder, a MemoryRetriever already holding
+ * `documents`, and EchoLLM (or the llm you pass).
+ *
+ * @example
+ * const { createTestPipeline } = require('@devilsdev/rag-pipeline-utils/testing');
+ * const { pipeline, llm } = await createTestPipeline();
+ * const result = await pipeline.run({ query: 'How many vacation days?' });
+ * // result.results[0].id === 'vacation'; llm.calls.length === 1
+ *
+ * @param {Object} [options]
+ * @param {Array<{id: string, content: string, metadata?: Object}>} [options.documents=fixtures.documents]
+ * @param {Object} [options.embedder] - Defaults to a new HashEmbedder
+ * @param {Object|null} [options.llm] - Defaults to a new EchoLLM; null for retrieval only
+ * @param {Object} [options.pipeline] - Extra createRagPipeline options (e.g. reranker)
+ * @returns {Promise<{ pipeline: Object, embedder: Object, retriever: MemoryRetriever, llm: Object|null }>}
+ */
+export function createTestPipeline({ documents, embedder, llm, pipeline: pipelineOptions, }?: {
+    documents?: Array<{
+        id: string;
+        content: string;
+        metadata?: any;
+    }>;
+    embedder?: any;
+    llm?: any | null;
+    pipeline?: any;
+}): Promise<{
+    pipeline: any;
+    embedder: any;
+    retriever: MemoryRetriever;
+    llm: any | null;
+}>;
+import { MemoryRetriever } from "./connectors/memory-retriever";

package/dist/types/utils/embed-query.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Embed a single query string with any supported embedder shape.
+ *
+ * Prefers embedQuery(); falls back to embed(). Accepts the common return
+ * shapes: a vector, a list of vectors, or [{ values | vector | embedding }].
+ *
+ * @param {object} embedder - Object with embedQuery() or embed()
+ * @param {string} query - Text to embed
+ * @param {{ signal?: AbortSignal }} [options] - Passed to the embedder as its
+ *   second argument (only when given, so embedders keep their usual arity)
+ * @returns {Promise<number[]|undefined>} The query vector
+ */
+export function embedQuery(embedder: object, query: string, options?: {
+    signal?: AbortSignal;
+}): Promise<number[] | undefined>;
+/**
+ * @param {object} [embedder]
+ * @returns {boolean} Whether embedQuery() can use this embedder
+ */
+export function canEmbed(embedder?: object): boolean;

package/dist/types/utils/logger.d.ts

@@ -0,0 +1,50 @@
+export const logger: StructuredLogger;
+/**
+ * Create a logger with correlation ID for request tracing
+ */
+export function createLogger(correlationId: any, context?: {}): StructuredLogger;
+/**
+ * Middleware function for Express.js to add correlation IDs to requests
+ */
+export function correlationMiddleware(req: any, res: any, next: any): void;
+/**
+ * Enhanced logger with correlation ID support and structured context
+ */
+export class StructuredLogger {
+    constructor(baseLogger: any);
+    logger: any;
+    correlationId: any;
+    context: {};
+    /**
+     * Create a child logger with correlation ID and context
+     */
+    child(context?: {}): StructuredLogger;
+    /**
+     * Set correlation ID for distributed tracing
+     */
+    setCorrelationId(correlationId: any): this;
+    /**
+     * Add persistent context to all log messages
+     */
+    setContext(context: any): this;
+    /**
+     * Log with automatic correlation ID and context injection
+     */
+    _log(level: any, message: any, meta?: {}): void;
+    info(message: any, meta: any): void;
+    error(message: any, meta: any): void;
+    warn(message: any, meta: any): void;
+    debug(message: any, meta: any): void;
+    /**
+     * Security audit logging with enhanced metadata
+     */
+    audit(action: any, meta?: {}): void;
+    /**
+     * Performance logging with timing information
+     */
+    performance(operation: any, duration: any, meta?: {}): void;
+    /**
+     * Tenant-specific logging for multi-tenant applications
+     */
+    tenant(tenantId: any, message: any, meta?: {}): void;
+}