@devilsdev/rag-pipeline-utils 2.4.5 → 2.5.0

package/dist/types/streaming/stream-router.d.ts

@@ -0,0 +1,88 @@
+/**
+ * Routes pipeline streaming output to SSE or WebSocket transports.
+ *
+ * Provides both an Express/Node-HTTP-compatible request handler via
+ * {@link StreamRouter#createHandler} and lower-level convenience methods
+ * ({@link StreamRouter#streamSSE}, {@link StreamRouter#streamWS}) for
+ * manual integration.
+ *
+ * @extends EventEmitter
+ *
+ * @example
+ *   const router = new StreamRouter({ enableCORS: true });
+ *   const handler = router.createHandler(pipeline);
+ *   app.post('/api/query', handler);
+ */
+export class StreamRouter extends EventEmitter<[never]> {
+    /**
+     * @param {Object} [options] - Configuration overrides
+     * @param {string} [options.defaultTransport='sse'] - Fallback transport type
+     * @param {boolean} [options.enableCORS=true] - Enable CORS headers
+     * @param {string} [options.corsOrigin='*'] - CORS origin value
+     * @param {Object} [options.sseOptions] - Options forwarded to SSEAdapter
+     * @param {Object} [options.wsOptions] - Options forwarded to WebSocketAdapter
+     */
+    constructor(options?: {
+        defaultTransport?: string;
+        enableCORS?: boolean;
+        corsOrigin?: string;
+        sseOptions?: any;
+        wsOptions?: any;
+    });
+    config: any;
+    sseAdapter: SSEAdapter;
+    wsAdapter: WebSocketAdapter;
+    /**
+     * Create an HTTP request handler that runs the pipeline in streaming mode
+     * and routes output to the appropriate transport.
+     *
+     * The returned function is compatible with Node's `http.createServer`,
+     * Express, and similar frameworks.
+     *
+     * Transport detection:
+     * - `Accept: text/event-stream` header selects SSE
+     * - `Upgrade: websocket` header selects WebSocket (note: actual upgrade
+     *   negotiation is typically handled at a higher level)
+     * - Otherwise falls back to {@link DEFAULT_CONFIG.defaultTransport}
+     *
+     * @param {Object} pipeline - A pipeline object with a `.run()` method
+     * @returns {Function} Async request handler `(req, res) => Promise<void>`
+     */
+    createHandler(pipeline: any): Function;
+    /**
+     * Run the pipeline in streaming mode and pipe tokens to an HTTP response
+     * via Server-Sent Events.
+     *
+     * If the pipeline returns a non-iterable result, it is sent as a single
+     * SSE data frame followed by the done sentinel.
+     *
+     * @param {Object} pipeline - Pipeline with a `.run()` method
+     * @param {string} query - User query string
+     * @param {import('http').ServerResponse} httpResponse - Writable HTTP response
+     * @param {Object} [options={}] - Additional pipeline run options
+     * @returns {Promise<void>}
+     */
+    streamSSE(pipeline: any, query: string, httpResponse: import("http").ServerResponse, options?: any): Promise<void>;
+    /**
+     * Run the pipeline in streaming mode and pipe tokens over a WebSocket
+     * connection.
+     *
+     * If the pipeline returns a non-iterable result, it is sent as a single
+     * `result` message followed by a `done` message.
+     *
+     * @param {Object} pipeline - Pipeline with a `.run()` method
+     * @param {string} query - User query string
+     * @param {Object} wsConnection - WebSocket-like connection with `.send()`
+     * @param {Object} [options={}] - Additional pipeline run options
+     * @returns {Promise<void>}
+     */
+    streamWS(pipeline: any, query: string, wsConnection: any, options?: any): Promise<void>;
+}
+/**
+ * Default configuration for the stream router
+ * @type {Object}
+ */
+export const DEFAULT_CONFIG: any;
+import { EventEmitter } from "events";
+import { SSEAdapter } from "./sse-adapter";
+import { WebSocketAdapter } from "./websocket-adapter";

package/dist/types/streaming/streaming-embedder.d.ts

@@ -0,0 +1,99 @@
+/**
+ * EventEmitter-based wrapper around {@link createEmbeddingStream}.
+ *
+ * @extends EventEmitter
+ *
+ * @fires StreamingEmbedder#start
+ * @fires StreamingEmbedder#embedding
+ * @fires StreamingEmbedder#batch
+ * @fires StreamingEmbedder#backpressure
+ * @fires StreamingEmbedder#done
+ * @fires StreamingEmbedder#error
+ *
+ * @example
+ *   const se = new StreamingEmbedder(myEmbedder, { batchSize: 25 });
+ *   se.on('embedding', (e) => console.log(e.id, e.progress));
+ *   se.on('done', (stats) => console.log('finished', stats));
+ *   const all = await se.embedAll(documents);
+ */
+export class StreamingEmbedder extends EventEmitter<[never]> {
+    /**
+     * @param {Object} embedder - Embedder instance with `.embed()` or `.embedQuery()`
+     * @param {Object} [options] - Override any key from {@link DEFAULT_CONFIG}
+     */
+    constructor(embedder: any, options?: any);
+    /** @type {Object} */
+    embedder: any;
+    /** @type {Object} */
+    config: any;
+    /** @type {{ processed: number, failed: number, totalTime: number }} */
+    stats: {
+        processed: number;
+        failed: number;
+        totalTime: number;
+    };
+    /**
+     * Async generator that yields embeddings while emitting lifecycle events.
+     *
+     * @param {Array<{id: string, content: string}>} documents - Documents to embed
+     * @yields {{ id: string, vector: number[], index: number, progress: number }}
+     *
+     * @fires StreamingEmbedder#start
+     * @fires StreamingEmbedder#embedding
+     * @fires StreamingEmbedder#batch
+     * @fires StreamingEmbedder#backpressure
+     * @fires StreamingEmbedder#done
+     * @fires StreamingEmbedder#error
+     */
+    embedStream(documents: Array<{
+        id: string;
+        content: string;
+    }>): AsyncGenerator<{
+        backpressure?: boolean;
+        id: string;
+        vector: any;
+        index: number;
+        progress: number;
+        error?: undefined;
+    } | {
+        id: string;
+        error: any;
+        index: number;
+        progress: number;
+    }, void, unknown>;
+    /**
+     * Convenience method that collects all embeddings into an array.
+     *
+     * @param {Array<{id: string, content: string}>} documents - Documents to embed
+     * @returns {Promise<Array<{ id: string, vector: number[], index: number, progress: number }>>}
+     */
+    embedAll(documents: Array<{
+        id: string;
+        content: string;
+    }>): Promise<Array<{
+        id: string;
+        vector: number[];
+        index: number;
+        progress: number;
+    }>>;
+    /**
+     * Returns a snapshot of the cumulative statistics.
+     *
+     * @returns {{ processed: number, failed: number, totalTime: number }}
+     */
+    getStats(): {
+        processed: number;
+        failed: number;
+        totalTime: number;
+    };
+    /**
+     * Resets the cumulative statistics to zero.
+     */
+    resetStats(): void;
+}
+/**
+ * Default configuration for the StreamingEmbedder
+ * @type {Object}
+ */
+export const DEFAULT_CONFIG: any;
+import { EventEmitter } from "events";

package/dist/types/streaming/websocket-adapter.d.ts

@@ -0,0 +1,71 @@
+/**
+ * Adapts an AsyncGenerator to a WebSocket connection.
+ *
+ * Sends JSON-framed messages for each token, a completion message when
+ * the generator is exhausted, and an error message on failure. Handles
+ * premature connection closure gracefully.
+ *
+ * @extends EventEmitter
+ *
+ * @example
+ *   const adapter = new WebSocketAdapter();
+ *   adapter.on('done', () => console.log('stream complete'));
+ *   await adapter.stream(asyncTokenGenerator, ws);
+ */
+export class WebSocketAdapter extends EventEmitter<[never]> {
+    /**
+     * @param {Object} [options] - Configuration overrides
+     * @param {string} [options.messageType='token'] - Type field for token messages
+     * @param {string} [options.doneType='done'] - Type field for completion message
+     * @param {string} [options.errorType='error'] - Type field for error messages
+     */
+    constructor(options?: {
+        messageType?: string;
+        doneType?: string;
+        errorType?: string;
+    });
+    config: any;
+    /**
+     * Stream tokens from an AsyncGenerator over a WebSocket connection.
+     *
+     * Each yielded value should be an object with at least a `token` property
+     * and an optional `done` boolean. Messages are sent as JSON strings via
+     * `wsConnection.send()`.
+     *
+     * @param {AsyncGenerator<{token: string, done?: boolean}>} asyncGenerator - Token source
+     * @param {Object} wsConnection - WebSocket-like object with a `send(data)` method
+     * @param {Function} wsConnection.send - Sends a string message over the socket
+     * @param {number} [wsConnection.readyState] - Optional W3C readyState property
+     * @returns {Promise<void>} Resolves when the stream ends or the connection closes
+     * @throws {TypeError} If wsConnection lacks a send method
+     */
+    stream(asyncGenerator: AsyncGenerator<{
+        token: string;
+        done?: boolean;
+    }>, wsConnection: {
+        send: Function;
+        readyState?: number;
+    }): Promise<void>;
+    /**
+     * Check whether the WebSocket connection is still open.
+     *
+     * @param {Object} wsConnection
+     * @returns {boolean}
+     * @private
+     */
+    private _isOpen;
+    /**
+     * Send a JSON-serialised message over the WebSocket.
+     *
+     * @param {Object} wsConnection
+     * @param {Object} payload
+     * @private
+     */
+    private _send;
+}
+/**
+ * Default configuration for WebSocket streaming
+ * @type {Object}
+ */
+export const DEFAULT_CONFIG: any;
+import { EventEmitter } from "events";

package/dist/types/templates/scaffold.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Creates complete project directories from registered templates.
+ *
+ * @extends EventEmitter
+ * @fires ProjectScaffolder#created
+ *
+ * @example
+ * const scaffolder = new ProjectScaffolder();
+ * const result = scaffolder.create('document-qa', '/path/to/my-project');
+ * console.log(result.files); // list of created files
+ */
+export class ProjectScaffolder extends EventEmitter<[never]> {
+    /**
+     * @param {Object} [options={}] - Scaffolder configuration
+     * @param {boolean} [options.installDependencies=false] - Whether to auto-run npm install
+     * @param {boolean} [options.overwrite=false] - Allow overwriting existing directories
+     */
+    constructor(options?: {
+        installDependencies?: boolean;
+        overwrite?: boolean;
+    });
+    config: any;
+    registry: TemplateRegistry;
+    /**
+     * Create a new project from a template.
+     *
+     * @param {string} templateKey - Key of the template to use
+     * @param {string} projectDir - Target directory for the new project
+     * @param {Object} [options={}] - Per-call overrides
+     * @param {boolean} [options.overwrite] - Override the instance-level overwrite setting
+     * @returns {{projectDir: string, files: string[], template: string}} Creation result
+     * @throws {Error} If template is not found or directory already exists
+     */
+    create(templateKey: string, projectDir: string, options?: {
+        overwrite?: boolean;
+    }): {
+        projectDir: string;
+        files: string[];
+        template: string;
+    };
+    /**
+     * List all available templates.
+     *
+     * @returns {Array<{key: string, name: string, description: string}>} Available templates
+     */
+    listTemplates(): Array<{
+        key: string;
+        name: string;
+        description: string;
+    }>;
+}
+/** @type {Object} Default configuration for the project scaffolder */
+export const DEFAULT_CONFIG: any;
+import { EventEmitter } from "events";
+import { TemplateRegistry } from "./template-registry";

package/dist/types/templates/template-registry.d.ts

@@ -0,0 +1,63 @@
+/**
+ * Registry of built-in and custom project templates.
+ *
+ * @example
+ * const registry = new TemplateRegistry();
+ * const templates = registry.list();
+ * const template = registry.get('document-qa');
+ */
+export class TemplateRegistry {
+    /** @type {Map<string, Object>} */
+    templates: Map<string, any>;
+    /**
+     * List all available templates.
+     *
+     * @returns {Array<{key: string, name: string, description: string}>} Summary of each template
+     */
+    list(): Array<{
+        key: string;
+        name: string;
+        description: string;
+    }>;
+    /**
+     * Get a full template definition by key.
+     *
+     * @param {string} templateKey - Template identifier
+     * @returns {Object|null} Full template object, or null if not found
+     */
+    get(templateKey: string): any | null;
+    /**
+     * Register a custom template.
+     *
+     * @param {string} key - Template identifier
+     * @param {Object} template - Template definition
+     * @param {string} template.name - Display name
+     * @param {string} template.description - Short description
+     * @param {Object} template.dependencies - npm dependencies
+     * @param {Object} template.config - Pipeline configuration
+     * @param {Object} template.files - Map of filename to file content
+     * @returns {void}
+     */
+    register(key: string, template: {
+        name: string;
+        description: string;
+        dependencies: any;
+        config: any;
+        files: any;
+    }): void;
+    /**
+     * Check if a template exists.
+     *
+     * @param {string} key - Template identifier
+     * @returns {boolean} Whether the template is registered
+     */
+    has(key: string): boolean;
+}
+/**
+ * @module templates/template-registry
+ * @description Registry of built-in project templates for RAG pipeline scaffolding.
+ */
+/** @type {Object<string, Object>} Built-in template definitions */
+export const TEMPLATES: {
+    [x: string]: any;
+};

package/dist/types/utils/embed-query.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Embed a single query string with any supported embedder shape.
+ *
+ * Prefers embedQuery(); falls back to embed(). Accepts the common return
+ * shapes: a vector, a list of vectors, or [{ values | vector | embedding }].
+ *
+ * @param {object} embedder - Object with embedQuery() or embed()
+ * @param {string} query - Text to embed
+ * @param {{ signal?: AbortSignal }} [options] - Passed to the embedder as its
+ *   second argument (only when given, so embedders keep their usual arity)
+ * @returns {Promise<number[]|undefined>} The query vector
+ */
+export function embedQuery(embedder: object, query: string, options?: {
+    signal?: AbortSignal;
+}): Promise<number[] | undefined>;
+/**
+ * @param {object} [embedder]
+ * @returns {boolean} Whether embedQuery() can use this embedder
+ */
+export function canEmbed(embedder?: object): boolean;

package/dist/types/utils/logger.d.ts

@@ -0,0 +1,50 @@
+export const logger: StructuredLogger;
+/**
+ * Create a logger with correlation ID for request tracing
+ */
+export function createLogger(correlationId: any, context?: {}): StructuredLogger;
+/**
+ * Middleware function for Express.js to add correlation IDs to requests
+ */
+export function correlationMiddleware(req: any, res: any, next: any): void;
+/**
+ * Enhanced logger with correlation ID support and structured context
+ */
+export class StructuredLogger {
+    constructor(baseLogger: any);
+    logger: any;
+    correlationId: any;
+    context: {};
+    /**
+     * Create a child logger with correlation ID and context
+     */
+    child(context?: {}): StructuredLogger;
+    /**
+     * Set correlation ID for distributed tracing
+     */
+    setCorrelationId(correlationId: any): this;
+    /**
+     * Add persistent context to all log messages
+     */
+    setContext(context: any): this;
+    /**
+     * Log with automatic correlation ID and context injection
+     */
+    _log(level: any, message: any, meta?: {}): void;
+    info(message: any, meta: any): void;
+    error(message: any, meta: any): void;
+    warn(message: any, meta: any): void;
+    debug(message: any, meta: any): void;
+    /**
+     * Security audit logging with enhanced metadata
+     */
+    audit(action: any, meta?: {}): void;
+    /**
+     * Performance logging with timing information
+     */
+    performance(operation: any, duration: any, meta?: {}): void;
+    /**
+     * Tenant-specific logging for multi-tenant applications
+     */
+    tenant(tenantId: any, message: any, meta?: {}): void;
+}

package/dist/types/utils/rate-limiter.d.ts

@@ -0,0 +1,66 @@
+export = RateLimiter;
+declare class RateLimiter {
+    /**
+     * Create a rate limiter instance
+     * @param {Object} options - Configuration options
+     * @param {number} options.maxAttempts - Maximum attempts allowed (default: 5)
+     * @param {number} options.windowMs - Time window in milliseconds (default: 15 minutes)
+     * @param {number} options.blockDurationMs - How long to block after exceeding limit (default: 1 hour)
+     * @param {number} options.cleanupIntervalMs - How often to cleanup expired entries (default: 5 minutes)
+     */
+    constructor(options?: {
+        maxAttempts: number;
+        windowMs: number;
+        blockDurationMs: number;
+        cleanupIntervalMs: number;
+    });
+    maxAttempts: number;
+    windowMs: number;
+    blockDurationMs: number;
+    cleanupIntervalMs: number;
+    store: Map<any, any>;
+    /**
+     * Check if a request should be allowed
+     * @param {string} identifier - Unique identifier (IP, userId, tenantId, etc.)
+     * @returns {Object} { allowed: boolean, remaining: number, resetAt: Date, blockedUntil: Date|null }
+     */
+    allowRequest(identifier: string): any;
+    /**
+     * Manually reset rate limit for an identifier
+     * @param {string} identifier - Identifier to reset
+     */
+    reset(identifier: string): void;
+    /**
+     * Get current status for an identifier
+     * @param {string} identifier - Identifier to check
+     * @returns {Object} Current rate limit status
+     */
+    getStatus(identifier: string): any;
+    /**
+     * Clean up expired entries
+     */
+    cleanup(): {
+        cleaned: number;
+        remaining: number;
+    };
+    /**
+     * Get statistics about rate limiter usage
+     * @returns {Object} Statistics
+     */
+    getStats(): any;
+    /**
+     * Stop the rate limiter and cleanup
+     */
+    destroy(): void;
+    _cleanupInterval: NodeJS.Timeout;
+    /**
+     * Hash identifier for storage (privacy)
+     * @private
+     */
+    private _hashIdentifier;
+    /**
+     * Start automatic cleanup timer
+     * @private
+     */
+    private _startCleanup;
+}

package/dist/types/utils/retry.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Retry a function with exponential backoff.
+ *
+ * @param {Function} fn - async or sync function to call; receives (attempt)
+ * @param {Object} [opts]
+ * @param {number} [opts.retries=3] - number of *retries* (total attempts = retries + 1)
+ * @param {number} [opts.baseDelay=100] - initial delay in ms
+ * @param {number} [opts.multiplier=2] - backoff multiplier
+ * @param {boolean} [opts.jitter=false] - add +/-50% jitter if true
+ * @param {Function} [opts.sleep=defaultSleep] - injected sleeper for tests
+ * @param {Function} [opts.onDelay] - callback (delayMs, attempt, error) before each sleep
+ * @returns {Promise<*>}
+ */
+export function retry(fn: Function, opts?: {
+    retries?: number;
+    baseDelay?: number;
+    multiplier?: number;
+    jitter?: boolean;
+    sleep?: Function;
+    onDelay?: Function;
+}): Promise<any>;
+export function withRetry(_fn: any, { retries, initialDelay, label }?: {
+    retries?: number;
+    initialDelay?: number;
+    label?: string;
+}): Promise<any>;
+export function defaultSleep(ms: any): Promise<any>;

package/dist/types/utils/run-abort.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Per-run cancellation: one AbortController that is aborted when `timeout`
+ * elapses or when the caller's `signal` aborts. Its `signal` is handed to
+ * every component call; `race()` makes the run settle as soon as it is
+ * aborted even if a component ignores the signal; `throwIfAborted()` stops
+ * work between stages.
+ *
+ * @param {object} [options]
+ * @param {number} [options.timeout] - Milliseconds before the run is aborted
+ * @param {AbortSignal} [options.signal] - Caller's signal to follow
+ * @returns {{
+ *   signal: AbortSignal,
+ *   race: (promise: Promise<any>) => Promise<any>,
+ *   throwIfAborted: () => void,
+ *   clearTimer: () => void,
+ *   detach: () => void,
+ *   cleanup: () => void
+ * }}
+ */
+export function createRunAbort({ timeout, signal }?: {
+    timeout?: number;
+    signal?: AbortSignal;
+}): {
+    signal: AbortSignal;
+    race: (promise: Promise<any>) => Promise<any>;
+    throwIfAborted: () => void;
+    clearTimer: () => void;
+    detach: () => void;
+    cleanup: () => void;
+};
+/** Message used for a run that exceeded its timeout. */
+export const TIMEOUT_MESSAGE: "Execution timeout";
+/** Message used for a run cancelled through the caller's AbortSignal. */
+export const ABORTED_MESSAGE: "Execution aborted";

package/dist/types/utils/secure-logging.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Redact sensitive fields in an object
+ * @param {any} obj - Object to redact
+ * @param {number} depth - Current recursion depth
+ * @param {number} maxDepth - Maximum recursion depth
+ * @returns {any} Redacted object
+ */
+export function redactObject(obj: any, depth?: number, maxDepth?: number): any;
+/**
+ * Redact a string value if it contains secrets
+ * @param {string} value - String value to redact
+ * @returns {string} Redacted value if sensitive, original otherwise
+ */
+export function redactStringValue(value: string): string;
+/**
+ * Redact sensitive information from log data
+ * @param {string} message - Log message
+ * @param {object} meta - Log metadata object
+ * @returns {object} Redacted message and metadata
+ */
+export function redactLogData(message: string, meta?: object): object;
+/**
+ * Create a redacting wrapper for any logger
+ * @param {object} logger - Logger instance to wrap
+ * @returns {object} Wrapped logger with automatic redaction
+ */
+export function createSecureLogger(logger: object): object;
+/**
+ * Check if a field name is sensitive
+ * @param {string} fieldName - Field name to check
+ * @returns {boolean} True if sensitive
+ */
+export function isSensitiveFieldName(fieldName: string): boolean;
+/**
+ * Check if a value matches a secret pattern
+ * @param {string} value - Value to check
+ * @returns {boolean} True if matches secret pattern
+ */
+export function matchesSecretPattern(value: string): boolean;
+/**
+ * Sensitive field names that should be redacted
+ */
+export const SENSITIVE_FIELD_NAMES: Set<string>;
+/**
+ * Patterns that match secret-like values
+ */
+export const SECRET_PATTERNS: RegExp[];
+/**
+ * Redaction marker
+ */
+export const REDACTED: "[REDACTED]";