@devilsdev/rag-pipeline-utils 2.4.4 → 2.5.0

package/dist/types/security/jwt-validator.d.ts

@@ -0,0 +1,195 @@
+/**
+ * JWT Validator with hardened security
+ */
+export class JWTValidator extends EventEmitter<[never]> {
+    constructor(options?: {});
+    config: {
+        algorithm: any;
+        allowedAlgorithms: any;
+        secret: any;
+        publicKey: any;
+        privateKey: any;
+        issuer: any;
+        audience: any;
+        clockTolerance: any;
+        maxTokenAge: any;
+        notBeforeTolerance: any;
+        requireJti: boolean;
+        requireSub: boolean;
+        allowNoneAlgorithm: boolean;
+        strictValidation: boolean;
+        enableJtiTracking: boolean;
+        jtiExpirationMs: any;
+        singleUseTokens: boolean;
+        maxTrackedJtis: any;
+        validateClaims: any;
+        enableAuditLog: boolean;
+    };
+    usedJtis: Map<any, any>;
+    selfSignedJtis: Set<any>;
+    stats: {
+        tokensGenerated: number;
+        tokensValidated: number;
+        validationFailures: number;
+        algorithmMismatch: number;
+        expiredTokens: number;
+        invalidSignature: number;
+        replayAttempts: number;
+    };
+    /**
+     * Validate configuration on initialization
+     * @private
+     */
+    private _validateConfig;
+    /**
+     * Generate a secure JWT token
+     *
+     * @param {Object} payload - Token payload
+     * @param {Object} options - Additional signing options
+     * @returns {string} Signed JWT token
+     *
+     * @example
+     * const token = validator.sign({
+     *   sub: 'user-123',
+     *   roles: ['admin'],
+     *   tenant: 'acme-corp'
+     * });
+     */
+    sign(payload: any, options?: any): string;
+    /**
+     * Verify and decode a JWT token with hardened validation
+     *
+     * @param {string} token - JWT token to verify
+     * @param {Object} options - Verification options
+     * @returns {Object} Decoded and validated payload
+     *
+     * @example
+     * try {
+     *   const payload = validator.verify(token);
+     *   console.log('Valid token for user:', payload.sub);
+     * } catch (error) {
+     *   console.error('Invalid token:', error.message);
+     * }
+     */
+    verify(token: string, options?: any): any;
+    /**
+     * Decode token header without verification (for algorithm checking)
+     * @private
+     */
+    private _decodeTokenHeader;
+    /**
+     * Validate required claims are present
+     * @private
+     */
+    private _validateRequiredClaims;
+    /**
+     * Validate custom claims
+     * @private
+     */
+    private _validateCustomClaims;
+    /**
+     * Expiry time (ms) of a JTI store entry, including clock tolerance.
+     * @private
+     */
+    private _jtiExpiry;
+    /**
+     * Remove a JTI from both tracking structures.
+     * @private
+     */
+    private _forgetJti;
+    /**
+     * Look up a JTI, dropping it if it has expired and refreshing its LRU
+     * position otherwise.
+     * @private
+     */
+    private _getJtiEntry;
+    /**
+     * Record a JTI in the bounded store. When the store is full, expired
+     * entries are purged first; if it is still full the least-recently-used
+     * entry is evicted.
+     * @private
+     */
+    private _trackJti;
+    /**
+     * Drop every expired JTI from the store.
+     * @private
+     * @returns {number} Number of entries removed
+     */
+    private _purgeExpiredJtis;
+    /**
+     * Start periodic cleanup of expired JTIs
+     * @private
+     */
+    private _startJtiCleanup;
+    jtiCleanupInterval: NodeJS.Timeout;
+    /**
+     * Decode token without verification (use with caution)
+     *
+     * @param {string} token - JWT token to decode
+     * @returns {Object} Decoded payload (unverified)
+     */
+    decode(token: string): any;
+    /**
+     * Get validator statistics
+     *
+     * @returns {Object} Statistics object
+     */
+    getStats(): any;
+    /**
+     * Reset statistics
+     */
+    resetStats(): void;
+    /**
+     * Clear JTI tracking
+     */
+    clearJtiTracking(): void;
+    /**
+     * Destroy validator and cleanup resources
+     */
+    destroy(): void;
+}
+export namespace SECURE_ALGORITHMS {
+    let HS256: string;
+    let HS384: string;
+    let HS512: string;
+    let RS256: string;
+    let RS384: string;
+    let RS512: string;
+    let ES256: string;
+    let ES384: string;
+    let ES512: string;
+    let PS256: string;
+    let PS384: string;
+    let PS512: string;
+}
+export namespace MIN_KEY_SIZES {
+    let HS256_1: number;
+    export { HS256_1 as HS256 };
+    let HS384_1: number;
+    export { HS384_1 as HS384 };
+    let HS512_1: number;
+    export { HS512_1 as HS512 };
+    let RS256_1: number;
+    export { RS256_1 as RS256 };
+    let RS384_1: number;
+    export { RS384_1 as RS384 };
+    let RS512_1: number;
+    export { RS512_1 as RS512 };
+    let ES256_1: number;
+    export { ES256_1 as ES256 };
+    let ES384_1: number;
+    export { ES384_1 as ES384 };
+    let ES512_1: number;
+    export { ES512_1 as ES512 };
+    let PS256_1: number;
+    export { PS256_1 as PS256 };
+    let PS384_1: number;
+    export { PS384_1 as PS384 };
+    let PS512_1: number;
+    export { PS512_1 as PS512 };
+}
+/**
+ * Required JWT claims for validation
+ */
+export const REQUIRED_CLAIMS: string[];
+import { EventEmitter } from "events";

package/dist/types/security/plugin-sandbox.d.ts

@@ -0,0 +1,158 @@
+/**
+ * Secure plugin sandbox for executing third-party plugins with strict isolation.
+ *
+ * Features:
+ * - VM-based execution isolation
+ * - Resource limits (memory, CPU time, I/O)
+ * - Permission-based access control
+ * - Audit logging for security compliance
+ * - Plugin signature verification
+ * - Fail-closed security model
+ *
+ * @class PluginSandbox
+ * @extends EventEmitter
+ */
+export class PluginSandbox extends EventEmitter<[never]> {
+    constructor(options?: {});
+    config: {
+        timeout: any;
+        memoryLimit: any;
+        allowEval: boolean;
+        allowWasm: boolean;
+        allowAsync: any;
+        permissions: any;
+        maxFileSize: any;
+        maxNetworkRequests: any;
+        maxCpuTime: any;
+        auditEnabled: boolean;
+        logLevel: any;
+    };
+    signatureVerifier: any;
+    activeSandboxes: Map<any, any>;
+    executionMetrics: Map<any, any>;
+    /**
+     * Execute a plugin within the secure sandbox environment.
+     *
+     * @param {Object} plugin - Plugin configuration and code
+     * @param {Object} context - Execution context and input data
+     * @param {Object} options - Override options for this execution
+     * @returns {Promise<any>} Plugin execution result
+     * @throws {SecurityError} On security policy violations
+     * @throws {ResourceError} On resource limit exceeded
+     * @throws {ValidationError} On plugin validation failure
+     */
+    execute(plugin: any, context?: any, options?: any): Promise<any>;
+    /**
+     * Enforce rate limiting
+     */
+    _enforceRateLimit(customRateLimit?: {}): void;
+    /**
+     * Validate plugin structure and security
+     */
+    _validatePlugin(plugin: any): void;
+    /**
+     * Execute plugin in isolated VM context using isolated-vm
+     */
+    _executeInSandbox(plugin: any, prompt: any, executionId: any): Promise<any>;
+    /**
+     * Create secure globals for isolated-vm
+     */
+    _createSecureGlobals(executionId: any): {
+        console: {
+            log: () => void;
+            error: () => void;
+            warn: () => void;
+            info: () => void;
+        };
+        setTimeout: (fn: any, delay: any) => NodeJS.Timeout;
+        clearTimeout: typeof clearTimeout;
+        Math: Math;
+        JSON: JSON;
+        Date: DateConstructor;
+        __SANDBOX_ID__: any;
+    };
+    /**
+     * Create secure sandbox context (legacy method for compatibility)
+     */
+    _createSandboxContext(executionId: any): {
+        console: {
+            log: () => void;
+            error: () => void;
+            warn: () => void;
+            info: () => void;
+        };
+        setTimeout: (fn: any, delay: any) => NodeJS.Timeout;
+        clearTimeout: typeof clearTimeout;
+        setImmediate: typeof setImmediate;
+        process: {
+            env: {};
+            memoryUsage: any;
+            hrtime: any;
+        };
+        Buffer: {
+            from: any;
+            alloc: (size: any) => Buffer<ArrayBuffer>;
+        };
+        Math: Math;
+        JSON: JSON;
+        Date: DateConstructor;
+        require: () => never;
+        import: () => never;
+        eval: () => never;
+        Function: () => never;
+        fetch: typeof fetch;
+        fs: typeof fs;
+        global: any;
+        globalThis: any;
+        window: any;
+        __SANDBOX_ID__: any;
+    };
+    /**
+     * Sanitize execution result
+     */
+    _sanitizeResult(result: any): any;
+    /**
+     * Get sandbox statistics
+     */
+    getStats(): {
+        activeExecutions: any;
+        rateLimitEntries: any;
+        options: any;
+    };
+    /**
+     * Clean up rate limit entries
+     */
+    cleanup(): void;
+}
+/**
+ * Legacy sanitizer for backward compatibility
+ */
+export function sanitizeOutput(obj: any): any;
+/**
+ * Input sanitizer for security validation
+ */
+export class InputSanitizer {
+    static sanitizeInput(input: any): string;
+    static validateInput(input: any): boolean;
+}
+/**
+ * Output sanitizer for data protection
+ */
+export class OutputSanitizer {
+    static sanitizeOutput(output: any): any;
+    static sanitizeText(text: any): any;
+}
+/**
+ * Simple sandbox runner with timeout and abort support
+ * @param {Function} fn - Function to execute
+ * @param {Object} options - Options object
+ * @param {number} options.timeoutMs - Timeout in milliseconds (default: 5000)
+ * @param {AbortSignal} options.signal - AbortSignal for cancellation
+ * @returns {Promise<Object>} Result object with success/error
+ */
+export function run(fn: Function, { timeoutMs, signal }?: {
+    timeoutMs: number;
+    signal: AbortSignal;
+}): Promise<any>;
+import { EventEmitter } from "events";
+import fs = require("fs");

package/dist/types/security/plugin-signature-verifier.d.ts

@@ -0,0 +1,139 @@
+/**
+ * Plugin signature verification with Ed25519
+ */
+export class PluginSignatureVerifier {
+    constructor(options?: {});
+    enabled: boolean;
+    trustStore: any;
+    auditLogger: any;
+    failClosed: boolean;
+    requireContentDigest: boolean;
+    /**
+     * Load trusted public keys from configuration.
+     * Without a keys file the trust store stays empty (no built-in keys).
+     * @param {string} keysPath - Path to trusted keys file
+     */
+    _loadTrustedKeys(keysPath: string): void;
+    /**
+     * Verify plugin signature
+     * @param {object} manifest - Plugin manifest
+     * @param {string} signature - Base64 encoded signature
+     * @param {string} signerId - Signer identifier
+     * @param {object} [content] - Plugin content to check against
+     *   manifest.contentDigest: `{ files, baseDir, contents }` (see
+     *   computeContentDigest)
+     * @returns {Promise<object>} Verification result
+     */
+    verifyPluginSignature(manifest: object, signature: string, signerId: string, content?: object): Promise<object>;
+    /**
+     * Check the supplied plugin content against the signed contentDigest.
+     * @param {object} canonicalManifest - Signed manifest
+     * @param {object|null} content - `{ files, baseDir, contents }`
+     * @returns {boolean} true when the content digest was checked and matched
+     * @private
+     */
+    private _verifyContentDigest;
+    /**
+     * Verify Ed25519 signature
+     * @param {Buffer} data - Data that was signed
+     * @param {Buffer} signature - Signature bytes
+     * @param {crypto.KeyObject|Buffer|string} publicKey - Ed25519 public key
+     *   (raw hex, PEM, DER or KeyObject)
+     * @returns {Promise<boolean>} Verification result
+     */
+    _verifyEd25519Signature(data: Buffer, signature: Buffer, publicKey: crypto.KeyObject | Buffer | string): Promise<boolean>;
+    /**
+     * Canonicalize manifest for consistent signing
+     * @param {object} manifest - Plugin manifest
+     * @returns {object} Canonicalized manifest
+     */
+    _canonicalizeManifest(manifest: object): object;
+    /**
+     * Bytes that a signer must sign for a manifest (canonical JSON).
+     * @param {object} manifest - Plugin manifest
+     * @returns {Buffer}
+     */
+    getSigningPayload(manifest: object): Buffer;
+    /**
+     * Sort object keys recursively
+     * @param {any} obj - Object to sort
+     * @returns {any} Object with sorted keys
+     */
+    _sortObjectKeys(obj: any): any;
+    /**
+     * Hash manifest for audit trail
+     * @param {object} manifest - Canonicalized manifest
+     * @returns {string} SHA-256 hash
+     */
+    _hashManifest(manifest: object): string;
+    /**
+     * Default audit logger
+     * @param {object} result - Verification result
+     * @param {object} manifest - Plugin manifest
+     */
+    _defaultAuditLogger(result: object, manifest: object): void;
+    /**
+     * Audit verification attempt
+     * @param {object} result - Verification result
+     * @param {object} manifest - Plugin manifest
+     */
+    _auditVerification(result: object, manifest: object): void;
+    /**
+     * Add trusted signer
+     * @param {string} signerId - Signer identifier
+     * @param {string|Buffer|crypto.KeyObject} publicKey - Ed25519 public key:
+     *   64 hex characters (raw 32 bytes), PEM, DER (SPKI) or a KeyObject
+     * @param {string} name - Human-readable name
+     * @param {object} metadata - Additional metadata
+     */
+    addTrustedSigner(signerId: string, publicKey: string | Buffer | crypto.KeyObject, name: string, metadata?: object): void;
+    /**
+     * Remove trusted signer
+     * @param {string} signerId - Signer identifier
+     */
+    removeTrustedSigner(signerId: string): any;
+    /**
+     * List trusted signers
+     * @returns {Array} List of trusted signers
+     */
+    listTrustedSigners(): any[];
+}
+/**
+ * Create plugin signature verifier instance
+ * @param {object} options - Configuration options
+ * @returns {PluginSignatureVerifier} Verifier instance
+ */
+export function createPluginVerifier(options?: object): PluginSignatureVerifier;
+/**
+ * Compute the content digest of a plugin.
+ *
+ * The digest is `sha256-<hex>` of the lines `<relativePath>\0<sha256 of
+ * file>\n`, sorted by relative path. Paths are relative to `baseDir` and use
+ * forward slashes, so the digest is identical on Windows and POSIX.
+ *
+ * @param {Object} options
+ * @param {string[]} [options.files] - File paths (absolute or relative to baseDir)
+ * @param {string} [options.baseDir] - Plugin root; files must be inside it
+ * @param {Object<string, Buffer|string>} [options.contents] - In-memory
+ *   files keyed by relative path (alternative or addition to `files`)
+ * @returns {string} `sha256-<hex>`
+ */
+export function computeContentDigest({ files, baseDir, contents }?: {
+    files?: string[];
+    baseDir?: string;
+    contents?: {
+        [x: string]: Buffer | string;
+    };
+}): string;
+/**
+ * Convert supported public key encodings to an Ed25519 KeyObject.
+ *
+ * Accepted: a KeyObject; a 64-character hex string or 32-byte Buffer (raw
+ * key); a PEM string ("-----BEGIN PUBLIC KEY-----"); a DER (SPKI) Buffer or
+ * base64 string.
+ *
+ * @param {crypto.KeyObject|Buffer|string} publicKey
+ * @returns {crypto.KeyObject}
+ */
+export function toEd25519PublicKey(publicKey: crypto.KeyObject | Buffer | string): crypto.KeyObject;
+import crypto = require("crypto");

package/dist/types/security/validator.d.ts

@@ -0,0 +1,5 @@
+export function validateInput(input: any, opts?: {}): {
+    valid: boolean;
+    reasons: string[];
+    sanitized: string;
+};

package/dist/types/streaming/embedding-stream.d.ts

@@ -0,0 +1,65 @@
+/**
+ * Async generator that streams embedding results for a collection of documents.
+ *
+ * Documents are processed in batches of `batchSize`. Before each batch the
+ * generator checks heap memory; when usage exceeds
+ * `backpressureThreshold * maxMemoryMB` it pauses briefly and retries,
+ * giving the GC time to reclaim memory.
+ *
+ * Each yielded value is an object containing the document id, the computed
+ * vector, the document index, and a progress fraction (0-1).
+ *
+ * Errors within a batch are caught and yielded as objects with an `error`
+ * property so that consumers can decide how to handle partial failures
+ * without losing the rest of the stream.
+ *
+ * @param {Array<{id: string, content: string}>} documents - Documents to embed
+ * @param {Object} embedder - Embedder with `.embed()` or `.embedQuery()`
+ * @param {Object} [options] - Override any key from {@link DEFAULT_CONFIG}
+ * @yields {{ id: string, vector: number[], index: number, progress: number }
+ *          | { id: string, error: Error, index: number, progress: number }}
+ *
+ * @example
+ *   const docs = [{ id: '1', content: 'hello' }, { id: '2', content: 'world' }];
+ *   for await (const emb of createEmbeddingStream(docs, myEmbedder)) {
+ *     console.log(emb.id, emb.vector.length);
+ *   }
+ */
+export function createEmbeddingStream(documents: Array<{
+    id: string;
+    content: string;
+}>, embedder: any, options?: any): AsyncGenerator<{
+    backpressure?: boolean;
+    id: string;
+    vector: any;
+    index: number;
+    progress: number;
+    error?: undefined;
+} | {
+    id: string;
+    error: any;
+    index: number;
+    progress: number;
+}, void, unknown>;
+/**
+ * Streaming Embeddings Generator
+ *
+ * Async generator that yields embeddings chunk by chunk with backpressure
+ * support. Processes documents in configurable batches while monitoring
+ * memory usage to prevent OOM conditions in large-scale pipelines.
+ *
+ * Features:
+ * - Batch-based document embedding with configurable size
+ * - Memory-aware backpressure (pauses when heap exceeds threshold)
+ * - Per-batch error isolation (failures don't abort the stream)
+ * - Progress tracking via yielded metadata
+ * - Support for both `.embed()` and `.embedQuery()` embedder APIs
+ *
+ * @module streaming/embedding-stream
+ * @since 2.5.0
+ */
+/**
+ * Default configuration for the embedding stream
+ * @type {Object}
+ */
+export const DEFAULT_CONFIG: any;

package/dist/types/streaming/sse-adapter.d.ts

@@ -0,0 +1,59 @@
+/**
+ * Adapts an AsyncGenerator to the Server-Sent Events protocol.
+ *
+ * Writes well-formed SSE frames to an HTTP response, manages heartbeat
+ * keep-alive comments, and handles client disconnects gracefully.
+ *
+ * @extends EventEmitter
+ *
+ * @example
+ *   const adapter = new SSEAdapter({ heartbeatIntervalMs: 10000 });
+ *   adapter.on('token', (t) => console.log('sent', t));
+ *   await adapter.stream(asyncTokenGenerator, res);
+ */
+export class SSEAdapter extends EventEmitter<[never]> {
+    /**
+     * @param {Object} [options] - Configuration overrides
+     * @param {string} [options.eventName='token'] - SSE event name for token frames
+     * @param {string} [options.doneEvent='[DONE]'] - Sentinel value for the final data frame
+     * @param {number} [options.heartbeatIntervalMs=15000] - Heartbeat interval in milliseconds
+     */
+    constructor(options?: {
+        eventName?: string;
+        doneEvent?: string;
+        heartbeatIntervalMs?: number;
+    });
+    config: any;
+    /**
+     * Stream tokens from an AsyncGenerator to an HTTP response as SSE.
+     *
+     * Sets the required SSE headers, starts a heartbeat timer, and iterates
+     * over the generator. Each yielded value should be an object with at least
+     * a `token` property and an optional `done` boolean.
+     *
+     * @param {AsyncGenerator<{token: string, done?: boolean}>} asyncGenerator - Token source
+     * @param {import('http').ServerResponse} httpResponse - Writable HTTP response
+     * @returns {Promise<void>} Resolves when the stream ends or the client disconnects
+     */
+    stream(asyncGenerator: AsyncGenerator<{
+        token: string;
+        done?: boolean;
+    }>, httpResponse: import("http").ServerResponse): Promise<void>;
+    /**
+     * Format a single SSE event string.
+     *
+     * Multi-line data is split so each line is prefixed with `data: ` as
+     * required by the SSE specification.
+     *
+     * @param {string} eventName - The SSE event type
+     * @param {*} data - Payload (will be JSON-stringified)
+     * @returns {string} Fully formatted SSE frame including trailing blank line
+     */
+    formatEvent(eventName: string, data: any): string;
+}
+/**
+ * Default configuration for SSE streaming
+ * @type {Object}
+ */
+export const DEFAULT_CONFIG: any;
+import { EventEmitter } from "events";