@devilsdev/rag-pipeline-utils 2.4.5 → 2.6.0

package/dist/types/agentic/self-critique.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Answer verification against retrieved source documents.
+ * Uses CitationTracker (lazy-loaded) to compute groundedness and detect hallucinations.
+ *
+ * @extends EventEmitter
+ */
+export class SelfCritiqueChecker extends EventEmitter<[never]> {
+    /**
+     * @param {object} [options] - Override default configuration
+     * @param {number} [options.approvalThreshold=0.5] - Minimum groundedness score to approve an answer
+     */
+    constructor(options?: {
+        approvalThreshold?: number;
+    });
+    config: any;
+    /**
+     * Check an answer against retrieved documents for groundedness.
+     *
+     * @param {string} answer - The LLM-generated answer to verify
+     * @param {Array<object>} retrievedDocs - Source documents to check against
+     * @returns {{
+     *   approved: boolean,
+     *   score: number,
+     *   issues: Array<{sentence: string, classification: string}>,
+     *   citationResult: object
+     * }}
+     * @throws {Error} If answer or retrievedDocs are invalid
+     * @fires SelfCritiqueChecker#checked
+     */
+    check(answer: string, retrievedDocs: Array<object>): {
+        approved: boolean;
+        score: number;
+        issues: Array<{
+            sentence: string;
+            classification: string;
+        }>;
+        citationResult: object;
+    };
+}
+/**
+ * Default configuration for the SelfCritiqueChecker.
+ * @type {object}
+ */
+export const DEFAULT_CONFIG: object;
+import { EventEmitter } from "events";

package/dist/types/ai/adaptive-retrieval.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Initialize or update user profile
+ * @param {string} userId - User identifier
+ * @param {object} profile - Profile data
+ * @returns {Promise<object>} Created/updated profile
+ */
+export function initializeUserProfile(userId: string, profile?: object): Promise<object>;
+/**
+ * Record feedback for a query/item pair
+ * @param {string} userId - User identifier
+ * @param {object} feedback - Feedback object with query, results, ratings, etc.
+ * @returns {Promise<void>}
+ */
+export function recordFeedback(userId: string, feedback: object): Promise<void>;
+/**
+ * Perform adaptive retrieval with deterministic scoring
+ * @param {string} userId - User identifier
+ * @param {string} query - Search query
+ * @param {object} options - Retrieval options
+ * @returns {Promise<object>} Object with documents array and adaptationMetadata
+ */
+export function adaptiveRetrieve(userId: string, query: string, options?: object): Promise<object>;
+/**
+ * Get user profile
+ * @param {string} userId - User identifier
+ * @returns {Promise<object|null>} User profile or null if not found
+ */
+export function getUserProfile(userId: string): Promise<object | null>;
+/**
+ * Personalize ranking for a set of documents
+ * @param {string} userId - User ID
+ * @param {array} documents - Array of documents to personalize
+ * @returns {array} Documents with personalized scores and ranking factors
+ */
+export function personalizeRanking(userId: string, documents: any[]): any[];
+/**
+ * Get statistics about users and feedback
+ * @returns {object} Stats object with users and interactions
+ */
+export function getStats(): object;
+/**
+ * Clear all stored data
+ */
+export function clear(): void;

package/dist/types/ai/federation/federated-learning-coordinator.d.ts

@@ -0,0 +1,62 @@
+export class FederatedLearningCoordinator extends EventEmitter<[never]> {
+    constructor(options?: {});
+    config: {
+        minParticipants: any;
+        maxParticipants: any;
+        roundDuration: any;
+        convergenceThreshold: any;
+        maxRounds: any;
+    };
+    federations: Map<any, any>;
+    participants: Map<any, any>;
+    globalModels: Map<any, any>;
+    createFederation(tenantId: any, modelConfig: any, federationConfig?: {}): Promise<`${string}-${string}-${string}-${string}-${string}`>;
+    registerParticipant(federationId: any, participantInfo: any): Promise<`${string}-${string}-${string}-${string}-${string}`>;
+    startFederatedTraining(federationId: any): Promise<{
+        federationId: any;
+        status: any;
+        rounds: any;
+        participants: any;
+    }>;
+    getFederationStatus(federationId: any): Promise<{
+        id: any;
+        status: any;
+        currentRound: any;
+        participantCount: any;
+        convergenceHistory: any;
+        createdAt: any;
+        startedAt: any;
+        completedAt: any;
+    }>;
+    getParticipantStatus(participantId: any): Promise<any>;
+    updateParticipantModel(participantId: any, modelUpdate: any): Promise<{
+        participantId: any;
+        updateAccepted: boolean;
+        newPerformance: any;
+    }>;
+    aggregateModels(federationId: any, participantUpdates: any): Promise<{
+        id: `${string}-${string}-${string}-${string}-${string}`;
+        federationId: any;
+        round: any;
+        participantCount: any;
+        aggregationMethod: string;
+        weights: number[];
+        performance: {
+            accuracy: number;
+            loss: number;
+            convergence: number;
+        };
+        createdAt: number;
+    }>;
+    _executeTrainingRound(federation: any, roundNumber: any): Promise<{
+        round: any;
+        accuracy: number;
+        loss: number;
+        convergence: number;
+        participantCount: any;
+        timestamp: number;
+    }>;
+    getGlobalModel(federationId: any): Promise<any>;
+    removeParticipant(participantId: any): Promise<boolean>;
+}
+import { EventEmitter } from "events";

package/dist/types/ai/multimodal-processing.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Process multimodal content and store with embedding
+ * @param {string} tenantId - Tenant identifier
+ * @param {object} content - Content object with type and data
+ * @returns {Promise<object>} Processed content record with ID, modalities, metadata
+ */
+export function processContent(tenantId: string, content: object): Promise<object>;
+/**
+ * Generate description for content by ID
+ * @param {string} contentId - Content ID
+ * @returns {Promise<object>} Object with modality-specific descriptions and unified description
+ */
+export function describeContent(contentId: string): Promise<object>;
+/**
+ * Search multimodal content using query object
+ * @param {string} tenantId - Tenant identifier
+ * @param {object} queryObj - Query object
+ * @param {object} options - Search options
+ * @returns {Promise<object>} Object with results array and metadata
+ */
+export function search(tenantId: string, queryObj: object, options?: object): Promise<object>;
+/**
+ * Find similar content to given ID
+ * @param {string} contentId - Source content ID
+ * @param {object} options - Options with threshold
+ * @returns {Promise<Array>} Array of similar items with id and score
+ */
+export function findSimilar(contentId: string, options?: object): Promise<any[]>;
+/**
+ * Clear all stored content (for testing)
+ */
+export function clear(): void;

package/dist/types/ai/multimodal.d.ts

@@ -0,0 +1,39 @@
+/**
+ * MultiModalProcessor class
+ * Processes and integrates multiple content modalities
+ */
+export class MultiModalProcessor {
+    supportedTypes: Set<string>;
+    processors: Map<any, any>;
+    embeddings: Map<any, any>;
+    /**
+     * Process multimodal content
+     * @param {string} tenantId - Tenant identifier
+     * @param {Array} content - Array of content objects with type and data
+     * @param {object} options - Processing options
+     * @returns {Promise<object>} Processing results
+     */
+    processContent(tenantId: string, content: any[], options?: object): Promise<object>;
+    /**
+     * Generate embeddings for multimodal content
+     * @param {string} tenantId - Tenant identifier
+     * @param {object} content - Content to embed
+     * @returns {Promise<Array>} Generated embeddings
+     */
+    generateEmbeddings(tenantId: string, content: object): Promise<any[]>;
+    /**
+     * Align embeddings across modalities
+     * @param {string} tenantId - Tenant identifier
+     * @param {Array} embeddings - Multi-modal embeddings
+     * @returns {Promise<Array>} Aligned embeddings
+     */
+    alignEmbeddings(tenantId: string, embeddings: any[]): Promise<any[]>;
+    /**
+     * Perform cross-modal search
+     * @param {string} tenantId - Tenant identifier
+     * @param {object} query - Query object with modality
+     * @param {object} options - Search options
+     * @returns {Promise<Array>} Search results
+     */
+    crossModalSearch(tenantId: string, query: object, options?: object): Promise<any[]>;
+}

package/dist/types/ai/retrieval-engine.d.ts

@@ -0,0 +1,37 @@
+/**
+ * AdaptiveRetrievalEngine class
+ * Manages intelligent document retrieval with performance optimization
+ */
+export class AdaptiveRetrievalEngine {
+    algorithms: Map<any, any>;
+    performanceMetrics: Map<any, any>;
+    adaptationThreshold: number;
+    /**
+     * Process retrieval query with adaptive algorithms
+     * @param {string} tenantId - Tenant identifier
+     * @param {string} query - Search query
+     * @param {object} options - Retrieval options
+     * @returns {Promise<object>} Retrieval results
+     */
+    retrieveDocuments(tenantId: string, query: string, options?: object): Promise<object>;
+    /**
+     * Optimize retrieval algorithm based on performance data
+     * @param {string} tenantId - Tenant identifier
+     * @param {object} performanceData - Performance metrics
+     * @returns {Promise<object>} Optimization results
+     */
+    optimizeRetrieval(tenantId: string, performanceData: object): Promise<object>;
+    /**
+     * Get current algorithm configuration
+     * @param {string} tenantId - Tenant identifier
+     * @returns {Promise<object>} Current algorithm configuration
+     */
+    getAlgorithmConfig(tenantId: string): Promise<object>;
+    /**
+     * Update retrieval strategy
+     * @param {string} tenantId - Tenant identifier
+     * @param {string} strategy - New retrieval strategy
+     * @returns {Promise<boolean>} Success status
+     */
+    updateStrategy(tenantId: string, strategy: string): Promise<boolean>;
+}

package/dist/types/ai/training/model-training-orchestrator.d.ts

@@ -0,0 +1,31 @@
+export class ModelTrainingOrchestrator extends EventEmitter<[never]> {
+    constructor(options?: {});
+    config: {
+        batchSize: any;
+        learningRate: any;
+        epochs: any;
+    };
+    trainingJobs: Map<any, any>;
+    deployments: Map<any, any>;
+    optimizations: Map<any, any>;
+    createTrainingJob(tenantId: any, config?: {}): Promise<`${string}-${string}-${string}-${string}-${string}`>;
+    startTraining(jobId: any, data?: any, config?: {}): Promise<{
+        jobId: any;
+        status: string;
+    }>;
+    getTrainingStatus(jobId: any): any;
+    stopTraining(jobId: any): Promise<boolean>;
+    deployModel(jobId: any, deploymentConfig?: {}): Promise<`${string}-${string}-${string}-${string}-${string}`>;
+    getDeploymentStatus(deploymentId: any): Promise<any>;
+    optimizeHyperparameters(tenantId: any, optimizationConfig: any): Promise<`${string}-${string}-${string}-${string}-${string}`>;
+    getOptimizationResults(optimizationId: any): Promise<{
+        id: any;
+        status: any;
+        bestConfiguration: any;
+        bestScore: any;
+        trials: any;
+        startedAt: any;
+        completedAt: any;
+    }>;
+}
+import { EventEmitter } from "events";

package/dist/types/cache/component-keys.d.ts

@@ -0,0 +1,7 @@
+/**
+ * @param {string} kind - e.g. 'openai'
+ * @param {Object} fields - Settings that change the output
+ * @returns {string}
+ */
+export function componentCacheKey(kind: string, fields: any): string;
+export function digest(value: any): string;

package/dist/types/cache/memory-cache.d.ts

@@ -0,0 +1,54 @@
+/**
+ * In-process cache with least-recently-used eviction and per-entry time to
+ * live. Implements the CacheBackend contract:
+ *
+ *   get(key) -> Promise<value | undefined>
+ *   set(key, value, { ttlSeconds }) -> Promise<void>
+ *   delete(key) -> Promise<void>
+ *   stats() -> Promise<{ hits, misses, size }>
+ *
+ * Values are stored as JSON copies, so later changes to a returned value do
+ * not change the cache (the same behaviour as RedisCache).
+ */
+export class MemoryCache {
+    /**
+     * @param {Object} [options]
+     * @param {number} [options.maxEntries=1000] - Entries kept before the least
+     *   recently used is evicted
+     * @param {number} [options.ttlSeconds] - Default time to live; none means
+     *   entries never expire on their own
+     */
+    constructor({ maxEntries, ttlSeconds }?: {
+        maxEntries?: number;
+        ttlSeconds?: number;
+    });
+    maxEntries: number;
+    ttlSeconds: number;
+    _entries: Map<any, any>;
+    _hits: number;
+    _misses: number;
+    /**
+     * @param {string} key
+     * @returns {Promise<unknown|undefined>}
+     */
+    get(key: string): Promise<unknown | undefined>;
+    /**
+     * @param {string} key
+     * @param {unknown} value - JSON-serialisable value
+     * @param {{ ttlSeconds?: number }} [options]
+     * @returns {Promise<void>}
+     */
+    set(key: string, value: unknown, { ttlSeconds }?: {
+        ttlSeconds?: number;
+    }): Promise<void>;
+    /** @param {string} key */
+    delete(key: string): Promise<void>;
+    /** Remove every entry. */
+    clear(): Promise<void>;
+    /** @returns {Promise<{ hits: number, misses: number, size: number }>} */
+    stats(): Promise<{
+        hits: number;
+        misses: number;
+        size: number;
+    }>;
+}

package/dist/types/cache/pipeline-cache.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Normalise the `cache` option of createRagPipeline.
+ *
+ * - `true`: all three layers in one in-memory cache
+ * - a CacheBackend: all three layers in it
+ * - `{ embeddings, retrieval, generation, ttlSeconds }`: per layer, each
+ *   `true` (in-memory), `false`/absent (off) or a CacheBackend;
+ *   `ttlSeconds: { embeddings, retrieval, generation }` overrides the
+ *   defaults (30 days, 1 hour, 24 hours)
+ *
+ * @returns {Object|null} Layer config, or null when caching is off
+ */
+export function normalizeCacheOption(option: any): any | null;
+/**
+ * Per-pipeline cache. Every method returns undefined and does nothing for a
+ * layer that is off or whose component has no cache key.
+ */
+export class PipelineCache {
+    /**
+     * @param {Object} layers - normalizeCacheOption() output
+     * @param {{ embedder?: Object, retriever?: Object, llm?: Object }} components
+     */
+    constructor(layers: any, components: {
+        embedder?: any;
+        retriever?: any;
+        llm?: any;
+    });
+    layers: any;
+    components: {
+        embedder?: any;
+        retriever?: any;
+        llm?: any;
+    };
+    /** @private */
+    private _key;
+    /** @private */
+    private _get;
+    /** @private */
+    private _set;
+    embeddingKey(query: any): string;
+    retrievalKey({ query, queryVector, topK }: {
+        query: any;
+        queryVector: any;
+        topK: any;
+    }): string;
+    /**
+     * @returns {string|null} The key, or null to skip (temperature above 0,
+     *   or no llm cache key)
+     */
+    generationKey({ query, docs, options }: {
+        query: any;
+        docs: any;
+        options: any;
+    }): string | null;
+    get(layer: any, key: any, status: any): Promise<any>;
+    set(layer: any, key: any, value: any): Promise<void>;
+    /** Hits, misses and skips per layer since the pipeline was created. */
+    stats(): {};
+}
+/** A component's identity for a role, or null when it has none. */
+export function componentKey(component: any, role: any): any;
+export namespace DEFAULT_TTL_SECONDS {
+    let embeddings: number;
+    let retrieval: number;
+    let generation: number;
+}

package/dist/types/cache/redis-cache.d.ts

@@ -0,0 +1,46 @@
+/**
+ * CacheBackend on Redis. Pass a connected client from `redis` (node-redis 4
+ * and later) or `ioredis`; neither is a dependency of this package.
+ *
+ * Values are stored as JSON under `prefix + key`, with a millisecond expiry
+ * when a time to live applies.
+ */
+export class RedisCache {
+    /**
+     * @param {Object} options
+     * @param {Object} options.client - Connected node-redis or ioredis client
+     * @param {string} [options.prefix='rag:'] - Key prefix
+     * @param {number} [options.ttlSeconds] - Default time to live
+     */
+    constructor({ client, prefix, ttlSeconds }?: {
+        client: any;
+        prefix?: string;
+        ttlSeconds?: number;
+    });
+    client: any;
+    prefix: string;
+    ttlSeconds: number;
+    _nodeRedis: boolean;
+    _hits: number;
+    _misses: number;
+    /**
+     * @param {string} key
+     * @returns {Promise<unknown|undefined>}
+     */
+    get(key: string): Promise<unknown | undefined>;
+    /**
+     * @param {string} key
+     * @param {unknown} value - JSON-serialisable value
+     * @param {{ ttlSeconds?: number }} [options]
+     */
+    set(key: string, value: unknown, { ttlSeconds }?: {
+        ttlSeconds?: number;
+    }): Promise<void>;
+    /** @param {string} key */
+    delete(key: string): Promise<void>;
+    /** Hits and misses seen by this instance (size is not tracked). */
+    stats(): Promise<{
+        hits: number;
+        misses: number;
+    }>;
+}

package/dist/types/chunking/chunking-engine.d.ts

@@ -0,0 +1,70 @@
+/**
+ * ChunkingEngine orchestrates text chunking using pluggable strategies.
+ * Extends EventEmitter to emit 'chunk' events after each chunking operation.
+ *
+ * @extends EventEmitter
+ */
+export class ChunkingEngine extends EventEmitter<[never]> {
+    /**
+     * Create a ChunkingEngine instance.
+     *
+     * @param {object} [options={}] - Configuration options merged with DEFAULT_CONFIG.
+     * @param {string} [options.strategy='recursive'] - Default strategy name.
+     * @param {number} [options.chunkSize=512] - Default chunk size.
+     * @param {number} [options.chunkOverlap=50] - Default chunk overlap.
+     * @param {number} [options.minChunkSize=50] - Default minimum chunk size.
+     * @param {string[]} [options.separators] - Default separators for recursive strategy.
+     * @param {number} [options.similarityThreshold=0.3] - Default similarity threshold for semantic strategy.
+     */
+    constructor(options?: {
+        strategy?: string;
+        chunkSize?: number;
+        chunkOverlap?: number;
+        minChunkSize?: number;
+        separators?: string[];
+        similarityThreshold?: number;
+    });
+    _config: any;
+    _strategies: Map<any, any>;
+    /**
+     * Chunk text using the configured or specified strategy.
+     *
+     * @param {string} text - The input text to chunk. Must be a non-empty string.
+     * @param {object} [options={}] - Per-call options that override the engine config.
+     * @param {string} [options.strategy] - Strategy name to use for this call.
+     * @returns {string[]} Array of text chunks.
+     * @throws {Error} If text is not a non-empty string.
+     * @throws {Error} If the specified strategy is not registered.
+     * @fires ChunkingEngine#chunk
+     */
+    chunk(text: string, options?: {
+        strategy?: string;
+    }): string[];
+    /**
+     * Register a custom chunking strategy.
+     *
+     * @param {string} name - Strategy name (used to select it via options.strategy).
+     * @param {function} fn - Strategy function with signature (text, options) => string[].
+     * @throws {Error} If name is not a non-empty string.
+     * @throws {Error} If fn is not a function.
+     */
+    registerStrategy(name: string, fn: Function): void;
+    /**
+     * List all registered strategy names.
+     *
+     * @returns {string[]} Array of strategy names.
+     */
+    listStrategies(): string[];
+    /**
+     * Get the current engine configuration.
+     *
+     * @returns {object} A copy of the current configuration.
+     */
+    getConfig(): object;
+}
+/**
+ * Default configuration for the chunking engine.
+ * @type {object}
+ */
+export const DEFAULT_CONFIG: object;
+import { EventEmitter } from "events";

package/dist/types/chunking/strategies/fixed-size.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Fixed-size chunking strategy.
+ * Splits text by character count with configurable overlap.
+ * Attempts to break at word boundaries by looking back up to 20 characters.
+ *
+ * @param {string} text - The input text to chunk.
+ * @param {object} [options={}] - Chunking options.
+ * @param {number} [options.chunkSize=512] - Target chunk size in characters.
+ * @param {number} [options.chunkOverlap=50] - Number of overlapping characters between chunks.
+ * @param {number} [options.minChunkSize=50] - Minimum chunk size; smaller chunks are filtered out.
+ * @returns {string[]} Array of text chunks.
+ */
+export function fixedSizeChunk(text: string, options?: {
+    chunkSize?: number;
+    chunkOverlap?: number;
+    minChunkSize?: number;
+}): string[];

package/dist/types/chunking/strategies/recursive.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Recursive text chunking strategy.
+ * Splits text using a hierarchy of separators, merging consecutive pieces
+ * into chunks up to chunkSize. Pieces exceeding chunkSize are recursively
+ * split with the next separator in the hierarchy.
+ *
+ * @param {string} text - The input text to chunk.
+ * @param {object} [options={}] - Chunking options.
+ * @param {number} [options.chunkSize=512] - Maximum chunk size in characters.
+ * @param {number} [options.chunkOverlap=50] - Number of overlapping characters between chunks.
+ * @param {string[]} [options.separators] - Separator hierarchy for splitting.
+ * @param {number} [options.minChunkSize=50] - Minimum chunk size; smaller chunks are filtered out.
+ * @returns {string[]} Array of text chunks.
+ */
+export function recursiveChunk(text: string, options?: {
+    chunkSize?: number;
+    chunkOverlap?: number;
+    separators?: string[];
+    minChunkSize?: number;
+}): string[];

package/dist/types/chunking/strategies/semantic.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Semantic chunking strategy using topic-boundary detection.
+ * Splits text into sentences, computes Jaccard token overlap between
+ * consecutive sentence pairs, and inserts chunk boundaries where
+ * similarity drops below the threshold. Oversized segments fall back
+ * to recursive splitting.
+ *
+ * @param {string} text - The input text to chunk.
+ * @param {object} [options={}] - Chunking options.
+ * @param {number} [options.chunkSize=512] - Maximum chunk size in characters.
+ * @param {number} [options.similarityThreshold=0.3] - Minimum similarity to keep sentences in the same chunk.
+ * @param {number} [options.minChunkSize=50] - Minimum chunk size; smaller chunks are filtered out.
+ * @returns {string[]} Array of text chunks.
+ */
+export function semanticChunk(text: string, options?: {
+    chunkSize?: number;
+    similarityThreshold?: number;
+    minChunkSize?: number;
+}): string[];

package/dist/types/chunking/strategies/sentence.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Sentence-based chunking strategy.
+ * Extracted from MarkdownLoader._chunkText for backward compatibility.
+ * Splits text on sentence boundaries and merges into chunks up to maxLen.
+ *
+ * @param {string} text - The input text to chunk.
+ * @param {object} [options={}] - Chunking options.
+ * @param {number} [options.maxLen=500] - Maximum character length per chunk.
+ * @returns {string[]} Array of text chunks.
+ */
+export function sentenceChunk(text: string, options?: {
+    maxLen?: number;
+}): string[];

package/dist/types/chunking/strategies/structure-aware.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Structure-aware chunking strategy.
+ * Detects document format (Markdown, HTML, or plain) and respects
+ * structural boundaries. For Markdown, splits on headers and preserves
+ * code blocks. For HTML, splits on heading/section tags. Oversized
+ * sections are further split via the recursive strategy.
+ *
+ * @param {string} text - The input text to chunk.
+ * @param {object} [options={}] - Chunking options.
+ * @param {number} [options.chunkSize=512] - Maximum chunk size in characters.
+ * @param {number} [options.chunkOverlap=50] - Number of overlapping characters between chunks.
+ * @param {string} [options.format] - Document format: 'markdown', 'html', 'plain', or auto-detect.
+ * @returns {string[]} Array of text chunks.
+ */
+export function structureAwareChunk(text: string, options?: {
+    chunkSize?: number;
+    chunkOverlap?: number;
+    format?: string;
+}): string[];