@juspay/neurolink 9.1.1 → 9.3.0

package/dist/lib/rag/ChunkerRegistry.js

@@ -0,0 +1,422 @@
+/**
+ * Chunker Registry
+ *
+ * Centralized registry for all chunking strategies with metadata
+ * and discovery capabilities. Follows the BaseRegistry pattern.
+ */
+import { BaseRegistry } from "../core/infrastructure/index.js";
+import { logger } from "../utils/logger.js";
+import { ChunkingError, RAGErrorCodes } from "./errors/RAGError.js";
+/**
+ * Default chunker metadata entries
+ */
+const DEFAULT_CHUNKER_METADATA = {
+    character: {
+        description: "Splits text into fixed-size character chunks with optional overlap",
+        defaultConfig: {
+            maxSize: 1000,
+            overlap: 100,
+        },
+        supportedOptions: ["maxSize", "overlap", "minSize"],
+        useCases: [
+            "Simple text processing",
+            "Fixed-size chunks needed",
+            "Language-agnostic splitting",
+        ],
+        aliases: ["char", "fixed-size", "fixed"],
+    },
+    recursive: {
+        description: "Recursively splits text using ordered separators (paragraphs, sentences, etc.)",
+        defaultConfig: {
+            maxSize: 1000,
+            overlap: 100,
+            separators: ["\n\n", "\n", ". ", " ", ""],
+        },
+        supportedOptions: [
+            "maxSize",
+            "overlap",
+            "separators",
+            "keepSeparators",
+            "minSize",
+        ],
+        useCases: [
+            "General text documents",
+            "Preserving semantic boundaries",
+            "Default choice for most use cases",
+        ],
+        aliases: ["recursive-character", "langchain-default"],
+    },
+    sentence: {
+        description: "Splits text by sentence boundaries for semantically meaningful chunks",
+        defaultConfig: {
+            maxSize: 1000,
+            overlap: 1,
+        },
+        supportedOptions: [
+            "maxSize",
+            "overlap",
+            "boundaryDetection",
+            "maxSentences",
+        ],
+        useCases: [
+            "Q&A applications",
+            "Sentence-level analysis",
+            "Preserving complete thoughts",
+        ],
+        aliases: ["sent", "sentence-based"],
+    },
+    token: {
+        description: "Splits text by token count using a specific tokenizer (GPT, Claude, etc.)",
+        defaultConfig: {
+            maxSize: 512,
+            overlap: 50,
+        },
+        supportedOptions: ["maxSize", "overlap", "tokenizer", "maxTokens"],
+        useCases: [
+            "Token-aware splitting",
+            "Optimal for specific models",
+            "Precise token budget management",
+        ],
+        aliases: ["tok", "tokenized"],
+    },
+    markdown: {
+        description: "Splits markdown content by headers and structural elements",
+        defaultConfig: {
+            maxSize: 1000,
+            overlap: 0,
+        },
+        supportedOptions: [
+            "maxSize",
+            "overlap",
+            "headerLevels",
+            "splitCodeBlocks",
+            "preserveMetadata",
+        ],
+        useCases: [
+            "Documentation processing",
+            "README files",
+            "Technical documentation",
+        ],
+        aliases: ["md", "markdown-header"],
+    },
+    html: {
+        description: "Splits HTML content by semantic tags while optionally stripping markup",
+        defaultConfig: {
+            maxSize: 1000,
+            overlap: 0,
+        },
+        supportedOptions: [
+            "maxSize",
+            "overlap",
+            "splitTags",
+            "stripTags",
+            "preserveAttributes",
+        ],
+        useCases: ["Web content processing", "HTML documents", "Web scraping"],
+        aliases: ["html-tag", "web"],
+    },
+    json: {
+        description: "Splits JSON documents by object boundaries and nested structures",
+        defaultConfig: {
+            maxSize: 1000,
+            overlap: 0,
+        },
+        supportedOptions: ["maxSize", "overlap", "maxDepth", "chunkKeys"],
+        useCases: [
+            "API response processing",
+            "Structured data",
+            "Configuration files",
+        ],
+        aliases: ["json-object", "structured"],
+    },
+    latex: {
+        description: "Splits LaTeX documents by sections, environments, and math blocks",
+        defaultConfig: {
+            maxSize: 1000,
+            overlap: 0,
+        },
+        supportedOptions: [
+            "maxSize",
+            "overlap",
+            "environments",
+            "splitMathBlocks",
+            "preserveMetadata",
+        ],
+        useCases: [
+            "Academic papers",
+            "Scientific documents",
+            "Mathematical content",
+        ],
+        aliases: ["tex", "latex-section"],
+    },
+    semantic: {
+        description: "Uses LLM to identify semantically meaningful split points",
+        defaultConfig: {
+            maxSize: 1000,
+            overlap: 100,
+        },
+        supportedOptions: [
+            "maxSize",
+            "overlap",
+            "modelName",
+            "provider",
+            "similarityThreshold",
+        ],
+        useCases: [
+            "Advanced semantic understanding",
+            "Context-aware splitting",
+            "AI-enhanced chunking",
+        ],
+        aliases: ["llm", "ai-semantic"],
+    },
+    "semantic-markdown": {
+        description: "Combines markdown splitting with semantic similarity for intelligent merging",
+        defaultConfig: {
+            maxSize: 1000,
+            overlap: 100,
+        },
+        supportedOptions: [
+            "maxSize",
+            "overlap",
+            "similarityThreshold",
+            "maxMergeSize",
+            "preserveMetadata",
+        ],
+        useCases: [
+            "Context-aware documentation",
+            "Knowledge base creation",
+            "Semantic search preparation",
+        ],
+        aliases: ["semantic-md", "smart-markdown"],
+    },
+};
+/**
+ * Chunker Registry
+ *
+ * Manages registration and discovery of all chunking strategies.
+ * Extends BaseRegistry for consistent lifecycle management.
+ */
+export class ChunkerRegistry extends BaseRegistry {
+    static instance = null;
+    aliasMap = new Map();
+    constructor() {
+        super();
+    }
+    /**
+     * Get singleton instance
+     */
+    static getInstance() {
+        if (!ChunkerRegistry.instance) {
+            ChunkerRegistry.instance = new ChunkerRegistry();
+        }
+        return ChunkerRegistry.instance;
+    }
+    /**
+     * Reset singleton (for testing)
+     */
+    static resetInstance() {
+        if (ChunkerRegistry.instance) {
+            ChunkerRegistry.instance.clear();
+            ChunkerRegistry.instance = null;
+        }
+    }
+    /**
+     * Register all default chunkers
+     */
+    async registerAll() {
+        // Register character chunker
+        this.registerChunker("character", async () => {
+            const { CharacterChunker } = await import("./chunkers/CharacterChunker.js");
+            return new CharacterChunker();
+        }, DEFAULT_CHUNKER_METADATA.character);
+        // Register recursive chunker
+        this.registerChunker("recursive", async () => {
+            const { RecursiveChunker } = await import("./chunkers/RecursiveChunker.js");
+            return new RecursiveChunker();
+        }, DEFAULT_CHUNKER_METADATA.recursive);
+        // Register sentence chunker
+        this.registerChunker("sentence", async () => {
+            const { SentenceChunker } = await import("./chunkers/SentenceChunker.js");
+            return new SentenceChunker();
+        }, DEFAULT_CHUNKER_METADATA.sentence);
+        // Register token chunker
+        this.registerChunker("token", async () => {
+            const { TokenChunker } = await import("./chunkers/TokenChunker.js");
+            return new TokenChunker();
+        }, DEFAULT_CHUNKER_METADATA.token);
+        // Register markdown chunker
+        this.registerChunker("markdown", async () => {
+            const { MarkdownChunker } = await import("./chunkers/MarkdownChunker.js");
+            return new MarkdownChunker();
+        }, DEFAULT_CHUNKER_METADATA.markdown);
+        // Register HTML chunker
+        this.registerChunker("html", async () => {
+            const { HTMLChunker } = await import("./chunkers/HTMLChunker.js");
+            return new HTMLChunker();
+        }, DEFAULT_CHUNKER_METADATA.html);
+        // Register JSON chunker
+        this.registerChunker("json", async () => {
+            const { JSONChunker } = await import("./chunkers/JSONChunker.js");
+            return new JSONChunker();
+        }, DEFAULT_CHUNKER_METADATA.json);
+        // Register LaTeX chunker
+        this.registerChunker("latex", async () => {
+            const { LaTeXChunker } = await import("./chunkers/LaTeXChunker.js");
+            return new LaTeXChunker();
+        }, DEFAULT_CHUNKER_METADATA.latex);
+        // Register semantic chunker
+        this.registerChunker("semantic", async () => {
+            const { SemanticChunker } = await import("./chunking/semanticChunker.js");
+            return new SemanticChunker();
+        }, DEFAULT_CHUNKER_METADATA.semantic);
+        // Register semantic-markdown chunker
+        this.registerChunker("semantic-markdown", async () => {
+            const { SemanticMarkdownChunker } = await import("./chunkers/SemanticMarkdownChunker.js");
+            return new SemanticMarkdownChunker();
+        }, DEFAULT_CHUNKER_METADATA["semantic-markdown"]);
+        logger.debug(`[ChunkerRegistry] Registered ${this.items.size} chunking strategies`);
+    }
+    /**
+     * Register a chunker with aliases
+     */
+    registerChunker(strategy, factory, metadata) {
+        this.register(strategy, factory, metadata);
+        // Register aliases
+        if (metadata.aliases) {
+            for (const alias of metadata.aliases) {
+                this.aliasMap.set(alias.toLowerCase(), strategy);
+                logger.debug(`[ChunkerRegistry] Registered alias '${alias}' -> '${strategy}'`);
+            }
+        }
+    }
+    /**
+     * Resolve strategy name from alias
+     */
+    resolveStrategy(nameOrAlias) {
+        const lower = nameOrAlias.toLowerCase();
+        // Check if it's a direct strategy name
+        if (this.items.has(lower)) {
+            return lower;
+        }
+        // Check aliases
+        const resolved = this.aliasMap.get(lower);
+        if (resolved) {
+            return resolved;
+        }
+        throw new ChunkingError(`Unknown chunking strategy: '${nameOrAlias}'. Available strategies: ${this.list()
+            .map((item) => item.id)
+            .join(", ")}`, {
+            code: RAGErrorCodes.CHUNKING_STRATEGY_NOT_FOUND,
+            details: {
+                requestedStrategy: nameOrAlias,
+                availableStrategies: this.list().map((item) => item.id),
+            },
+        });
+    }
+    /**
+     * Get a chunker by strategy name or alias
+     */
+    async getChunker(strategyOrAlias) {
+        await this.ensureInitialized();
+        const strategy = this.resolveStrategy(strategyOrAlias);
+        const chunker = await this.get(strategy);
+        if (!chunker) {
+            throw new ChunkingError(`Chunker not found: ${strategy}`, {
+                code: RAGErrorCodes.CHUNKING_STRATEGY_NOT_FOUND,
+                details: { strategy },
+            });
+        }
+        return chunker;
+    }
+    /**
+     * Get list of available chunker strategies
+     */
+    async getAvailableChunkers() {
+        await this.ensureInitialized();
+        return this.list().map((item) => item.id);
+    }
+    /**
+     * Get metadata for a specific chunker
+     */
+    getChunkerMetadata(strategyOrAlias) {
+        const strategy = this.resolveStrategy(strategyOrAlias);
+        const entry = this.list().find((item) => item.id === strategy);
+        return entry?.metadata;
+    }
+    /**
+     * Get all aliases for a strategy
+     */
+    getAliasesForStrategy(strategy) {
+        const metadata = DEFAULT_CHUNKER_METADATA[strategy];
+        return metadata?.aliases ?? [];
+    }
+    /**
+     * Get all registered aliases
+     */
+    getAllAliases() {
+        return new Map(this.aliasMap);
+    }
+    /**
+     * Check if a strategy or alias exists
+     */
+    hasChunker(strategyOrAlias) {
+        try {
+            this.resolveStrategy(strategyOrAlias);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Get chunkers by use case
+     */
+    getChunkersByUseCase(useCase) {
+        const matches = [];
+        const useCaseLower = useCase.toLowerCase();
+        for (const [strategy, metadata] of Object.entries(DEFAULT_CHUNKER_METADATA)) {
+            const hasMatchingUseCase = metadata.useCases?.some((uc) => uc.toLowerCase().includes(useCaseLower)) ?? false;
+            if (hasMatchingUseCase) {
+                matches.push(strategy);
+            }
+        }
+        return matches;
+    }
+    /**
+     * Get default configuration for a chunker
+     */
+    getDefaultConfig(strategyOrAlias) {
+        const metadata = this.getChunkerMetadata(strategyOrAlias);
+        return metadata?.defaultConfig;
+    }
+    /**
+     * Clear the registry (also clears aliases)
+     */
+    clear() {
+        super.clear();
+        this.aliasMap.clear();
+    }
+}
+/**
+ * Global chunker registry singleton
+ */
+export const chunkerRegistry = ChunkerRegistry.getInstance();
+/**
+ * Convenience function to get available chunkers
+ */
+export async function getAvailableChunkers() {
+    return chunkerRegistry.getAvailableChunkers();
+}
+/**
+ * Convenience function to get chunker by strategy
+ */
+export async function getChunker(strategyOrAlias) {
+    return chunkerRegistry.getChunker(strategyOrAlias);
+}
+/**
+ * Convenience function to get chunker metadata
+ */
+export function getChunkerMetadata(strategyOrAlias) {
+    return chunkerRegistry.getChunkerMetadata(strategyOrAlias);
+}
+//# sourceMappingURL=ChunkerRegistry.js.map

package/dist/lib/rag/chunkers/BaseChunker.d.ts

@@ -0,0 +1,53 @@
+/**
+ * Base Chunker
+ *
+ * Abstract base class for all chunker implementations.
+ * Provides common functionality and interface contract.
+ */
+import type { Chunk, Chunker, ChunkerConfig, ChunkingStrategy } from "../types.js";
+/**
+ * Default chunker configuration
+ */
+export declare const DEFAULT_CHUNKER_CONFIG: ChunkerConfig;
+/**
+ * Base Chunker abstract class
+ *
+ * All chunker implementations should extend this class.
+ */
+export declare abstract class BaseChunker implements Chunker {
+    abstract readonly strategy: ChunkingStrategy;
+    protected config: ChunkerConfig;
+    constructor(config?: ChunkerConfig);
+    /**
+     * Get default configuration for this chunker
+     */
+    getDefaultConfig(): ChunkerConfig;
+    /**
+     * Validate chunker configuration
+     */
+    protected validateConfig(): void;
+    /**
+     * Chunk content into smaller pieces
+     */
+    chunk(content: string, config?: ChunkerConfig): Promise<Chunk[]>;
+    /**
+     * Perform the actual chunking (to be implemented by subclasses)
+     */
+    protected abstract doChunk(content: string, config: ChunkerConfig): Promise<Chunk[]>;
+    /**
+     * Filter chunks based on minimum size
+     */
+    protected filterChunks(chunks: Chunk[], config: ChunkerConfig): Chunk[];
+    /**
+     * Create a chunk object
+     */
+    protected createChunk(text: string, chunkIndex: number, startPosition: number, endPosition: number, documentId?: string, customMetadata?: Record<string, unknown>): Chunk;
+    /**
+     * Split content by size with overlap
+     */
+    protected splitBySizeWithOverlap(content: string, maxSize: number, overlap: number): Array<{
+        text: string;
+        start: number;
+        end: number;
+    }>;
+}

package/dist/lib/rag/chunkers/BaseChunker.js

@@ -0,0 +1,144 @@
+/**
+ * Base Chunker
+ *
+ * Abstract base class for all chunker implementations.
+ * Provides common functionality and interface contract.
+ */
+import { v4 as uuidv4 } from "uuid";
+import { ChunkingError, RAGErrorCodes } from "../errors/RAGError.js";
+/**
+ * Default chunker configuration
+ */
+export const DEFAULT_CHUNKER_CONFIG = {
+    maxSize: 1000,
+    overlap: 100,
+    minSize: 10,
+    preserveMetadata: true,
+};
+/**
+ * Base Chunker abstract class
+ *
+ * All chunker implementations should extend this class.
+ */
+export class BaseChunker {
+    config;
+    constructor(config) {
+        this.config = { ...this.getDefaultConfig(), ...config };
+        this.validateConfig();
+    }
+    /**
+     * Get default configuration for this chunker
+     */
+    getDefaultConfig() {
+        return { ...DEFAULT_CHUNKER_CONFIG };
+    }
+    /**
+     * Validate chunker configuration
+     */
+    validateConfig() {
+        if (this.config.maxSize !== undefined && this.config.maxSize <= 0) {
+            throw new ChunkingError("maxSize must be positive", {
+                code: RAGErrorCodes.CHUNKING_INVALID_CONFIG,
+                details: { maxSize: this.config.maxSize },
+            });
+        }
+        if (this.config.overlap !== undefined && this.config.overlap < 0) {
+            throw new ChunkingError("overlap cannot be negative", {
+                code: RAGErrorCodes.CHUNKING_INVALID_CONFIG,
+                details: { overlap: this.config.overlap },
+            });
+        }
+        if (this.config.maxSize !== undefined &&
+            this.config.overlap !== undefined &&
+            this.config.overlap >= this.config.maxSize) {
+            throw new ChunkingError("overlap must be less than maxSize", {
+                code: RAGErrorCodes.CHUNKING_INVALID_CONFIG,
+                details: {
+                    maxSize: this.config.maxSize,
+                    overlap: this.config.overlap,
+                },
+            });
+        }
+    }
+    /**
+     * Chunk content into smaller pieces
+     */
+    async chunk(content, config) {
+        const effectiveConfig = { ...this.config, ...config };
+        if (!content || content.trim().length === 0) {
+            throw new ChunkingError("Content is empty", {
+                code: RAGErrorCodes.CHUNKING_EMPTY_CONTENT,
+                strategy: this.strategy,
+                contentLength: 0,
+            });
+        }
+        try {
+            const chunks = await this.doChunk(content, effectiveConfig);
+            return this.filterChunks(chunks, effectiveConfig);
+        }
+        catch (error) {
+            if (error instanceof ChunkingError) {
+                throw error;
+            }
+            throw new ChunkingError(`Chunking failed: ${error instanceof Error ? error.message : String(error)}`, {
+                code: RAGErrorCodes.CHUNKING_ERROR,
+                cause: error instanceof Error ? error : undefined,
+                strategy: this.strategy,
+                contentLength: content.length,
+            });
+        }
+    }
+    /**
+     * Filter chunks based on minimum size
+     */
+    filterChunks(chunks, config) {
+        const minSize = config.minSize ?? 0;
+        return chunks.filter((chunk) => chunk.text.length >= minSize);
+    }
+    /**
+     * Create a chunk object
+     */
+    createChunk(text, chunkIndex, startPosition, endPosition, documentId = "unknown", customMetadata) {
+        const metadata = {
+            documentId,
+            chunkIndex,
+            startPosition,
+            endPosition,
+            custom: this.config.preserveMetadata ? customMetadata : undefined,
+        };
+        return {
+            id: uuidv4(),
+            text,
+            metadata,
+        };
+    }
+    /**
+     * Split content by size with overlap
+     */
+    splitBySizeWithOverlap(content, maxSize, overlap) {
+        const result = [];
+        let start = 0;
+        while (start < content.length) {
+            const end = Math.min(start + maxSize, content.length);
+            result.push({
+                text: content.slice(start, end),
+                start,
+                end,
+            });
+            // If we've reached the end of content, stop
+            if (end >= content.length) {
+                break;
+            }
+            // Move start position, accounting for overlap
+            // Ensure start always moves forward by at least 1 character
+            const nextStart = end - overlap;
+            start = Math.max(nextStart, start + 1);
+            // Prevent infinite loop if overlap >= chunk size
+            if (start >= end) {
+                break;
+            }
+        }
+        return result;
+    }
+}
+//# sourceMappingURL=BaseChunker.js.map

package/dist/lib/rag/chunkers/CharacterChunker.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Character Chunker
+ *
+ * Splits text into fixed-size character chunks with optional overlap.
+ * The simplest chunking strategy for language-agnostic processing.
+ */
+import type { Chunk, ChunkerConfig, ChunkingStrategy } from "../types.js";
+import { BaseChunker } from "./BaseChunker.js";
+/**
+ * Character Chunker
+ *
+ * Splits content into fixed-size character chunks.
+ */
+export declare class CharacterChunker extends BaseChunker {
+    readonly strategy: ChunkingStrategy;
+    getDefaultConfig(): ChunkerConfig;
+    protected doChunk(content: string, config: ChunkerConfig): Promise<Chunk[]>;
+}

package/dist/lib/rag/chunkers/CharacterChunker.js

@@ -0,0 +1,29 @@
+/**
+ * Character Chunker
+ *
+ * Splits text into fixed-size character chunks with optional overlap.
+ * The simplest chunking strategy for language-agnostic processing.
+ */
+import { BaseChunker, DEFAULT_CHUNKER_CONFIG } from "./BaseChunker.js";
+/**
+ * Character Chunker
+ *
+ * Splits content into fixed-size character chunks.
+ */
+export class CharacterChunker extends BaseChunker {
+    strategy = "character";
+    getDefaultConfig() {
+        return {
+            ...DEFAULT_CHUNKER_CONFIG,
+            maxSize: 1000,
+            overlap: 100,
+        };
+    }
+    async doChunk(content, config) {
+        const maxSize = config.maxSize ?? 1000;
+        const overlap = config.overlap ?? 100;
+        const segments = this.splitBySizeWithOverlap(content, maxSize, overlap);
+        return segments.map((segment, index) => this.createChunk(segment.text, index, segment.start, segment.end));
+    }
+}
+//# sourceMappingURL=CharacterChunker.js.map

package/dist/lib/rag/chunkers/HTMLChunker.d.ts

@@ -0,0 +1,19 @@
+/**
+ * HTML Chunker
+ *
+ * Splits HTML content by semantic tags.
+ */
+import type { Chunk, ChunkerConfig, ChunkingStrategy } from "../types.js";
+import { BaseChunker } from "./BaseChunker.js";
+/**
+ * HTML Chunker
+ */
+export declare class HTMLChunker extends BaseChunker {
+    readonly strategy: ChunkingStrategy;
+    getDefaultConfig(): ChunkerConfig;
+    protected doChunk(content: string, config: ChunkerConfig): Promise<Chunk[]>;
+    /**
+     * Strip HTML tags from content
+     */
+    private stripHtml;
+}