@storepress/llm-md-text-splitter 0.0.1

package/package.json

@@ -0,0 +1,79 @@
+{
+  "name": "@storepress/llm-md-text-splitter",
+  "version": "0.0.1",
+  "description": "High-performance streaming Markdown text splitter for LLM pipelines and RAG systems. Zero sequence loss for code blocks, tables, links, and videos. 5 built-in strategies + custom. Zero dependencies.",
+  "type": "module",
+  "main": "./src/MarkdownTextSplitter.js",
+  "module": "./src/MarkdownTextSplitter.js",
+  "publishConfig": {
+    "access": "public"
+  },
+  "exports": {
+    ".": {
+      "import": "./src/MarkdownTextSplitter.js",
+      "default": "./src/MarkdownTextSplitter.js"
+    }
+  },
+  "browser": "./src/MarkdownTextSplitter.js",
+  "types": "./src/MarkdownTextSplitter.d.ts",
+  "files": [
+    "src/**",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "test": "node --test test/test.mjs",
+    "test:watch": "node --test --watch test/test.mjs",
+    "demo": "npx serve .",
+    "lint": "eslint src/",
+    "prepublishOnly": "npm test"
+  },
+  "keywords": [
+    "markdown",
+    "text-splitter",
+    "llm",
+    "rag",
+    "chunking",
+    "streaming",
+    "zero-loss",
+    "code-blocks",
+    "semantic",
+    "tokenizer",
+    "langchain",
+    "vector-database",
+    "embeddings",
+    "ai",
+    "openai",
+    "anthropic",
+    "gpt",
+    "claude",
+    "pinecone",
+    "weaviate",
+    "chromadb",
+    "text-processing",
+    "document-processing",
+    "markdown-parser",
+    "context-window",
+    "chunk",
+    "splitter",
+    "delimiter",
+    "word-limit",
+    "char-limit",
+    "token-limit",
+    "browser",
+    "esm"
+  ],
+  "author": "StorePress",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/EmranAhmed/storepress-llm-md-text-splitter.git"
+  },
+  "bugs": {
+    "url": "https://github.com/EmranAhmed/storepress-llm-md-text-splitter/issues"
+  },
+  "homepage": "https://emranahmed.github.io/storepress-llm-md-text-splitter/",
+  "sideEffects": false,
+  "dependencies": {},
+  "devDependencies": {}
+}

package/src/MarkdownTextSplitter.d.ts

@@ -0,0 +1,304 @@
+/**
+ * md-text-splitter — TypeScript Declarations
+ * @module md-text-splitter
+ * @version 3.0.0
+ */
+
+// ── Block Types ──
+
+export declare const BlockType: Readonly<{
+    HEADING: "heading";
+    PARAGRAPH: "paragraph";
+    CODE_BLOCK: "code_block";
+    LIST: "list";
+    BLOCKQUOTE: "blockquote";
+    TABLE: "table";
+    LINK_REF: "link_reference";
+    VIDEO_EMBED: "video_embed";
+    HR: "hr";
+    EMPTY: "empty";
+    FRONTMATTER: "frontmatter";
+    HTML_BLOCK: "html_block";
+    IMAGE: "image";
+}>;
+
+export type BlockTypeName = (typeof BlockType)[keyof typeof BlockType];
+
+// ── Configuration ──
+
+export interface StrategyOptions {
+    /** DelimiterStrategy: string to split on (default: '---') */
+    delimiter?: string;
+    /** DelimiterStrategy: include delimiter in output (default: false) */
+    keepDelimiter?: boolean;
+    /** DelimiterStrategy: trim whitespace from chunk edges (default: true) */
+    trimChunks?: boolean;
+    /** CharLimitStrategy: max characters per chunk (default: maxChunkTokens × charsPerToken) */
+    charLimit?: number;
+    /** CharLimitStrategy / WordLimitStrategy: overlap amount */
+    overlap?: number;
+    /** WordLimitStrategy: max words per chunk (default: 1000) */
+    wordLimit?: number;
+    /** TokenLimitStrategy: max tokens per chunk (default: maxChunkTokens) */
+    tokenLimit?: number;
+
+    /** Custom strategy options (any key-value pairs) */
+    [key: string]: unknown;
+}
+
+export interface SplitterConfig {
+    /** Target max tokens per chunk (~4 chars/token). Default: 1500 */
+    maxChunkTokens?: number;
+    /** Tokens of overlap between consecutive chunks. Default: 150 */
+    overlapTokens?: number;
+    /** Characters-per-token ratio for estimation. Default: 4 */
+    charsPerToken?: number;
+    /** HTTP fetch timeout in milliseconds. Default: 60000 */
+    fetchTimeoutMs?: number;
+    /** Group code blocks with surrounding text (semantic). Default: true */
+    preserveCodeContext?: boolean;
+    /** Group reference links with their sections (semantic). Default: true */
+    preserveLinks?: boolean;
+    /** Group video embeds with their context (semantic). Default: true */
+    preserveVideos?: boolean;
+    /** Prefix for generated chunk IDs. Default: 'chunk' */
+    chunkIdPrefix?: string;
+    /** Regex to detect video embed lines */
+    videoPattern?: RegExp;
+    /** Regex to detect reference-style link definitions */
+    linkRefPattern?: RegExp;
+    /** Active strategy name. Default: 'semantic' */
+    strategy?: "semantic" | "delimiter" | "char" | "word" | "token" | string;
+    /** Strategy-specific options */
+    strategyOptions?: StrategyOptions;
+}
+
+export declare const DEFAULT_CONFIG: Readonly<Required<SplitterConfig>>;
+
+// ── Chunk Output ──
+
+export interface ExtractedLink {
+    text: string;
+    url: string;
+}
+
+export interface ExtractedVideo {
+    platform: "youtube" | "vimeo";
+    url: string;
+    videoId: string;
+}
+
+export interface ChunkLines {
+    start: number;
+    end: number;
+}
+
+export interface Chunk {
+    /** Deterministic FNV-1a hash-based ID */
+    id: string;
+    /** Sequential position (0-based) */
+    index: number;
+    /** The actual text content */
+    content: string;
+    /** Approximate LLM token count */
+    tokenEstimate: number;
+    /** Tokens repeated from previous chunk */
+    overlapTokens: number;
+    /** Exact character count */
+    charCount: number;
+    /** Word count */
+    wordCount: number;
+    /** Original line numbers */
+    lines: ChunkLines;
+    /** Nearest heading text (semantic strategy) */
+    heading: string | null;
+    /** Full heading breadcrumb trail */
+    headingPath: string[];
+    /** Heading depth (1–6) */
+    headingLevel: number | null;
+    /** Contains fenced code blocks */
+    hasCode: boolean;
+    /** Contains markdown tables */
+    hasTable: boolean;
+    /** Contains video embeds */
+    hasVideo: boolean;
+    /** Programming languages detected in code blocks */
+    languages: string[];
+    /** Extracted inline and reference links */
+    links: ExtractedLink[];
+    /** Extracted video embed metadata */
+    videos: ExtractedVideo[];
+    /** Exceeds 1.5× target chunk size */
+    isOversized: boolean;
+    /** Contains atomic blocks (code/table/video) */
+    containsAtomicBlock: boolean;
+    /** Semantic block types present */
+    blockTypes: BlockTypeName[];
+    /** Which strategy produced this chunk */
+    strategy: string;
+    /** Extensible metadata */
+    metadata: Record<string, unknown>;
+}
+
+// ── Statistics ──
+
+export interface SplitterStats {
+    totalChunks: number;
+    totalTokens: number;
+    totalChars: number;
+    totalWords: number;
+    oversizedChunks: number;
+    codeBlockChunks: number;
+    tableChunks: number;
+    videoChunks: number;
+    processingTimeMs: number;
+    source: string;
+}
+
+// ── Line Iterator ──
+
+export interface LineObject {
+    lineNumber: number;
+    text: string;
+}
+
+// ── Semantic Block ──
+
+export interface SemanticBlock {
+    type: BlockTypeName;
+    content: string;
+    lines: ChunkLines;
+    heading: string | null;
+    headingLevel: number | null;
+    headingPath: string[];
+    language: string | null;
+    isAtomic: boolean;
+    links: ExtractedLink[];
+    videos: ExtractedVideo[];
+    metadata: Record<string, unknown>;
+}
+
+// ── Strategy Interface ──
+
+export interface SplittingStrategy {
+    name: string;
+
+    process(lineIterator: AsyncIterable<LineObject>): AsyncGenerator<Chunk>;
+}
+
+export interface SplittingStrategyConstructor {
+    new(config: Required<SplitterConfig>): SplittingStrategy;
+}
+
+// ── Strategy Classes ──
+
+export declare class SemanticStrategy implements SplittingStrategy {
+    name: "semantic";
+
+    constructor(config: Required<SplitterConfig>);
+
+    process(lineIterator: AsyncIterable<LineObject>): AsyncGenerator<Chunk>;
+}
+
+export declare class DelimiterStrategy implements SplittingStrategy {
+    name: "delimiter";
+
+    constructor(config: Required<SplitterConfig>);
+
+    process(lineIterator: AsyncIterable<LineObject>): AsyncGenerator<Chunk>;
+}
+
+export declare class CharLimitStrategy implements SplittingStrategy {
+    name: "char";
+
+    constructor(config: Required<SplitterConfig>);
+
+    process(lineIterator: AsyncIterable<LineObject>): AsyncGenerator<Chunk>;
+}
+
+export declare class WordLimitStrategy implements SplittingStrategy {
+    name: "word";
+
+    constructor(config: Required<SplitterConfig>);
+
+    process(lineIterator: AsyncIterable<LineObject>): AsyncGenerator<Chunk>;
+}
+
+export declare class TokenLimitStrategy extends CharLimitStrategy {
+    name: "token";
+
+    constructor(config: Required<SplitterConfig>);
+}
+
+// ── Parser ──
+
+export declare class SemanticParser {
+    constructor(config: Required<SplitterConfig>);
+
+    parse(lineIterator: AsyncIterable<LineObject>): AsyncGenerator<SemanticBlock>;
+
+    getLinkDefinitions(): Map<string, string>;
+}
+
+// ── Main Class ──
+
+export declare class MarkdownTextSplitter {
+    config: Required<SplitterConfig>;
+    strategy: SplittingStrategy;
+    stats: SplitterStats;
+
+    constructor(config?: SplitterConfig);
+
+    /** Split a markdown string into chunks */
+    splitFromString(markdown: string): Promise<Chunk[]>;
+
+    /** Split a remote markdown file via streaming HTTP */
+    splitFromUrl(url: string, fetchOptions?: RequestInit): Promise<Chunk[]>;
+
+    /** Split a browser File or Blob */
+    splitFromFile(file: File | Blob): Promise<Chunk[]>;
+
+    /** Stream chunks one-at-a-time from a string */
+    streamFromString(markdown: string): AsyncGenerator<Chunk>;
+
+    /** Stream chunks one-at-a-time from a URL */
+    streamFromUrl(url: string, fetchOptions?: RequestInit): AsyncGenerator<Chunk>;
+
+    /** Stream chunks one-at-a-time from a File/Blob */
+    streamFromFile(file: File | Blob): AsyncGenerator<Chunk>;
+
+    /** Switch the active strategy at runtime */
+    setStrategy(name: string, options?: StrategyOptions): void;
+
+    /** Get processing statistics from the last split */
+    getStats(): SplitterStats;
+
+    /** Reset internal state for reuse */
+    reset(): void;
+
+    /** Register a custom splitting strategy globally */
+    static registerStrategy(name: string, strategyClass: SplittingStrategyConstructor): void;
+
+    /** List all available strategy names (built-in + custom) */
+    static getAvailableStrategies(): string[];
+}
+
+// ── Utility Functions ──
+
+export declare function estimateTokens(text: string, charsPerToken?: number): number;
+
+export declare function countWords(text: string): number;
+
+export declare function generateChunkId(content: string, index: number, prefix?: string): string;
+
+export declare function extractLinks(text: string): ExtractedLink[];
+
+export declare function extractVideos(text: string): ExtractedVideo[];
+
+export declare function streamToLines(byteStream: ReadableStream<Uint8Array>): AsyncGenerator<LineObject>;
+
+export declare function stringToLines(text: string): AsyncGenerator<LineObject>;
+
+// ── Default Export ──
+
+export default MarkdownTextSplitter;