@nodetool-ai/base-nodes 0.7.0-rc.22 → 0.8.1

package/dist/nodes/document.js

@@ -1,984 +0,0 @@
-var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
-    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
-    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
-    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
-    return c > 3 && r && Object.defineProperty(target, key, r), r;
-};
-import { BaseNode, prop } from "@nodetool-ai/node-sdk";
-import { promises as fs } from "node:fs";
-import path from "node:path";
-function documentSourceId(refOrPath) {
-    if (typeof refOrPath === "string" && refOrPath) {
-        return refOrPath;
-    }
-    if (refOrPath && typeof refOrPath === "object") {
-        const ref = refOrPath;
-        if (typeof ref.uri === "string" && ref.uri)
-            return ref.uri;
-    }
-    return "document";
-}
-function asBytes(data) {
-    if (!data)
-        return new Uint8Array();
-    if (data instanceof Uint8Array)
-        return data;
-    return Uint8Array.from(Buffer.from(data, "base64"));
-}
-function toFilePath(uriOrPath) {
-    if (uriOrPath.startsWith("file://")) {
-        return uriOrPath.slice("file://".length);
-    }
-    return uriOrPath;
-}
-function wildcardToRegExp(pattern) {
-    const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&");
-    return new RegExp(`^${escaped.replaceAll("*", ".*")}$`);
-}
-function textChunk(text, sourceId, startIndex) {
-    return {
-        chunk: text,
-        text,
-        source_id: sourceId,
-        start_index: startIndex
-    };
-}
-function splitByChunk(text, chunkSize, overlap) {
-    if (!text)
-        return [];
-    const size = Math.max(1, chunkSize);
-    const step = Math.max(1, size - Math.max(0, overlap));
-    const out = [];
-    for (let i = 0; i < text.length; i += step) {
-        out.push(text.slice(i, i + size));
-        if (i + size >= text.length)
-            break;
-    }
-    return out;
-}
-async function readDocumentText(refOrPath, context) {
-    if (typeof refOrPath === "string" && refOrPath) {
-        if (context?.storage) {
-            const stored = await context.storage.retrieve(refOrPath);
-            if (stored !== null)
-                return Buffer.from(stored).toString("utf8");
-        }
-        return fs.readFile(toFilePath(refOrPath), "utf8");
-    }
-    if (refOrPath && typeof refOrPath === "object") {
-        const ref = refOrPath;
-        if (typeof ref.text === "string")
-            return ref.text;
-        if (ref.data) {
-            const bytes = asBytes(ref.data);
-            return Buffer.from(bytes).toString("utf8");
-        }
-        if (typeof ref.uri === "string" && ref.uri) {
-            if (context?.storage) {
-                const stored = await context.storage.retrieve(ref.uri);
-                if (stored !== null)
-                    return Buffer.from(stored).toString("utf8");
-            }
-            if (ref.uri.startsWith("file://") || !ref.uri.startsWith("http")) {
-                return fs.readFile(toFilePath(ref.uri), "utf8");
-            }
-        }
-    }
-    return "";
-}
-export class LoadDocumentFileNode extends BaseNode {
-    static nodeType = "nodetool.document.LoadDocumentFile";
-    static title = "Load Document File";
-    static description = "Read a document from disk.\n    files, document, read, input, load, file";
-    static metadataOutputTypes = {
-        output: "document"
-    };
-    async process() {
-        const p = String(this.path ?? this.path ?? "");
-        const full = toFilePath(p);
-        const bytes = new Uint8Array(await fs.readFile(full));
-        return {
-            output: {
-                uri: `file://${full}`,
-                data: Buffer.from(bytes).toString("base64")
-            }
-        };
-    }
-}
-__decorate([
-    prop({
-        type: "str",
-        default: "",
-        title: "Path",
-        description: "Path to the document to read"
-    })
-], LoadDocumentFileNode.prototype, "path", void 0);
-export class SaveDocumentFileNode extends BaseNode {
-    static nodeType = "nodetool.document.SaveDocumentFile";
-    static title = "Save Document File";
-    static description = "Write a document to disk.\n    files, document, write, output, save, file\n\n    The filename can include time and date variables:\n    %Y - Year, %m - Month, %d - Day\n    %H - Hour, %M - Minute, %S - Second";
-    async process() {
-        const document = (this.document ?? this.document ?? {});
-        const p = String(this.path ?? "");
-        const full = toFilePath(p);
-        await fs.mkdir(path.dirname(full), { recursive: true });
-        if (document.data) {
-            await fs.writeFile(full, asBytes(document.data));
-        }
-        else if (typeof document.text === "string") {
-            await fs.writeFile(full, document.text, "utf8");
-        }
-        else if (document.uri) {
-            await fs.copyFile(toFilePath(document.uri), full);
-        }
-        else {
-            await fs.writeFile(full, "", "utf8");
-        }
-        return { output: full };
-    }
-}
-__decorate([
-    prop({
-        type: "document",
-        default: {
-            type: "document",
-            uri: "",
-            asset_id: null,
-            data: null,
-            metadata: null
-        },
-        title: "Document",
-        description: "The document to save"
-    })
-], SaveDocumentFileNode.prototype, "document", void 0);
-__decorate([
-    prop({
-        type: "str",
-        default: "",
-        title: "Folder",
-        description: "Folder where the file will be saved"
-    })
-], SaveDocumentFileNode.prototype, "folder", void 0);
-__decorate([
-    prop({
-        type: "str",
-        default: "",
-        title: "Filename",
-        description: "Name of the file to save. Supports strftime format codes."
-    })
-], SaveDocumentFileNode.prototype, "filename", void 0);
-export class ListDocumentsNode extends BaseNode {
-    static nodeType = "nodetool.document.ListDocuments";
-    static title = "List Documents";
-    static description = "List documents in a directory.\n    files, list, directory";
-    static metadataOutputTypes = {
-        document: "document",
-        documents: "list"
-    };
-    static isStreamingOutput = true;
-    async process() {
-        const collected = [];
-        for await (const item of this._listDocuments()) {
-            collected.push(item.document);
-        }
-        return {
-            document: collected[0] ?? { uri: "" },
-            documents: collected
-        };
-    }
-    async *_listDocuments() {
-        const folder = String(this.folder ?? this.folder ?? ".");
-        const pattern = String(this.pattern ?? this.pattern ?? "*");
-        const recursive = Boolean(this.recursive ?? this.recursive ?? false);
-        const allowed = new Set([
-            ".txt",
-            ".md",
-            ".markdown",
-            ".json",
-            ".html",
-            ".pdf",
-            ".docx"
-        ]);
-        const matches = wildcardToRegExp(pattern);
-        const visit = async function* (dir) {
-            const entries = await fs.readdir(dir, { withFileTypes: true });
-            for (const entry of entries) {
-                const full = path.join(dir, entry.name);
-                if (entry.isDirectory()) {
-                    if (recursive) {
-                        yield* visit(full);
-                    }
-                    continue;
-                }
-                yield full;
-            }
-        };
-        for await (const full of visit(folder)) {
-            if (allowed.has(path.extname(full).toLowerCase()) &&
-                matches.test(path.basename(full))) {
-                yield { document: { uri: `file://${full}` } };
-            }
-        }
-    }
-    async *genProcess() {
-        const collected = [];
-        for await (const item of this._listDocuments()) {
-            collected.push(item.document);
-            yield { document: item.document };
-        }
-        yield { documents: collected };
-    }
-}
-__decorate([
-    prop({
-        type: "str",
-        default: "~",
-        title: "Folder",
-        description: "Directory to scan"
-    })
-], ListDocumentsNode.prototype, "folder", void 0);
-__decorate([
-    prop({
-        type: "str",
-        default: "*",
-        title: "Pattern",
-        description: "File pattern to match (e.g. *.txt)"
-    })
-], ListDocumentsNode.prototype, "pattern", void 0);
-__decorate([
-    prop({
-        type: "bool",
-        default: false,
-        title: "Recursive",
-        description: "Search subdirectories"
-    })
-], ListDocumentsNode.prototype, "recursive", void 0);
-export class SplitDocumentNode extends BaseNode {
-    static nodeType = "nodetool.document.SplitDocument";
-    static title = "Split Document";
-    static description = "Split text semantically.\n    chroma, embedding, collection, RAG, index, text, markdown, semantic";
-    static metadataOutputTypes = {
-        text: "str",
-        source_id: "str",
-        start_index: "int",
-        chunks: "list"
-    };
-    static recommendedModels = [
-        {
-            id: "embeddinggemma",
-            type: "embedding_model",
-            name: "Embedding Gemma",
-            repo_id: "embeddinggemma",
-            description: "Embedding model for semantic splitting"
-        },
-        {
-            id: "nomic-embed-text",
-            type: "embedding_model",
-            name: "Nomic Embed Text",
-            repo_id: "nomic-embed-text",
-            description: "Embedding model for semantic splitting"
-        },
-        {
-            id: "mxbai-embed-large",
-            type: "embedding_model",
-            name: "MXBai Embed Large",
-            repo_id: "mxbai-embed-large",
-            description: "Embedding model for semantic splitting"
-        },
-        {
-            id: "bge-m3",
-            type: "embedding_model",
-            name: "BGE M3",
-            repo_id: "bge-m3",
-            description: "Embedding model for semantic splitting"
-        },
-        {
-            id: "all-minilm",
-            type: "embedding_model",
-            name: "All Minilm",
-            repo_id: "all-minilm",
-            description: "Embedding model for semantic splitting"
-        }
-    ];
-    static isStreamingOutput = true;
-    async process(context) {
-        const collected = [];
-        for await (const item of this.genProcess(context)) {
-            if ("chunks" in item)
-                continue;
-            collected.push(item);
-        }
-        return {
-            text: collected[0]?.text ?? "",
-            source_id: collected[0]?.source_id ?? "",
-            start_index: collected[0]?.start_index ?? 0,
-            chunks: collected
-        };
-    }
-    async *genProcess(context) {
-        const document = this.document ?? this.document;
-        const text = await readDocumentText(document, context);
-        const sourceId = documentSourceId(document);
-        const chunkSize = Number(this.chunk_size ?? 1200);
-        const overlap = Number(this.chunk_overlap ?? 100);
-        const collected = [];
-        let startIndex = 0;
-        for (const chunk of splitByChunk(text, chunkSize, overlap)) {
-            const idx = text.indexOf(chunk, startIndex);
-            const resolvedIndex = idx >= 0 ? idx : startIndex;
-            const item = textChunk(chunk, sourceId, resolvedIndex);
-            collected.push(item);
-            yield item;
-            startIndex = resolvedIndex + Math.max(chunk.length - overlap, 1);
-        }
-        yield { chunks: collected };
-    }
-}
-__decorate([
-    prop({
-        type: "language_model",
-        default: {
-            type: "language_model",
-            provider: "ollama",
-            id: "embeddinggemma",
-            name: "",
-            path: null,
-            supported_tasks: []
-        },
-        title: "Embed Model",
-        description: "Embedding model to use"
-    })
-], SplitDocumentNode.prototype, "embed_model", void 0);
-__decorate([
-    prop({
-        type: "document",
-        default: {
-            type: "document",
-            uri: "",
-            asset_id: null,
-            data: null,
-            metadata: null
-        },
-        title: "Document",
-        description: "Document ID to associate with the text content"
-    })
-], SplitDocumentNode.prototype, "document", void 0);
-__decorate([
-    prop({
-        type: "int",
-        default: 1,
-        title: "Buffer Size",
-        description: "Buffer size for semantic splitting",
-        min: 1,
-        max: 100
-    })
-], SplitDocumentNode.prototype, "buffer_size", void 0);
-__decorate([
-    prop({
-        type: "int",
-        default: 95,
-        title: "Threshold",
-        description: "Breakpoint percentile threshold for semantic splitting",
-        min: 0,
-        max: 100
-    })
-], SplitDocumentNode.prototype, "threshold", void 0);
-export class SplitHTMLNode extends BaseNode {
-    static nodeType = "nodetool.document.SplitHTML";
-    static title = "Split HTML";
-    static description = "Split HTML content into semantic chunks based on HTML tags.\n    html, text, semantic, tags, parsing";
-    static metadataOutputTypes = {
-        text: "str",
-        source_id: "str",
-        start_index: "int",
-        chunks: "list"
-    };
-    static isStreamingOutput = true;
-    async process(context) {
-        const collected = [];
-        for await (const item of this.genProcess(context)) {
-            if ("chunks" in item)
-                continue;
-            collected.push(item);
-        }
-        return {
-            text: collected[0]?.text ?? "",
-            source_id: collected[0]?.source_id ?? "",
-            start_index: collected[0]?.start_index ?? 0,
-            chunks: collected
-        };
-    }
-    async *genProcess(context) {
-        const document = this.document ?? this.document;
-        const html = await readDocumentText(document, context);
-        const sourceId = documentSourceId(document);
-        const chunkSize = Number(this.chunk_size ?? 1200);
-        const overlap = Number(this.chunk_overlap ?? 100);
-        // Split on block-level HTML tags
-        const blockTags = "p|div|h[1-6]|li|section|article|blockquote|tr|pre|header|footer|nav|main|aside";
-        const blockPattern = new RegExp(`<(?:${blockTags})(?:\\s[^>]*)?>([\\s\\S]*?)<\\/(?:${blockTags})>`, "gi");
-        const sections = [];
-        let lastIndex = 0;
-        let match;
-        while ((match = blockPattern.exec(html)) !== null) {
-            // Capture any text between block elements
-            if (match.index > lastIndex) {
-                const between = html
-                    .slice(lastIndex, match.index)
-                    .replace(/<[^>]+>/g, " ")
-                    .replace(/\s+/g, " ")
-                    .trim();
-                if (between)
-                    sections.push(between);
-            }
-            // Extract text content from the block element
-            const content = match[1]
-                .replace(/<[^>]+>/g, " ")
-                .replace(/\s+/g, " ")
-                .trim();
-            if (content)
-                sections.push(content);
-            lastIndex = match.index + match[0].length;
-        }
-        // Any remaining text after the last block element
-        if (lastIndex < html.length) {
-            const remaining = html
-                .slice(lastIndex)
-                .replace(/<[^>]+>/g, " ")
-                .replace(/\s+/g, " ")
-                .trim();
-            if (remaining)
-                sections.push(remaining);
-        }
-        // If no block elements found, fall back to stripping all tags
-        if (sections.length === 0) {
-            const text = html
-                .replace(/<[^>]+>/g, " ")
-                .replace(/\s+/g, " ")
-                .trim();
-            if (text)
-                sections.push(text);
-        }
-        // Merge small sections and split large ones respecting chunk_size
-        const collected = [];
-        let startIndex = 0;
-        let chunkIdx = 0;
-        let buffer = "";
-        for (const section of sections) {
-            if (buffer && (buffer.length + 1 + section.length) > chunkSize) {
-                // Emit buffer
-                for (const chunk of splitByChunk(buffer, chunkSize, overlap)) {
-                    const item = textChunk(chunk, `${sourceId}:${chunkIdx}`, startIndex);
-                    collected.push(item);
-                    yield item;
-                    startIndex += Math.max(chunk.length - overlap, 1);
-                    chunkIdx++;
-                }
-                buffer = "";
-            }
-            buffer = buffer ? buffer + "\n" + section : section;
-        }
-        // Emit remaining buffer
-        if (buffer) {
-            for (const chunk of splitByChunk(buffer, chunkSize, overlap)) {
-                const item = textChunk(chunk, `${sourceId}:${chunkIdx}`, startIndex);
-                collected.push(item);
-                yield item;
-                startIndex += Math.max(chunk.length - overlap, 1);
-                chunkIdx++;
-            }
-        }
-        yield { chunks: collected };
-    }
-}
-__decorate([
-    prop({
-        type: "document",
-        default: {
-            type: "document",
-            uri: "",
-            asset_id: null,
-            data: null,
-            metadata: null
-        },
-        title: "Document",
-        description: "Document ID to associate with the HTML content"
-    })
-], SplitHTMLNode.prototype, "document", void 0);
-export class SplitJSONNode extends BaseNode {
-    static nodeType = "nodetool.document.SplitJSON";
-    static title = "Split JSON";
-    static description = "Split JSON content into semantic chunks.\n    json, parsing, semantic, structured";
-    static metadataOutputTypes = {
-        text: "str",
-        source_id: "str",
-        start_index: "int",
-        chunks: "list"
-    };
-    static isStreamingOutput = true;
-    async process(context) {
-        const collected = [];
-        for await (const item of this.genProcess(context)) {
-            if ("chunks" in item)
-                continue;
-            collected.push(item);
-        }
-        return {
-            text: collected[0]?.text ?? "",
-            source_id: collected[0]?.source_id ?? "",
-            start_index: collected[0]?.start_index ?? 0,
-            chunks: collected
-        };
-    }
-    async *genProcess(context) {
-        const document = this.document ?? this.document;
-        const raw = await readDocumentText(document, context);
-        const sourceId = documentSourceId(document);
-        const chunkSize = Number(this.chunk_size ?? 1200);
-        const overlap = Number(this.chunk_overlap ?? 100);
-        const collected = [];
-        let parsed;
-        try {
-            parsed = JSON.parse(raw);
-        }
-        catch {
-            // Fallback to character-based splitting for invalid JSON
-            let startIndex = 0;
-            for (const chunk of splitByChunk(raw, chunkSize, overlap)) {
-                const idx = raw.indexOf(chunk, startIndex);
-                const resolvedIndex = idx >= 0 ? idx : startIndex;
-                const item = textChunk(chunk, sourceId, resolvedIndex);
-                collected.push(item);
-                yield item;
-                startIndex = resolvedIndex + Math.max(chunk.length - overlap, 1);
-            }
-            yield { chunks: collected };
-            return;
-        }
-        // Split by JSON structure
-        let elements;
-        if (Array.isArray(parsed)) {
-            // Top-level array: split into individual elements
-            elements = parsed.map((item) => JSON.stringify(item, null, 2));
-        }
-        else if (parsed && typeof parsed === "object") {
-            // Top-level object: split by top-level keys
-            elements = Object.entries(parsed).map(([key, value]) => JSON.stringify({ [key]: value }, null, 2));
-        }
-        else {
-            elements = [JSON.stringify(parsed, null, 2)];
-        }
-        // Merge small elements and split large ones respecting chunk_size
-        let chunkIdx = 0;
-        let buffer = "";
-        for (const element of elements) {
-            if (element.length > chunkSize) {
-                // Emit buffer first if any
-                if (buffer) {
-                    const item = textChunk(buffer, `${sourceId}:${chunkIdx}`, chunkIdx);
-                    collected.push(item);
-                    yield item;
-                    chunkIdx++;
-                    buffer = "";
-                }
-                // Split large element by characters
-                for (const chunk of splitByChunk(element, chunkSize, overlap)) {
-                    const item = textChunk(chunk, `${sourceId}:${chunkIdx}`, chunkIdx);
-                    collected.push(item);
-                    yield item;
-                    chunkIdx++;
-                }
-            }
-            else if (buffer && (buffer.length + 2 + element.length) > chunkSize) {
-                // Buffer would exceed chunk_size, emit it
-                const item = textChunk(buffer, `${sourceId}:${chunkIdx}`, chunkIdx);
-                collected.push(item);
-                yield item;
-                chunkIdx++;
-                buffer = element;
-            }
-            else {
-                buffer = buffer ? buffer + ",\n" + element : element;
-            }
-        }
-        if (buffer) {
-            const item = textChunk(buffer, `${sourceId}:${chunkIdx}`, chunkIdx);
-            collected.push(item);
-            yield item;
-        }
-        yield { chunks: collected };
-    }
-}
-__decorate([
-    prop({
-        type: "document",
-        default: {
-            type: "document",
-            uri: "",
-            asset_id: null,
-            data: null,
-            metadata: null
-        },
-        title: "Document",
-        description: "Document ID to associate with the JSON content"
-    })
-], SplitJSONNode.prototype, "document", void 0);
-__decorate([
-    prop({
-        type: "bool",
-        default: true,
-        title: "Include Metadata",
-        description: "Whether to include metadata in nodes"
-    })
-], SplitJSONNode.prototype, "include_metadata", void 0);
-__decorate([
-    prop({
-        type: "bool",
-        default: true,
-        title: "Include Prev Next Rel",
-        description: "Whether to include prev/next relationships"
-    })
-], SplitJSONNode.prototype, "include_prev_next_rel", void 0);
-export class SplitRecursivelyNode extends BaseNode {
-    static nodeType = "nodetool.document.SplitRecursively";
-    static title = "Split Recursively";
-    static description = "Splits text recursively using LangChain's RecursiveCharacterTextSplitter.\n    text, split, chunks\n\n    Use cases:\n    - Splitting documents while preserving semantic relationships\n    - Creating chunks for language model processing\n    - Handling text in languages with/without word boundaries";
-    static metadataOutputTypes = {
-        text: "str",
-        source_id: "str",
-        start_index: "int",
-        chunks: "list"
-    };
-    static isStreamingOutput = true;
-    async process(context) {
-        const collected = [];
-        for await (const item of this.genProcess(context)) {
-            if ("chunks" in item)
-                continue;
-            collected.push(item);
-        }
-        return {
-            text: collected[0]?.text ?? "",
-            source_id: collected[0]?.source_id ?? "",
-            start_index: collected[0]?.start_index ?? 0,
-            chunks: collected
-        };
-    }
-    async *genProcess(context) {
-        const document = this.document ?? this.document;
-        const text = await readDocumentText(document, context);
-        const sourceId = documentSourceId(document);
-        const chunkSize = Number(this.chunk_size ?? this.chunk_size ?? 1000);
-        const overlap = Number(this.chunk_overlap ?? this.chunk_overlap ?? 200);
-        const separators = Array.isArray(this.separators ?? this.separators)
-            ? (this.separators ?? this.separators).map((s) => String(s))
-            : ["\n\n", "\n", "."];
-        // Truly recursive splitting: for each separator in order,
-        // split text and recursively split chunks that are too large
-        // using the next separator.
-        function recursiveSplit(input, sepIndex) {
-            if (input.length <= chunkSize)
-                return [input];
-            if (sepIndex >= separators.length) {
-                // No more separators — fall back to character splitting
-                return splitByChunk(input, chunkSize, overlap);
-            }
-            const sep = separators[sepIndex];
-            if (!input.includes(sep)) {
-                // This separator doesn't exist in the text, try next
-                return recursiveSplit(input, sepIndex + 1);
-            }
-            const rawParts = input.split(sep);
-            const merged = [];
-            let current = "";
-            for (let i = 0; i < rawParts.length; i++) {
-                const part = rawParts[i];
-                const candidate = current ? current + sep + part : part;
-                if (candidate.length <= chunkSize) {
-                    current = candidate;
-                }
-                else {
-                    if (current)
-                        merged.push(current);
-                    if (part.length <= chunkSize) {
-                        current = part;
-                    }
-                    else {
-                        // Part is still too large — recurse with next separator
-                        const subChunks = recursiveSplit(part, sepIndex + 1);
-                        for (let j = 0; j < subChunks.length - 1; j++) {
-                            merged.push(subChunks[j]);
-                        }
-                        current = subChunks[subChunks.length - 1];
-                    }
-                }
-            }
-            if (current)
-                merged.push(current);
-            // Apply overlap by prepending end of previous chunk
-            if (overlap > 0 && merged.length > 1) {
-                const withOverlap = [merged[0]];
-                for (let i = 1; i < merged.length; i++) {
-                    const prev = merged[i - 1];
-                    const overlapText = prev.slice(-overlap);
-                    withOverlap.push(overlapText + merged[i]);
-                }
-                return withOverlap;
-            }
-            return merged;
-        }
-        const chunks = recursiveSplit(text, 0);
-        const collected = [];
-        let chunkIndex = 0;
-        let searchFrom = 0;
-        for (const chunk of chunks) {
-            // Find position accounting for overlap (search from where we expect)
-            const idx = text.indexOf(chunk, searchFrom);
-            const resolvedIndex = idx >= 0 ? idx : searchFrom;
-            const item = textChunk(chunk, `${sourceId}:${chunkIndex}`, resolvedIndex);
-            collected.push(item);
-            yield item;
-            // Move search forward, but not past the overlap region
-            searchFrom = resolvedIndex + Math.max(chunk.length - overlap, 1);
-            chunkIndex++;
-        }
-        yield { chunks: collected };
-    }
-}
-__decorate([
-    prop({
-        type: "document",
-        default: {
-            type: "document",
-            uri: "",
-            asset_id: null,
-            data: null,
-            metadata: null
-        },
-        title: "Document"
-    })
-], SplitRecursivelyNode.prototype, "document", void 0);
-__decorate([
-    prop({
-        type: "int",
-        default: 1000,
-        title: "Chunk Size",
-        description: "Maximum size of each chunk in characters"
-    })
-], SplitRecursivelyNode.prototype, "chunk_size", void 0);
-__decorate([
-    prop({
-        type: "int",
-        default: 200,
-        title: "Chunk Overlap",
-        description: "Number of characters to overlap between chunks"
-    })
-], SplitRecursivelyNode.prototype, "chunk_overlap", void 0);
-__decorate([
-    prop({
-        type: "list[str]",
-        default: ["\n\n", "\n", "."],
-        title: "Separators",
-        description: "List of separators to use for splitting, in order of preference"
-    })
-], SplitRecursivelyNode.prototype, "separators", void 0);
-export class SplitMarkdownNode extends BaseNode {
-    static nodeType = "nodetool.document.SplitMarkdown";
-    static title = "Split Markdown";
-    static description = "Splits markdown text by headers while preserving header hierarchy in metadata.\n    markdown, split, headers\n\n    Use cases:\n    - Splitting markdown documentation while preserving structure\n    - Processing markdown files for semantic search\n    - Creating context-aware chunks from markdown content";
-    static metadataOutputTypes = {
-        text: "str",
-        source_id: "str",
-        start_index: "int",
-        chunks: "list"
-    };
-    static isStreamingOutput = true;
-    async process(context) {
-        const collected = [];
-        for await (const item of this.genProcess(context)) {
-            if ("chunks" in item)
-                continue;
-            collected.push(item);
-        }
-        return {
-            text: collected[0]?.text ?? "",
-            source_id: collected[0]?.source_id ?? "",
-            start_index: collected[0]?.start_index ?? 0,
-            chunks: collected
-        };
-    }
-    async *genProcess(context) {
-        const document = this.document ?? this.document;
-        const markdown = await readDocumentText(document, context);
-        const sourceId = documentSourceId(document);
-        const stripHeaders = Boolean(this.strip_headers ?? this.strip_headers ?? true);
-        const returnEachLine = Boolean(this.return_each_line ?? this.return_each_line ?? false);
-        const chunkSize = Number(this.chunk_size ?? this.chunk_size ?? 1000);
-        const overlap = Number(this.chunk_overlap ?? this.chunk_overlap ?? 30);
-        // Parse headers_to_split_on config
-        const headersConfig = Array.isArray(this.headers_to_split_on)
-            ? this.headers_to_split_on
-            : [["#", "Header 1"], ["##", "Header 2"], ["###", "Header 3"]];
-        // Build header patterns sorted by level (longest first so ## matches before #)
-        const headerPatterns = headersConfig
-            .map(([symbol, name]) => ({ symbol, name, level: symbol.length }))
-            .sort((a, b) => b.level - a.level);
-        const sections = [];
-        let currentContent = [];
-        const activeHeaders = {};
-        for (const line of markdown.split("\n")) {
-            const trimmedLine = line.trim();
-            // Check if this line is a header we care about
-            let matchedHeader = null;
-            for (const hp of headerPatterns) {
-                if (trimmedLine.startsWith(hp.symbol + " ") && !trimmedLine.startsWith(hp.symbol + "#")) {
-                    matchedHeader = hp;
-                    break;
-                }
-            }
-            if (matchedHeader) {
-                // Emit previous section
-                if (currentContent.length > 0) {
-                    sections.push({
-                        content: [...currentContent],
-                        metadata: { ...activeHeaders }
-                    });
-                    currentContent = [];
-                }
-                // Update header hierarchy: set this header and clear deeper levels
-                const headerText = trimmedLine.slice(matchedHeader.symbol.length).trim();
-                activeHeaders[matchedHeader.name] = headerText;
-                // Clear any deeper headers
-                for (const hp of headerPatterns) {
-                    if (hp.level > matchedHeader.level) {
-                        delete activeHeaders[hp.name];
-                    }
-                }
-                if (!stripHeaders) {
-                    currentContent.push(line);
-                }
-            }
-            else {
-                currentContent.push(line);
-            }
-        }
-        // Emit final section
-        if (currentContent.length > 0) {
-            sections.push({
-                content: [...currentContent],
-                metadata: { ...activeHeaders }
-            });
-        }
-        // Yield sections
-        const collected = [];
-        let chunkIdx = 0;
-        for (const section of sections) {
-            const sectionText = section.content.join("\n").trim();
-            if (!sectionText)
-                continue;
-            if (returnEachLine) {
-                for (const line of section.content) {
-                    const trimmed = line.trim();
-                    if (trimmed) {
-                        const result = textChunk(trimmed, sourceId, chunkIdx);
-                        result.metadata = section.metadata;
-                        collected.push(result);
-                        yield result;
-                        chunkIdx++;
-                    }
-                }
-            }
-            else {
-                const sectionChunks = splitByChunk(sectionText, chunkSize, overlap);
-                for (const chunk of sectionChunks) {
-                    const result = textChunk(chunk, sourceId, chunkIdx);
-                    result.metadata = section.metadata;
-                    collected.push(result);
-                    yield result;
-                    chunkIdx++;
-                }
-            }
-        }
-        // Fallback if no sections were created
-        if (sections.length === 0) {
-            let startIndex = 0;
-            for (const chunk of splitByChunk(markdown, chunkSize, overlap)) {
-                const idx = markdown.indexOf(chunk, startIndex);
-                const resolvedIndex = idx >= 0 ? idx : startIndex;
-                const item = textChunk(chunk, sourceId, resolvedIndex);
-                collected.push(item);
-                yield item;
-                startIndex = resolvedIndex + Math.max(chunk.length - overlap, 1);
-            }
-        }
-        yield { chunks: collected };
-    }
-}
-__decorate([
-    prop({
-        type: "document",
-        default: {
-            type: "document",
-            uri: "",
-            asset_id: null,
-            data: null,
-            metadata: null
-        },
-        title: "Document"
-    })
-], SplitMarkdownNode.prototype, "document", void 0);
-__decorate([
-    prop({
-        type: "list[tuple[str, str]]",
-        default: [
-            ["#", "Header 1"],
-            ["##", "Header 2"],
-            ["###", "Header 3"]
-        ],
-        title: "Headers To Split On",
-        description: "List of tuples containing (header_symbol, header_name)"
-    })
-], SplitMarkdownNode.prototype, "headers_to_split_on", void 0);
-__decorate([
-    prop({
-        type: "bool",
-        default: true,
-        title: "Strip Headers",
-        description: "Whether to remove headers from the output content"
-    })
-], SplitMarkdownNode.prototype, "strip_headers", void 0);
-__decorate([
-    prop({
-        type: "bool",
-        default: false,
-        title: "Return Each Line",
-        description: "Whether to split into individual lines instead of header sections"
-    })
-], SplitMarkdownNode.prototype, "return_each_line", void 0);
-__decorate([
-    prop({
-        type: "int",
-        default: 1000,
-        title: "Chunk Size",
-        description: "Optional maximum chunk size for further splitting"
-    })
-], SplitMarkdownNode.prototype, "chunk_size", void 0);
-__decorate([
-    prop({
-        type: "int",
-        default: 30,
-        title: "Chunk Overlap",
-        description: "Overlap size when using chunk_size"
-    })
-], SplitMarkdownNode.prototype, "chunk_overlap", void 0);
-export const DOCUMENT_NODES = [
-    LoadDocumentFileNode,
-    SaveDocumentFileNode,
-    ListDocumentsNode,
-    SplitDocumentNode,
-    SplitHTMLNode,
-    SplitJSONNode,
-    SplitRecursivelyNode,
-    SplitMarkdownNode
-];
-//# sourceMappingURL=document.js.map