@agentionai/agents 0.11.0 → 0.12.0-beta

package/dist/chunkers/ElementChunker.d.ts

@@ -0,0 +1,100 @@
+import { Chunker } from "./Chunker";
+import { Chunk, ChunkerConfig, ChunkOptions } from "./types";
+import { ParsedElement } from "../parsers/types";
+/**
+ * Configuration for {@link ElementChunker}.
+ */
+export interface ElementChunkerConfig extends ChunkerConfig {
+    /**
+     * Element types to skip entirely.
+     * Useful for dropping decorative or non-content elements.
+     * @example ["Image", "PageBreak", "Header", "Footer"]
+     */
+    excludeTypes?: string[];
+    /**
+     * Element types that always start a new chunk, even if there is room
+     * in the current one. Use this to keep headings at the top of their
+     * section's chunk.
+     * @default ["Title"]
+     */
+    breakOnTypes?: string[];
+}
+/**
+ * Chunks a document by grouping its **structured elements** rather than
+ * splitting raw text. Designed for use with parsers that return element
+ * lists (e.g. {@link UnstructuredLocalParser}, {@link UnstructuredAPIParser}).
+ *
+ * **How it works:**
+ * 1. Adjacent elements are merged into a single chunk until the combined
+ *    character count would exceed `chunkSize`.
+ * 2. A `breakOnTypes` element (default: `"Title"`) always starts a fresh
+ *    chunk so that headings introduce their section's content.
+ * 3. A single element whose text exceeds `chunkSize` is split recursively
+ *    using separator heuristics (paragraphs → sentences → words → characters).
+ * 4. Element types are stored in `chunk.metadata.element_types`; page number
+ *    is stored in `chunk.metadata.page` when available.
+ *
+ * Use via {@link IngestionPipeline.ingestFile} — the pipeline automatically
+ * calls `chunkElements()` instead of `chunk()` when this chunker is used and
+ * the parser returns a structured element list.
+ *
+ * @example
+ * ```typescript
+ * import { ElementChunker } from '@agentionai/agents/chunkers';
+ * import { UnstructuredLocalParser } from '@agentionai/agents/parsers/unstructured-local';
+ *
+ * const pipeline = new IngestionPipeline(
+ *   new ElementChunker({ chunkSize: 1000 }),
+ *   embeddings,
+ *   store,
+ * );
+ *
+ * await pipeline.ingestFile('/docs/report.pdf', new UnstructuredLocalParser(), {
+ *   strategy: 'hi_res',
+ * });
+ * ```
+ */
+export declare class ElementChunker extends Chunker {
+    readonly name = "ElementChunker";
+    private readonly excludeTypes;
+    private readonly breakOnTypes;
+    constructor(config: ElementChunkerConfig);
+    /**
+     * Chunk a list of structured elements into {@link Chunk} objects.
+     *
+     * This is the primary entry point when using this chunker with a parser.
+     * Called automatically by {@link IngestionPipeline.ingestFile} when
+     * the parsed document has an `elements` array.
+     *
+     * @param elements - Parsed elements from a {@link DocumentParser}
+     * @param options  - Source tracking and custom metadata
+     */
+    chunkElements(elements: ParsedElement[], options?: ChunkOptions): Promise<Chunk[]>;
+    /**
+     * Fallback text splitting used when {@link Chunker.chunk} is called directly
+     * (i.e. without a structured element list). Splits on double newlines first,
+     * then sentences, then words.
+     */
+    protected splitText(text: string): string[];
+    /**
+     * Build a {@link Chunk} from a group of elements.
+     */
+    private buildChunk;
+    /**
+     * Split a single large element text using separator heuristics.
+     */
+    private splitLargeText;
+    /**
+     * Greedily merge string parts into windows of at most `maxSize` characters.
+     */
+    private mergeToSize;
+    /**
+     * Hard character-count split when no separator works.
+     */
+    private forceSplit;
+    /**
+     * Apply character-level overlap between already-split strings.
+     */
+    private applyCharOverlap;
+}
+//# sourceMappingURL=ElementChunker.d.ts.map

package/dist/chunkers/ElementChunker.js

@@ -0,0 +1,242 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ElementChunker = void 0;
+const Chunker_1 = require("./Chunker");
+/**
+ * Chunks a document by grouping its **structured elements** rather than
+ * splitting raw text. Designed for use with parsers that return element
+ * lists (e.g. {@link UnstructuredLocalParser}, {@link UnstructuredAPIParser}).
+ *
+ * **How it works:**
+ * 1. Adjacent elements are merged into a single chunk until the combined
+ *    character count would exceed `chunkSize`.
+ * 2. A `breakOnTypes` element (default: `"Title"`) always starts a fresh
+ *    chunk so that headings introduce their section's content.
+ * 3. A single element whose text exceeds `chunkSize` is split recursively
+ *    using separator heuristics (paragraphs → sentences → words → characters).
+ * 4. Element types are stored in `chunk.metadata.element_types`; page number
+ *    is stored in `chunk.metadata.page` when available.
+ *
+ * Use via {@link IngestionPipeline.ingestFile} — the pipeline automatically
+ * calls `chunkElements()` instead of `chunk()` when this chunker is used and
+ * the parser returns a structured element list.
+ *
+ * @example
+ * ```typescript
+ * import { ElementChunker } from '@agentionai/agents/chunkers';
+ * import { UnstructuredLocalParser } from '@agentionai/agents/parsers/unstructured-local';
+ *
+ * const pipeline = new IngestionPipeline(
+ *   new ElementChunker({ chunkSize: 1000 }),
+ *   embeddings,
+ *   store,
+ * );
+ *
+ * await pipeline.ingestFile('/docs/report.pdf', new UnstructuredLocalParser(), {
+ *   strategy: 'hi_res',
+ * });
+ * ```
+ */
+class ElementChunker extends Chunker_1.Chunker {
+    constructor(config) {
+        super(config);
+        this.name = "ElementChunker";
+        this.excludeTypes = new Set(config.excludeTypes ?? []);
+        this.breakOnTypes = new Set(config.breakOnTypes ?? ["Title"]);
+    }
+    // ─── element-aware primary path ──────────────────────────────────────────
+    /**
+     * Chunk a list of structured elements into {@link Chunk} objects.
+     *
+     * This is the primary entry point when using this chunker with a parser.
+     * Called automatically by {@link IngestionPipeline.ingestFile} when
+     * the parsed document has an `elements` array.
+     *
+     * @param elements - Parsed elements from a {@link DocumentParser}
+     * @param options  - Source tracking and custom metadata
+     */
+    async chunkElements(elements, options) {
+        const { chunkSize } = this.config;
+        const chunks = [];
+        // Working group — elements accumulated into the next chunk
+        let groupElements = [];
+        let groupSize = 0;
+        const flush = () => {
+            if (groupElements.length === 0)
+                return;
+            const content = groupElements
+                .map((el) => el.text)
+                .filter(Boolean)
+                .join("\n\n");
+            if (content.trim()) {
+                chunks.push(this.buildChunk(content, groupElements, chunks.length, options));
+            }
+            groupElements = [];
+            groupSize = 0;
+        };
+        for (const el of elements) {
+            if (this.excludeTypes.has(el.type))
+                continue;
+            const text = el.text?.trim() ?? "";
+            if (!text)
+                continue;
+            // Break-on-type: flush current group before adding this element
+            if (this.breakOnTypes.has(el.type) && groupElements.length > 0) {
+                flush();
+            }
+            if (text.length > chunkSize) {
+                // Flush current group first, then split the large element into sub-chunks
+                flush();
+                const subTexts = this.splitLargeText(text);
+                for (const subText of subTexts) {
+                    chunks.push(this.buildChunk(subText, [el], chunks.length, options));
+                }
+            }
+            else if (groupSize + text.length > chunkSize && groupElements.length > 0) {
+                // Adding this element would overflow — flush and start fresh
+                flush();
+                groupElements.push(el);
+                groupSize = text.length;
+            }
+            else {
+                groupElements.push(el);
+                groupSize += text.length;
+            }
+        }
+        flush();
+        if (chunks.length === 0)
+            return [];
+        // Set correct total and link chunks
+        for (const chunk of chunks) {
+            chunk.metadata.total = chunks.length;
+        }
+        this.linkChunks(chunks);
+        if (this.config.chunkProcessor) {
+            return this.applyProcessor(chunks);
+        }
+        return chunks;
+    }
+    // ─── text fallback path (required by Chunker) ────────────────────────────
+    /**
+     * Fallback text splitting used when {@link Chunker.chunk} is called directly
+     * (i.e. without a structured element list). Splits on double newlines first,
+     * then sentences, then words.
+     */
+    splitText(text) {
+        return this.splitLargeText(text);
+    }
+    // ─── helpers ─────────────────────────────────────────────────────────────
+    /**
+     * Build a {@link Chunk} from a group of elements.
+     */
+    buildChunk(content, sourceElements, index, options) {
+        const elementTypes = [...new Set(sourceElements.map((el) => el.type))];
+        // Use page_number from the first element that provides it
+        const page = sourceElements
+            .map((el) => el.metadata?.["page_number"])
+            .find((p) => p != null);
+        const metadata = {
+            index,
+            total: 0, // set after all chunks are built
+            prev_id: null,
+            next_id: null,
+            start: 0,
+            end: content.length,
+            source_id: options?.sourceId,
+            source_path: options?.sourcePath,
+            char_count: content.length,
+            hash: this.computeHash(content),
+            section: this.detectSectionTitle(content),
+            page,
+            element_types: elementTypes,
+            ...options?.metadata,
+        };
+        return {
+            id: this.generateId(content, index, options?.sourceId),
+            content,
+            metadata,
+        };
+    }
+    /**
+     * Split a single large element text using separator heuristics.
+     */
+    splitLargeText(text) {
+        const { chunkSize, chunkOverlap = 0 } = this.config;
+        if (text.length <= chunkSize)
+            return [text];
+        const separators = ["\n\n", "\n", ". ", " "];
+        for (const sep of separators) {
+            const parts = text.split(sep).filter((s) => s.trim());
+            if (parts.length <= 1)
+                continue;
+            const merged = this.mergeToSize(parts, sep, chunkSize);
+            // Apply overlap if configured
+            if (chunkOverlap > 0 && merged.length > 1) {
+                return this.applyCharOverlap(merged, chunkOverlap);
+            }
+            return merged;
+        }
+        return this.forceSplit(text, chunkSize, chunkOverlap);
+    }
+    /**
+     * Greedily merge string parts into windows of at most `maxSize` characters.
+     */
+    mergeToSize(parts, sep, maxSize) {
+        const result = [];
+        let current = "";
+        for (const part of parts) {
+            const addition = current ? sep + part : part;
+            if (current.length + addition.length <= maxSize) {
+                current = current + addition;
+            }
+            else {
+                if (current)
+                    result.push(current);
+                if (part.length > maxSize) {
+                    result.push(...this.forceSplit(part, maxSize, 0));
+                    current = "";
+                }
+                else {
+                    current = part;
+                }
+            }
+        }
+        if (current)
+            result.push(current);
+        return result;
+    }
+    /**
+     * Hard character-count split when no separator works.
+     */
+    forceSplit(text, size, overlap) {
+        const chunks = [];
+        const step = size - overlap;
+        let start = 0;
+        while (start < text.length) {
+            const end = Math.min(start + size, text.length);
+            const slice = text.slice(start, end);
+            if (slice.trim())
+                chunks.push(slice);
+            if (end >= text.length)
+                break;
+            start += step;
+        }
+        return chunks;
+    }
+    /**
+     * Apply character-level overlap between already-split strings.
+     */
+    applyCharOverlap(chunks, overlap) {
+        if (chunks.length <= 1)
+            return chunks;
+        const result = [chunks[0]];
+        for (let i = 1; i < chunks.length; i++) {
+            const prev = chunks[i - 1];
+            const tail = prev.length > overlap ? prev.slice(prev.length - overlap) : prev;
+            result.push(tail + " " + chunks[i]);
+        }
+        return result;
+    }
+}
+exports.ElementChunker = ElementChunker;
+//# sourceMappingURL=ElementChunker.js.map

package/dist/chunkers/index.d.ts

@@ -3,4 +3,5 @@ export { Chunker } from "./Chunker";
 export { TextChunker } from "./TextChunker";
 export { RecursiveChunker } from "./RecursiveChunker";
 export { TokenChunker } from "./TokenChunker";
+export { ElementChunker, type ElementChunkerConfig } from "./ElementChunker";
 //# sourceMappingURL=index.d.ts.map

package/dist/chunkers/index.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.TokenChunker = exports.RecursiveChunker = exports.TextChunker = exports.Chunker = void 0;
+exports.ElementChunker = exports.TokenChunker = exports.RecursiveChunker = exports.TextChunker = exports.Chunker = void 0;
 // Base class
 var Chunker_1 = require("./Chunker");
 Object.defineProperty(exports, "Chunker", { enumerable: true, get: function () { return Chunker_1.Chunker; } });
@@ -11,4 +11,6 @@ var RecursiveChunker_1 = require("./RecursiveChunker");
 Object.defineProperty(exports, "RecursiveChunker", { enumerable: true, get: function () { return RecursiveChunker_1.RecursiveChunker; } });
 var TokenChunker_1 = require("./TokenChunker");
 Object.defineProperty(exports, "TokenChunker", { enumerable: true, get: function () { return TokenChunker_1.TokenChunker; } });
+var ElementChunker_1 = require("./ElementChunker");
+Object.defineProperty(exports, "ElementChunker", { enumerable: true, get: function () { return ElementChunker_1.ElementChunker; } });
 //# sourceMappingURL=index.js.map

package/dist/ingestion/IngestionPipeline.d.ts

@@ -2,6 +2,8 @@ import { Chunk, ChunkOptions } from "../chunkers/types";
 import { Chunker } from "../chunkers/Chunker";
 import { Embeddings } from "../embeddings/Embeddings";
 import { VectorStore } from "../vectorstore/VectorStore";
+import { DocumentParser } from "../parsers/DocumentParser";
+import { ParseOptions } from "../parsers/types";
 import { IngestionOptions, IngestionResult, DocumentInput } from "./types";
 /**
  * Pipeline for ingesting documents into a vector store.
@@ -31,7 +33,15 @@ export declare class IngestionPipeline {
     private chunker;
     private embeddings;
     private store;
-    constructor(chunker: Chunker, embeddings: Embeddings, store: VectorStore);
+    private parser?;
+    /**
+     * @param chunker    - Chunker to split parsed/raw text into chunks
+     * @param embeddings - Embeddings provider
+     * @param store      - Vector store for persistence
+     * @param parser     - Optional default parser used by {@link ingestFile} and
+     *                     {@link ingestFiles} when no parser is passed at call time
+     */
+    constructor(chunker: Chunker, embeddings: Embeddings, store: VectorStore, parser?: DocumentParser);
     /**
      * Ingest a single document into the vector store.
      *
@@ -48,6 +58,64 @@ export declare class IngestionPipeline {
      * @returns Aggregated result of all ingestions
      */
     ingestMany(documents: DocumentInput[], options?: IngestionOptions): Promise<IngestionResult>;
+    /**
+     * Parse a file and ingest it into the vector store.
+     *
+     * Combines parsing + chunking + embedding + storing in a single call.
+     * When the pipeline's chunker is an {@link ElementChunker} and the parser
+     * returns structured elements, chunking is done on element boundaries
+     * instead of raw text.
+     *
+     * The `parser` argument is optional when one was configured on the pipeline
+     * constructor; it is required otherwise.
+     *
+     * @example Using a pipeline-level parser:
+     * ```typescript
+     * const pipeline = new IngestionPipeline(
+     *   new ElementChunker({ chunkSize: 1000 }),
+     *   embeddings,
+     *   store,
+     *   new UnstructuredLocalParser(),
+     * );
+     * await pipeline.ingestFile("/docs/report.pdf", { strategy: "hi_res" });
+     * ```
+     *
+     * @example Passing a parser per call:
+     * ```typescript
+     * await pipeline.ingestFile("/docs/report.pdf", new UnstructuredLocalParser(), {
+     *   strategy: "hi_res",
+     *   sourceId: "report-2024",
+     * });
+     * ```
+     */
+    ingestFile(filePath: string, options?: ParseOptions & ChunkOptions & IngestionOptions): Promise<IngestionResult>;
+    ingestFile(filePath: string, parser: DocumentParser, options?: ParseOptions & ChunkOptions & IngestionOptions): Promise<IngestionResult>;
+    /**
+     * Parse and ingest multiple files.
+     *
+     * Files are parsed sequentially; all chunks are batched together for
+     * embedding and storage. When the pipeline uses an {@link ElementChunker}
+     * and the parser returns structured elements, element-aware chunking is
+     * applied per file (preserving `element_types` and `page` metadata).
+     * The `parser` argument is optional when one was set on the pipeline
+     * constructor.
+     *
+     * @example Using a pipeline-level parser:
+     * ```typescript
+     * await pipeline.ingestFiles(["/a.pdf", "/b.docx"], { skipDuplicates: true });
+     * ```
+     *
+     * @example Passing a parser per call:
+     * ```typescript
+     * await pipeline.ingestFiles(
+     *   ["/docs/a.pdf", "/docs/b.docx"],
+     *   new UnstructuredAPIParser({ serverUrl: "http://localhost:8000" }),
+     *   { strategy: "auto", skipDuplicates: true }
+     * );
+     * ```
+     */
+    ingestFiles(filePaths: string[], options?: ParseOptions & ChunkOptions & IngestionOptions): Promise<IngestionResult>;
+    ingestFiles(filePaths: string[], parser: DocumentParser, options?: ParseOptions & ChunkOptions & IngestionOptions): Promise<IngestionResult>;
     /**
      * Ingest pre-chunked data into the vector store.
      * Useful when chunking is done separately.
@@ -82,5 +150,9 @@ export declare class IngestionPipeline {
      * Get the vector store used by this pipeline.
      */
     getStore(): VectorStore;
+    /**
+     * Get the default parser configured on this pipeline, if any.
+     */
+    getParser(): DocumentParser | undefined;
 }
 //# sourceMappingURL=IngestionPipeline.d.ts.map

package/dist/ingestion/IngestionPipeline.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.IngestionPipeline = void 0;
+const ElementChunker_1 = require("../chunkers/ElementChunker");
 /**
  * Pipeline for ingesting documents into a vector store.
  * Orchestrates the flow: chunk → batch embed → store
@@ -26,10 +27,18 @@ exports.IngestionPipeline = void 0;
  * ```
  */
 class IngestionPipeline {
-    constructor(chunker, embeddings, store) {
+    /**
+     * @param chunker    - Chunker to split parsed/raw text into chunks
+     * @param embeddings - Embeddings provider
+     * @param store      - Vector store for persistence
+     * @param parser     - Optional default parser used by {@link ingestFile} and
+     *                     {@link ingestFiles} when no parser is passed at call time
+     */
+    constructor(chunker, embeddings, store, parser) {
         this.chunker = chunker;
         this.embeddings = embeddings;
         this.store = store;
+        this.parser = parser;
     }
     /**
      * Ingest a single document into the vector store.
@@ -95,6 +104,100 @@ class IngestionPipeline {
         // Process all chunks together
         return this.processChunks(allChunks, options ?? {}, startTime);
     }
+    async ingestFile(filePath, parserOrOptions, options) {
+        let parser;
+        let opts;
+        if (parserOrOptions != null && typeof parserOrOptions.parse === "function") {
+            parser = parserOrOptions;
+            opts = options;
+        }
+        else {
+            parser = this.parser;
+            opts = parserOrOptions;
+        }
+        if (!parser) {
+            throw new Error("No parser provided. Pass a DocumentParser to ingestFile() or set one in the IngestionPipeline constructor.");
+        }
+        const parseOptions = {
+            strategy: opts?.strategy,
+            languages: opts?.languages,
+        };
+        const parsed = await parser.parse(filePath, parseOptions);
+        const chunkOptions = {
+            sourceId: opts?.sourceId,
+            sourcePath: opts?.sourcePath ?? filePath,
+            metadata: opts?.metadata,
+        };
+        const ingestionOptions = {
+            batchSize: opts?.batchSize,
+            onProgress: opts?.onProgress,
+            onError: opts?.onError,
+            skipDuplicates: opts?.skipDuplicates,
+        };
+        // When the pipeline uses an ElementChunker and the parser returned
+        // structured elements, chunk on element boundaries instead of raw text.
+        if (this.chunker instanceof ElementChunker_1.ElementChunker && parsed.elements?.length) {
+            const startTime = Date.now();
+            const chunks = await this.chunker.chunkElements(parsed.elements, chunkOptions);
+            return this.processChunks(chunks, ingestionOptions, startTime);
+        }
+        return this.ingest(parsed.text, { ...chunkOptions, ...ingestionOptions });
+    }
+    async ingestFiles(filePaths, parserOrOptions, options) {
+        let parser;
+        let opts;
+        if (parserOrOptions != null && typeof parserOrOptions.parse === "function") {
+            parser = parserOrOptions;
+            opts = options;
+        }
+        else {
+            parser = this.parser;
+            opts = parserOrOptions;
+        }
+        if (!parser) {
+            throw new Error("No parser provided. Pass a DocumentParser to ingestFiles() or set one in the IngestionPipeline constructor.");
+        }
+        const parseOptions = {
+            strategy: opts?.strategy,
+            languages: opts?.languages,
+        };
+        const ingestionOptions = {
+            batchSize: opts?.batchSize,
+            onProgress: opts?.onProgress,
+            onError: opts?.onError,
+            skipDuplicates: opts?.skipDuplicates,
+        };
+        const startTime = Date.now();
+        const allChunks = [];
+        this.emitProgress(ingestionOptions.onProgress, {
+            phase: "chunking",
+            processed: 0,
+            total: filePaths.length,
+        });
+        for (let i = 0; i < filePaths.length; i++) {
+            const filePath = filePaths[i];
+            const parsed = await parser.parse(filePath, parseOptions);
+            const chunkOptions = {
+                sourceId: opts?.sourceId,
+                sourcePath: filePath,
+                metadata: opts?.metadata,
+            };
+            let fileChunks;
+            if (this.chunker instanceof ElementChunker_1.ElementChunker && parsed.elements?.length) {
+                fileChunks = await this.chunker.chunkElements(parsed.elements, chunkOptions);
+            }
+            else {
+                fileChunks = await this.chunker.chunk(parsed.text, chunkOptions);
+            }
+            allChunks.push(...fileChunks);
+            this.emitProgress(ingestionOptions.onProgress, {
+                phase: "chunking",
+                processed: i + 1,
+                total: filePaths.length,
+            });
+        }
+        return this.processChunks(allChunks, ingestionOptions, startTime);
+    }
     /**
      * Ingest pre-chunked data into the vector store.
      * Useful when chunking is done separately.
@@ -261,6 +364,12 @@ class IngestionPipeline {
     getStore() {
         return this.store;
     }
+    /**
+     * Get the default parser configured on this pipeline, if any.
+     */
+    getParser() {
+        return this.parser;
+    }
 }
 exports.IngestionPipeline = IngestionPipeline;
 //# sourceMappingURL=IngestionPipeline.js.map

package/dist/parsers/DocumentParser.d.ts

@@ -0,0 +1,36 @@
+import { ParsedDocument, ParsedElement, ParseOptions } from "./types";
+/**
+ * Abstract base class for document parsers.
+ *
+ * Implementations wrap third-party libraries (Unstructured, LlamaIndex, etc.)
+ * and normalise their output into a {@link ParsedDocument} that can be fed
+ * directly into an {@link IngestionPipeline}.
+ *
+ * All peer dependencies are loaded lazily via dynamic import so that
+ * users only need to install what they actually use.
+ *
+ * @example
+ * ```typescript
+ * const parser = new UnstructuredLocalParser();
+ * const doc = await parser.parse("/path/to/report.pdf");
+ * console.log(doc.elements?.length, "elements parsed");
+ * await pipeline.ingestFile("/path/to/report.pdf", parser);
+ * ```
+ */
+export declare abstract class DocumentParser {
+    /** Human-readable parser identifier (e.g. "unstructured-local") */
+    abstract readonly name: string;
+    /**
+     * Parse a document file and return its content.
+     *
+     * @param filePath - Absolute or relative path to the file
+     * @param options  - Optional parsing hints (strategy, languages, etc.)
+     */
+    abstract parse(filePath: string, options?: ParseOptions): Promise<ParsedDocument>;
+    /**
+     * Join elements into a single plain-text string.
+     * Filters out empty strings and separates elements with a blank line.
+     */
+    protected elementsToText(elements: ParsedElement[]): string;
+}
+//# sourceMappingURL=DocumentParser.d.ts.map

package/dist/parsers/DocumentParser.js

@@ -0,0 +1,35 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DocumentParser = void 0;
+/**
+ * Abstract base class for document parsers.
+ *
+ * Implementations wrap third-party libraries (Unstructured, LlamaIndex, etc.)
+ * and normalise their output into a {@link ParsedDocument} that can be fed
+ * directly into an {@link IngestionPipeline}.
+ *
+ * All peer dependencies are loaded lazily via dynamic import so that
+ * users only need to install what they actually use.
+ *
+ * @example
+ * ```typescript
+ * const parser = new UnstructuredLocalParser();
+ * const doc = await parser.parse("/path/to/report.pdf");
+ * console.log(doc.elements?.length, "elements parsed");
+ * await pipeline.ingestFile("/path/to/report.pdf", parser);
+ * ```
+ */
+class DocumentParser {
+    /**
+     * Join elements into a single plain-text string.
+     * Filters out empty strings and separates elements with a blank line.
+     */
+    elementsToText(elements) {
+        return elements
+            .map((el) => el.text)
+            .filter(Boolean)
+            .join("\n\n");
+    }
+}
+exports.DocumentParser = DocumentParser;
+//# sourceMappingURL=DocumentParser.js.map