@agentionai/agents 0.12.0-beta → 0.12.0

package/dist/chunkers/ElementChunker.js

@@ -1,242 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ElementChunker = void 0;
-const Chunker_1 = require("./Chunker");
-/**
- * Chunks a document by grouping its **structured elements** rather than
- * splitting raw text. Designed for use with parsers that return element
- * lists (e.g. {@link UnstructuredLocalParser}, {@link UnstructuredAPIParser}).
- *
- * **How it works:**
- * 1. Adjacent elements are merged into a single chunk until the combined
- *    character count would exceed `chunkSize`.
- * 2. A `breakOnTypes` element (default: `"Title"`) always starts a fresh
- *    chunk so that headings introduce their section's content.
- * 3. A single element whose text exceeds `chunkSize` is split recursively
- *    using separator heuristics (paragraphs → sentences → words → characters).
- * 4. Element types are stored in `chunk.metadata.element_types`; page number
- *    is stored in `chunk.metadata.page` when available.
- *
- * Use via {@link IngestionPipeline.ingestFile} — the pipeline automatically
- * calls `chunkElements()` instead of `chunk()` when this chunker is used and
- * the parser returns a structured element list.
- *
- * @example
- * ```typescript
- * import { ElementChunker } from '@agentionai/agents/chunkers';
- * import { UnstructuredLocalParser } from '@agentionai/agents/parsers/unstructured-local';
- *
- * const pipeline = new IngestionPipeline(
- *   new ElementChunker({ chunkSize: 1000 }),
- *   embeddings,
- *   store,
- * );
- *
- * await pipeline.ingestFile('/docs/report.pdf', new UnstructuredLocalParser(), {
- *   strategy: 'hi_res',
- * });
- * ```
- */
-class ElementChunker extends Chunker_1.Chunker {
-    constructor(config) {
-        super(config);
-        this.name = "ElementChunker";
-        this.excludeTypes = new Set(config.excludeTypes ?? []);
-        this.breakOnTypes = new Set(config.breakOnTypes ?? ["Title"]);
-    }
-    // ─── element-aware primary path ──────────────────────────────────────────
-    /**
-     * Chunk a list of structured elements into {@link Chunk} objects.
-     *
-     * This is the primary entry point when using this chunker with a parser.
-     * Called automatically by {@link IngestionPipeline.ingestFile} when
-     * the parsed document has an `elements` array.
-     *
-     * @param elements - Parsed elements from a {@link DocumentParser}
-     * @param options  - Source tracking and custom metadata
-     */
-    async chunkElements(elements, options) {
-        const { chunkSize } = this.config;
-        const chunks = [];
-        // Working group — elements accumulated into the next chunk
-        let groupElements = [];
-        let groupSize = 0;
-        const flush = () => {
-            if (groupElements.length === 0)
-                return;
-            const content = groupElements
-                .map((el) => el.text)
-                .filter(Boolean)
-                .join("\n\n");
-            if (content.trim()) {
-                chunks.push(this.buildChunk(content, groupElements, chunks.length, options));
-            }
-            groupElements = [];
-            groupSize = 0;
-        };
-        for (const el of elements) {
-            if (this.excludeTypes.has(el.type))
-                continue;
-            const text = el.text?.trim() ?? "";
-            if (!text)
-                continue;
-            // Break-on-type: flush current group before adding this element
-            if (this.breakOnTypes.has(el.type) && groupElements.length > 0) {
-                flush();
-            }
-            if (text.length > chunkSize) {
-                // Flush current group first, then split the large element into sub-chunks
-                flush();
-                const subTexts = this.splitLargeText(text);
-                for (const subText of subTexts) {
-                    chunks.push(this.buildChunk(subText, [el], chunks.length, options));
-                }
-            }
-            else if (groupSize + text.length > chunkSize && groupElements.length > 0) {
-                // Adding this element would overflow — flush and start fresh
-                flush();
-                groupElements.push(el);
-                groupSize = text.length;
-            }
-            else {
-                groupElements.push(el);
-                groupSize += text.length;
-            }
-        }
-        flush();
-        if (chunks.length === 0)
-            return [];
-        // Set correct total and link chunks
-        for (const chunk of chunks) {
-            chunk.metadata.total = chunks.length;
-        }
-        this.linkChunks(chunks);
-        if (this.config.chunkProcessor) {
-            return this.applyProcessor(chunks);
-        }
-        return chunks;
-    }
-    // ─── text fallback path (required by Chunker) ────────────────────────────
-    /**
-     * Fallback text splitting used when {@link Chunker.chunk} is called directly
-     * (i.e. without a structured element list). Splits on double newlines first,
-     * then sentences, then words.
-     */
-    splitText(text) {
-        return this.splitLargeText(text);
-    }
-    // ─── helpers ─────────────────────────────────────────────────────────────
-    /**
-     * Build a {@link Chunk} from a group of elements.
-     */
-    buildChunk(content, sourceElements, index, options) {
-        const elementTypes = [...new Set(sourceElements.map((el) => el.type))];
-        // Use page_number from the first element that provides it
-        const page = sourceElements
-            .map((el) => el.metadata?.["page_number"])
-            .find((p) => p != null);
-        const metadata = {
-            index,
-            total: 0, // set after all chunks are built
-            prev_id: null,
-            next_id: null,
-            start: 0,
-            end: content.length,
-            source_id: options?.sourceId,
-            source_path: options?.sourcePath,
-            char_count: content.length,
-            hash: this.computeHash(content),
-            section: this.detectSectionTitle(content),
-            page,
-            element_types: elementTypes,
-            ...options?.metadata,
-        };
-        return {
-            id: this.generateId(content, index, options?.sourceId),
-            content,
-            metadata,
-        };
-    }
-    /**
-     * Split a single large element text using separator heuristics.
-     */
-    splitLargeText(text) {
-        const { chunkSize, chunkOverlap = 0 } = this.config;
-        if (text.length <= chunkSize)
-            return [text];
-        const separators = ["\n\n", "\n", ". ", " "];
-        for (const sep of separators) {
-            const parts = text.split(sep).filter((s) => s.trim());
-            if (parts.length <= 1)
-                continue;
-            const merged = this.mergeToSize(parts, sep, chunkSize);
-            // Apply overlap if configured
-            if (chunkOverlap > 0 && merged.length > 1) {
-                return this.applyCharOverlap(merged, chunkOverlap);
-            }
-            return merged;
-        }
-        return this.forceSplit(text, chunkSize, chunkOverlap);
-    }
-    /**
-     * Greedily merge string parts into windows of at most `maxSize` characters.
-     */
-    mergeToSize(parts, sep, maxSize) {
-        const result = [];
-        let current = "";
-        for (const part of parts) {
-            const addition = current ? sep + part : part;
-            if (current.length + addition.length <= maxSize) {
-                current = current + addition;
-            }
-            else {
-                if (current)
-                    result.push(current);
-                if (part.length > maxSize) {
-                    result.push(...this.forceSplit(part, maxSize, 0));
-                    current = "";
-                }
-                else {
-                    current = part;
-                }
-            }
-        }
-        if (current)
-            result.push(current);
-        return result;
-    }
-    /**
-     * Hard character-count split when no separator works.
-     */
-    forceSplit(text, size, overlap) {
-        const chunks = [];
-        const step = size - overlap;
-        let start = 0;
-        while (start < text.length) {
-            const end = Math.min(start + size, text.length);
-            const slice = text.slice(start, end);
-            if (slice.trim())
-                chunks.push(slice);
-            if (end >= text.length)
-                break;
-            start += step;
-        }
-        return chunks;
-    }
-    /**
-     * Apply character-level overlap between already-split strings.
-     */
-    applyCharOverlap(chunks, overlap) {
-        if (chunks.length <= 1)
-            return chunks;
-        const result = [chunks[0]];
-        for (let i = 1; i < chunks.length; i++) {
-            const prev = chunks[i - 1];
-            const tail = prev.length > overlap ? prev.slice(prev.length - overlap) : prev;
-            result.push(tail + " " + chunks[i]);
-        }
-        return result;
-    }
-}
-exports.ElementChunker = ElementChunker;
-//# sourceMappingURL=ElementChunker.js.map

package/dist/parsers/DocumentParser.d.ts

@@ -1,36 +0,0 @@
-import { ParsedDocument, ParsedElement, ParseOptions } from "./types";
-/**
- * Abstract base class for document parsers.
- *
- * Implementations wrap third-party libraries (Unstructured, LlamaIndex, etc.)
- * and normalise their output into a {@link ParsedDocument} that can be fed
- * directly into an {@link IngestionPipeline}.
- *
- * All peer dependencies are loaded lazily via dynamic import so that
- * users only need to install what they actually use.
- *
- * @example
- * ```typescript
- * const parser = new UnstructuredLocalParser();
- * const doc = await parser.parse("/path/to/report.pdf");
- * console.log(doc.elements?.length, "elements parsed");
- * await pipeline.ingestFile("/path/to/report.pdf", parser);
- * ```
- */
-export declare abstract class DocumentParser {
-    /** Human-readable parser identifier (e.g. "unstructured-local") */
-    abstract readonly name: string;
-    /**
-     * Parse a document file and return its content.
-     *
-     * @param filePath - Absolute or relative path to the file
-     * @param options  - Optional parsing hints (strategy, languages, etc.)
-     */
-    abstract parse(filePath: string, options?: ParseOptions): Promise<ParsedDocument>;
-    /**
-     * Join elements into a single plain-text string.
-     * Filters out empty strings and separates elements with a blank line.
-     */
-    protected elementsToText(elements: ParsedElement[]): string;
-}
-//# sourceMappingURL=DocumentParser.d.ts.map

package/dist/parsers/DocumentParser.js

@@ -1,35 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DocumentParser = void 0;
-/**
- * Abstract base class for document parsers.
- *
- * Implementations wrap third-party libraries (Unstructured, LlamaIndex, etc.)
- * and normalise their output into a {@link ParsedDocument} that can be fed
- * directly into an {@link IngestionPipeline}.
- *
- * All peer dependencies are loaded lazily via dynamic import so that
- * users only need to install what they actually use.
- *
- * @example
- * ```typescript
- * const parser = new UnstructuredLocalParser();
- * const doc = await parser.parse("/path/to/report.pdf");
- * console.log(doc.elements?.length, "elements parsed");
- * await pipeline.ingestFile("/path/to/report.pdf", parser);
- * ```
- */
-class DocumentParser {
-    /**
-     * Join elements into a single plain-text string.
-     * Filters out empty strings and separates elements with a blank line.
-     */
-    elementsToText(elements) {
-        return elements
-            .map((el) => el.text)
-            .filter(Boolean)
-            .join("\n\n");
-    }
-}
-exports.DocumentParser = DocumentParser;
-//# sourceMappingURL=DocumentParser.js.map

package/dist/parsers/LlamaIndexParser.d.ts

@@ -1,58 +0,0 @@
-import { DocumentParser } from "./DocumentParser";
-import { ParsedDocument, ParseOptions } from "./types";
-/**
- * A LlamaIndex reader instance.
- * Matches the `BaseReader` interface from `llamaindex` and `@llamaindex/readers`.
- */
-export interface LlamaIndexReader {
-    loadData(filePath: string, ...args: unknown[]): Promise<Array<{
-        text: string;
-        metadata?: Record<string, unknown>;
-    }>>;
-}
-/**
- * Document parser that delegates to any **LlamaIndex reader**.
- *
- * Pass any reader from `llamaindex` or `@llamaindex/readers` — e.g.
- * `PDFReader`, `DocxReader`, `HTMLReader`, `LlamaParseReader`, etc. — and
- * this class normalises the output into a {@link ParsedDocument}.
- *
- * **Peer dependency:** `llamaindex` and/or `@llamaindex/readers`
- *
- * @example
- * ```typescript
- * import { PDFReader } from "@llamaindex/readers/pdf";
- * import { LlamaIndexParser } from "@agentionai/agents/parsers";
- *
- * const parser = new LlamaIndexParser(new PDFReader());
- * const doc = await parser.parse("/path/to/report.pdf");
- * await pipeline.ingestFile("/path/to/report.pdf", parser);
- * ```
- *
- * @example Using LlamaParse (cloud OCR / layout AI)
- * ```typescript
- * import { LlamaParseReader } from "llamaindex";
- *
- * const parser = new LlamaIndexParser(
- *   new LlamaParseReader({ resultType: "markdown" })
- * );
- * ```
- */
-export declare class LlamaIndexParser extends DocumentParser {
-    private readonly reader;
-    readonly name: string;
-    /**
-     * @param reader      - Any LlamaIndex reader instance
-     * @param readerName  - Optional label used in {@link name}; defaults to the
-     *                      reader's constructor name
-     */
-    constructor(reader: LlamaIndexReader, readerName?: string);
-    /**
-     * Parse a file using the configured LlamaIndex reader.
-     *
-     * @param filePath - Path to the document file
-     * @param options  - Currently unused; kept for interface compatibility
-     */
-    parse(filePath: string, _options?: ParseOptions): Promise<ParsedDocument>;
-}
-//# sourceMappingURL=LlamaIndexParser.d.ts.map

package/dist/parsers/LlamaIndexParser.js

@@ -1,71 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.LlamaIndexParser = void 0;
-const DocumentParser_1 = require("./DocumentParser");
-/**
- * Document parser that delegates to any **LlamaIndex reader**.
- *
- * Pass any reader from `llamaindex` or `@llamaindex/readers` — e.g.
- * `PDFReader`, `DocxReader`, `HTMLReader`, `LlamaParseReader`, etc. — and
- * this class normalises the output into a {@link ParsedDocument}.
- *
- * **Peer dependency:** `llamaindex` and/or `@llamaindex/readers`
- *
- * @example
- * ```typescript
- * import { PDFReader } from "@llamaindex/readers/pdf";
- * import { LlamaIndexParser } from "@agentionai/agents/parsers";
- *
- * const parser = new LlamaIndexParser(new PDFReader());
- * const doc = await parser.parse("/path/to/report.pdf");
- * await pipeline.ingestFile("/path/to/report.pdf", parser);
- * ```
- *
- * @example Using LlamaParse (cloud OCR / layout AI)
- * ```typescript
- * import { LlamaParseReader } from "llamaindex";
- *
- * const parser = new LlamaIndexParser(
- *   new LlamaParseReader({ resultType: "markdown" })
- * );
- * ```
- */
-class LlamaIndexParser extends DocumentParser_1.DocumentParser {
-    /**
-     * @param reader      - Any LlamaIndex reader instance
-     * @param readerName  - Optional label used in {@link name}; defaults to the
-     *                      reader's constructor name
-     */
-    constructor(reader, readerName) {
-        super();
-        this.reader = reader;
-        this.name = `llamaindex:${readerName ?? reader.constructor?.name ?? "reader"}`;
-    }
-    /**
-     * Parse a file using the configured LlamaIndex reader.
-     *
-     * @param filePath - Path to the document file
-     * @param options  - Currently unused; kept for interface compatibility
-     */
-    async parse(filePath, _options) {
-        let docs;
-        try {
-            docs = await this.reader.loadData(filePath);
-        }
-        catch (err) {
-            const msg = err instanceof Error ? err.message : String(err);
-            throw new Error(`LlamaIndexParser (${this.name}) failed to load "${filePath}": ${msg}`);
-        }
-        const elements = docs.map((doc, i) => ({
-            type: "Document",
-            text: doc.text ?? "",
-            metadata: { ...doc.metadata, doc_index: i },
-        }));
-        return {
-            text: this.elementsToText(elements),
-            elements,
-        };
-    }
-}
-exports.LlamaIndexParser = LlamaIndexParser;
-//# sourceMappingURL=LlamaIndexParser.js.map

package/dist/parsers/OllamaOCRParser.d.ts

@@ -1,98 +0,0 @@
-import { DocumentParser } from "./DocumentParser";
-import { ParsedDocument, ParseOptions } from "./types";
-/**
- * Configuration for {@link OllamaOCRParser}.
- */
-export interface OllamaOCRParserConfig {
-    /**
-     * Ollama model to use for OCR.
-     * @default "glm-ocr"
-     */
-    model?: string;
-    /**
-     * Base URL of the local Ollama server.
-     * @default "http://localhost:11434"
-     */
-    baseUrl?: string;
-    /**
-     * Prompt sent alongside each image.
-     * @default "Extract and transcribe all text from this image. Preserve the original structure, headings, and formatting as much as possible. Output only the extracted text."
-     */
-    prompt?: string;
-    /**
-     * Scale factor for rendering PDF pages to images.
-     * 1.0 = 72 DPI, 2.0 = 144 DPI. Lower is faster; higher improves OCR accuracy.
-     * @default 2.0
-     */
-    pdfScale?: number;
-    /**
-     * Number of pages to OCR in parallel.
-     * Higher values are faster but use more memory and GPU.
-     * @default 3
-     */
-    concurrency?: number;
-    /**
-     * Called after each PDF page is OCR'd.
-     * With concurrency > 1 pages may complete out of order,
-     * but the final document is always in the correct page order.
-     */
-    onProgress?: (completed: number, total: number) => void;
-}
-/**
- * Document parser that uses a locally-running **Ollama** vision model (e.g. `glm-ocr`)
- * to perform OCR on image files and PDF documents.
- *
- * **Supported file types:**
- * - Images: `.jpg`, `.jpeg`, `.png`, `.gif`, `.webp`, `.bmp` — no extra dependencies
- * - PDF: requires the optional peer dependency `pdf-to-img` (`npm install pdf-to-img`)
- *
- * **Ollama must be running** with the model pulled:
- * ```bash
- * ollama pull glm-ocr
- * ollama serve   # if not already running
- * ```
- *
- * @example
- * ```typescript
- * import { OllamaOCRParser } from "@agentionai/agents/parsers/ollama-ocr";
- *
- * const parser = new OllamaOCRParser({
- *   model: "glm-ocr",
- *   pdfScale: 1.5,
- *   onProgress: (page, total) => console.log(`OCR page ${page}/${total}...`),
- * });
- *
- * // Parse an image
- * const doc = await parser.parse("/path/to/scan.png");
- *
- * // Parse a PDF (requires: npm install pdf-to-img)
- * const pdf = await parser.parse("/path/to/report.pdf");
- *
- * // Use with IngestionPipeline
- * await pipeline.ingestFile("/path/to/scan.png", parser);
- * ```
- */
-export declare class OllamaOCRParser extends DocumentParser {
-    readonly name = "ollama-ocr";
-    private readonly model;
-    private readonly baseUrl;
-    private readonly prompt;
-    private readonly pdfScale;
-    private readonly concurrency;
-    private readonly onProgress?;
-    constructor(config?: OllamaOCRParserConfig);
-    /**
-     * Parse a document file using Ollama OCR.
-     *
-     * @param filePath - Path to the image or PDF file
-     * @param options  - Optional hints (unused by this parser; provided for interface compatibility)
-     */
-    parse(filePath: string, _options?: ParseOptions): Promise<ParsedDocument>;
-    private parseImageFile;
-    private parsePdf;
-    /**
-     * Send a base64-encoded image to the Ollama chat API and return the extracted text.
-     */
-    private runOCR;
-}
-//# sourceMappingURL=OllamaOCRParser.d.ts.map