@framers/agentos 0.1.101 → 0.1.102

package/dist/memory/ingestion/HtmlLoader.d.ts

@@ -0,0 +1,49 @@
+/**
+ * @fileoverview HtmlLoader — basic HTML-to-text document loader.
+ *
+ * Converts `.html` and `.htm` files into plain text using lightweight regex
+ * transformations.  This is intentionally a *simple* loader — it covers the
+ * common case of stripping tag soup and decoding standard HTML entities.  For
+ * complex documents (nested frames, JavaScript-rendered content) a headless
+ * browser or DOM-parsing library would be more appropriate.
+ *
+ * Supported extensions: `.html`, `.htm`
+ *
+ * @module memory/ingestion/HtmlLoader
+ */
+import type { IDocumentLoader } from './IDocumentLoader.js';
+import type { LoadOptions, LoadedDocument } from '../facade/types.js';
+/**
+ * Basic document loader for HTML (`.html`, `.htm`) files.
+ *
+ * ### Text extraction strategy
+ * 1. `<script>` and `<style>` blocks are removed entirely.
+ * 2. Block-level elements (`<p>`, `<div>`, `<h1>`–`<h6>`, etc.) are replaced
+ *    with newline characters to preserve paragraph structure.
+ * 3. All remaining HTML tags are stripped.
+ * 4. A common subset of HTML entities is decoded.
+ * 5. Excessive whitespace is collapsed.
+ *
+ * ### Metadata
+ * - `title` — extracted from the `<title>` element when present.
+ * - `wordCount` — approximate count of words in the extracted text.
+ * - `source` — absolute file path (when loaded from disk).
+ *
+ * @implements {IDocumentLoader}
+ *
+ * @example
+ * ```ts
+ * const loader = new HtmlLoader();
+ * const doc = await loader.load('/public/index.html');
+ * console.log(doc.metadata.title); // e.g. 'Welcome to AgentOS'
+ * ```
+ */
+export declare class HtmlLoader implements IDocumentLoader {
+    /** @inheritdoc */
+    readonly supportedExtensions: string[];
+    /** @inheritdoc */
+    canLoad(source: string | Buffer): boolean;
+    /** @inheritdoc */
+    load(source: string | Buffer, _options?: LoadOptions): Promise<LoadedDocument>;
+}
+//# sourceMappingURL=HtmlLoader.d.ts.map

package/dist/memory/ingestion/HtmlLoader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"HtmlLoader.d.ts","sourceRoot":"","sources":["../../../src/memory/ingestion/HtmlLoader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAIH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAE,WAAW,EAAE,cAAc,EAAoB,MAAM,oBAAoB,CAAC;AAmIxF;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,qBAAa,UAAW,YAAW,eAAe;IAChD,kBAAkB;IAClB,QAAQ,CAAC,mBAAmB,EAAE,MAAM,EAAE,CAA6B;IAMnE,kBAAkB;IAClB,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO;IAWzC,kBAAkB;IACZ,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,QAAQ,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,cAAc,CAAC;CAiCrF"}

package/dist/memory/ingestion/HtmlLoader.js

@@ -0,0 +1,202 @@
+/**
+ * @fileoverview HtmlLoader — basic HTML-to-text document loader.
+ *
+ * Converts `.html` and `.htm` files into plain text using lightweight regex
+ * transformations.  This is intentionally a *simple* loader — it covers the
+ * common case of stripping tag soup and decoding standard HTML entities.  For
+ * complex documents (nested frames, JavaScript-rendered content) a headless
+ * browser or DOM-parsing library would be more appropriate.
+ *
+ * Supported extensions: `.html`, `.htm`
+ *
+ * @module memory/ingestion/HtmlLoader
+ */
+import fs from 'node:fs/promises';
+import path from 'node:path';
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+/** Extensions handled by this loader, each with a leading dot. */
+const SUPPORTED_EXTENSIONS = ['.html', '.htm'];
+// ---------------------------------------------------------------------------
+// Regex helpers
+// ---------------------------------------------------------------------------
+/**
+ * Strip all HTML/XML tags from a string.
+ *
+ * Handles multi-line tags and self-closing tags.  Does **not** attempt to
+ * parse the HTML into a DOM; purely textual.
+ *
+ * @param html - Raw HTML string.
+ */
+function stripTags(html) {
+    // Remove `<script>…</script>` and `<style>…</style>` blocks entirely so
+    // their content doesn't leak into the extracted text.
+    let text = html
+        .replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, ' ')
+        .replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, ' ');
+    // Replace block-level elements with newlines to preserve paragraph breaks.
+    text = text.replace(/<\/?(p|div|section|article|header|footer|h[1-6]|li|tr|blockquote)[^>]*>/gi, '\n');
+    // Replace `<br>` and `<hr>` with newlines.
+    text = text.replace(/<br\s*\/?>/gi, '\n');
+    text = text.replace(/<hr\s*\/?>/gi, '\n');
+    // Strip remaining tags.
+    text = text.replace(/<[^>]+>/g, '');
+    return text;
+}
+/**
+ * Decode a small but common set of named and numeric HTML entities.
+ *
+ * Covers the most frequently encountered entities in prose documents.
+ * Full entity decoding would require an external library; this subset handles
+ * ~95% of real-world cases without dependencies.
+ *
+ * @param text - Text containing HTML entity references.
+ */
+function decodeEntities(text) {
+    return text
+        // Named character entities (most common subset).
+        .replace(/&amp;/gi, '&')
+        .replace(/&lt;/gi, '<')
+        .replace(/&gt;/gi, '>')
+        .replace(/&quot;/gi, '"')
+        .replace(/&apos;/gi, "'")
+        .replace(/&nbsp;/gi, ' ')
+        .replace(/&mdash;/gi, '—')
+        .replace(/&ndash;/gi, '–')
+        .replace(/&hellip;/gi, '…')
+        .replace(/&lsquo;/gi, '\u2018')
+        .replace(/&rsquo;/gi, '\u2019')
+        .replace(/&ldquo;/gi, '\u201C')
+        .replace(/&rdquo;/gi, '\u201D')
+        .replace(/&copy;/gi, '©')
+        .replace(/&reg;/gi, '®')
+        .replace(/&trade;/gi, '™')
+        // Numeric decimal entities: &#160; &#8212; etc.
+        .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(parseInt(code, 10)))
+        // Numeric hexadecimal entities: &#x00A0; &#x2014; etc.
+        .replace(/&#x([0-9a-f]+);/gi, (_, hex) => String.fromCodePoint(parseInt(hex, 16)));
+}
+/**
+ * Extract the text content of the first `<title>` element, if present.
+ *
+ * Returns `undefined` when no `<title>` tag is found.
+ *
+ * @param html - Raw HTML string.
+ */
+function extractTitle(html) {
+    const match = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(html);
+    if (!match)
+        return undefined;
+    // Decode entities inside the title and trim surrounding whitespace.
+    return decodeEntities(match[1]).trim() || undefined;
+}
+/**
+ * Collapse repeated whitespace and blank lines in extracted text.
+ *
+ * Runs of more than two consecutive newlines are folded to two (paragraph
+ * boundary).  Runs of horizontal whitespace within a line are collapsed to a
+ * single space.
+ *
+ * @param text - Text with raw whitespace inherited from the HTML source.
+ */
+function normaliseWhitespace(text) {
+    return text
+        .replace(/[ \t]+/g, ' ') // Collapse horizontal runs.
+        .replace(/\n{3,}/g, '\n\n') // Collapse blank-line runs.
+        .trim();
+}
+/**
+ * Approximate word count.
+ *
+ * @param text - Plain text string.
+ */
+function wordCount(text) {
+    return text.trim() === '' ? 0 : text.trim().split(/\s+/).length;
+}
+/**
+ * Returns the lower-cased extension of a file path.
+ *
+ * @param filePath - File path string.
+ */
+function extOf(filePath) {
+    return path.extname(filePath).toLowerCase();
+}
+// ---------------------------------------------------------------------------
+// HtmlLoader
+// ---------------------------------------------------------------------------
+/**
+ * Basic document loader for HTML (`.html`, `.htm`) files.
+ *
+ * ### Text extraction strategy
+ * 1. `<script>` and `<style>` blocks are removed entirely.
+ * 2. Block-level elements (`<p>`, `<div>`, `<h1>`–`<h6>`, etc.) are replaced
+ *    with newline characters to preserve paragraph structure.
+ * 3. All remaining HTML tags are stripped.
+ * 4. A common subset of HTML entities is decoded.
+ * 5. Excessive whitespace is collapsed.
+ *
+ * ### Metadata
+ * - `title` — extracted from the `<title>` element when present.
+ * - `wordCount` — approximate count of words in the extracted text.
+ * - `source` — absolute file path (when loaded from disk).
+ *
+ * @implements {IDocumentLoader}
+ *
+ * @example
+ * ```ts
+ * const loader = new HtmlLoader();
+ * const doc = await loader.load('/public/index.html');
+ * console.log(doc.metadata.title); // e.g. 'Welcome to AgentOS'
+ * ```
+ */
+export class HtmlLoader {
+    constructor() {
+        /** @inheritdoc */
+        this.supportedExtensions = [...SUPPORTED_EXTENSIONS];
+    }
+    // -------------------------------------------------------------------------
+    // canLoad
+    // -------------------------------------------------------------------------
+    /** @inheritdoc */
+    canLoad(source) {
+        if (Buffer.isBuffer(source)) {
+            return false;
+        }
+        return SUPPORTED_EXTENSIONS.includes(extOf(source));
+    }
+    // -------------------------------------------------------------------------
+    // load
+    // -------------------------------------------------------------------------
+    /** @inheritdoc */
+    async load(source, _options) {
+        let html;
+        let resolvedPath;
+        if (Buffer.isBuffer(source)) {
+            html = source.toString('utf8');
+        }
+        else {
+            resolvedPath = source;
+            const bytes = await fs.readFile(resolvedPath);
+            html = bytes.toString('utf8');
+        }
+        // ---- Extract title before stripping tags ----
+        const title = extractTitle(html);
+        // ---- Convert HTML to plain text ----
+        const rawText = stripTags(html);
+        const decoded = decodeEntities(rawText);
+        const content = normaliseWhitespace(decoded);
+        // ---- Assemble metadata ----
+        const meta = {
+            ...(title !== undefined ? { title } : {}),
+            wordCount: wordCount(content),
+            ...(resolvedPath ? { source: resolvedPath } : {}),
+        };
+        return {
+            content,
+            metadata: meta,
+            format: 'html',
+        };
+    }
+}
+//# sourceMappingURL=HtmlLoader.js.map

package/dist/memory/ingestion/HtmlLoader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"HtmlLoader.js","sourceRoot":"","sources":["../../../src/memory/ingestion/HtmlLoader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAI7B,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E,kEAAkE;AAClE,MAAM,oBAAoB,GAAG,CAAC,OAAO,EAAE,MAAM,CAAU,CAAC;AAExD,8EAA8E;AAC9E,gBAAgB;AAChB,8EAA8E;AAE9E;;;;;;;GAOG;AACH,SAAS,SAAS,CAAC,IAAY;IAC7B,wEAAwE;IACxE,sDAAsD;IACtD,IAAI,IAAI,GAAG,IAAI;SACZ,OAAO,CAAC,qCAAqC,EAAE,GAAG,CAAC;SACnD,OAAO,CAAC,mCAAmC,EAAE,GAAG,CAAC,CAAC;IAErD,2EAA2E;IAC3E,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,2EAA2E,EAAE,IAAI,CAAC,CAAC;IAEvG,2CAA2C;IAC3C,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;IAC1C,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;IAE1C,wBAAwB;IACxB,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAEpC,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,IAAI;QACT,iDAAiD;SAChD,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC;SACzB,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC;SACzB,OAAO,CAAC,YAAY,EAAE,GAAG,CAAC;SAC1B,OAAO,CAAC,WAAW,EAAE,QAAQ,CAAC;SAC9B,OAAO,CAAC,WAAW,EAAE,QAAQ,CAAC;SAC9B,OAAO,CAAC,WAAW,EAAE,QAAQ,CAAC;SAC9B,OAAO,CAAC,WAAW,EAAE,QAAQ,CAAC;SAC9B,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC;QAC1B,gDAAgD;SAC/C,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,IAAY,EAAE,EAAE,CACxC,MAAM,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CACzC;QACD,uDAAuD;SACtD,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAAC,EAAE,GAAW,EAAE,EAAE,CAC/C,MAAM,CAAC,aAAa,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CACxC,CAAC;AACN,CAAC;AAED;;;;;;GAMG;AACH,SAAS,YAAY,CAAC,IAAY;IAChC,MAAM,KAAK,GAAG,kCAAkC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5D,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,oEAAoE;IACpE,OAAO,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC;AACtD,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,mBAAmB,CAAC,IAAY;IACvC,OAAO,IAAI;SACR,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAU,4BAA4B;SAC7D,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAO,4BAA4B;SAC7D,IAAI,EAAE,CAAC;AACZ,CAAC;AAED;;;;GAIG;AACH,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;AAClE,CAAC;AAED;;;;GAIG;AACH,SAAS,KAAK,CAAC,QAAgB;IAC7B,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;AAC9C,CAAC;AAED,8EAA8E;AAC9E,aAAa;AACb,8EAA8E;AAE9E;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,OAAO,UAAU;IAAvB;QACE,kBAAkB;QACT,wBAAmB,GAAa,CAAC,GAAG,oBAAoB,CAAC,CAAC;IAoDrE,CAAC;IAlDC,4EAA4E;IAC5E,UAAU;IACV,4EAA4E;IAE5E,kBAAkB;IAClB,OAAO,CAAC,MAAuB;QAC7B,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,OAAO,KAAK,CAAC;QACf,CAAC;QACD,OAAQ,oBAA0C,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC7E,CAAC;IAED,4EAA4E;IAC5E,OAAO;IACP,4EAA4E;IAE5E,kBAAkB;IAClB,KAAK,CAAC,IAAI,CAAC,MAAuB,EAAE,QAAsB;QACxD,IAAI,IAAY,CAAC;QACjB,IAAI,YAAgC,CAAC;QAErC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC;aAAM,CAAC;YACN,YAAY,GAAG,MAAM,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;YAC9C,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAChC,CAAC;QAED,gDAAgD;QAChD,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QAEjC,uCAAuC;QACvC,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAChC,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;QACxC,MAAM,OAAO,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;QAE7C,8BAA8B;QAC9B,MAAM,IAAI,GAAqB;YAC7B,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzC,SAAS,EAAE,SAAS,CAAC,OAAO,CAAC;YAC7B,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClD,CAAC;QAEF,OAAO;YACL,OAAO;YACP,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,MAAM;SACf,CAAC;IACJ,CAAC;CACF"}

package/dist/memory/ingestion/IDocumentLoader.d.ts

@@ -0,0 +1,63 @@
+/**
+ * @fileoverview IDocumentLoader — contract for all document loading strategies.
+ *
+ * Every loader targets one or more file extensions and exposes a uniform
+ * `load()` primitive that normalises diverse document formats into a single
+ * {@link LoadedDocument} shape ready for chunking and indexing.
+ *
+ * @module memory/ingestion/IDocumentLoader
+ */
+import type { LoadOptions, LoadedDocument } from '../facade/types.js';
+/**
+ * Unified interface for all document loaders in the AgentOS ingestion
+ * pipeline.
+ *
+ * Implementations handle a specific set of file extensions and are
+ * responsible for:
+ * 1. Reading raw bytes from a file path or in-memory `Buffer`.
+ * 2. Extracting plain-text `content` and structured `metadata`.
+ * 3. Returning a {@link LoadedDocument} ready for downstream chunking.
+ *
+ * @example
+ * ```ts
+ * const loader: IDocumentLoader = new MarkdownLoader();
+ * if (loader.canLoad('README.md')) {
+ *   const doc = await loader.load('README.md');
+ *   console.log(doc.metadata.title);
+ * }
+ * ```
+ */
+export interface IDocumentLoader {
+    /**
+     * File extensions this loader handles, each with a leading dot.
+     *
+     * Used by {@link LoaderRegistry} to route file paths to the correct loader.
+     *
+     * @example ['.md', '.mdx']
+     */
+    readonly supportedExtensions: string[];
+    /**
+     * Parses `source` and returns a normalised {@link LoadedDocument}.
+     *
+     * When `source` is a `string` the loader treats it as an absolute (or
+     * resolvable) file path and reads the file from disk.  When `source` is a
+     * `Buffer` the loader parses the bytes directly and derives as much
+     * metadata as possible from the buffer content alone.
+     *
+     * @param source  - Absolute file path OR raw document bytes.
+     * @param options - Optional hints such as a format override.
+     * @returns A promise resolving to the fully-populated {@link LoadedDocument}.
+     * @throws When the file cannot be read or the format is not parsable.
+     */
+    load(source: string | Buffer, options?: LoadOptions): Promise<LoadedDocument>;
+    /**
+     * Returns `true` when this loader is capable of handling `source`.
+     *
+     * For string sources the check is purely extension-based.  For `Buffer`
+     * sources the loader may inspect magic bytes when relevant.
+     *
+     * @param source - Absolute file path or raw bytes.
+     */
+    canLoad(source: string | Buffer): boolean;
+}
+//# sourceMappingURL=IDocumentLoader.d.ts.map

package/dist/memory/ingestion/IDocumentLoader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"IDocumentLoader.d.ts","sourceRoot":"","sources":["../../../src/memory/ingestion/IDocumentLoader.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAEtE;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,WAAW,eAAe;IAC9B;;;;;;OAMG;IACH,QAAQ,CAAC,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAEvC;;;;;;;;;;;;OAYG;IACH,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IAE9E;;;;;;;OAOG;IACH,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;CAC3C"}

package/dist/memory/ingestion/IDocumentLoader.js

@@ -0,0 +1,11 @@
+/**
+ * @fileoverview IDocumentLoader — contract for all document loading strategies.
+ *
+ * Every loader targets one or more file extensions and exposes a uniform
+ * `load()` primitive that normalises diverse document formats into a single
+ * {@link LoadedDocument} shape ready for chunking and indexing.
+ *
+ * @module memory/ingestion/IDocumentLoader
+ */
+export {};
+//# sourceMappingURL=IDocumentLoader.js.map

package/dist/memory/ingestion/IDocumentLoader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"IDocumentLoader.js","sourceRoot":"","sources":["../../../src/memory/ingestion/IDocumentLoader.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG"}

package/dist/memory/ingestion/LoaderRegistry.d.ts

@@ -0,0 +1,140 @@
+/**
+ * @fileoverview LoaderRegistry — extension-to-loader routing table.
+ *
+ * The registry maintains a map of file extensions to {@link IDocumentLoader}
+ * implementations and provides a convenience `loadFile()` method that
+ * auto-detects the format from a file path before delegating to the
+ * appropriate loader.
+ *
+ * On construction the registry pre-registers five built-in loaders:
+ * {@link TextLoader}, {@link MarkdownLoader}, {@link HtmlLoader},
+ * {@link PdfLoader}, and {@link DocxLoader}.  In addition, the optional
+ * {@link OcrPdfLoader} and {@link DoclingLoader} are registered when their
+ * respective factories return non-null values (i.e. when `tesseract.js` and
+ * `python3 -m docling` are available in the environment).
+ *
+ * Additional loaders can be added at runtime via {@link LoaderRegistry.register}.
+ *
+ * @module memory/ingestion/LoaderRegistry
+ */
+import type { IDocumentLoader } from './IDocumentLoader.js';
+import type { LoadOptions, LoadedDocument } from '../facade/types.js';
+/**
+ * Central registry mapping file extensions to {@link IDocumentLoader}
+ * implementations.
+ *
+ * ### Built-in loaders (registered automatically)
+ * | Extensions                                         | Loader                |
+ * |----------------------------------------------------|-----------------------|
+ * | `.txt`, `.csv`, `.tsv`, `.json`, `.yaml`, `.yml`  | {@link TextLoader}    |
+ * | `.md`, `.mdx`                                     | {@link MarkdownLoader} |
+ * | `.html`, `.htm`                                   | {@link HtmlLoader}    |
+ * | `.pdf`                                            | {@link PdfLoader}     |
+ * | `.docx`                                           | {@link DocxLoader}    |
+ *
+ * ### Conditional loaders (registered when available)
+ * | Condition                     | Loader                              |
+ * |-------------------------------|-------------------------------------|
+ * | `tesseract.js` installed      | {@link OcrPdfLoader} (overrides PDF) |
+ * | `python3 -m docling` available | {@link DoclingLoader} (overrides PDF + DOCX) |
+ *
+ * ### Registering a custom loader
+ * ```ts
+ * const registry = new LoaderRegistry();
+ * registry.register(new PdfLoader());
+ * const doc = await registry.loadFile('/reports/q3.pdf');
+ * ```
+ *
+ * ### Using loadFile
+ * ```ts
+ * const registry = new LoaderRegistry();
+ * const doc = await registry.loadFile('/notes/meeting.md');
+ * console.log(doc.metadata.title);
+ * ```
+ */
+export declare class LoaderRegistry {
+    /**
+     * Internal map from lower-cased extension (with dot) to the loader
+     * responsible for that extension.
+     *
+     * When multiple loaders claim the same extension the last one registered
+     * wins (newest-registration-wins semantics), allowing callers to override
+     * built-in loaders.
+     */
+    private readonly _loaders;
+    /**
+     * Creates a new registry pre-populated with the built-in loaders.
+     *
+     * Loader registration order determines conflict resolution: later
+     * registrations override earlier ones for the same extension.
+     *
+     * Registration order:
+     * 1. {@link TextLoader}, {@link MarkdownLoader}, {@link HtmlLoader} — core text formats.
+     * 2. {@link PdfLoader} (with injected OCR + Docling loaders) — PDF extraction.
+     * 3. {@link DocxLoader} — DOCX extraction.
+     * 4. Optional: an {@link OcrPdfLoader} override when `tesseract.js` is installed.
+     * 5. Optional: a {@link DoclingLoader} override when Python Docling is available.
+     *    DoclingLoader supports both `.pdf` and `.docx`, so it supersedes both
+     *    PdfLoader and DocxLoader when present.
+     */
+    constructor();
+    /**
+     * Register a loader for all extensions it declares.
+     *
+     * If a previously registered loader already handles one of the extension,
+     * it is replaced.  This makes it trivial to swap in a higher-fidelity
+     * implementation for any format.
+     *
+     * @param loader - The loader instance to register.
+     *
+     * @example
+     * ```ts
+     * registry.register(new PdfLoader());
+     * ```
+     */
+    register(loader: IDocumentLoader): void;
+    /**
+     * Retrieve the loader registered for `extensionOrPath`.
+     *
+     * Both bare extensions (`.md`, `md`) and full file paths
+     * (`/docs/guide.md`) are accepted.
+     *
+     * @param extensionOrPath - File extension or full path.
+     * @returns The matching {@link IDocumentLoader}, or `undefined` when no
+     *          loader is registered for the detected extension.
+     *
+     * @example
+     * ```ts
+     * const loader = registry.getLoader('.md');
+     * const loader2 = registry.getLoader('README.md');
+     * ```
+     */
+    getLoader(extensionOrPath: string): IDocumentLoader | undefined;
+    /**
+     * Return a sorted array of all extensions currently registered.
+     *
+     * Each extension is returned with a leading dot in lower-case, e.g.
+     * `['.csv', '.htm', '.html', '.json', '.md', …]`.
+     *
+     * @returns Sorted array of registered extension strings.
+     */
+    getSupportedExtensions(): string[];
+    /**
+     * Convenience method: detect format from `filePath`, find the matching
+     * loader, and delegate to its `load()` method.
+     *
+     * @param filePath - Absolute (or resolvable relative) file path.
+     * @param options  - Optional load hints forwarded to the loader.
+     * @returns A promise resolving to the {@link LoadedDocument}.
+     *
+     * @throws {Error} When no loader is registered for the file's extension.
+     * @throws {Error} When the underlying loader's `load()` throws.
+     *
+     * @example
+     * ```ts
+     * const doc = await registry.loadFile('/notes/architecture.md');
+     * ```
+     */
+    loadFile(filePath: string, options?: LoadOptions): Promise<LoadedDocument>;
+}
+//# sourceMappingURL=LoaderRegistry.d.ts.map

package/dist/memory/ingestion/LoaderRegistry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"LoaderRegistry.d.ts","sourceRoot":"","sources":["../../../src/memory/ingestion/LoaderRegistry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAkDtE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,qBAAa,cAAc;IACzB;;;;;;;OAOG;IACH,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA2C;IAEpE;;;;;;;;;;;;;;OAcG;;IA6BH;;;;;;;;;;;;;OAaG;IACH,QAAQ,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI;IAUvC;;;;;;;;;;;;;;;OAeG;IACH,SAAS,CAAC,eAAe,EAAE,MAAM,GAAG,eAAe,GAAG,SAAS;IAS/D;;;;;;;OAOG;IACH,sBAAsB,IAAI,MAAM,EAAE;IAQlC;;;;;;;;;;;;;;;OAeG;IACG,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,cAAc,CAAC;CAajF"}