@framers/agentos 0.1.101 → 0.1.102

package/dist/memory/ingestion/TextLoader.js

@@ -0,0 +1,207 @@
+/**
+ * @fileoverview TextLoader — loads plain-text, CSV/TSV, JSON, and YAML files.
+ *
+ * This is the most general-purpose loader in the AgentOS ingestion pipeline.
+ * It handles six extensions that all share the same fundamental operation:
+ * read raw text and attach lightweight metadata derived from the file content
+ * and extension.
+ *
+ * Supported extensions: `.txt`, `.csv`, `.tsv`, `.json`, `.yaml`, `.yml`
+ *
+ * @module memory/ingestion/TextLoader
+ */
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+/** Extensions handled by this loader, all lower-cased with a leading dot. */
+const SUPPORTED_EXTENSIONS = ['.txt', '.csv', '.tsv', '.json', '.yaml', '.yml'];
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/**
+ * Returns the lower-cased extension (with dot) of a file path, or an empty
+ * string when the path has no extension.
+ *
+ * @param filePath - Absolute or relative file path string.
+ */
+function extOf(filePath) {
+    return path.extname(filePath).toLowerCase();
+}
+/**
+ * Count the approximate number of words in a string.
+ *
+ * Splits on runs of whitespace — fast and allocation-light for the typical
+ * document sizes encountered during ingestion.
+ *
+ * @param text - Raw text to count.
+ */
+function wordCount(text) {
+    return text.trim() === '' ? 0 : text.trim().split(/\s+/).length;
+}
+/**
+ * Maps a file extension to a human-readable format label returned inside
+ * `LoadedDocument.format`.
+ *
+ * @param ext - Lower-cased extension including leading dot.
+ */
+function formatLabel(ext) {
+    switch (ext) {
+        case '.txt':
+            return 'txt';
+        case '.csv':
+            return 'csv';
+        case '.tsv':
+            return 'tsv';
+        case '.json':
+            return 'json';
+        case '.yaml':
+        case '.yml':
+            return 'yaml';
+        default:
+            return 'txt';
+    }
+}
+/**
+ * For JSON files: parse, then re-serialise with two-space indentation.
+ *
+ * This makes the stored `content` more human-readable and consistent
+ * regardless of how the source file was originally formatted.
+ *
+ * Returns `raw` unchanged when parsing fails (e.g. when a `.json` file
+ * contains invalid JSON) so the loader never throws on bad input.
+ *
+ * @param raw - Raw UTF-8 content of the JSON file.
+ */
+function prettyJson(raw) {
+    try {
+        return JSON.stringify(JSON.parse(raw), null, 2);
+    }
+    catch {
+        // Invalid JSON — return raw text so the caller still gets something.
+        return raw;
+    }
+}
+// ---------------------------------------------------------------------------
+// TextLoader
+// ---------------------------------------------------------------------------
+/**
+ * Loader for plain-text, CSV, TSV, JSON, and YAML files.
+ *
+ * The loader performs minimal transformation:
+ * - **`.json`** — re-serialises with pretty-printing so stored content is
+ *   consistently formatted.
+ * - **`.yaml` / `.yml`** — the `yaml` package is used to parse and re-dump
+ *   for consistent formatting; falls back to raw text on parse error.
+ * - All other extensions — content is returned as-is.
+ *
+ * Metadata includes the approximate `wordCount` and a `format` label derived
+ * from the file extension.
+ *
+ * @implements {IDocumentLoader}
+ *
+ * @example
+ * ```ts
+ * const loader = new TextLoader();
+ * const doc = await loader.load('/data/notes.txt');
+ * console.log(doc.metadata.wordCount); // e.g. 312
+ * ```
+ */
+export class TextLoader {
+    constructor() {
+        /** @inheritdoc */
+        this.supportedExtensions = [...SUPPORTED_EXTENSIONS];
+    }
+    // -------------------------------------------------------------------------
+    // canLoad
+    // -------------------------------------------------------------------------
+    /** @inheritdoc */
+    canLoad(source) {
+        // Buffer sources: we have no extension to check, so we conservatively
+        // return false unless the caller passes a string path.
+        if (Buffer.isBuffer(source)) {
+            return false;
+        }
+        return SUPPORTED_EXTENSIONS.includes(extOf(source));
+    }
+    // -------------------------------------------------------------------------
+    // load
+    // -------------------------------------------------------------------------
+    /** @inheritdoc */
+    async load(source, _options) {
+        let raw;
+        let resolvedPath;
+        let ext;
+        if (Buffer.isBuffer(source)) {
+            // In-memory buffer: treat as plain text, no extension info available.
+            raw = source.toString('utf8');
+            ext = '.txt';
+        }
+        else {
+            // File path: read from disk.
+            resolvedPath = source;
+            ext = extOf(source);
+            const bytes = await fs.readFile(resolvedPath);
+            raw = bytes.toString('utf8');
+        }
+        // ---- Content normalisation ----
+        const content = this._normalise(raw, ext);
+        // ---- Metadata assembly ----
+        const meta = {
+            wordCount: wordCount(raw),
+            format: formatLabel(ext),
+            ...(resolvedPath ? { source: resolvedPath } : {}),
+        };
+        return {
+            content,
+            metadata: meta,
+            format: formatLabel(ext),
+        };
+    }
+    // -------------------------------------------------------------------------
+    // Private helpers
+    // -------------------------------------------------------------------------
+    /**
+     * Normalises raw file content based on the detected extension.
+     *
+     * - JSON files are pretty-printed.
+     * - YAML files are parsed and re-dumped for consistent formatting.
+     * - All other formats are returned unchanged.
+     *
+     * @param raw - Raw UTF-8 string read from the source.
+     * @param ext - Lower-cased extension with leading dot.
+     */
+    _normalise(raw, ext) {
+        switch (ext) {
+            case '.json':
+                return prettyJson(raw);
+            case '.yaml':
+            case '.yml':
+                return this._prettyYaml(raw);
+            default:
+                // .txt, .csv, .tsv — return raw content unchanged.
+                return raw;
+        }
+    }
+    /**
+     * Parse and re-serialise YAML content for consistent formatting.
+     *
+     * Uses the `yaml` package that is already a production dependency of the
+     * `@framers/agentos` package.  Falls back to the original raw string on
+     * any parse error so the loader never throws on malformed YAML.
+     *
+     * @param raw - Raw YAML string.
+     */
+    _prettyYaml(raw) {
+        try {
+            const parsed = parseYaml(raw);
+            return stringifyYaml(parsed);
+        }
+        catch {
+            return raw;
+        }
+    }
+}
+//# sourceMappingURL=TextLoader.js.map

package/dist/memory/ingestion/TextLoader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TextLoader.js","sourceRoot":"","sources":["../../../src/memory/ingestion/TextLoader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,SAAS,IAAI,aAAa,EAAE,MAAM,MAAM,CAAC;AAItE,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E,6EAA6E;AAC7E,MAAM,oBAAoB,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,CAAU,CAAC;AAKzF,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E;;;;;GAKG;AACH,SAAS,KAAK,CAAC,QAAgB;IAC7B,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;AAC9C,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;AAClE,CAAC;AAED;;;;;GAKG;AACH,SAAS,WAAW,CAAC,GAA0B;IAC7C,QAAQ,GAAmB,EAAE,CAAC;QAC5B,KAAK,MAAM;YACT,OAAO,KAAK,CAAC;QACf,KAAK,MAAM;YACT,OAAO,KAAK,CAAC;QACf,KAAK,MAAM;YACT,OAAO,KAAK,CAAC;QACf,KAAK,OAAO;YACV,OAAO,MAAM,CAAC;QAChB,KAAK,OAAO,CAAC;QACb,KAAK,MAAM;YACT,OAAO,MAAM,CAAC;QAChB;YACE,OAAO,KAAK,CAAC;IACjB,CAAC;AACH,CAAC;AAED;;;;;;;;;;GAUG;AACH,SAAS,UAAU,CAAC,GAAW;IAC7B,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAClD,CAAC;IAAC,MAAM,CAAC;QACP,qEAAqE;QACrE,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,aAAa;AACb,8EAA8E;AAE9E;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,OAAO,UAAU;IAAvB;QACE,kBAAkB;QACT,wBAAmB,GAAa,CAAC,GAAG,oBAAoB,CAAC,CAAC;IAqGrE,CAAC;IAnGC,4EAA4E;IAC5E,UAAU;IACV,4EAA4E;IAE5E,kBAAkB;IAClB,OAAO,CAAC,MAAuB;QAC7B,sEAAsE;QACtE,uDAAuD;QACvD,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,OAAO,KAAK,CAAC;QACf,CAAC;QACD,OAAQ,oBAA0C,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC7E,CAAC;IAED,4EAA4E;IAC5E,OAAO;IACP,4EAA4E;IAE5E,kBAAkB;IAClB,KAAK,CAAC,IAAI,CAAC,MAAuB,EAAE,QAAsB;QACxD,IAAI,GAAW,CAAC;QAChB,IAAI,YAAgC,CAAC;QACrC,IAAI,GAAW,CAAC;QAEhB,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,sEAAsE;YACtE,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC9B,GAAG,GAAG,MAAM,CAAC;QACf,CAAC;aAAM,CAAC;YACN,6BAA6B;YAC7B,YAAY,GAAG,MAAM,CAAC;YACtB,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;YACpB,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;YAC9C,GAAG,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC/B,CAAC;QAED,kCAAkC;QAClC,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,GAAmB,CAAC,CAAC;QAE1D,8BAA8B;QAC9B,MAAM,IAAI,GAAqB;YAC7B,SAAS,EAAE,SAAS,CAAC,GAAG,CAAC;YACzB,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC;YACxB,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClD,CAAC;QAEF,OAAO;YACL,OAAO;YACP,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC;SACzB,CAAC;IACJ,CAAC;IAED,4EAA4E;IAC5E,kBAAkB;IAClB,4EAA4E;IAE5E;;;;;;;;;OASG;IACK,UAAU,CAAC,GAAW,EAAE,GAA0B;QACxD,QAAQ,GAAmB,EAAE,CAAC;YAC5B,KAAK,OAAO;gBACV,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC;YAEzB,KAAK,OAAO,CAAC;YACb,KAAK,MAAM;gBACT,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YAE/B;gBACE,mDAAmD;gBACnD,OAAO,GAAG,CAAC;QACf,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACK,WAAW,CAAC,GAAW;QAC7B,IAAI,CAAC;YACH,MAAM,MAAM,GAAY,SAAS,CAAC,GAAG,CAAC,CAAC;YACvC,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,GAAG,CAAC;QACb,CAAC;IACH,CAAC;CACF"}

package/dist/memory/ingestion/UrlLoader.d.ts

@@ -0,0 +1,95 @@
+/**
+ * @fileoverview UrlLoader — fetch-and-delegate loader for HTTP/HTTPS URLs.
+ *
+ * `UrlLoader` implements {@link IDocumentLoader} and handles `http://` and
+ * `https://` sources.  It fetches the remote resource, inspects the
+ * `Content-Type` response header, and delegates to the most appropriate
+ * registered loader:
+ *
+ * - `text/html` → {@link HtmlLoader} (via the registry)
+ * - `application/pdf` → {@link PdfLoader} (via the registry)
+ * - Anything else → raw UTF-8 text, format `'text'`
+ *
+ * Because URLs have no file extension in the traditional sense,
+ * `supportedExtensions` is deliberately empty.  Routing to `UrlLoader` must
+ * be done explicitly — either by calling `UrlLoader.load()` directly or by
+ * checking `UrlLoader.canLoad()` before dispatching.
+ *
+ * @module memory/ingestion/UrlLoader
+ */
+import type { IDocumentLoader } from './IDocumentLoader.js';
+import type { LoadOptions, LoadedDocument } from '../facade/types.js';
+import type { LoaderRegistry } from './LoaderRegistry.js';
+/**
+ * An {@link IDocumentLoader} that fetches a remote URL and delegates parsing
+ * to the appropriate registered loader based on the response `Content-Type`.
+ *
+ * ### Supported content types
+ * | Content-Type          | Delegates to          |
+ * |-----------------------|-----------------------|
+ * | `text/html`           | HtmlLoader (registry) |
+ * | `application/pdf`     | PdfLoader  (registry) |
+ * | Everything else       | Plain UTF-8 text      |
+ *
+ * ### Example
+ * ```ts
+ * const registry = new LoaderRegistry();
+ * const urlLoader = new UrlLoader(registry);
+ *
+ * // Register so the registry also dispatches URLs via canLoad checks.
+ * // (Optional — UrlLoader can be used standalone too.)
+ *
+ * if (urlLoader.canLoad('https://example.com/report.pdf')) {
+ *   const doc = await urlLoader.load('https://example.com/report.pdf');
+ *   console.log(doc.format); // 'pdf'
+ * }
+ * ```
+ *
+ * @implements {IDocumentLoader}
+ */
+export declare class UrlLoader implements IDocumentLoader {
+    private readonly registry;
+    /**
+     * URLs have no file extension so this array is always empty.
+     *
+     * Routing to this loader must be performed via {@link canLoad} rather than
+     * the registry's extension-based lookup.
+     */
+    readonly supportedExtensions: string[];
+    /**
+     * @param registry - The {@link LoaderRegistry} used to resolve format-specific
+     *                   loaders once the remote content type is known.
+     */
+    constructor(registry: LoaderRegistry);
+    /**
+     * Returns `true` when `source` is a string that starts with `http://` or
+     * `https://`.
+     *
+     * Buffer sources are always rejected — raw bytes cannot be a URL.
+     *
+     * @param source - Absolute file path, URL string, or raw bytes.
+     */
+    canLoad(source: string | Buffer): boolean;
+    /**
+     * Fetch `source` over HTTP/HTTPS and return a {@link LoadedDocument}.
+     *
+     * The response body is buffered in memory and then handed to the appropriate
+     * sub-loader according to the `Content-Type` header:
+     *
+     * - `text/html` → fetched as text, passed to the HTML loader as a `Buffer`.
+     * - `application/pdf` → fetched as bytes, passed to the PDF loader as a
+     *    `Buffer`.
+     * - Anything else → returned as plain text with format `'text'` and
+     *   `source` metadata set to the URL.
+     *
+     * @param source  - HTTP/HTTPS URL string.
+     * @param options - Optional load hints forwarded to the delegated loader.
+     * @returns A promise resolving to the {@link LoadedDocument}.
+     *
+     * @throws {Error} When `source` is a `Buffer` (URLs must be strings).
+     * @throws {Error} When the HTTP request fails (network error or non-2xx
+     *                 status).
+     */
+    load(source: string | Buffer, options?: LoadOptions): Promise<LoadedDocument>;
+}
+//# sourceMappingURL=UrlLoader.d.ts.map

package/dist/memory/ingestion/UrlLoader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"UrlLoader.d.ts","sourceRoot":"","sources":["../../../src/memory/ingestion/UrlLoader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACtE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAa1D;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,qBAAa,SAAU,YAAW,eAAe;IAanC,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAZrC;;;;;OAKG;IACH,QAAQ,CAAC,mBAAmB,EAAE,MAAM,EAAE,CAAM;IAE5C;;;OAGG;gBAC0B,QAAQ,EAAE,cAAc;IAMrD;;;;;;;OAOG;IACH,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO;IASzC;;;;;;;;;;;;;;;;;;;OAmBG;IACG,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,cAAc,CAAC;CAiFpF"}

package/dist/memory/ingestion/UrlLoader.js

@@ -0,0 +1,174 @@
+/**
+ * @fileoverview UrlLoader — fetch-and-delegate loader for HTTP/HTTPS URLs.
+ *
+ * `UrlLoader` implements {@link IDocumentLoader} and handles `http://` and
+ * `https://` sources.  It fetches the remote resource, inspects the
+ * `Content-Type` response header, and delegates to the most appropriate
+ * registered loader:
+ *
+ * - `text/html` → {@link HtmlLoader} (via the registry)
+ * - `application/pdf` → {@link PdfLoader} (via the registry)
+ * - Anything else → raw UTF-8 text, format `'text'`
+ *
+ * Because URLs have no file extension in the traditional sense,
+ * `supportedExtensions` is deliberately empty.  Routing to `UrlLoader` must
+ * be done explicitly — either by calling `UrlLoader.load()` directly or by
+ * checking `UrlLoader.canLoad()` before dispatching.
+ *
+ * @module memory/ingestion/UrlLoader
+ */
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+/** Prefix patterns that identify an HTTP/HTTPS URL. */
+const URL_PREFIXES = ['http://', 'https://'];
+// ---------------------------------------------------------------------------
+// UrlLoader
+// ---------------------------------------------------------------------------
+/**
+ * An {@link IDocumentLoader} that fetches a remote URL and delegates parsing
+ * to the appropriate registered loader based on the response `Content-Type`.
+ *
+ * ### Supported content types
+ * | Content-Type          | Delegates to          |
+ * |-----------------------|-----------------------|
+ * | `text/html`           | HtmlLoader (registry) |
+ * | `application/pdf`     | PdfLoader  (registry) |
+ * | Everything else       | Plain UTF-8 text      |
+ *
+ * ### Example
+ * ```ts
+ * const registry = new LoaderRegistry();
+ * const urlLoader = new UrlLoader(registry);
+ *
+ * // Register so the registry also dispatches URLs via canLoad checks.
+ * // (Optional — UrlLoader can be used standalone too.)
+ *
+ * if (urlLoader.canLoad('https://example.com/report.pdf')) {
+ *   const doc = await urlLoader.load('https://example.com/report.pdf');
+ *   console.log(doc.format); // 'pdf'
+ * }
+ * ```
+ *
+ * @implements {IDocumentLoader}
+ */
+export class UrlLoader {
+    /**
+     * @param registry - The {@link LoaderRegistry} used to resolve format-specific
+     *                   loaders once the remote content type is known.
+     */
+    constructor(registry) {
+        this.registry = registry;
+        /**
+         * URLs have no file extension so this array is always empty.
+         *
+         * Routing to this loader must be performed via {@link canLoad} rather than
+         * the registry's extension-based lookup.
+         */
+        this.supportedExtensions = [];
+    }
+    // -------------------------------------------------------------------------
+    // canLoad
+    // -------------------------------------------------------------------------
+    /**
+     * Returns `true` when `source` is a string that starts with `http://` or
+     * `https://`.
+     *
+     * Buffer sources are always rejected — raw bytes cannot be a URL.
+     *
+     * @param source - Absolute file path, URL string, or raw bytes.
+     */
+    canLoad(source) {
+        if (Buffer.isBuffer(source))
+            return false;
+        return URL_PREFIXES.some((prefix) => source.startsWith(prefix));
+    }
+    // -------------------------------------------------------------------------
+    // load
+    // -------------------------------------------------------------------------
+    /**
+     * Fetch `source` over HTTP/HTTPS and return a {@link LoadedDocument}.
+     *
+     * The response body is buffered in memory and then handed to the appropriate
+     * sub-loader according to the `Content-Type` header:
+     *
+     * - `text/html` → fetched as text, passed to the HTML loader as a `Buffer`.
+     * - `application/pdf` → fetched as bytes, passed to the PDF loader as a
+     *    `Buffer`.
+     * - Anything else → returned as plain text with format `'text'` and
+     *   `source` metadata set to the URL.
+     *
+     * @param source  - HTTP/HTTPS URL string.
+     * @param options - Optional load hints forwarded to the delegated loader.
+     * @returns A promise resolving to the {@link LoadedDocument}.
+     *
+     * @throws {Error} When `source` is a `Buffer` (URLs must be strings).
+     * @throws {Error} When the HTTP request fails (network error or non-2xx
+     *                 status).
+     */
+    async load(source, options) {
+        if (Buffer.isBuffer(source)) {
+            throw new Error('UrlLoader: source must be a URL string, not a Buffer.');
+        }
+        const url = source;
+        // Fetch the remote resource.
+        const response = await fetch(url);
+        if (!response.ok) {
+            throw new Error(`UrlLoader: HTTP ${response.status} ${response.statusText} for URL "${url}".`);
+        }
+        // Determine content type from the response header, stripping parameters
+        // such as `; charset=utf-8`.
+        const contentTypeHeader = response.headers.get('content-type') ?? '';
+        const contentType = contentTypeHeader.split(';')[0].trim().toLowerCase();
+        // ------------------------------------------------------------------
+        // Delegate based on content type.
+        // ------------------------------------------------------------------
+        if (contentType.includes('text/html')) {
+            // Fetch as text and pass as a UTF-8 Buffer to the HTML loader.
+            const text = await response.text();
+            const htmlBuffer = Buffer.from(text, 'utf8');
+            const htmlLoader = this.registry.getLoader('.html');
+            if (htmlLoader) {
+                const doc = await htmlLoader.load(htmlBuffer, options);
+                // Attach the URL as the source metadata since the loader receives a
+                // Buffer and cannot derive the origin URL itself.
+                return {
+                    ...doc,
+                    metadata: { ...doc.metadata, source: url },
+                };
+            }
+            // Fallback: return raw HTML text if no HTML loader is registered.
+            return {
+                content: text,
+                metadata: { source: url, wordCount: text.trim().split(/\s+/).length },
+                format: 'html',
+            };
+        }
+        if (contentType.includes('application/pdf')) {
+            // Fetch as bytes and pass as a Buffer to the PDF loader.
+            const bytes = await response.arrayBuffer();
+            const pdfBuffer = Buffer.from(bytes);
+            const pdfLoader = this.registry.getLoader('.pdf');
+            if (pdfLoader) {
+                const doc = await pdfLoader.load(pdfBuffer, options);
+                return {
+                    ...doc,
+                    metadata: { ...doc.metadata, source: url },
+                };
+            }
+            // Fallback: cannot parse PDF without a loader.
+            throw new Error(`UrlLoader: received application/pdf from "${url}" but no PDF loader is registered.`);
+        }
+        // Default: treat the response body as plain UTF-8 text.
+        const text = await response.text();
+        return {
+            content: text,
+            metadata: {
+                source: url,
+                wordCount: text.trim() === '' ? 0 : text.trim().split(/\s+/).length,
+            },
+            format: 'text',
+        };
+    }
+}
+//# sourceMappingURL=UrlLoader.js.map

package/dist/memory/ingestion/UrlLoader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"UrlLoader.js","sourceRoot":"","sources":["../../../src/memory/ingestion/UrlLoader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAMH,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E,uDAAuD;AACvD,MAAM,YAAY,GAAG,CAAC,SAAS,EAAE,UAAU,CAAU,CAAC;AAEtD,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,OAAO,SAAS;IASpB;;;OAGG;IACH,YAA6B,QAAwB;QAAxB,aAAQ,GAAR,QAAQ,CAAgB;QAZrD;;;;;WAKG;QACM,wBAAmB,GAAa,EAAE,CAAC;IAMY,CAAC;IAEzD,4EAA4E;IAC5E,UAAU;IACV,4EAA4E;IAE5E;;;;;;;OAOG;IACH,OAAO,CAAC,MAAuB;QAC7B,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,KAAK,CAAC;QAC1C,OAAO,YAAY,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;IAClE,CAAC;IAED,4EAA4E;IAC5E,OAAO;IACP,4EAA4E;IAE5E;;;;;;;;;;;;;;;;;;;OAmBG;IACH,KAAK,CAAC,IAAI,CAAC,MAAuB,EAAE,OAAqB;QACvD,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;QAC3E,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,CAAC;QAEnB,6BAA6B;QAC7B,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAElC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CACb,mBAAmB,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,aAAa,GAAG,IAAI,CAC9E,CAAC;QACJ,CAAC;QAED,wEAAwE;QACxE,6BAA6B;QAC7B,MAAM,iBAAiB,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QACrE,MAAM,WAAW,GAAG,iBAAiB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAEzE,qEAAqE;QACrE,kCAAkC;QAClC,qEAAqE;QAErE,IAAI,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YACtC,+DAA+D;YAC/D,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAE7C,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YACpD,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;gBACvD,oEAAoE;gBACpE,kDAAkD;gBAClD,OAAO;oBACL,GAAG,GAAG;oBACN,QAAQ,EAAE,EAAE,GAAG,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE;iBAC3C,CAAC;YACJ,CAAC;YAED,kEAAkE;YAClE,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE;gBACrE,MAAM,EAAE,MAAM;aACf,CAAC;QACJ,CAAC;QAED,IAAI,WAAW,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;YAC5C,yDAAyD;YACzD,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC;YAC3C,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAErC,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAClD,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;gBACrD,OAAO;oBACL,GAAG,GAAG;oBACN,QAAQ,EAAE,EAAE,GAAG,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE;iBAC3C,CAAC;YACJ,CAAC;YAED,+CAA+C;YAC/C,MAAM,IAAI,KAAK,CACb,6CAA6C,GAAG,oCAAoC,CACrF,CAAC;QACJ,CAAC;QAED,wDAAwD;QACxD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAEnC,OAAO;YACL,OAAO,EAAE,IAAI;YACb,QAAQ,EAAE;gBACR,MAAM,EAAE,GAAG;gBACX,SAAS,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM;aACpE;YACD,MAAM,EAAE,MAAM;SACf,CAAC;IACJ,CAAC;CACF"}

package/dist/memory/io/ChatGptImporter.d.ts

@@ -0,0 +1,85 @@
+/**
+ * @fileoverview ChatGPT export importer for AgentOS memory brain.
+ *
+ * Parses the `conversations.json` file produced by ChatGPT's "Export data"
+ * feature and imports each conversation into the target `SqliteBrain`.
+ *
+ * ## Import strategy
+ *
+ * For each conversation object in `conversations.json`:
+ * 1. A row is inserted into the `conversations` table (deduped by title + created_at).
+ * 2. Every user/assistant message pair is encoded as an **episodic memory trace**
+ *    whose content is `"[user]: {user_text}\n[assistant]: {assistant_text}"`.
+ *    This preserves conversational context in a single retrievable unit.
+ * 3. System messages and tool messages are skipped (they are not episodic memories).
+ *
+ * ## ChatGPT export format
+ * ```json
+ * [
+ *   {
+ *     "title": "Conversation title",
+ *     "create_time": 1711234567.89,
+ *     "mapping": {
+ *       "node-id": {
+ *         "message": {
+ *           "author": { "role": "user" },
+ *           "content": { "parts": ["Hello!"] }
+ *         },
+ *         "children": ["next-node-id"]
+ *       }
+ *     }
+ *   }
+ * ]
+ * ```
+ *
+ * @module memory/io/ChatGptImporter
+ */
+import type { ImportResult } from '../facade/types.js';
+import type { SqliteBrain } from '../store/SqliteBrain.js';
+/**
+ * Imports a ChatGPT `conversations.json` export into a `SqliteBrain`.
+ *
+ * **Usage:**
+ * ```ts
+ * const importer = new ChatGptImporter(brain);
+ * const result = await importer.import('/path/to/conversations.json');
+ * ```
+ */
+export declare class ChatGptImporter {
+    private readonly brain;
+    /**
+     * @param brain - The target `SqliteBrain` to import into.
+     */
+    constructor(brain: SqliteBrain);
+    /**
+     * Parse `conversations.json` and import all conversations and message pairs.
+     *
+     * @param sourcePath - Absolute path to the ChatGPT `conversations.json` file.
+     * @returns `ImportResult` with counts of imported traces, skipped duplicates,
+     *   and any per-item error messages.
+     */
+    import(sourcePath: string): Promise<ImportResult>;
+    /**
+     * Import a single ChatGPT conversation object.
+     *
+     * Creates a `conversations` row and then iterates through the message
+     * mapping in tree order (BFS), pairing adjacent user/assistant messages
+     * into episodic memory traces.
+     *
+     * @param convo  - Raw ChatGPT conversation object.
+     * @param result - Mutable result accumulator.
+     */
+    private _importConversation;
+    /**
+     * Insert a single episodic memory trace derived from a message pair.
+     *
+     * Deduplication is based on SHA-256 of the combined `content` string.
+     *
+     * @param content        - The `[user]:...\n[assistant]:...` content string.
+     * @param createdAt      - Unix timestamp (ms) of the user message.
+     * @param conversationId - ID of the parent conversation row.
+     * @param result         - Mutable result accumulator.
+     */
+    private _insertEpisodicTrace;
+}
+//# sourceMappingURL=ChatGptImporter.d.ts.map

package/dist/memory/io/ChatGptImporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ChatGptImporter.d.ts","sourceRoot":"","sources":["../../../src/memory/io/ChatGptImporter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAKH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AA4C3D;;;;;;;;GAQG;AACH,qBAAa,eAAe;IAId,OAAO,CAAC,QAAQ,CAAC,KAAK;IAHlC;;OAEG;gBAC0B,KAAK,EAAE,WAAW;IAM/C;;;;;;OAMG;IACG,MAAM,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAyCvD;;;;;;;;;OASG;IACH,OAAO,CAAC,mBAAmB;IAuG3B;;;;;;;;;OASG;IACH,OAAO,CAAC,oBAAoB;CA4C7B"}