@eidentic/rag 0.1.0

package/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or Derivative
+          Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work, excluding
+          those notices that do not pertain to any part of the Derivative
+          Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and do
+          not modify the License. You may add Your own attribution notices
+          within Derivative Works that You distribute, alongside or as an
+          addendum to the NOTICE text from the Work, provided that such
+          additional attribution notices cannot be construed as modifying
+          the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2026 Baran Özdemir
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

package/README.md

@@ -0,0 +1,50 @@
+# @eidentic/rag
+
+RAG pipeline utilities for Eidentic — chunk text into overlapping windows, ingest
+documents from URLs or typed content sources, and load Markdown, HTML, and PDF files
+into a format ready for agent memory. Works with any `Memory` instance.
+
+## Install
+
+```bash
+pnpm add @eidentic/rag
+```
+
+For PDF loading, also install `pdf-parse`:
+
+```bash
+pnpm add pdf-parse
+```
+
+## Usage
+
+```ts
+import { ingestDocument, loadMarkdown, chunkText } from "@eidentic/rag";
+import { Memory } from "@eidentic/memory";
+
+// Ingest a URL directly into agent memory
+await ingestDocument({
+  source: { type: "url", url: "https://docs.example.com/guide" },
+  memory,
+  scope: { userId: "u-1" },
+});
+
+// Or load and chunk manually
+const doc = await loadMarkdown({ path: "./docs/guide.md" });
+const chunks = chunkText(doc.text, { chunkSize: 512, overlap: 64 });
+
+// Or ingest typed content
+await ingestDocument({
+  source: { type: "text", text: "...", mimeType: "text/plain", title: "My Doc" },
+  memory,
+  scope: { userId: "u-1" },
+});
+```
+
+## Links
+
+- [GitHub](https://github.com/eidentic/eidentic)
+- [Issue tracker](https://github.com/eidentic/eidentic/issues)
+- [Root README](https://github.com/eidentic/eidentic#readme)
+
+Apache-2.0

package/dist/index.cjs

@@ -0,0 +1,417 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  chunkText: () => chunkText,
+  ingestDocument: () => ingestDocument,
+  loadHtml: () => loadHtml,
+  loadMarkdown: () => loadMarkdown,
+  loadPdf: () => loadPdf
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/chunk.ts
+function splitByWordBoundary(text, size) {
+  if (text.length <= size) return text ? [text] : [];
+  const segments = [];
+  let pos = 0;
+  while (pos < text.length) {
+    if (pos + size >= text.length) {
+      segments.push(text.slice(pos));
+      break;
+    }
+    let cut = pos + size;
+    while (cut > pos && !/\s/.test(text[cut])) cut--;
+    if (cut === pos) {
+      cut = pos + size;
+    } else {
+      while (cut < text.length && /\s/.test(text[cut])) cut++;
+    }
+    segments.push(text.slice(pos, cut).trimEnd());
+    pos = cut;
+  }
+  return segments.filter((s) => s.length > 0);
+}
+function splitIntoParagraphs(text) {
+  return text.split(/\n\s*\n/).map((p) => p.trim()).filter((p) => p.length > 0);
+}
+function splitIntoSentences(text) {
+  const parts = text.split(/(?<=[.!?])\s+/);
+  return parts.map((p) => p.trim()).filter((p) => p.length > 0);
+}
+function packSegments(segments, source, size, overlap, separator = "\n\n") {
+  const segmentOffsets = [];
+  let searchFrom = 0;
+  for (const seg of segments) {
+    const idx = source.indexOf(seg, searchFrom);
+    if (idx === -1) {
+      const prev = segmentOffsets[segmentOffsets.length - 1];
+      const fallbackStart = prev ? prev.end : 0;
+      segmentOffsets.push({ start: fallbackStart, end: fallbackStart + seg.length, text: seg });
+      searchFrom = fallbackStart + seg.length;
+    } else {
+      segmentOffsets.push({ start: idx, end: idx + seg.length, text: seg });
+      searchFrom = idx + seg.length;
+    }
+  }
+  const chunks = [];
+  let current = [];
+  let currentLen = 0;
+  const flush = () => {
+    if (current.length === 0) return;
+    const start = current[0].start;
+    const end = current[current.length - 1].end;
+    const text = current.map((s) => s.text).join(separator);
+    chunks.push({ text, start, end });
+    current = [];
+    currentLen = 0;
+  };
+  for (const seg of segmentOffsets) {
+    if (currentLen > 0 && currentLen + separator.length + seg.text.length > size) {
+      flush();
+    }
+    if (seg.text.length > size) {
+      flush();
+      const subSegs = splitByWordBoundary(seg.text, size);
+      let subOffset = seg.start;
+      for (const sub of subSegs) {
+        const subStart = source.indexOf(sub, subOffset);
+        const actualStart = subStart === -1 ? subOffset : subStart;
+        chunks.push({ text: sub, start: actualStart, end: actualStart + sub.length });
+        subOffset = actualStart + sub.length;
+      }
+      continue;
+    }
+    current.push(seg);
+    currentLen += (currentLen > 0 ? separator.length : 0) + seg.text.length;
+  }
+  flush();
+  return chunks.map((chunk, i) => {
+    if (i === 0 || overlap === 0) return chunk;
+    const prev = chunks[i - 1];
+    const prefix = prev.text.slice(-overlap);
+    return { text: prefix + chunk.text, start: chunk.start, end: chunk.end };
+  });
+}
+function chunkText(text, opts) {
+  const size = opts?.size ?? 1e3;
+  const overlap = Math.min(opts?.overlap ?? 150, size - 1);
+  const strategy = opts?.strategy ?? "fixed";
+  const normalized = text.trim();
+  if (normalized.length === 0) return [];
+  if (strategy === "paragraph" || strategy === "sentence") {
+    const naturalSegments = strategy === "paragraph" ? splitIntoParagraphs(normalized) : splitIntoSentences(normalized);
+    const separator = strategy === "sentence" ? " " : "\n\n";
+    const packed = packSegments(naturalSegments, normalized, size, overlap, separator);
+    return packed.map((c, i) => ({ text: c.text, index: i, start: c.start, end: c.end }));
+  }
+  const chunks = [];
+  let pos = 0;
+  let index = 0;
+  while (pos < normalized.length) {
+    let end = pos + size;
+    if (end >= normalized.length) {
+      end = normalized.length;
+    } else {
+      let cut = end;
+      while (cut > pos && !/\s/.test(normalized[cut])) cut--;
+      if (cut === pos) {
+        cut = end;
+      }
+      end = cut;
+    }
+    const chunkText2 = normalized.slice(pos, end).trim();
+    if (chunkText2.length > 0) {
+      let fullText = chunkText2;
+      if (index > 0 && overlap > 0) {
+        const prev = chunks[chunks.length - 1];
+        const prefix = prev.text.slice(-overlap);
+        fullText = prefix + chunkText2;
+      }
+      chunks.push({ text: fullText, index, start: pos, end });
+      index++;
+    }
+    let next = end;
+    while (next < normalized.length && /\s/.test(normalized[next])) next++;
+    if (next === pos) {
+      break;
+    }
+    pos = next;
+  }
+  return chunks;
+}
+
+// src/ingest.ts
+var import_tools = require("@eidentic/tools");
+
+// src/loaders.ts
+var import_node_module = require("node:module");
+var import_node_html_parser = require("node-html-parser");
+var import_meta = {};
+function loadMarkdown(content, opts) {
+  const source = opts?.source ?? "markdown";
+  let text = content;
+  text = text.replace(/```[\s\S]*?```/g, "");
+  text = text.replace(/^(?:    |\t).+$/gm, "");
+  text = text.replace(/<[^>]+>/g, "");
+  text = text.replace(/^#{1,6}\s+(.+)$/gm, "$1");
+  text = text.replace(/^[=-]{2,}\s*$/gm, "");
+  text = text.replace(/^(?:[*_-][\s]*){3,}$/gm, "");
+  text = text.replace(/!\[([^\]]*)\]\([^)]*\)/g, "$1");
+  text = text.replace(/\[([^\]]+)\]\([^)]*\)/g, "$1");
+  text = text.replace(/\[([^\]]+)\]\[[^\]]*\]/g, "$1");
+  text = text.replace(/^\[[^\]]+\]:\s+\S+(?:\s+"[^"]*")?$/gm, "");
+  text = text.replace(/\*{1,2}([^*\n]+)\*{1,2}/g, "$1");
+  text = text.replace(/_{1,2}([^_\n]+)_{1,2}/g, "$1");
+  text = text.replace(/`([^`]+)`/g, "$1");
+  text = text.replace(/~~([^~]+)~~/g, "$1");
+  text = text.replace(/^>\s?/gm, "");
+  text = text.replace(/^[\s]*[-*+]\s+/gm, "");
+  text = text.replace(/^[\s]*\d+\.\s+/gm, "");
+  text = text.replace(/\n{3,}/g, "\n\n");
+  text = text.trim();
+  return { text, metadata: { source } };
+}
+var BLOCK_TAGS = /* @__PURE__ */ new Set([
+  "p",
+  "div",
+  "section",
+  "article",
+  "aside",
+  "main",
+  "header",
+  "footer",
+  "nav",
+  "h1",
+  "h2",
+  "h3",
+  "h4",
+  "h5",
+  "h6",
+  "li",
+  "dt",
+  "dd",
+  "ul",
+  "ol",
+  "blockquote",
+  "pre",
+  "figure",
+  "figcaption",
+  "br",
+  "hr",
+  "tr",
+  "th",
+  "td",
+  "table"
+]);
+function loadHtml(html, opts) {
+  const source = opts?.source ?? "html";
+  const root = (0, import_node_html_parser.parse)(html, {
+    lowerCaseTagName: true,
+    comment: false,
+    blockTextElements: {
+      script: false,
+      style: false,
+      noscript: false,
+      pre: true
+    }
+  });
+  for (const tag of ["script", "style", "noscript", "head"]) {
+    for (const el of root.querySelectorAll(tag)) {
+      el.remove();
+    }
+  }
+  function extractText(node) {
+    if (node.nodeType === 3) {
+      return node.rawText ?? node.text ?? "";
+    }
+    const tag = node.tagName?.toLowerCase() ?? "";
+    const isBlock = BLOCK_TAGS.has(tag);
+    const childText = node.childNodes.map((c) => extractText(c)).join("");
+    return isBlock ? `
+${childText}
+` : childText;
+  }
+  let text = extractText(root);
+  text = text.replace(/[^\S\n]+/g, " ");
+  text = text.replace(/\n{3,}/g, "\n\n");
+  text = text.trim();
+  return { text, metadata: { source } };
+}
+function loadPdfParseModule() {
+  const req = typeof require === "function" ? require : (0, import_node_module.createRequire)(import_meta.url);
+  try {
+    const mod = req("pdf-parse");
+    const fn = typeof mod === "function" ? mod : mod.default;
+    if (typeof fn !== "function") {
+      throw new Error("unexpected module shape");
+    }
+    return fn;
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    if (msg.includes("Cannot find module") || msg.includes("unexpected module shape")) {
+      throw new Error(
+        "[eidentic/rag] loadPdf requires the `pdf-parse` package.\nInstall it in your project:\n  npm install pdf-parse\n  # or\n  pnpm add pdf-parse"
+      );
+    }
+    throw err;
+  }
+}
+async function loadPdf(buf, opts) {
+  const source = opts?.source ?? "pdf";
+  const parse = opts?._parser ?? loadPdfParseModule();
+  const result = await parse(buf);
+  let text = result.text;
+  text = text.replace(/[^\S\n]+/g, " ");
+  text = text.replace(/\n{3,}/g, "\n\n");
+  text = text.trim();
+  return {
+    text,
+    metadata: { source, pages: result.numpages }
+  };
+}
+
+// src/ingest.ts
+function slugFromUrl(url) {
+  try {
+    const u = new URL(url);
+    const parts = (u.hostname + u.pathname).toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/(^-|-$)/g, "");
+    return parts.slice(0, 64) || "doc";
+  } catch {
+    return "doc";
+  }
+}
+function hashSlug(text) {
+  let h = 5381;
+  for (let i = 0; i < Math.min(text.length, 2048); i++) {
+    h = (h << 5) + h + text.charCodeAt(i) >>> 0;
+  }
+  return `doc-${h.toString(16)}`;
+}
+var MAX_REDIRECT_HOPS = 5;
+function normalizeUrl(url) {
+  try {
+    const u = new URL(url);
+    return `${u.protocol}//${u.host}${u.pathname.replace(/\/+$/, "")}${u.search}`;
+  } catch {
+    return url;
+  }
+}
+async function ingestDocument(source, opts) {
+  let rawText;
+  let docId;
+  let extraMeta = {};
+  if (typeof source === "string") {
+    if (source.startsWith("http://") || source.startsWith("https://")) {
+      console.warn(
+        `[ingestDocument] source looks like a URL but was passed as a plain string \u2014 it will be ingested as raw text, NOT fetched. If you meant to fetch this URL, use { url: "${source}" } instead.`
+      );
+    }
+    rawText = source;
+    docId = opts.docId ?? hashSlug(rawText);
+  } else if ("type" in source) {
+    const typedSource = source;
+    let loaded;
+    if (typedSource.type === "markdown") {
+      loaded = loadMarkdown(typedSource.data, { source: typedSource.source });
+    } else if (typedSource.type === "html") {
+      loaded = loadHtml(typedSource.data, { source: typedSource.source });
+    } else {
+      loaded = await loadPdf(typedSource.data, {
+        source: typedSource.source,
+        _parser: typedSource._parser
+      });
+    }
+    rawText = loaded.text;
+    extraMeta = loaded.metadata;
+    docId = opts.docId ?? hashSlug(rawText);
+  } else {
+    (0, import_tools.assertFetchableUrl)(source.url, { allowlist: opts.allowlist });
+    const doFetch = opts.fetchImpl ?? globalThis.fetch;
+    let currentUrl = source.url;
+    let hops = 0;
+    let finalRes;
+    while (true) {
+      const res = await (0, import_tools.resilientFetch)(currentUrl, { redirect: "manual" }, { fetchImpl: doFetch });
+      const autoFollowed = res.redirected === true || res.status < 300 && typeof res.url === "string" && res.url !== "" && normalizeUrl(res.url) !== normalizeUrl(currentUrl);
+      if (autoFollowed) {
+        throw new Error(
+          'ingestDocument: the supplied fetchImpl auto-followed a redirect instead of honouring redirect:"manual". Replace it with a fetch implementation that returns 3xx responses rather than following them, so every redirect hop can be re-validated by the SSRF guard.'
+        );
+      }
+      if (res.status >= 300 && res.status < 400) {
+        hops++;
+        if (hops > MAX_REDIRECT_HOPS) {
+          throw new Error(
+            `ingestDocument: too many redirects (> ${MAX_REDIRECT_HOPS} hops) starting from ${source.url}`
+          );
+        }
+        const location = res.headers.get("location");
+        if (location === null) {
+          throw new Error(`ingestDocument: redirect has no Location header (hop ${hops})`);
+        }
+        let nextUrl;
+        try {
+          nextUrl = new URL(location, currentUrl);
+        } catch {
+          throw new Error(`ingestDocument: invalid redirect target at hop ${hops} (URL omitted for security)`);
+        }
+        (0, import_tools.assertFetchableUrl)(nextUrl.toString(), { allowlist: opts.allowlist });
+        currentUrl = nextUrl.toString();
+      } else {
+        finalRes = res;
+        break;
+      }
+    }
+    if (!finalRes.ok) {
+      throw new Error(`ingestDocument: failed to fetch ${source.url} \u2014 HTTP ${finalRes.status}`);
+    }
+    rawText = await finalRes.text();
+    docId = opts.docId ?? slugFromUrl(source.url);
+  }
+  const chunks = chunkText(rawText, opts.chunk);
+  if (chunks.length === 0) return { chunks: 0 };
+  let citationSource;
+  if (typeof source === "string") {
+    citationSource = docId;
+  } else if ("type" in source) {
+    citationSource = extraMeta.source ?? docId;
+  } else {
+    citationSource = source.url;
+  }
+  const events = chunks.map((chunk, i) => ({
+    id: `${docId}:chunk:${i}`,
+    scope: opts.scope,
+    text: chunk.text,
+    // Merge loader-provided metadata (source, pages, etc.) into chunk metadata.
+    metadata: { ...extraMeta, source: citationSource }
+  }));
+  await opts.memory.ingest(events);
+  return { chunks: chunks.length };
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  chunkText,
+  ingestDocument,
+  loadHtml,
+  loadMarkdown,
+  loadPdf
+});