@eidentic/rag 0.1.0

package/dist/index.js

@@ -0,0 +1,392 @@
+var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
+  get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
+}) : x)(function(x) {
+  if (typeof require !== "undefined") return require.apply(this, arguments);
+  throw Error('Dynamic require of "' + x + '" is not supported');
+});
+
+// src/chunk.ts
+function splitByWordBoundary(text, size) {
+  if (text.length <= size) return text ? [text] : [];
+  const segments = [];
+  let pos = 0;
+  while (pos < text.length) {
+    if (pos + size >= text.length) {
+      segments.push(text.slice(pos));
+      break;
+    }
+    let cut = pos + size;
+    while (cut > pos && !/\s/.test(text[cut])) cut--;
+    if (cut === pos) {
+      cut = pos + size;
+    } else {
+      while (cut < text.length && /\s/.test(text[cut])) cut++;
+    }
+    segments.push(text.slice(pos, cut).trimEnd());
+    pos = cut;
+  }
+  return segments.filter((s) => s.length > 0);
+}
+function splitIntoParagraphs(text) {
+  return text.split(/\n\s*\n/).map((p) => p.trim()).filter((p) => p.length > 0);
+}
+function splitIntoSentences(text) {
+  const parts = text.split(/(?<=[.!?])\s+/);
+  return parts.map((p) => p.trim()).filter((p) => p.length > 0);
+}
+function packSegments(segments, source, size, overlap, separator = "\n\n") {
+  const segmentOffsets = [];
+  let searchFrom = 0;
+  for (const seg of segments) {
+    const idx = source.indexOf(seg, searchFrom);
+    if (idx === -1) {
+      const prev = segmentOffsets[segmentOffsets.length - 1];
+      const fallbackStart = prev ? prev.end : 0;
+      segmentOffsets.push({ start: fallbackStart, end: fallbackStart + seg.length, text: seg });
+      searchFrom = fallbackStart + seg.length;
+    } else {
+      segmentOffsets.push({ start: idx, end: idx + seg.length, text: seg });
+      searchFrom = idx + seg.length;
+    }
+  }
+  const chunks = [];
+  let current = [];
+  let currentLen = 0;
+  const flush = () => {
+    if (current.length === 0) return;
+    const start = current[0].start;
+    const end = current[current.length - 1].end;
+    const text = current.map((s) => s.text).join(separator);
+    chunks.push({ text, start, end });
+    current = [];
+    currentLen = 0;
+  };
+  for (const seg of segmentOffsets) {
+    if (currentLen > 0 && currentLen + separator.length + seg.text.length > size) {
+      flush();
+    }
+    if (seg.text.length > size) {
+      flush();
+      const subSegs = splitByWordBoundary(seg.text, size);
+      let subOffset = seg.start;
+      for (const sub of subSegs) {
+        const subStart = source.indexOf(sub, subOffset);
+        const actualStart = subStart === -1 ? subOffset : subStart;
+        chunks.push({ text: sub, start: actualStart, end: actualStart + sub.length });
+        subOffset = actualStart + sub.length;
+      }
+      continue;
+    }
+    current.push(seg);
+    currentLen += (currentLen > 0 ? separator.length : 0) + seg.text.length;
+  }
+  flush();
+  return chunks.map((chunk, i) => {
+    if (i === 0 || overlap === 0) return chunk;
+    const prev = chunks[i - 1];
+    const prefix = prev.text.slice(-overlap);
+    return { text: prefix + chunk.text, start: chunk.start, end: chunk.end };
+  });
+}
+function chunkText(text, opts) {
+  const size = opts?.size ?? 1e3;
+  const overlap = Math.min(opts?.overlap ?? 150, size - 1);
+  const strategy = opts?.strategy ?? "fixed";
+  const normalized = text.trim();
+  if (normalized.length === 0) return [];
+  if (strategy === "paragraph" || strategy === "sentence") {
+    const naturalSegments = strategy === "paragraph" ? splitIntoParagraphs(normalized) : splitIntoSentences(normalized);
+    const separator = strategy === "sentence" ? " " : "\n\n";
+    const packed = packSegments(naturalSegments, normalized, size, overlap, separator);
+    return packed.map((c, i) => ({ text: c.text, index: i, start: c.start, end: c.end }));
+  }
+  const chunks = [];
+  let pos = 0;
+  let index = 0;
+  while (pos < normalized.length) {
+    let end = pos + size;
+    if (end >= normalized.length) {
+      end = normalized.length;
+    } else {
+      let cut = end;
+      while (cut > pos && !/\s/.test(normalized[cut])) cut--;
+      if (cut === pos) {
+        cut = end;
+      }
+      end = cut;
+    }
+    const chunkText2 = normalized.slice(pos, end).trim();
+    if (chunkText2.length > 0) {
+      let fullText = chunkText2;
+      if (index > 0 && overlap > 0) {
+        const prev = chunks[chunks.length - 1];
+        const prefix = prev.text.slice(-overlap);
+        fullText = prefix + chunkText2;
+      }
+      chunks.push({ text: fullText, index, start: pos, end });
+      index++;
+    }
+    let next = end;
+    while (next < normalized.length && /\s/.test(normalized[next])) next++;
+    if (next === pos) {
+      break;
+    }
+    pos = next;
+  }
+  return chunks;
+}
+
+// src/ingest.ts
+import { resilientFetch, assertFetchableUrl } from "@eidentic/tools";
+
+// src/loaders.ts
+import { createRequire } from "node:module";
+import { parse as parseHtml } from "node-html-parser";
+function loadMarkdown(content, opts) {
+  const source = opts?.source ?? "markdown";
+  let text = content;
+  text = text.replace(/```[\s\S]*?```/g, "");
+  text = text.replace(/^(?:    |\t).+$/gm, "");
+  text = text.replace(/<[^>]+>/g, "");
+  text = text.replace(/^#{1,6}\s+(.+)$/gm, "$1");
+  text = text.replace(/^[=-]{2,}\s*$/gm, "");
+  text = text.replace(/^(?:[*_-][\s]*){3,}$/gm, "");
+  text = text.replace(/!\[([^\]]*)\]\([^)]*\)/g, "$1");
+  text = text.replace(/\[([^\]]+)\]\([^)]*\)/g, "$1");
+  text = text.replace(/\[([^\]]+)\]\[[^\]]*\]/g, "$1");
+  text = text.replace(/^\[[^\]]+\]:\s+\S+(?:\s+"[^"]*")?$/gm, "");
+  text = text.replace(/\*{1,2}([^*\n]+)\*{1,2}/g, "$1");
+  text = text.replace(/_{1,2}([^_\n]+)_{1,2}/g, "$1");
+  text = text.replace(/`([^`]+)`/g, "$1");
+  text = text.replace(/~~([^~]+)~~/g, "$1");
+  text = text.replace(/^>\s?/gm, "");
+  text = text.replace(/^[\s]*[-*+]\s+/gm, "");
+  text = text.replace(/^[\s]*\d+\.\s+/gm, "");
+  text = text.replace(/\n{3,}/g, "\n\n");
+  text = text.trim();
+  return { text, metadata: { source } };
+}
+var BLOCK_TAGS = /* @__PURE__ */ new Set([
+  "p",
+  "div",
+  "section",
+  "article",
+  "aside",
+  "main",
+  "header",
+  "footer",
+  "nav",
+  "h1",
+  "h2",
+  "h3",
+  "h4",
+  "h5",
+  "h6",
+  "li",
+  "dt",
+  "dd",
+  "ul",
+  "ol",
+  "blockquote",
+  "pre",
+  "figure",
+  "figcaption",
+  "br",
+  "hr",
+  "tr",
+  "th",
+  "td",
+  "table"
+]);
+function loadHtml(html, opts) {
+  const source = opts?.source ?? "html";
+  const root = parseHtml(html, {
+    lowerCaseTagName: true,
+    comment: false,
+    blockTextElements: {
+      script: false,
+      style: false,
+      noscript: false,
+      pre: true
+    }
+  });
+  for (const tag of ["script", "style", "noscript", "head"]) {
+    for (const el of root.querySelectorAll(tag)) {
+      el.remove();
+    }
+  }
+  function extractText(node) {
+    if (node.nodeType === 3) {
+      return node.rawText ?? node.text ?? "";
+    }
+    const tag = node.tagName?.toLowerCase() ?? "";
+    const isBlock = BLOCK_TAGS.has(tag);
+    const childText = node.childNodes.map((c) => extractText(c)).join("");
+    return isBlock ? `
+${childText}
+` : childText;
+  }
+  let text = extractText(root);
+  text = text.replace(/[^\S\n]+/g, " ");
+  text = text.replace(/\n{3,}/g, "\n\n");
+  text = text.trim();
+  return { text, metadata: { source } };
+}
+function loadPdfParseModule() {
+  const req = typeof __require === "function" ? __require : createRequire(import.meta.url);
+  try {
+    const mod = req("pdf-parse");
+    const fn = typeof mod === "function" ? mod : mod.default;
+    if (typeof fn !== "function") {
+      throw new Error("unexpected module shape");
+    }
+    return fn;
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    if (msg.includes("Cannot find module") || msg.includes("unexpected module shape")) {
+      throw new Error(
+        "[eidentic/rag] loadPdf requires the `pdf-parse` package.\nInstall it in your project:\n  npm install pdf-parse\n  # or\n  pnpm add pdf-parse"
+      );
+    }
+    throw err;
+  }
+}
+async function loadPdf(buf, opts) {
+  const source = opts?.source ?? "pdf";
+  const parse = opts?._parser ?? loadPdfParseModule();
+  const result = await parse(buf);
+  let text = result.text;
+  text = text.replace(/[^\S\n]+/g, " ");
+  text = text.replace(/\n{3,}/g, "\n\n");
+  text = text.trim();
+  return {
+    text,
+    metadata: { source, pages: result.numpages }
+  };
+}
+
+// src/ingest.ts
+function slugFromUrl(url) {
+  try {
+    const u = new URL(url);
+    const parts = (u.hostname + u.pathname).toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/(^-|-$)/g, "");
+    return parts.slice(0, 64) || "doc";
+  } catch {
+    return "doc";
+  }
+}
+function hashSlug(text) {
+  let h = 5381;
+  for (let i = 0; i < Math.min(text.length, 2048); i++) {
+    h = (h << 5) + h + text.charCodeAt(i) >>> 0;
+  }
+  return `doc-${h.toString(16)}`;
+}
+var MAX_REDIRECT_HOPS = 5;
+function normalizeUrl(url) {
+  try {
+    const u = new URL(url);
+    return `${u.protocol}//${u.host}${u.pathname.replace(/\/+$/, "")}${u.search}`;
+  } catch {
+    return url;
+  }
+}
+async function ingestDocument(source, opts) {
+  let rawText;
+  let docId;
+  let extraMeta = {};
+  if (typeof source === "string") {
+    if (source.startsWith("http://") || source.startsWith("https://")) {
+      console.warn(
+        `[ingestDocument] source looks like a URL but was passed as a plain string \u2014 it will be ingested as raw text, NOT fetched. If you meant to fetch this URL, use { url: "${source}" } instead.`
+      );
+    }
+    rawText = source;
+    docId = opts.docId ?? hashSlug(rawText);
+  } else if ("type" in source) {
+    const typedSource = source;
+    let loaded;
+    if (typedSource.type === "markdown") {
+      loaded = loadMarkdown(typedSource.data, { source: typedSource.source });
+    } else if (typedSource.type === "html") {
+      loaded = loadHtml(typedSource.data, { source: typedSource.source });
+    } else {
+      loaded = await loadPdf(typedSource.data, {
+        source: typedSource.source,
+        _parser: typedSource._parser
+      });
+    }
+    rawText = loaded.text;
+    extraMeta = loaded.metadata;
+    docId = opts.docId ?? hashSlug(rawText);
+  } else {
+    assertFetchableUrl(source.url, { allowlist: opts.allowlist });
+    const doFetch = opts.fetchImpl ?? globalThis.fetch;
+    let currentUrl = source.url;
+    let hops = 0;
+    let finalRes;
+    while (true) {
+      const res = await resilientFetch(currentUrl, { redirect: "manual" }, { fetchImpl: doFetch });
+      const autoFollowed = res.redirected === true || res.status < 300 && typeof res.url === "string" && res.url !== "" && normalizeUrl(res.url) !== normalizeUrl(currentUrl);
+      if (autoFollowed) {
+        throw new Error(
+          'ingestDocument: the supplied fetchImpl auto-followed a redirect instead of honouring redirect:"manual". Replace it with a fetch implementation that returns 3xx responses rather than following them, so every redirect hop can be re-validated by the SSRF guard.'
+        );
+      }
+      if (res.status >= 300 && res.status < 400) {
+        hops++;
+        if (hops > MAX_REDIRECT_HOPS) {
+          throw new Error(
+            `ingestDocument: too many redirects (> ${MAX_REDIRECT_HOPS} hops) starting from ${source.url}`
+          );
+        }
+        const location = res.headers.get("location");
+        if (location === null) {
+          throw new Error(`ingestDocument: redirect has no Location header (hop ${hops})`);
+        }
+        let nextUrl;
+        try {
+          nextUrl = new URL(location, currentUrl);
+        } catch {
+          throw new Error(`ingestDocument: invalid redirect target at hop ${hops} (URL omitted for security)`);
+        }
+        assertFetchableUrl(nextUrl.toString(), { allowlist: opts.allowlist });
+        currentUrl = nextUrl.toString();
+      } else {
+        finalRes = res;
+        break;
+      }
+    }
+    if (!finalRes.ok) {
+      throw new Error(`ingestDocument: failed to fetch ${source.url} \u2014 HTTP ${finalRes.status}`);
+    }
+    rawText = await finalRes.text();
+    docId = opts.docId ?? slugFromUrl(source.url);
+  }
+  const chunks = chunkText(rawText, opts.chunk);
+  if (chunks.length === 0) return { chunks: 0 };
+  let citationSource;
+  if (typeof source === "string") {
+    citationSource = docId;
+  } else if ("type" in source) {
+    citationSource = extraMeta.source ?? docId;
+  } else {
+    citationSource = source.url;
+  }
+  const events = chunks.map((chunk, i) => ({
+    id: `${docId}:chunk:${i}`,
+    scope: opts.scope,
+    text: chunk.text,
+    // Merge loader-provided metadata (source, pages, etc.) into chunk metadata.
+    metadata: { ...extraMeta, source: citationSource }
+  }));
+  await opts.memory.ingest(events);
+  return { chunks: chunks.length };
+}
+export {
+  chunkText,
+  ingestDocument,
+  loadHtml,
+  loadMarkdown,
+  loadPdf
+};

package/package.json

@@ -0,0 +1,65 @@
+{
+  "name": "@eidentic/rag",
+  "version": "0.1.0",
+  "type": "module",
+  "license": "Apache-2.0",
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/eidentic/eidentic.git",
+    "directory": "packages/rag"
+  },
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
+    }
+  },
+  "sideEffects": false,
+  "files": [
+    "dist",
+    "LICENSE",
+    "README.md"
+  ],
+  "dependencies": {
+    "node-html-parser": "^7.1.0",
+    "@eidentic/tools": "0.1.0",
+    "@eidentic/types": "0.1.0"
+  },
+  "peerDependencies": {
+    "pdf-parse": "^1.1.1"
+  },
+  "peerDependenciesMeta": {
+    "pdf-parse": {
+      "optional": true
+    }
+  },
+  "description": "RAG pipeline utilities for Eidentic — chunk text, ingest documents (PDF / HTML / Markdown), and load content into agent memory.",
+  "keywords": [
+    "ai",
+    "agents",
+    "typescript",
+    "eidentic",
+    "rag",
+    "document-loading",
+    "chunking",
+    "memory"
+  ],
+  "homepage": "https://github.com/eidentic/eidentic#readme",
+  "bugs": {
+    "url": "https://github.com/eidentic/eidentic/issues"
+  },
+  "engines": {
+    "node": ">=22"
+  },
+  "scripts": {
+    "build": "tsup src/index.ts --format esm,cjs --dts --clean",
+    "typecheck": "tsc --noEmit"
+  }
+}