npm - @warlock.js/core - Versions diffs - 4.7.0 → 4.8.1 - Mend

@warlock.js/core 4.7.0 → 4.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (812) hide show

package/esm/ai/src/rag/loaders/load-html.mjs DELETED Viewed

@@ -1,138 +0,0 @@
-//#region ../@warlock.js/ai/src/rag/loaders/load-html.ts
-/** Default `id` when the caller supplies none. */
-const DEFAULT_ID = "document";
-/**
-* Elements whose *content* is not human-readable text and must be removed
-* wholesale (open tag → close tag → everything in between) before tags are
-* stripped. `script` / `style` would otherwise leak code into the chunked
-* text; `noscript` / `template` / `head` / `svg` are non-prose noise.
-*/
-const STRIPPED_ELEMENTS = [
-	"script",
-	"style",
-	"noscript",
-	"template",
-	"head",
-	"svg"
-];
-/**
-* Block-level tags that imply a line break in the readable text. Replacing
-* them with `\n` BEFORE the generic tag strip keeps paragraph / list / table
-* structure (so the recursive splitter still sees `\n\n` boundaries) instead
-* of collapsing the whole page onto one line.
-*/
-const BLOCK_TAGS = /<\/?(?:p|div|section|article|header|footer|main|aside|nav|h[1-6]|ul|ol|li|table|tr|td|th|thead|tbody|blockquote|pre|hr|br)\b[^>]*>/gi;
-/** Named HTML entities common in prose. Numeric entities are decoded generically. */
-const NAMED_ENTITIES = {
-	amp: "&",
-	lt: "<",
-	gt: ">",
-	quot: "\"",
-	apos: "'",
-	nbsp: " ",
-	copy: "©",
-	reg: "®",
-	trade: "™",
-	hellip: "…",
-	mdash: "—",
-	ndash: "–",
-	lsquo: "‘",
-	rsquo: "’",
-	ldquo: "“",
-	rdquo: "”",
-	laquo: "«",
-	raquo: "»",
-	middot: "·",
-	bull: "•"
-};
-/**
-* Decode the HTML entities that survive tag stripping: named (`&amp;`),
-* decimal (`&#169;`), and hex (`&#xA9;`). Unknown named entities are left
-* verbatim rather than dropped, so unusual markup never silently loses text.
-*/
-function decodeEntities(text) {
-	return text.replace(/&(#x?[0-9a-f]+|[a-z][a-z0-9]*);/gi, (match, body) => {
-		if (body[0] === "#") {
-			const codePoint = body[1] === "x" || body[1] === "X" ? Number.parseInt(body.slice(2), 16) : Number.parseInt(body.slice(1), 10);
-			if (Number.isNaN(codePoint) || codePoint < 0 || codePoint > 1114111) return match;
-			try {
-				return String.fromCodePoint(codePoint);
-			} catch {
-				return match;
-			}
-		}
-		return NAMED_ENTITIES[body.toLowerCase()] ?? match;
-	});
-}
-/**
-* Pull the `<title>` text out of the document, decoded and trimmed, or
-* `undefined` when there is none. Read BEFORE `<head>` is stripped.
-*/
-function extractTitle(html) {
-	const match = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(html);
-	if (!match) return;
-	const title = decodeEntities(match[1]).replace(/\s+/g, " ").trim();
-	return title.length > 0 ? title : void 0;
-}
-/**
-* Strip HTML markup down to readable plain text — a lightweight,
-* dependency-free pass (no DOM parser): drop comments and non-prose elements
-* (`script` / `style` / `head` / `svg` / …) content-and-all, convert block
-* tags to line breaks to preserve paragraph structure, remove every
-* remaining tag, decode entities, then collapse runs of whitespace while
-* keeping blank-line paragraph separators.
-*/
-function htmlToText(html) {
-	let text = html;
-	text = text.replace(/<!--[\s\S]*?-->/g, " ");
-	for (const tag of STRIPPED_ELEMENTS) {
-		const element = new RegExp(`<${tag}\\b[^>]*>[\\s\\S]*?<\\/${tag}>`, "gi");
-		text = text.replace(element, " ");
-		text = text.replace(new RegExp(`<\\/?${tag}\\b[^>]*>`, "gi"), " ");
-	}
-	text = text.replace(BLOCK_TAGS, "\n");
-	text = text.replace(/<[^>]+>/g, "");
-	text = decodeEntities(text);
-	text = text.replace(/[^\S\n]+/g, " ").replace(/[ \t]*\n[ \t]*/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
-	return text;
-}
-/**
-* Load an HTML string into a single {@link RagDocument} of readable text.
-* Scripts, styles, and other non-prose elements are dropped content-and-all,
-* block tags become line breaks (so paragraph structure survives for the
-* splitter), remaining tags are stripped, and HTML entities are decoded — a
-* lightweight regex pass, no heavy DOM dependency.
-*
-* The document's `metadata.title` is set from the page's `<title>` (unless
-* the caller overrode it), and `metadata.loader` is `"html"`. The output is
-* the exact shape `index()` consumes.
-*
-* @example
-* const kb = ai.rag({ embedder, store });
-* await kb.index(loadHtml(rawHtmlString, { id: "landing-page" }));
-*
-* @param html - The raw HTML markup.
-* @param options - Shared `id` / `metadata` / `tags` ({@link LoadHtmlOptions}).
-* @returns A {@link RagLoaderResult} (one document) ready for `rag.index()`.
-*/
-function loadHtml(html, options = {}) {
-	const id = options.id ?? DEFAULT_ID;
-	const title = extractTitle(html);
-	const text = htmlToText(html);
-	if (text.length === 0) return [];
-	return [{
-		id,
-		text,
-		metadata: {
-			source: id,
-			loader: "html",
-			...title !== void 0 ? { title } : {},
-			...options.metadata
-		},
-		tags: options.tags
-	}];
-}
-//#endregion
-export { extractTitle, htmlToText, loadHtml };
-//# sourceMappingURL=load-html.mjs.map

package/esm/ai/src/rag/loaders/load-html.mjs.map DELETED Viewed

@@ -1 +0,0 @@

- {"version":3,"file":"load-html.mjs","names":[],"sources":["../../../../../../../../../../@warlock.js/ai/src/rag/loaders/load-html.ts"],"sourcesContent":["import type { RagDocument } from \"../contracts/rag-document.type\";\nimport type { LoadHtmlOptions, RagLoaderResult } from \"./loader.type\";\n\n/** Default `id` when the caller supplies none. */\nconst DEFAULT_ID = \"document\";\n\n/**\n * Elements whose *content* is not human-readable text and must be removed\n * wholesale (open tag → close tag → everything in between) before tags are\n * stripped. `script` / `style` would otherwise leak code into the chunked\n * text; `noscript` / `template` / `head` / `svg` are non-prose noise.\n */\nconst STRIPPED_ELEMENTS = [\n \"script\",\n \"style\",\n \"noscript\",\n \"template\",\n \"head\",\n \"svg\",\n];\n\n/**\n * Block-level tags that imply a line break in the readable text. Replacing\n * them with `\\n` BEFORE the generic tag strip keeps paragraph / list / table\n * structure (so the recursive splitter still sees `\\n\\n` boundaries) instead\n * of collapsing the whole page onto one line.\n */\nconst BLOCK_TAGS =\n /<\\/?(?:p|div|section|article|header|footer|main|aside|nav|h[1-6]|ul|ol|li|table|tr|td|th|thead|tbody|blockquote|pre|hr|br)\\b[^>]*>/gi;\n\n/** Named HTML entities common in prose. Numeric entities are decoded generically. */\nconst NAMED_ENTITIES: Record<string, string> = {\n amp: \"&\",\n lt: \"<\",\n gt: \">\",\n quot: '\"',\n apos: \"'\",\n nbsp: \" \",\n copy: \"©\",\n reg: \"®\",\n trade: \"™\",\n hellip: \"…\",\n mdash: \"—\",\n ndash: \"–\",\n lsquo: \"‘\",\n rsquo: \"’\",\n ldquo: \"“\",\n rdquo: \"”\",\n laquo: \"«\",\n raquo: \"»\",\n middot: \"·\",\n bull: \"•\",\n};\n\n/**\n * Decode the HTML entities that survive tag stripping: named (`&`),\n * decimal (`©`), and hex (`©`). Unknown named entities are left\n * verbatim rather than dropped, so unusual markup never silently loses text.\n */\nfunction decodeEntities(text: string): string {\n return text.replace(/&(#x?[0-9a-f]+|[a-z][a-z0-9]*);/gi, (match, body: string) => {\n if (body[0] === \"#\") {\n const codePoint =\n body[1] === \"x\" || body[1] === \"X\"\n ? Number.parseInt(body.slice(2), 16)\n : Number.parseInt(body.slice(1), 10);\n\n if (Number.isNaN(codePoint) || codePoint < 0 || codePoint > 0x10ffff) {\n return match;\n }\n\n try {\n return String.fromCodePoint(codePoint);\n } catch {\n return match;\n }\n }\n\n const named = NAMED_ENTITIES[body.toLowerCase()];\n\n return named ?? match;\n });\n}\n\n/**\n * Pull the `<title>` text out of the document, decoded and trimmed, or\n * `undefined` when there is none. Read BEFORE `<head>` is stripped.\n */\nfunction extractTitle(html: string): string | undefined {\n const match = /<title[^>]*>([\\s\\S]*?)<\\/title>/i.exec(html);\n\n if (!match) {\n return undefined;\n }\n\n const title = decodeEntities(match[1]).replace(/\\s+/g, \" \").trim();\n\n return title.length > 0 ? title : undefined;\n}\n\n/**\n * Strip HTML markup down to readable plain text — a lightweight,\n * dependency-free pass (no DOM parser): drop comments and non-prose elements\n * (`script` / `style` / `head` / `svg` / …) content-and-all, convert block\n * tags to line breaks to preserve paragraph structure, remove every\n * remaining tag, decode entities, then collapse runs of whitespace while\n * keeping blank-line paragraph separators.\n */\nfunction htmlToText(html: string): string {\n let text = html;\n\n // 1. Comments first — a commented-out `<script>` must not survive.\n text = text.replace(//g, \" \");\n\n // 2. Non-prose elements, content and all.\n for (const tag of STRIPPED_ELEMENTS) {\n const element = new RegExp(`<${tag}\\\\b[^>]*>[\\\\s\\\\S]*?<\\\\/${tag}>`, \"gi\");\n text = text.replace(element, \" \");\n // Defensively drop a self-closing / unterminated open tag too.\n text = text.replace(new RegExp(`<\\\\/?${tag}\\\\b[^>]*>`, \"gi\"), \" \");\n }\n\n // 3. Block tags → newlines, so paragraph / list structure survives.\n text = text.replace(BLOCK_TAGS, \"\\n\");\n\n // 4. Every remaining tag → gone.\n text = text.replace(/<[^>]+>/g, \"\");\n\n // 5. Entities → characters.\n text = decodeEntities(text);\n\n // 6. Normalize whitespace: trim each line, drop blank runs to a single\n // blank line (a paragraph separator the recursive splitter honors).\n text = text\n .replace(/[^\\S\\n]+/g, \" \")\n .replace(/[ \\t]*\\n[ \\t]*/g, \"\\n\")\n .replace(/\\n{3,}/g, \"\\n\\n\")\n .trim();\n\n return text;\n}\n\n/**\n * Load an HTML string into a single {@link RagDocument} of readable text.\n * Scripts, styles, and other non-prose elements are dropped content-and-all,\n * block tags become line breaks (so paragraph structure survives for the\n * splitter), remaining tags are stripped, and HTML entities are decoded — a\n * lightweight regex pass, no heavy DOM dependency.\n *\n * The document's `metadata.title` is set from the page's `<title>` (unless\n * the caller overrode it), and `metadata.loader` is `\"html\"`. The output is\n * the exact shape `index()` consumes.\n *\n * @example\n * const kb = ai.rag({ embedder, store });\n * await kb.index(loadHtml(rawHtmlString, { id: \"landing-page\" }));\n *\n * @param html - The raw HTML markup.\n * @param options - Shared `id` / `metadata` / `tags` ({@link LoadHtmlOptions}).\n * @returns A {@link RagLoaderResult} (one document) ready for `rag.index()`.\n */\nexport function loadHtml(\n html: string,\n options: LoadHtmlOptions = {},\n): RagLoaderResult {\n const id = options.id ?? DEFAULT_ID;\n const title = extractTitle(html);\n const text = htmlToText(html);\n\n // An all-markup / empty page strips to nothing; emit no document so\n // index() never receives a no-op record (matches loadText's behavior).\n if (text.length === 0) {\n return [];\n }\n\n // Derived keys (source, loader, title) sit UNDER the caller's metadata so\n // an explicit override always wins.\n const doc: RagDocument = {\n id,\n text,\n metadata: {\n source: id,\n loader: \"html\",\n ...(title !== undefined ? { title } : {}),\n ...options.metadata,\n },\n tags: options.tags,\n };\n\n return [doc];\n}\n\n/** Internal — exported for the web loader so it shares the exact strip pass. */\nexport { htmlToText, extractTitle };\n"],"mappings":";;AAIA,MAAM,aAAa;;;;;;;AAQnB,MAAM,oBAAoB;CACxB;CACA;CACA;CACA;CACA;CACA;AACF;;;;;;;AAQA,MAAM,aACJ;;AAGF,MAAM,iBAAyC;CAC7C,KAAK;CACL,IAAI;CACJ,IAAI;CACJ,MAAM;CACN,MAAM;CACN,MAAM;CACN,MAAM;CACN,KAAK;CACL,OAAO;CACP,QAAQ;CACR,OAAO;CACP,OAAO;CACP,OAAO;CACP,OAAO;CACP,OAAO;CACP,OAAO;CACP,OAAO;CACP,OAAO;CACP,QAAQ;CACR,MAAM;AACR;;;;;;AAOA,SAAS,eAAe,MAAsB;CAC5C,OAAO,KAAK,QAAQ,sCAAsC,OAAO,SAAiB;EAChF,IAAI,KAAK,OAAO,KAAK;GACnB,MAAM,YACJ,KAAK,OAAO,OAAO,KAAK,OAAO,MAC3B,OAAO,SAAS,KAAK,MAAM,CAAC,GAAG,EAAE,IACjC,OAAO,SAAS,KAAK,MAAM,CAAC,GAAG,EAAE;GAEvC,IAAI,OAAO,MAAM,SAAS,KAAK,YAAY,KAAK,YAAY,SAC1D,OAAO;GAGT,IAAI;IACF,OAAO,OAAO,cAAc,SAAS;GACvC,QAAQ;IACN,OAAO;GACT;EACF;EAIA,OAFc,eAAe,KAAK,YAAY,MAE9B;CAClB,CAAC;AACH;;;;;AAMA,SAAS,aAAa,MAAkC;CACtD,MAAM,QAAQ,mCAAmC,KAAK,IAAI;CAE1D,IAAI,CAAC,OACH;CAGF,MAAM,QAAQ,eAAe,MAAM,EAAE,CAAC,CAAC,QAAQ,QAAQ,GAAG,CAAC,CAAC,KAAK;CAEjE,OAAO,MAAM,SAAS,IAAI,QAAQ;AACpC;;;;;;;;;AAUA,SAAS,WAAW,MAAsB;CACxC,IAAI,OAAO;CAGX,OAAO,KAAK,QAAQ,oBAAoB,GAAG;CAG3C,KAAK,MAAM,OAAO,mBAAmB;EACnC,MAAM,UAAU,IAAI,OAAO,IAAI,IAAI,yBAAyB,IAAI,IAAI,IAAI;EACxE,OAAO,KAAK,QAAQ,SAAS,GAAG;EAEhC,OAAO,KAAK,QAAQ,IAAI,OAAO,QAAQ,IAAI,YAAY,IAAI,GAAG,GAAG;CACnE;CAGA,OAAO,KAAK,QAAQ,YAAY,IAAI;CAGpC,OAAO,KAAK,QAAQ,YAAY,EAAE;CAGlC,OAAO,eAAe,IAAI;CAI1B,OAAO,KACJ,QAAQ,aAAa,GAAG,CAAC,CACzB,QAAQ,mBAAmB,IAAI,CAAC,CAChC,QAAQ,WAAW,MAAM,CAAC,CAC1B,KAAK;CAER,OAAO;AACT;;;;;;;;;;;;;;;;;;;;AAqBA,SAAgB,SACd,MACA,UAA2B,CAAC,GACX;CACjB,MAAM,KAAK,QAAQ,MAAM;CACzB,MAAM,QAAQ,aAAa,IAAI;CAC/B,MAAM,OAAO,WAAW,IAAI;CAI5B,IAAI,KAAK,WAAW,GAClB,OAAO,CAAC;CAiBV,OAAO,CAAC;EAXN;EACA;EACA,UAAU;GACR,QAAQ;GACR,QAAQ;GACR,GAAI,UAAU,SAAY,EAAE,MAAM,IAAI,CAAC;GACvC,GAAG,QAAQ;EACb;EACA,MAAM,QAAQ;CAGN,CAAC;AACb"}

package/esm/ai/src/rag/loaders/load-pdf.mjs DELETED Viewed

@@ -1,150 +0,0 @@
-import { PDF_PARSE_INSTALL_INSTRUCTIONS } from "./errors.mjs";
-//#region ../@warlock.js/ai/src/rag/loaders/load-pdf.ts
-/** Default `id` when the caller supplies none. */
-const DEFAULT_ID = "document";
-let pdfParse;
-let isModuleExists;
-let loadingPromise;
-/**
-* Settle the lazy import of `pdf-parse` once, concurrency-safe. A bare
-* `catch` flips the flag to `false`; the curated
-* {@link PDF_PARSE_INSTALL_INSTRUCTIONS} surfaces at first
-* {@link loadPdf} call, never a raw module-resolution stack trace. Mirrors
-* the guard moderation detector's `loadOpenAi`.
-*/
-function loadPdfParse() {
-	if (isModuleExists !== void 0) return Promise.resolve();
-	if (loadingPromise) return loadingPromise;
-	loadingPromise = (async () => {
-		try {
-			const mod = await import("pdf-parse");
-			pdfParse = mod.default ?? mod;
-			isModuleExists = typeof pdfParse === "function";
-		} catch {
-			isModuleExists = false;
-		}
-	})();
-	return loadingPromise;
-}
-/**
-* Coerce a {@link RagDocument}-compatible binary input into a `Buffer` for
-* `pdf-parse`. Accepts a Node `Buffer`, an `ArrayBuffer`, or a typed array
-* (`Uint8Array`) — the shapes a file read / fetch body hands back.
-*/
-function toBuffer(input) {
-	if (Buffer.isBuffer(input)) return input;
-	if (input instanceof ArrayBuffer) return Buffer.from(input);
-	return Buffer.from(input.buffer, input.byteOffset, input.byteLength);
-}
-/**
-* Load a PDF's bytes into {@link RagDocument}(s) via the OPTIONAL `pdf-parse`
-* peer. The peer is resolved lazily on the FIRST call (not at import) so
-* importing `@warlock.js/ai` never forces it to be installed; when it is
-* absent the curated {@link PDF_PARSE_INSTALL_INSTRUCTIONS} is thrown as a
-* plain `Error` (a missing optional peer is an infrastructure fault, not a
-* content problem).
-*
-* By default the whole PDF becomes a single document carrying
-* `metadata.pageCount`. With `perPage: true`, each page becomes its own
-* document (`id` suffixed `#p<n>`, `metadata.page` set) so citations stay
-* page-precise. Document `metadata.title` comes from the PDF info
-* dictionary's `Title` (unless overridden), and `metadata.loader` is
-* `"pdf"`. The output is the exact shape `index()` consumes.
-*
-* @example
-* import { readFile } from "node:fs/promises";
-* const kb = ai.rag({ embedder, store });
-* await kb.index(await loadPdf(await readFile("guide.pdf"), { id: "guide" }));
-*
-* @example
-* // One document per page for page-precise citations:
-* await kb.index(await loadPdf(bytes, { id: "manual", perPage: true }));
-*
-* @param input - The PDF bytes (`Buffer`, `ArrayBuffer`, or `Uint8Array`).
-* @param options - `perPage` plus shared `id` / `metadata` / `tags`
-*   ({@link LoadPdfOptions}).
-* @returns A {@link RagLoaderResult} ready for `rag.index()`.
-* @throws {Error} carrying {@link PDF_PARSE_INSTALL_INSTRUCTIONS} when the
-*   `pdf-parse` peer is not installed.
-*/
-async function loadPdf(input, options = {}) {
-	await loadPdfParse();
-	if (!isModuleExists || !pdfParse) throw new Error(PDF_PARSE_INSTALL_INSTRUCTIONS);
-	const id = options.id ?? DEFAULT_ID;
-	if (options.perPage ?? false) return loadPerPage(input, id, options);
-	const parsed = await pdfParse(toBuffer(input));
-	const text = parsed.text.trim();
-	const title = parsed.info?.Title?.trim();
-	if (text.length === 0) return [];
-	return [{
-		id,
-		text,
-		metadata: {
-			source: id,
-			loader: "pdf",
-			pageCount: parsed.numpages,
-			...title ? { title } : {},
-			...options.metadata
-		},
-		tags: options.tags
-	}];
-}
-/**
-* Per-page variant: render each page separately via `pdf-parse`'s
-* `pagerender` hook, accumulating one document per non-empty page. Each
-* carries `metadata.page` (1-based) and `metadata.pageCount`, and its id is
-* the base id suffixed `#p<n>` so every page-document is distinctly
-* identified for citation.
-*
-* `pdf-parse` calls `pagerender` once per page in document order and
-* `await`s the returned string, so capturing each page's joined text content
-* here gives reliable page boundaries the concatenated `text` lacks.
-*/
-async function loadPerPage(input, id, options) {
-	const pages = [];
-	const parsed = await pdfParse(toBuffer(input), { pagerender: async (page) => {
-		const rendered = await renderPage(page);
-		pages.push(rendered);
-		return rendered;
-	} });
-	const title = parsed.info?.Title?.trim();
-	const docs = [];
-	pages.forEach((pageText, index) => {
-		const text = pageText.trim();
-		if (text.length === 0) return;
-		const pageNumber = index + 1;
-		docs.push({
-			id: `${id}#p${pageNumber}`,
-			text,
-			metadata: {
-				source: id,
-				loader: "pdf",
-				page: pageNumber,
-				pageCount: parsed.numpages,
-				...title ? { title } : {},
-				...options.metadata
-			},
-			tags: options.tags
-		});
-	});
-	return docs;
-}
-/**
-* Join a single page's text-layer items in reading order, inserting a space
-* between items so adjacent words do not run together. Mirrors the essence
-* of `pdf-parse`'s default renderer without depending on its internals, so
-* the per-page hook stays stable across `pdf-parse` versions. A page with no
-* text layer (scanned image) renders to an empty string and is dropped.
-*/
-async function renderPage(page) {
-	if (typeof page?.getTextContent !== "function") return "";
-	return (await page.getTextContent({
-		normalizeWhitespace: true,
-		disableCombineTextItems: false
-	})).items.map((item) => item.str).join(" ").replace(/\s+/g, " ").trim();
-}
-//#endregion
-export { loadPdf };
-//# sourceMappingURL=load-pdf.mjs.map

package/esm/ai/src/rag/loaders/load-pdf.mjs.map DELETED Viewed

@@ -1 +0,0 @@

- {"version":3,"file":"load-pdf.mjs","names":[],"sources":["../../../../../../../../../../@warlock.js/ai/src/rag/loaders/load-pdf.ts"],"sourcesContent":["import type { RagDocument } from \"../contracts/rag-document.type\";\nimport { PDF_PARSE_INSTALL_INSTRUCTIONS } from \"./errors\";\nimport type { LoadPdfOptions, RagLoaderResult } from \"./loader.type\";\n\n/** Default `id` when the caller supplies none. */\nconst DEFAULT_ID = \"document\";\n\n/**\n * The slice of `pdf-parse`'s result we consume. The peer returns more\n * (`info`, `metadata`, `version`); we only need the extracted `text` and\n * page count, so we type just those to keep the dependency at arm's length.\n */\ntype PdfParseResult = {\n /** Concatenated text of every page. */\n text: string;\n /** Number of pages in the document. */\n numpages: number;\n /** Document info dictionary — `Title` lifted into metadata when present. */\n info?: { Title?: string } & Record<string, unknown>;\n};\n\n/** The `pdf-parse` module's callable default export. */\ntype PdfParseFn = (\n data: Buffer | Uint8Array,\n options?: {\n /**\n * Per-page renderer `pdf-parse` calls once per page in document order and\n * `await`s — may return the page text synchronously or as a promise.\n */\n pagerender?: (page: unknown) => string | Promise<string>;\n },\n) => Promise<PdfParseResult>;\n\n// ============================================================\n// Lazily-loaded pdf-parse (OPTIONAL peer)\n// ============================================================\n\nlet pdfParse: PdfParseFn | undefined;\nlet isModuleExists: boolean | undefined;\nlet loadingPromise: Promise<void> | undefined;\n\n/**\n * Settle the lazy import of `pdf-parse` once, concurrency-safe. A bare\n * `catch` flips the flag to `false`; the curated\n * {@link PDF_PARSE_INSTALL_INSTRUCTIONS} surfaces at first\n * {@link loadPdf} call, never a raw module-resolution stack trace. Mirrors\n * the guard moderation detector's `loadOpenAi`.\n */\nfunction loadPdfParse(): Promise<void> {\n if (isModuleExists !== undefined) {\n return Promise.resolve();\n }\n\n if (loadingPromise) {\n return loadingPromise;\n }\n\n loadingPromise = (async () => {\n try {\n // Literal specifier so `vi.mock(\"pdf-parse\")` can intercept it in tests.\n // Typed via the ambient `pdf-parse` shim in this directory, so the bare\n // import resolves even though the OPTIONAL peer is not a dependency.\n const mod = (await import(\"pdf-parse\")) as {\n default?: PdfParseFn;\n } & Partial<PdfParseFn>;\n // pdf-parse ships CommonJS — the callable is `module.exports`, surfaced\n // as `default` under ESM interop. Fall back to the namespace itself for\n // bundlers that hoist the callable to the top level.\n pdfParse = mod.default ?? (mod as unknown as PdfParseFn);\n isModuleExists = typeof pdfParse === \"function\";\n } catch {\n isModuleExists = false;\n }\n })();\n\n return loadingPromise;\n}\n\n/**\n * Coerce a {@link RagDocument}-compatible binary input into a `Buffer` for\n * `pdf-parse`. Accepts a Node `Buffer`, an `ArrayBuffer`, or a typed array\n * (`Uint8Array`) — the shapes a file read / fetch body hands back.\n */\nfunction toBuffer(input: Buffer | ArrayBuffer | Uint8Array): Buffer {\n if (Buffer.isBuffer(input)) {\n return input;\n }\n\n if (input instanceof ArrayBuffer) {\n return Buffer.from(input);\n }\n\n return Buffer.from(input.buffer, input.byteOffset, input.byteLength);\n}\n\n/**\n * Load a PDF's bytes into {@link RagDocument}(s) via the OPTIONAL `pdf-parse`\n * peer. The peer is resolved lazily on the FIRST call (not at import) so\n * importing `@warlock.js/ai` never forces it to be installed; when it is\n * absent the curated {@link PDF_PARSE_INSTALL_INSTRUCTIONS} is thrown as a\n * plain `Error` (a missing optional peer is an infrastructure fault, not a\n * content problem).\n *\n * By default the whole PDF becomes a single document carrying\n * `metadata.pageCount`. With `perPage: true`, each page becomes its own\n * document (`id` suffixed `#p<n>`, `metadata.page` set) so citations stay\n * page-precise. Document `metadata.title` comes from the PDF info\n * dictionary's `Title` (unless overridden), and `metadata.loader` is\n * `\"pdf\"`. The output is the exact shape `index()` consumes.\n *\n * @example\n * import { readFile } from \"node:fs/promises\";\n * const kb = ai.rag({ embedder, store });\n * await kb.index(await loadPdf(await readFile(\"guide.pdf\"), { id: \"guide\" }));\n *\n * @example\n * // One document per page for page-precise citations:\n * await kb.index(await loadPdf(bytes, { id: \"manual\", perPage: true }));\n *\n * @param input - The PDF bytes (`Buffer`, `ArrayBuffer`, or `Uint8Array`).\n * @param options - `perPage` plus shared `id` / `metadata` / `tags`\n * ({@link LoadPdfOptions}).\n * @returns A {@link RagLoaderResult} ready for `rag.index()`.\n * @throws {Error} carrying {@link PDF_PARSE_INSTALL_INSTRUCTIONS} when the\n * `pdf-parse` peer is not installed.\n */\nexport async function loadPdf(\n input: Buffer | ArrayBuffer | Uint8Array,\n options: LoadPdfOptions = {},\n): Promise<RagLoaderResult> {\n await loadPdfParse();\n\n if (!isModuleExists || !pdfParse) {\n throw new Error(PDF_PARSE_INSTALL_INSTRUCTIONS);\n }\n\n const id = options.id ?? DEFAULT_ID;\n const perPage = options.perPage ?? false;\n\n if (perPage) {\n return loadPerPage(input, id, options);\n }\n\n const parsed = await pdfParse(toBuffer(input));\n const text = parsed.text.trim();\n const title = parsed.info?.Title?.trim();\n\n // An image-only / empty PDF extracts no text — emit nothing so index()\n // never receives a no-op record.\n if (text.length === 0) {\n return [];\n }\n\n const doc: RagDocument = {\n id,\n text,\n metadata: {\n source: id,\n loader: \"pdf\",\n pageCount: parsed.numpages,\n ...(title ? { title } : {}),\n ...options.metadata,\n },\n tags: options.tags,\n };\n\n return [doc];\n}\n\n/** One page of a parsed PDF — the text-layer item list `pagerender` sees. */\ntype PdfPage = {\n getTextContent: (\n options?: unknown,\n ) => Promise<{ items: { str: string }[] }>;\n};\n\n/**\n * Per-page variant: render each page separately via `pdf-parse`'s\n * `pagerender` hook, accumulating one document per non-empty page. Each\n * carries `metadata.page` (1-based) and `metadata.pageCount`, and its id is\n * the base id suffixed `#p<n>` so every page-document is distinctly\n * identified for citation.\n *\n * `pdf-parse` calls `pagerender` once per page in document order and\n * `await`s the returned string, so capturing each page's joined text content\n * here gives reliable page boundaries the concatenated `text` lacks.\n */\nasync function loadPerPage(\n input: Buffer | ArrayBuffer | Uint8Array,\n id: string,\n options: LoadPdfOptions,\n): Promise<RagDocument[]> {\n const pages: string[] = [];\n\n const parsed = await pdfParse!(toBuffer(input), {\n pagerender: async (page: unknown): Promise<string> => {\n const rendered = await renderPage(page as PdfPage);\n pages.push(rendered);\n return rendered;\n },\n });\n\n const title = parsed.info?.Title?.trim();\n const docs: RagDocument[] = [];\n\n pages.forEach((pageText, index) => {\n const text = pageText.trim();\n\n if (text.length === 0) {\n return;\n }\n\n const pageNumber = index + 1;\n\n docs.push({\n id: `${id}#p${pageNumber}`,\n text,\n metadata: {\n source: id,\n loader: \"pdf\",\n page: pageNumber,\n pageCount: parsed.numpages,\n ...(title ? { title } : {}),\n ...options.metadata,\n },\n tags: options.tags,\n });\n });\n\n return docs;\n}\n\n/**\n * Join a single page's text-layer items in reading order, inserting a space\n * between items so adjacent words do not run together. Mirrors the essence\n * of `pdf-parse`'s default renderer without depending on its internals, so\n * the per-page hook stays stable across `pdf-parse` versions. A page with no\n * text layer (scanned image) renders to an empty string and is dropped.\n */\nasync function renderPage(page: PdfPage): Promise<string> {\n if (typeof page?.getTextContent !== \"function\") {\n return \"\";\n }\n\n const content = await page.getTextContent({\n normalizeWhitespace: true,\n disableCombineTextItems: false,\n });\n\n return content.items\n .map((item) => item.str)\n .join(\" \")\n .replace(/\\s+/g, \" \")\n .trim();\n}\n"],"mappings":";;;;AAKA,MAAM,aAAa;AAgCnB,IAAI;AACJ,IAAI;AACJ,IAAI;;;;;;;;AASJ,SAAS,eAA8B;CACrC,IAAI,mBAAmB,QACrB,OAAO,QAAQ,QAAQ;CAGzB,IAAI,gBACF,OAAO;CAGT,kBAAkB,YAAY;EAC5B,IAAI;GAIF,MAAM,MAAO,MAAM,OAAO;GAM1B,WAAW,IAAI,WAAY;GAC3B,iBAAiB,OAAO,aAAa;EACvC,QAAQ;GACN,iBAAiB;EACnB;CACF,EAAC,CAAE;CAEH,OAAO;AACT;;;;;;AAOA,SAAS,SAAS,OAAkD;CAClE,IAAI,OAAO,SAAS,KAAK,GACvB,OAAO;CAGT,IAAI,iBAAiB,aACnB,OAAO,OAAO,KAAK,KAAK;CAG1B,OAAO,OAAO,KAAK,MAAM,QAAQ,MAAM,YAAY,MAAM,UAAU;AACrE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiCA,eAAsB,QACpB,OACA,UAA0B,CAAC,GACD;CAC1B,MAAM,aAAa;CAEnB,IAAI,CAAC,kBAAkB,CAAC,UACtB,MAAM,IAAI,MAAM,8BAA8B;CAGhD,MAAM,KAAK,QAAQ,MAAM;CAGzB,IAFgB,QAAQ,WAAW,OAGjC,OAAO,YAAY,OAAO,IAAI,OAAO;CAGvC,MAAM,SAAS,MAAM,SAAS,SAAS,KAAK,CAAC;CAC7C,MAAM,OAAO,OAAO,KAAK,KAAK;CAC9B,MAAM,QAAQ,OAAO,MAAM,OAAO,KAAK;CAIvC,IAAI,KAAK,WAAW,GAClB,OAAO,CAAC;CAgBV,OAAO,CAAC;EAZN;EACA;EACA,UAAU;GACR,QAAQ;GACR,QAAQ;GACR,WAAW,OAAO;GAClB,GAAI,QAAQ,EAAE,MAAM,IAAI,CAAC;GACzB,GAAG,QAAQ;EACb;EACA,MAAM,QAAQ;CAGN,CAAC;AACb;;;;;;;;;;;;AAoBA,eAAe,YACb,OACA,IACA,SACwB;CACxB,MAAM,QAAkB,CAAC;CAEzB,MAAM,SAAS,MAAM,SAAU,SAAS,KAAK,GAAG,EAC9C,YAAY,OAAO,SAAmC;EACpD,MAAM,WAAW,MAAM,WAAW,IAAe;EACjD,MAAM,KAAK,QAAQ;EACnB,OAAO;CACT,EACF,CAAC;CAED,MAAM,QAAQ,OAAO,MAAM,OAAO,KAAK;CACvC,MAAM,OAAsB,CAAC;CAE7B,MAAM,SAAS,UAAU,UAAU;EACjC,MAAM,OAAO,SAAS,KAAK;EAE3B,IAAI,KAAK,WAAW,GAClB;EAGF,MAAM,aAAa,QAAQ;EAE3B,KAAK,KAAK;GACR,IAAI,GAAG,GAAG,IAAI;GACd;GACA,UAAU;IACR,QAAQ;IACR,QAAQ;IACR,MAAM;IACN,WAAW,OAAO;IAClB,GAAI,QAAQ,EAAE,MAAM,IAAI,CAAC;IACzB,GAAG,QAAQ;GACb;GACA,MAAM,QAAQ;EAChB,CAAC;CACH,CAAC;CAED,OAAO;AACT;;;;;;;;AASA,eAAe,WAAW,MAAgC;CACxD,IAAI,OAAO,MAAM,mBAAmB,YAClC,OAAO;CAQT,QAAO,MALe,KAAK,eAAe;EACxC,qBAAqB;EACrB,yBAAyB;CAC3B,CAAC,EAEa,CAAC,MACZ,KAAK,SAAS,KAAK,GAAG,CAAC,CACvB,KAAK,GAAG,CAAC,CACT,QAAQ,QAAQ,GAAG,CAAC,CACpB,KAAK;AACV"}

package/esm/ai/src/rag/loaders/load-text.mjs DELETED Viewed

@@ -1,60 +0,0 @@
-//#region ../@warlock.js/ai/src/rag/loaders/load-text.ts
-/** Default `id` when the caller supplies neither `id` nor an item id. */
-const DEFAULT_ID = "document";
-/**
-* Load plain text into {@link RagDocument}(s) — the zero-dependency base
-* loader every other loader ultimately funnels into. Accepts a single
-* string, a single `{ id, text }` record, or an array mixing both; each
-* input becomes one document carrying `metadata.loader = "text"` plus a
-* `metadata.source` (the resolved id).
-*
-* Caller `metadata` always wins over the loader-derived keys, and per-item
-* `metadata` / `tags` (when an item is a record) layer on top of the shared
-* option values. Empty / whitespace-only items are dropped — they would
-* chunk to nothing anyway, so the result never carries a no-op document.
-*
-* The output is the exact shape `index()` consumes:
-*
-* @example
-* const kb = ai.rag({ embedder, store });
-* await kb.index(loadText("a long string of notes…"));
-*
-* @example
-* await kb.index(loadText([
-*   { id: "faq-1", text: "…", metadata: { section: "billing" } },
-*   { id: "faq-2", text: "…" },
-* ]));
-*
-* @param input - A string, a `{ id, text }` record, or an array of either.
-* @param options - Shared `id` / `metadata` / `tags` ({@link LoadTextOptions}).
-* @returns A {@link RagLoaderResult} ready to hand to `rag.index()`.
-*/
-function loadText(input, options = {}) {
-	const items = Array.isArray(input) ? input : [input];
-	const baseId = options.id ?? DEFAULT_ID;
-	const multiple = items.length > 1;
-	const docs = [];
-	items.forEach((item, index) => {
-		const text = typeof item === "string" ? item : item.text;
-		if (text.trim().length === 0) return;
-		const id = (typeof item === "string" ? void 0 : item.id) ?? (multiple ? `${baseId}#${index}` : baseId);
-		const itemMetadata = typeof item === "string" ? void 0 : item.metadata;
-		const itemTags = typeof item === "string" ? void 0 : item.tags;
-		docs.push({
-			id,
-			text,
-			metadata: {
-				source: id,
-				loader: "text",
-				...options.metadata,
-				...itemMetadata
-			},
-			tags: itemTags ?? options.tags
-		});
-	});
-	return docs;
-}
-//#endregion
-export { loadText };
-//# sourceMappingURL=load-text.mjs.map

package/esm/ai/src/rag/loaders/load-text.mjs.map DELETED Viewed

@@ -1 +0,0 @@

- {"version":3,"file":"load-text.mjs","names":[],"sources":["../../../../../../../../../../@warlock.js/ai/src/rag/loaders/load-text.ts"],"sourcesContent":["import type { RagDocument } from \"../contracts/rag-document.type\";\nimport type { LoadTextOptions, RagLoaderResult } from \"./loader.type\";\n\n/** Default `id` when the caller supplies neither `id` nor an item id. */\nconst DEFAULT_ID = \"document\";\n\n/**\n * One raw text item — a bare string, or a `{ id, text, … }` record giving the\n * item its own id / metadata / tags. Passing records lets a single\n * {@link loadText} call turn many strings into many distinctly-identified\n * {@link RagDocument}s.\n */\nexport type TextInput =\n | string\n | {\n /** Stable id for this item. Falls back to the option `id` + index. */\n id?: string;\n /** The text body. */\n text: string;\n /** Per-item metadata, merged under the shared option `metadata`. */\n metadata?: Record<string, unknown>;\n /** Per-item tags (override the shared option `tags` when present). */\n tags?: string[];\n };\n\n/**\n * Load plain text into {@link RagDocument}(s) — the zero-dependency base\n * loader every other loader ultimately funnels into. Accepts a single\n * string, a single `{ id, text }` record, or an array mixing both; each\n * input becomes one document carrying `metadata.loader = \"text\"` plus a\n * `metadata.source` (the resolved id).\n *\n * Caller `metadata` always wins over the loader-derived keys, and per-item\n * `metadata` / `tags` (when an item is a record) layer on top of the shared\n * option values. Empty / whitespace-only items are dropped — they would\n * chunk to nothing anyway, so the result never carries a no-op document.\n *\n * The output is the exact shape `index()` consumes:\n *\n * @example\n * const kb = ai.rag({ embedder, store });\n * await kb.index(loadText(\"a long string of notes…\"));\n *\n * @example\n * await kb.index(loadText([\n * { id: \"faq-1\", text: \"…\", metadata: { section: \"billing\" } },\n * { id: \"faq-2\", text: \"…\" },\n * ]));\n *\n * @param input - A string, a `{ id, text }` record, or an array of either.\n * @param options - Shared `id` / `metadata` / `tags` ({@link LoadTextOptions}).\n * @returns A {@link RagLoaderResult} ready to hand to `rag.index()`.\n */\nexport function loadText(\n input: TextInput | TextInput[],\n options: LoadTextOptions = {},\n): RagLoaderResult {\n const items = Array.isArray(input) ? input : [input];\n const baseId = options.id ?? DEFAULT_ID;\n const multiple = items.length > 1;\n\n const docs: RagDocument[] = [];\n\n items.forEach((item, index) => {\n const text = typeof item === \"string\" ? item : item.text;\n\n // Drop empties up front — they chunk to nothing, so emitting them would\n // only add a no-op document for index() to skip.\n if (text.trim().length === 0) {\n return;\n }\n\n const itemId =\n typeof item === \"string\" ? undefined : item.id;\n // A single input keeps the bare base id; multiple inputs are suffixed so\n // every emitted document has a distinct, stable id.\n const id = itemId ?? (multiple ? `${baseId}#${index}` : baseId);\n\n const itemMetadata =\n typeof item === \"string\" ? undefined : item.metadata;\n const itemTags = typeof item === \"string\" ? undefined : item.tags;\n\n docs.push({\n id,\n text,\n // Loader-derived keys first, then the shared option metadata, then the\n // per-item metadata — caller intent always overrides the derived keys.\n metadata: {\n source: id,\n loader: \"text\",\n ...options.metadata,\n ...itemMetadata,\n },\n tags: itemTags ?? options.tags,\n });\n });\n\n return docs;\n}\n"],"mappings":";;AAIA,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiDnB,SAAgB,SACd,OACA,UAA2B,CAAC,GACX;CACjB,MAAM,QAAQ,MAAM,QAAQ,KAAK,IAAI,QAAQ,CAAC,KAAK;CACnD,MAAM,SAAS,QAAQ,MAAM;CAC7B,MAAM,WAAW,MAAM,SAAS;CAEhC,MAAM,OAAsB,CAAC;CAE7B,MAAM,SAAS,MAAM,UAAU;EAC7B,MAAM,OAAO,OAAO,SAAS,WAAW,OAAO,KAAK;EAIpD,IAAI,KAAK,KAAK,CAAC,CAAC,WAAW,GACzB;EAOF,MAAM,MAHJ,OAAO,SAAS,WAAW,SAAY,KAAK,QAGxB,WAAW,GAAG,OAAO,GAAG,UAAU;EAExD,MAAM,eACJ,OAAO,SAAS,WAAW,SAAY,KAAK;EAC9C,MAAM,WAAW,OAAO,SAAS,WAAW,SAAY,KAAK;EAE7D,KAAK,KAAK;GACR;GACA;GAGA,UAAU;IACR,QAAQ;IACR,QAAQ;IACR,GAAG,QAAQ;IACX,GAAG;GACL;GACA,MAAM,YAAY,QAAQ;EAC5B,CAAC;CACH,CAAC;CAED,OAAO;AACT"}

package/esm/ai/src/rag/loaders/load-web.mjs DELETED Viewed

@@ -1,89 +0,0 @@
-import { OutboundPolicyError } from "../../errors/outbound-policy-error.mjs";
-import "../../errors/index.mjs";
-import { guardedFetch, readTextCapped, resolveOutboundPolicy } from "../../security/outbound-policy.mjs";
-import { extractTitle, htmlToText } from "./load-html.mjs";
-//#region ../@warlock.js/ai/src/rag/loaders/load-web.ts
-/** Browser-ish UA so servers that gate on it still return prose. */
-const DEFAULT_USER_AGENT = "warlock-ai-rag-loader/1.0 (+https://github.com/warlock-js)";
-/**
-* Whether a `content-type` header names an HTML document (so it is run
-* through the tag-strip pass) versus already-plain text (used verbatim).
-*/
-function isHtmlContentType(contentType) {
-	if (!contentType) return true;
-	const lower = contentType.toLowerCase();
-	return lower.includes("text/html") || lower.includes("application/xhtml");
-}
-/**
-* Fetch a URL through the SSRF-safe outbound policy and load it into a single
-* {@link RagDocument} of readable text. The fetch ALWAYS goes through
-* `guardedFetch` — never a raw `fetch` — so the scheme allowlist, host
-* allowlist, post-DNS private-IP guard, timeout, and response-size cap from
-* {@link LoadWebOptions.policy} (or the strict defaults) always apply.
-*
-* HTML responses are run through the same tag-strip pass as {@link loadHtml}
-* (scripts/styles dropped, entities decoded, paragraph structure kept);
-* non-HTML text responses (`text/plain`, markdown, …) are used verbatim.
-* The document's `metadata.source` is the resolved URL, `metadata.title` is
-* the page `<title>` (HTML only, unless overridden), `metadata.contentType`
-* is the server-reported type, and `metadata.loader` is `"web"`.
-*
-* The output is the exact shape `index()` consumes, so a load feeds straight
-* in:
-*
-* @example
-* const kb = ai.rag({ embedder, store });
-* await kb.index(await loadWeb("https://example.com/guide"));
-*
-* @example
-* // Tighten the SSRF policy to a single host:
-* await kb.index(await loadWeb(url, {
-*   policy: { hostAllowlist: ["docs.example.com"], maxBytes: 2_000_000 },
-*   tags: ["docs"],
-* }));
-*
-* @param url - The absolute URL to fetch. Validated by the outbound policy.
-* @param options - `policy` (the {@link OutboundPolicy}) plus shared
-*   `id` / `metadata` / `tags` ({@link LoadWebOptions}).
-* @returns A {@link RagLoaderResult} (one document) ready for `rag.index()`.
-* @throws {OutboundPolicyError} when the policy blocks the URL, the request
-*   times out, the body exceeds the cap, or the response is not OK.
-*/
-async function loadWeb(url, options = {}) {
-	const policy = resolveOutboundPolicy(options.policy);
-	const response = await guardedFetch(url, policy, {
-		headers: {
-			"user-agent": DEFAULT_USER_AGENT,
-			accept: "text/html,text/*"
-		},
-		redirect: "follow"
-	});
-	if (!response.ok) throw new OutboundPolicyError(`loadWeb: fetching "${url}" returned ${response.status} ${response.statusText}`, { context: {
-		url,
-		status: response.status
-	} });
-	const contentType = response.headers.get("content-type") ?? void 0;
-	const raw = await readTextCapped(response, policy.maxBytes);
-	const isHtml = isHtmlContentType(contentType);
-	const text = isHtml ? htmlToText(raw) : raw.trim();
-	const title = isHtml ? extractTitle(raw) : void 0;
-	const id = options.id ?? url;
-	if (text.length === 0) return [];
-	return [{
-		id,
-		text,
-		metadata: {
-			source: url,
-			loader: "web",
-			...title !== void 0 ? { title } : {},
-			...contentType !== void 0 ? { contentType } : {},
-			...options.metadata
-		},
-		tags: options.tags
-	}];
-}
-//#endregion
-export { loadWeb };
-//# sourceMappingURL=load-web.mjs.map

package/esm/ai/src/rag/loaders/load-web.mjs.map DELETED Viewed

@@ -1 +0,0 @@

- {"version":3,"file":"load-web.mjs","names":[],"sources":["../../../../../../../../../../@warlock.js/ai/src/rag/loaders/load-web.ts"],"sourcesContent":["import {\n guardedFetch,\n readTextCapped,\n resolveOutboundPolicy,\n} from \"../../security/outbound-policy\";\nimport { OutboundPolicyError } from \"../../errors\";\nimport type { RagDocument } from \"../contracts/rag-document.type\";\nimport { htmlToText, extractTitle } from \"./load-html\";\nimport type { LoadWebOptions, RagLoaderResult } from \"./loader.type\";\n\n/** Browser-ish UA so servers that gate on it still return prose. */\nconst DEFAULT_USER_AGENT =\n \"warlock-ai-rag-loader/1.0 (+https://github.com/warlock-js)\";\n\n/**\n * Whether a `content-type` header names an HTML document (so it is run\n * through the tag-strip pass) versus already-plain text (used verbatim).\n */\nfunction isHtmlContentType(contentType: string | undefined): boolean {\n if (!contentType) {\n // No header — assume HTML, the common case for a fetched URL.\n return true;\n }\n\n const lower = contentType.toLowerCase();\n\n return lower.includes(\"text/html\") || lower.includes(\"application/xhtml\");\n}\n\n/**\n * Fetch a URL through the SSRF-safe outbound policy and load it into a single\n * {@link RagDocument} of readable text. The fetch ALWAYS goes through\n * `guardedFetch` — never a raw `fetch` — so the scheme allowlist, host\n * allowlist, post-DNS private-IP guard, timeout, and response-size cap from\n * {@link LoadWebOptions.policy} (or the strict defaults) always apply.\n *\n * HTML responses are run through the same tag-strip pass as {@link loadHtml}\n * (scripts/styles dropped, entities decoded, paragraph structure kept);\n * non-HTML text responses (`text/plain`, markdown, …) are used verbatim.\n * The document's `metadata.source` is the resolved URL, `metadata.title` is\n * the page `<title>` (HTML only, unless overridden), `metadata.contentType`\n * is the server-reported type, and `metadata.loader` is `\"web\"`.\n *\n * The output is the exact shape `index()` consumes, so a load feeds straight\n * in:\n *\n * @example\n * const kb = ai.rag({ embedder, store });\n * await kb.index(await loadWeb(\"https://example.com/guide\"));\n *\n * @example\n * // Tighten the SSRF policy to a single host:\n * await kb.index(await loadWeb(url, {\n * policy: { hostAllowlist: [\"docs.example.com\"], maxBytes: 2_000_000 },\n * tags: [\"docs\"],\n * }));\n *\n * @param url - The absolute URL to fetch. Validated by the outbound policy.\n * @param options - `policy` (the {@link OutboundPolicy}) plus shared\n * `id` / `metadata` / `tags` ({@link LoadWebOptions}).\n * @returns A {@link RagLoaderResult} (one document) ready for `rag.index()`.\n * @throws {OutboundPolicyError} when the policy blocks the URL, the request\n * times out, the body exceeds the cap, or the response is not OK.\n */\nexport async function loadWeb(\n url: string,\n options: LoadWebOptions = {},\n): Promise<RagLoaderResult> {\n const policy = resolveOutboundPolicy(options.policy);\n\n const response = await guardedFetch(url, policy, {\n headers: { \"user-agent\": DEFAULT_USER_AGENT, accept: \"text/html,text/*\" },\n redirect: \"follow\",\n });\n\n if (!response.ok) {\n throw new OutboundPolicyError(\n `loadWeb: fetching \"${url}\" returned ${response.status} ${response.statusText}`,\n { context: { url, status: response.status } },\n );\n }\n\n const contentType = response.headers.get(\"content-type\") ?? undefined;\n const raw = await readTextCapped(response, policy.maxBytes);\n\n const isHtml = isHtmlContentType(contentType);\n const text = isHtml ? htmlToText(raw) : raw.trim();\n const title = isHtml ? extractTitle(raw) : undefined;\n\n const id = options.id ?? url;\n\n // An empty body / all-markup page yields no document, so index() never\n // receives a no-op record.\n if (text.length === 0) {\n return [];\n }\n\n // Derived keys sit UNDER the caller's metadata so an explicit override wins.\n const doc: RagDocument = {\n id,\n text,\n metadata: {\n source: url,\n loader: \"web\",\n ...(title !== undefined ? { title } : {}),\n ...(contentType !== undefined ? { contentType } : {}),\n ...options.metadata,\n },\n tags: options.tags,\n };\n\n return [doc];\n}\n"],"mappings":";;;;;;;AAWA,MAAM,qBACJ;;;;;AAMF,SAAS,kBAAkB,aAA0C;CACnE,IAAI,CAAC,aAEH,OAAO;CAGT,MAAM,QAAQ,YAAY,YAAY;CAEtC,OAAO,MAAM,SAAS,WAAW,KAAK,MAAM,SAAS,mBAAmB;AAC1E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqCA,eAAsB,QACpB,KACA,UAA0B,CAAC,GACD;CAC1B,MAAM,SAAS,sBAAsB,QAAQ,MAAM;CAEnD,MAAM,WAAW,MAAM,aAAa,KAAK,QAAQ;EAC/C,SAAS;GAAE,cAAc;GAAoB,QAAQ;EAAmB;EACxE,UAAU;CACZ,CAAC;CAED,IAAI,CAAC,SAAS,IACZ,MAAM,IAAI,oBACR,sBAAsB,IAAI,aAAa,SAAS,OAAO,GAAG,SAAS,cACnE,EAAE,SAAS;EAAE;EAAK,QAAQ,SAAS;CAAO,EAAE,CAC9C;CAGF,MAAM,cAAc,SAAS,QAAQ,IAAI,cAAc,KAAK;CAC5D,MAAM,MAAM,MAAM,eAAe,UAAU,OAAO,QAAQ;CAE1D,MAAM,SAAS,kBAAkB,WAAW;CAC5C,MAAM,OAAO,SAAS,WAAW,GAAG,IAAI,IAAI,KAAK;CACjD,MAAM,QAAQ,SAAS,aAAa,GAAG,IAAI;CAE3C,MAAM,KAAK,QAAQ,MAAM;CAIzB,IAAI,KAAK,WAAW,GAClB,OAAO,CAAC;CAiBV,OAAO,CAAC;EAZN;EACA;EACA,UAAU;GACR,QAAQ;GACR,QAAQ;GACR,GAAI,UAAU,SAAY,EAAE,MAAM,IAAI,CAAC;GACvC,GAAI,gBAAgB,SAAY,EAAE,YAAY,IAAI,CAAC;GACnD,GAAG,QAAQ;EACb;EACA,MAAM,QAAQ;CAGN,CAAC;AACb"}

package/esm/ai/src/rag/rag.mjs DELETED Viewed

@@ -1,126 +0,0 @@
-import { resolveDefaultStore } from "../config.mjs";
-import { chunk } from "./chunk/chunk.mjs";
-import { ragAsTool } from "./as-tool.mjs";
-import { retrieve } from "./retrieve.mjs";
-import { cacheVectorStore } from "./store/cache-vector-store.mjs";
-//#region ../@warlock.js/ai/src/rag/rag.ts
-const DEFAULT_NAME = "rag";
-const DEFAULT_NAMESPACE_PREFIX = "ai.rag";
-/**
-* Max chunk texts embedded per `embedder.embedMany()` call. One call is
-* one provider request, so a giant document is sub-batched to stay under
-* the provider's per-request token cap (the design's "chunk larger than
-* provider per-request cap" guard).
-*/
-const DEFAULT_MAX_BATCH = 96;
-/**
-* Create a RAG pipeline: **chunk → embed → vector store → retrieve →
-* rerank → cite**, reusing the app's `ai.embedder` for embedding, a
-* `@warlock.js/cache` `CacheDriver` as the vector store, and the
-* composite-as-tool engine to expose retrieval as a tool.
-*
-* Resolution is loud at construction (mirroring `memory()`):
-* - `embedder` is required — a provider with no embedder must be caught
-*   here, not at first index.
-* - `store` falls back to `ai.config({ defaultStore })`; if neither
-*   resolves, construction throws.
-*
-* `retrieve()` is return-only — it never auto-injects into a prompt; the
-* caller formats the cited chunks (or uses `asTool()` for the agent loop).
-* The reranker is OFF by default (cosine-only) unless `config.reranker`
-* is set.
-*
-* @example
-* import { ai } from "@warlock.js/ai";
-* import { MemoryCacheDriver } from "@warlock.js/cache";
-*
-* const kb = ai.rag({
-*   name: "docs",
-*   embedder: openai.embedder({ name: "text-embedding-3-small" }),
-*   store: new MemoryCacheDriver(),
-*   chunk: { type: "markdown", size: 800, overlap: 120 },
-* });
-*
-* await kb.index([{ id: "guide", text: longMarkdown, metadata: { url: "/guide" } }]);
-* const { chunks } = await kb.retrieve("how do I configure caching?", { topK: 4 });
-*/
-function rag(config) {
-	const name = config.name ?? DEFAULT_NAME;
-	if (!config.embedder) throw new Error(`rag("${name}"): an \`embedder\` is required — pass one from a provider that supports embeddings (e.g. openai.embedder({ name: "text-embedding-3-small" }))`);
-	const driver = config.store ?? resolveDefaultStore();
-	if (!driver) throw new Error(`rag("${name}"): no store — pass \`store\` (a vector-capable @warlock.js/cache CacheDriver) or call \`ai.config({ defaultStore })\` at app boot before constructing the rag`);
-	const store = cacheVectorStore(driver);
-	const namespace = config.namespace ?? `${DEFAULT_NAMESPACE_PREFIX}.${name}`;
-	const embedder = config.embedder;
-	let indexedDimensions;
-	const instance = {
-		name,
-		async index(docs, chunkOverride) {
-			const chunkOptions = chunkOverride ?? config.chunk;
-			const limits = config.limits;
-			if (limits?.maxDocuments !== void 0 && docs.length > limits.maxDocuments) throw new Error(`rag("${name}"): index() received ${docs.length} documents, exceeding the configured maxDocuments of ${limits.maxDocuments}`);
-			if (limits?.maxBytes !== void 0) {
-				const totalBytes = docs.reduce((sum, doc) => sum + Buffer.byteLength(doc.text ?? ""), 0);
-				if (totalBytes > limits.maxBytes) throw new Error(`rag("${name}"): index() received ${totalBytes} bytes of document text, exceeding the configured maxBytes of ${limits.maxBytes}`);
-			}
-			const records = [];
-			for (const doc of docs) {
-				const pieces = chunk(doc.text, chunkOptions);
-				for (const piece of pieces) {
-					const value = {
-						sourceId: doc.id,
-						chunkIndex: piece.index,
-						span: piece.span,
-						text: piece.text,
-						metadata: doc.metadata
-					};
-					records.push({
-						key: keyFor(namespace, doc.id, piece.index),
-						value,
-						text: piece.text,
-						tags: doc.tags
-					});
-				}
-			}
-			if (records.length === 0) return { chunks: 0 };
-			if (limits?.maxChunks !== void 0 && records.length > limits.maxChunks) throw new Error(`rag("${name}"): index() produced ${records.length} chunks, exceeding the configured maxChunks of ${limits.maxChunks}`);
-			for (let offset = 0; offset < records.length; offset += DEFAULT_MAX_BATCH) {
-				const batch = records.slice(offset, offset + DEFAULT_MAX_BATCH);
-				const { vectors, dimensions } = await embedder.embedMany(batch.map((record) => record.text));
-				if (indexedDimensions === void 0 && dimensions !== 0) indexedDimensions = dimensions;
-				await Promise.all(batch.map((record, position) => store.upsert(record.key, record.value, vectors[position], record.tags)));
-			}
-			return { chunks: records.length };
-		},
-		async retrieve(query, options) {
-			return retrieve(query, {
-				embedder,
-				store,
-				namespace,
-				reranker: config.reranker,
-				defaults: config.retrieve,
-				indexedDimensions
-			}, options);
-		},
-		async clear() {
-			await store.removeNamespace(namespace);
-		},
-		asTool(options) {
-			return ragAsTool(name, (query, retrieveOptions) => instance.retrieve(query, retrieveOptions), options);
-		}
-	};
-	return instance;
-}
-/**
-* Namespaced key for a stored chunk. Uses the `.` separator (matching
-* `SemanticMemory.keyFor`) so namespace-prefix filtering on the returned
-* `hit.key` stays aligned with the cache's `parseKey` normalization.
-*/
-function keyFor(namespace, sourceId, chunkIndex) {
-	return `${namespace}.${sourceId}.${chunkIndex}`;
-}
-//#endregion
-export { rag };
-//# sourceMappingURL=rag.mjs.map