@cogitator-ai/rag 0.1.10 → 0.2.0

package/dist/loaders/html-loader.js

@@ -9,6 +9,21 @@ async function loadCheerio() {
         throw new Error('cheerio is required for HTMLLoader. Install it: pnpm add cheerio');
     }
 }
+const NON_CONTENT_SELECTOR = 'script, style, noscript, template, svg, iframe, object, head';
+const BLOCK_SELECTOR = 'address, article, aside, blockquote, dd, div, dl, dt, fieldset, figcaption, figure, footer, ' +
+    'form, h1, h2, h3, h4, h5, h6, header, hr, li, main, nav, ol, p, pre, section, table, tr, ul';
+function normalizeWhitespace(text) {
+    return text
+        .replace(/\r\n?/g, '\n')
+        .replace(/[^\S\n]+/g, ' ')
+        .replace(/ *\n */g, '\n')
+        .replace(/\n{3,}/g, '\n\n')
+        .trim();
+}
+/**
+ * Extracts readable text from HTML: scripts, styles and other non-content elements are
+ * dropped, block elements become line breaks and whitespace is normalized.
+ */
 export class HTMLLoader {
     supportedTypes = ['html', 'htm'];
     selector;
@@ -23,8 +38,17 @@ export class HTMLLoader {
     async parseHTML(html, source, sourceType) {
         const cheerio = await loadCheerio();
         const $ = cheerio.load(html);
-        const content = $(this.selector).text().trim();
-        const title = $('title').text().trim() || undefined;
+        const title = $('title').first().text().trim() || undefined;
+        $(NON_CONTENT_SELECTOR).remove();
+        $('br').replaceWith('\n');
+        $(BLOCK_SELECTOR).each((_, element) => {
+            $(element).before('\n').after('\n');
+        });
+        $('td, th').after(' ');
+        const content = normalizeWhitespace($(this.selector)
+            .map((_, element) => $(element).text())
+            .get()
+            .join('\n\n'));
         const metadata = {};
         if (title)
             metadata.title = title;

package/dist/loaders/html-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"html-loader.js","sourceRoot":"","sources":["../../src/loaders/html-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,KAAK,UAAU,WAAW;IACxB,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;IACjC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;IACtF,CAAC;AACH,CAAC;AAMD,MAAM,OAAO,UAAU;IACZ,cAAc,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACjC,QAAQ,CAAS;IAE1B,YAAY,OAA2B;QACrC,IAAI,CAAC,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,MAAM,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC/C,OAAO,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,IAAY,EAAE,MAAc,EAAE,UAA0B;QACtE,MAAM,OAAO,GAAG,MAAM,WAAW,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE7B,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAC/C,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC;QAEpD,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,IAAI,KAAK;YAAE,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;QAElC,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM;YACN,UAAU;YACV,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;SACtD,CAAC;IACJ,CAAC;CACF"}
+{"version":3,"file":"html-loader.js","sourceRoot":"","sources":["../../src/loaders/html-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,KAAK,UAAU,WAAW;IACxB,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;IACjC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;IACtF,CAAC;AACH,CAAC;AAMD,MAAM,oBAAoB,GAAG,8DAA8D,CAAC;AAC5F,MAAM,cAAc,GAClB,8FAA8F;IAC9F,6FAA6F,CAAC;AAEhG,SAAS,mBAAmB,CAAC,IAAY;IACvC,OAAO,IAAI;SACR,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;SACvB,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC;SACzB,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC;SACxB,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;SAC1B,IAAI,EAAE,CAAC;AACZ,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,UAAU;IACZ,cAAc,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACjC,QAAQ,CAAS;IAE1B,YAAY,OAA2B;QACrC,IAAI,CAAC,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,MAAM,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC/C,OAAO,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,IAAY,EAAE,MAAc,EAAE,UAA0B;QACtE,MAAM,OAAO,GAAG,MAAM,WAAW,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE7B,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC;QAE5D,CAAC,CAAC,oBAAoB,CAAC,CAAC,MAAM,EAAE,CAAC;QACjC,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;YACpC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEvB,MAAM,OAAO,GAAG,mBAAmB,CACjC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC;aACb,GAAG,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;aACtC,GAAG,EAAE;aACL,IAAI,CAAC,MAAM,CAAC,CAChB,CAAC;QAEF,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,IAAI,KAAK;YAAE,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;QAElC,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM;YACN,UAAU;YACV,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;SACtD,CAAC;IACJ,CAAC;CACF"}

package/dist/loaders/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/loaders/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,YAAY,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACvD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,YAAY,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,YAAY,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACvD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,YAAY,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,YAAY,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/loaders/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,YAAY,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,YAAY,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,YAAY,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC"}

package/dist/loaders/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/loaders/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAE3C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAE3C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/loaders/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAEtD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC"}

package/dist/loaders/markdown-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"markdown-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/markdown-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAMvE,MAAM,WAAW,qBAAqB;IACpC,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,qBAAa,cAAe,YAAW,cAAc;IACnD,QAAQ,CAAC,cAAc,WAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAU;gBAE/B,OAAO,CAAC,EAAE,qBAAqB;IAIrC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YAWpC,QAAQ;IAsBtB,OAAO,CAAC,gBAAgB;YAWV,aAAa;CAiB5B"}
+{"version":3,"file":"markdown-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/markdown-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAWvE,MAAM,WAAW,qBAAqB;IACpC,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,qBAAa,cAAe,YAAW,cAAc;IACnD,QAAQ,CAAC,cAAc,WAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAU;gBAE/B,OAAO,CAAC,EAAE,qBAAqB;IAIrC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YAWpC,QAAQ;IAsBtB,OAAO,CAAC,gBAAgB;YAWV,aAAa;CAiB5B"}

package/dist/loaders/markdown-loader.js

@@ -3,6 +3,10 @@ import { join, resolve } from 'node:path';
 import { nanoid } from 'nanoid';
 const MD_EXTENSIONS = new Set(['md', 'mdx']);
 const FRONTMATTER_RE = /^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/;
+function unquote(value) {
+    const quoted = /^(['"])(.*)\1$/.exec(value);
+    return quoted ? quoted[2] : value;
+}
 export class MarkdownLoader {
     supportedTypes = ['md', 'mdx'];
     stripFrontmatter;
@@ -38,16 +42,16 @@ export class MarkdownLoader {
     }
     parseFrontmatter(raw) {
         const result = {};
-        for (const line of raw.split('\n')) {
+        for (const line of raw.split(/\r?\n/)) {
             const match = /^(\w[\w\s-]*?):\s*(.+)$/.exec(line);
             if (match) {
-                result[match[1].trim()] = match[2].trim();
+                result[match[1].trim()] = unquote(match[2].trim());
             }
         }
         return result;
     }
     async loadDirectory(dirPath) {
-        const entries = await readdir(dirPath);
+        const entries = (await readdir(dirPath)).sort();
         const docs = [];
         for (const entry of entries) {
             const ext = entry.split('.').pop()?.toLowerCase() ?? '';

package/dist/loaders/markdown-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"markdown-loader.js","sourceRoot":"","sources":["../../src/loaders/markdown-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;AAE7C,MAAM,cAAc,GAAG,wCAAwC,CAAC;AAMhE,MAAM,OAAO,cAAc;IAChB,cAAc,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACvB,gBAAgB,CAAU;IAE3C,YAAY,OAA+B;QACzC,IAAI,CAAC,gBAAgB,GAAG,OAAO,EAAE,gBAAgB,IAAI,KAAK,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;QAElC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,QAAgB;QACrC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9C,IAAI,OAAO,GAAG,GAAG,CAAC;QAClB,IAAI,QAA6C,CAAC;QAElD,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvC,IAAI,KAAK,EAAE,CAAC;gBACV,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC3C,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACvC,CAAC;QACH,CAAC;QAED,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,UAAU;YACtB,GAAG,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,CAAC;SAC9B,CAAC;IACJ,CAAC;IAEO,gBAAgB,CAAC,GAAW;QAClC,MAAM,MAAM,GAA4B,EAAE,CAAC;QAC3C,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnD,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC5C,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAe;QACzC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;QACvC,MAAM,IAAI,GAAkB,EAAE,CAAC;QAE/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YACxD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAEtC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClC,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;gBAClB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}
+{"version":3,"file":"markdown-loader.js","sourceRoot":"","sources":["../../src/loaders/markdown-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;AAE7C,MAAM,cAAc,GAAG,wCAAwC,CAAC;AAEhE,SAAS,OAAO,CAAC,KAAa;IAC5B,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC5C,OAAO,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,KAAK,CAAC;AACrC,CAAC;AAMD,MAAM,OAAO,cAAc;IAChB,cAAc,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACvB,gBAAgB,CAAU;IAE3C,YAAY,OAA+B;QACzC,IAAI,CAAC,gBAAgB,GAAG,OAAO,EAAE,gBAAgB,IAAI,KAAK,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;QAElC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,QAAgB;QACrC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9C,IAAI,OAAO,GAAG,GAAG,CAAC;QAClB,IAAI,QAA6C,CAAC;QAElD,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvC,IAAI,KAAK,EAAE,CAAC;gBACV,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC3C,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACvC,CAAC;QACH,CAAC;QAED,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,UAAU;YACtB,GAAG,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,CAAC;SAC9B,CAAC;IACJ,CAAC;IAEO,gBAAgB,CAAC,GAAW;QAClC,MAAM,MAAM,GAA4B,EAAE,CAAC;QAC3C,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YACtC,MAAM,KAAK,GAAG,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnD,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,CAAC;YACvD,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAe;QACzC,MAAM,OAAO,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,MAAM,IAAI,GAAkB,EAAE,CAAC;QAE/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YACxD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAEtC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClC,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;gBAClB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}

package/dist/loaders/text-loader.js

@@ -22,7 +22,7 @@ export class TextLoader {
         };
     }
     async loadDirectory(dirPath) {
-        const entries = await readdir(dirPath);
+        const entries = (await readdir(dirPath)).sort();
         const docs = [];
         for (const entry of entries) {
             const ext = entry.split('.').pop()?.toLowerCase() ?? '';

package/dist/loaders/text-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"text-loader.js","sourceRoot":"","sources":["../../src/loaders/text-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;AAEjD,MAAM,OAAO,UAAU;IACZ,cAAc,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAE1C,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;QAElC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,QAAgB;QACrC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAClD,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,MAAM;SACnB,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAe;QACzC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;QACvC,MAAM,IAAI,GAAkB,EAAE,CAAC;QAE/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YACxD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAExC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClC,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;gBAClB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}
+{"version":3,"file":"text-loader.js","sourceRoot":"","sources":["../../src/loaders/text-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;AAEjD,MAAM,OAAO,UAAU;IACZ,cAAc,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAE1C,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;QAElC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,QAAgB;QACrC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAClD,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,MAAM;SACnB,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAe;QACzC,MAAM,OAAO,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,MAAM,IAAI,GAAkB,EAAE,CAAC;QAE/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YACxD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAExC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClC,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;gBAClB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}

package/dist/loaders/web-loader.d.ts

@@ -1,18 +1,46 @@
+import { type LookupAddress, type LookupOptions } from 'node:dns';
 import type { DocumentLoader, RAGDocument } from '@cogitator-ai/types';
+/**
+ * Whether an IP address points to a loopback, private, link-local, CGNAT, multicast or
+ * otherwise non-public network (including IPv4-mapped / NAT64 IPv6 forms).
+ */
+export declare function isBlockedAddress(address: string): boolean;
+type LookupCallback = (err: NodeJS.ErrnoException | null, address: string | LookupAddress[], family?: number) => void;
+type Resolver = (hostname: string, callback: (err: NodeJS.ErrnoException | null, addresses: LookupAddress[]) => void) => void;
+/**
+ * DNS lookup for `http.request` that rejects private addresses at connect time,
+ * which also defeats DNS-rebinding between validation and connection.
+ */
+export declare function createGuardedLookup(resolver?: Resolver): (hostname: string, options: LookupOptions, callback: LookupCallback) => void;
 export interface WebLoaderOptions {
     selector?: string;
     headers?: Record<string, string>;
     timeoutMs?: number;
     maxResponseBytes?: number;
+    /**
+     * Allow fetching loopback / private-network hosts (e.g. an intranet wiki).
+     * Disabled by default to prevent SSRF when URLs come from untrusted input.
+     */
+    allowPrivateNetwork?: boolean;
 }
+/**
+ * Loads a web page over http(s). HTML is converted to text (scripts and styles removed);
+ * plain-text and JSON responses are kept as-is. Redirects are followed (max 5) and every
+ * hop — including DNS resolution at connect time — is checked against private networks.
+ */
 export declare class WebLoader implements DocumentLoader {
     readonly supportedTypes: string[];
     private readonly htmlLoader;
     private readonly headers?;
     private readonly timeoutMs;
     private readonly maxResponseBytes;
+    private readonly allowPrivateNetwork;
+    private readonly lookup;
     constructor(options?: WebLoaderOptions);
     load(source: string): Promise<RAGDocument[]>;
-    private readBodyWithLimit;
+    private fetchPage;
+    private request;
+    private readBody;
 }
+export {};
 //# sourceMappingURL=web-loader.d.ts.map

package/dist/loaders/web-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"web-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/web-loader.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAoEvE,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAKD,qBAAa,SAAU,YAAW,cAAc;IAC9C,QAAQ,CAAC,cAAc,WAAqB;IAC5C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAyB;IAClD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;gBAE9B,OAAO,CAAC,EAAE,gBAAgB;IAOhC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YAqCpC,iBAAiB;CA0ChC"}
+{"version":3,"file":"web-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/web-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAuB,KAAK,aAAa,EAAE,KAAK,aAAa,EAAE,MAAM,UAAU,CAAC;AAMvF,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAqDvE;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAUzD;AAED,KAAK,cAAc,GAAG,CACpB,GAAG,EAAE,MAAM,CAAC,cAAc,GAAG,IAAI,EACjC,OAAO,EAAE,MAAM,GAAG,aAAa,EAAE,EACjC,MAAM,CAAC,EAAE,MAAM,KACZ,IAAI,CAAC;AAEV,KAAK,QAAQ,GAAG,CACd,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,CAAC,GAAG,EAAE,MAAM,CAAC,cAAc,GAAG,IAAI,EAAE,SAAS,EAAE,aAAa,EAAE,KAAK,IAAI,KAC9E,IAAI,CAAC;AAMV;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,GAAE,QAAyB,IAC7D,UAAU,MAAM,EAAE,SAAS,aAAa,EAAE,UAAU,cAAc,KAAG,IAAI,CAsClF;AA2DD,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;OAGG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC/B;AAQD;;;;GAIG;AACH,qBAAa,SAAU,YAAW,cAAc;IAC9C,QAAQ,CAAC,cAAc,WAAqB;IAC5C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAyB;IAClD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAU;IAC9C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAyB;gBAEpC,OAAO,CAAC,EAAE,gBAAgB;IAQhC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YA0BpC,SAAS;IAkCvB,OAAO,CAAC,OAAO;IA4Bf,OAAO,CAAC,QAAQ;CA4BjB"}

package/dist/loaders/web-loader.js

@@ -1,36 +1,110 @@
-import { lookup } from 'node:dns/promises';
-import { isIP } from 'node:net';
+import { lookup as dnsLookup } from 'node:dns';
+import http from 'node:http';
+import https from 'node:https';
+import { BlockList, isIP } from 'node:net';
+import { createBrotliDecompress, createGunzip, createInflate } from 'node:zlib';
+import { nanoid } from 'nanoid';
 import { HTMLLoader } from './html-loader.js';
-const BLOCKED_HOSTNAMES = new Set([
-    'localhost',
-    '127.0.0.1',
-    '::1',
-    '[::1]',
-    '0.0.0.0',
-    '169.254.169.254',
-    'metadata.google.internal',
-]);
 const MAX_REDIRECTS = 5;
-function isPrivateIP(ip) {
-    if (ip.startsWith('10.'))
-        return true;
-    if (ip.startsWith('192.168.'))
-        return true;
-    if (ip === '0.0.0.0' || ip === '127.0.0.1' || ip === '::1')
-        return true;
-    if (ip.startsWith('169.254.'))
-        return true;
-    if (ip.startsWith('fc') || ip.startsWith('fd') || ip.startsWith('fe80'))
-        return true;
-    const match172 = /^172\.(\d+)\./.exec(ip);
-    if (match172) {
-        const second = parseInt(match172[1], 10);
-        if (second >= 16 && second <= 31)
-            return true;
+const DEFAULT_TIMEOUT_MS = 30_000;
+const DEFAULT_MAX_RESPONSE_BYTES = 50 * 1024 * 1024;
+const BLOCKED_HOSTNAMES = new Set(['localhost', 'metadata.google.internal', 'metadata']);
+const PRIVATE_RANGES = new BlockList();
+for (const [network, prefix] of [
+    ['0.0.0.0', 8],
+    ['10.0.0.0', 8],
+    ['100.64.0.0', 10],
+    ['127.0.0.0', 8],
+    ['169.254.0.0', 16],
+    ['172.16.0.0', 12],
+    ['192.0.0.0', 24],
+    ['192.168.0.0', 16],
+    ['198.18.0.0', 15],
+    ['224.0.0.0', 4],
+    ['240.0.0.0', 4],
+]) {
+    PRIVATE_RANGES.addSubnet(network, prefix, 'ipv4');
+}
+for (const [network, prefix] of [
+    ['::', 128],
+    ['::1', 128],
+    ['fc00::', 7],
+    ['fe80::', 10],
+    ['ff00::', 8],
+]) {
+    PRIVATE_RANGES.addSubnet(network, prefix, 'ipv6');
+}
+function embeddedIPv4(ipv6) {
+    const lower = ipv6.toLowerCase();
+    const prefixes = ['::ffff:', '64:ff9b::', '::'];
+    for (const prefix of prefixes) {
+        if (!lower.startsWith(prefix))
+            continue;
+        const rest = lower.slice(prefix.length);
+        if (isIP(rest) === 4)
+            return rest;
+        const hex = /^([0-9a-f]{1,4}):([0-9a-f]{1,4})$/.exec(rest);
+        if (hex) {
+            const high = parseInt(hex[1], 16);
+            const low = parseInt(hex[2], 16);
+            return [high >> 8, high & 0xff, low >> 8, low & 0xff].join('.');
+        }
+    }
+    return null;
+}
+/**
+ * Whether an IP address points to a loopback, private, link-local, CGNAT, multicast or
+ * otherwise non-public network (including IPv4-mapped / NAT64 IPv6 forms).
+ */
+export function isBlockedAddress(address) {
+    const ip = address.replace(/^\[|\]$/g, '');
+    const family = isIP(ip);
+    if (family === 4)
+        return PRIVATE_RANGES.check(ip, 'ipv4');
+    if (family === 6) {
+        const v4 = embeddedIPv4(ip);
+        if (v4)
+            return PRIVATE_RANGES.check(v4, 'ipv4');
+        return PRIVATE_RANGES.check(ip, 'ipv6');
     }
     return false;
 }
-async function validateUrl(urlString) {
+const systemResolver = (hostname, callback) => {
+    dnsLookup(hostname, { all: true, verbatim: true }, callback);
+};
+/**
+ * DNS lookup for `http.request` that rejects private addresses at connect time,
+ * which also defeats DNS-rebinding between validation and connection.
+ */
+export function createGuardedLookup(resolver = systemResolver) {
+    return (hostname, options, callback) => {
+        resolver(hostname, (err, addresses) => {
+            if (err) {
+                callback(err, []);
+                return;
+            }
+            const candidates = addresses.filter((entry) => !options.family || entry.family === options.family);
+            const blocked = candidates.find((entry) => isBlockedAddress(entry.address));
+            if (blocked) {
+                callback(Object.assign(new Error(`WebLoader: "${hostname}" resolves to private IP ${blocked.address} — blocked`), { code: 'EBLOCKED' }), []);
+                return;
+            }
+            if (candidates.length === 0) {
+                callback(Object.assign(new Error(`WebLoader: no addresses found for "${hostname}"`), {
+                    code: 'ENOTFOUND',
+                }), []);
+                return;
+            }
+            if (options.all) {
+                callback(null, candidates);
+            }
+            else {
+                callback(null, candidates[0].address, candidates[0].family);
+            }
+        });
+    };
+}
+function parseUrl(urlString, allowPrivateNetwork) {
     let url;
     try {
         url = new URL(urlString);
@@ -41,101 +115,162 @@ async function validateUrl(urlString) {
     if (url.protocol !== 'http:' && url.protocol !== 'https:') {
         throw new Error(`WebLoader: unsupported protocol "${url.protocol}" — only http and https are allowed`);
     }
-    const hostname = url.hostname.toLowerCase();
-    if (BLOCKED_HOSTNAMES.has(hostname)) {
+    if (allowPrivateNetwork)
+        return url;
+    const hostname = url.hostname
+        .toLowerCase()
+        .replace(/^\[|\]$/g, '')
+        .replace(/\.$/, '');
+    if (BLOCKED_HOSTNAMES.has(hostname) || hostname.endsWith('.localhost')) {
         throw new Error(`WebLoader: access to "${hostname}" is blocked (loopback / metadata endpoint)`);
     }
-    if (isIP(hostname) && isPrivateIP(hostname)) {
+    if (isIP(hostname) && isBlockedAddress(hostname)) {
         throw new Error(`WebLoader: access to private IP "${hostname}" is blocked`);
     }
-    if (!isIP(hostname)) {
+    return url;
+}
+function decodeBody(body, contentType) {
+    const charset = /charset\s*=\s*"?([\w.:-]+)"?/i.exec(contentType)?.[1];
+    if (charset) {
         try {
-            const { address } = await lookup(hostname);
-            if (isPrivateIP(address)) {
-                throw new Error(`WebLoader: "${hostname}" resolves to private IP ${address} — blocked`);
-            }
+            return new TextDecoder(charset).decode(body);
         }
-        catch (err) {
-            if (err instanceof Error && err.message.startsWith('WebLoader:'))
-                throw err;
+        catch {
+            return new TextDecoder('utf-8').decode(body);
         }
     }
-    return url;
+    return new TextDecoder('utf-8').decode(body);
 }
-const DEFAULT_TIMEOUT_MS = 30_000;
-const DEFAULT_MAX_RESPONSE_BYTES = 50 * 1024 * 1024;
+function decompress(response) {
+    const encoding = (response.headers['content-encoding'] ?? '').toLowerCase().trim();
+    switch (encoding) {
+        case 'gzip':
+        case 'x-gzip':
+            return response.pipe(createGunzip());
+        case 'deflate':
+            return response.pipe(createInflate());
+        case 'br':
+            return response.pipe(createBrotliDecompress());
+        default:
+            return response;
+    }
+}
+/**
+ * Loads a web page over http(s). HTML is converted to text (scripts and styles removed);
+ * plain-text and JSON responses are kept as-is. Redirects are followed (max 5) and every
+ * hop — including DNS resolution at connect time — is checked against private networks.
+ */
 export class WebLoader {
     supportedTypes = ['http', 'https'];
     htmlLoader;
     headers;
     timeoutMs;
     maxResponseBytes;
+    allowPrivateNetwork;
+    lookup = createGuardedLookup();
     constructor(options) {
         this.htmlLoader = new HTMLLoader({ selector: options?.selector });
         this.headers = options?.headers;
         this.timeoutMs = options?.timeoutMs ?? DEFAULT_TIMEOUT_MS;
         this.maxResponseBytes = options?.maxResponseBytes ?? DEFAULT_MAX_RESPONSE_BYTES;
+        this.allowPrivateNetwork = options?.allowPrivateNetwork ?? false;
     }
     async load(source) {
-        let currentUrl = source;
-        for (let redirectCount = 0; redirectCount <= MAX_REDIRECTS; redirectCount++) {
-            await validateUrl(currentUrl);
-            const response = await fetch(currentUrl, {
-                headers: this.headers ?? {},
-                redirect: 'manual',
-                signal: AbortSignal.timeout(this.timeoutMs),
-            });
-            if (response.status >= 300 && response.status < 400) {
-                const location = response.headers.get('location');
+        const page = await this.fetchPage(source);
+        const mime = page.contentType.split(';')[0].trim().toLowerCase();
+        if (mime === '' || mime.includes('html') || mime.endsWith('/xml') || mime.endsWith('+xml')) {
+            const doc = await this.htmlLoader.parseHTML(page.body, source, 'web');
+            return [
+                { ...doc, metadata: { ...doc.metadata, url: page.url, contentType: mime || 'text/html' } },
+            ];
+        }
+        if (mime.startsWith('text/') || mime === 'application/json' || mime.endsWith('+json')) {
+            return [
+                {
+                    id: nanoid(),
+                    content: page.body,
+                    source,
+                    sourceType: 'web',
+                    metadata: { url: page.url, contentType: mime },
+                },
+            ];
+        }
+        throw new Error(`WebLoader: unsupported content type "${mime}" from ${page.url}`);
+    }
+    async fetchPage(source) {
+        let current = parseUrl(source, this.allowPrivateNetwork);
+        for (let redirects = 0; redirects <= MAX_REDIRECTS; redirects++) {
+            const response = await this.request(current);
+            const status = response.statusCode ?? 0;
+            if (status >= 300 && status < 400) {
+                response.resume();
+                const location = response.headers.location;
                 if (!location) {
-                    throw new Error(`WebLoader: redirect ${response.status} without Location header from ${currentUrl}`);
+                    throw new Error(`WebLoader: redirect ${status} without Location header from ${current.href}`);
                 }
-                currentUrl = new URL(location, currentUrl).href;
+                current = parseUrl(new URL(location, current).href, this.allowPrivateNetwork);
                 continue;
             }
-            if (!response.ok) {
-                throw new Error(`WebLoader: failed to fetch ${currentUrl}: ${response.status} ${response.statusText}`);
+            if (status < 200 || status >= 300) {
+                response.resume();
+                throw new Error(`WebLoader: failed to fetch ${current.href}: ${status} ${response.statusMessage ?? ''}`.trim());
             }
-            const html = await this.readBodyWithLimit(response, currentUrl);
-            const doc = await this.htmlLoader.parseHTML(html, source, 'web');
-            return [doc];
+            const contentType = response.headers['content-type'] ?? '';
+            const body = await this.readBody(response, current.href);
+            return { url: current.href, contentType, body: decodeBody(body, contentType) };
         }
         throw new Error(`WebLoader: too many redirects (max ${MAX_REDIRECTS}) for ${source}`);
     }
-    async readBodyWithLimit(response, url) {
-        const body = response.body;
-        if (!body) {
-            const text = await response.text();
-            if (text.length > this.maxResponseBytes) {
-                throw new Error(`WebLoader: response from ${url} exceeds ${this.maxResponseBytes} bytes limit`);
-            }
-            return text;
-        }
-        const reader = body.getReader();
-        const chunks = [];
-        let totalBytes = 0;
-        try {
-            for (;;) {
-                const { done, value } = await reader.read();
-                if (done)
-                    break;
-                totalBytes += value.byteLength;
-                if (totalBytes > this.maxResponseBytes) {
-                    throw new Error(`WebLoader: response from ${url} exceeds ${this.maxResponseBytes} bytes limit`);
+    request(url) {
+        const transport = url.protocol === 'https:' ? https : http;
+        return new Promise((resolve, reject) => {
+            const req = transport.get(url, {
+                headers: {
+                    'user-agent': 'cogitator-rag-webloader',
+                    accept: 'text/html,application/xhtml+xml,text/plain;q=0.9,*/*;q=0.5',
+                    'accept-encoding': 'gzip, deflate, br',
+                    ...this.headers,
+                },
+                signal: AbortSignal.timeout(this.timeoutMs),
+                ...(!this.allowPrivateNetwork && { lookup: this.lookup }),
+            });
+            req.once('response', resolve);
+            req.once('error', (err) => {
+                if (err.message.startsWith('WebLoader:')) {
+                    reject(err);
                 }
-                chunks.push(value);
-            }
-        }
-        finally {
-            reader.releaseLock();
-        }
-        const merged = new Uint8Array(totalBytes);
-        let offset = 0;
-        for (const chunk of chunks) {
-            merged.set(chunk, offset);
-            offset += chunk.byteLength;
-        }
-        return new TextDecoder('utf-8').decode(merged);
+                else if (err.name === 'AbortError' || err.code === 'ABORT_ERR') {
+                    reject(new Error(`WebLoader: request to ${url.href} timed out after ${this.timeoutMs}ms`));
+                }
+                else {
+                    reject(new Error(`WebLoader: request to ${url.href} failed: ${err.message}`));
+                }
+            });
+        });
+    }
+    readBody(response, url) {
+        const stream = decompress(response);
+        return new Promise((resolve, reject) => {
+            const chunks = [];
+            let total = 0;
+            stream.on('data', (chunk) => {
+                total += chunk.length;
+                if (total > this.maxResponseBytes) {
+                    response.destroy();
+                    stream.destroy();
+                    reject(new Error(`WebLoader: response from ${url} exceeds ${this.maxResponseBytes} bytes limit`));
+                    return;
+                }
+                chunks.push(chunk);
+            });
+            stream.once('end', () => resolve(Buffer.concat(chunks)));
+            stream.once('error', (err) => {
+                reject(new Error(`WebLoader: failed to read response from ${url}: ${err.message}`));
+            });
+            response.once('error', (err) => {
+                reject(new Error(`WebLoader: failed to read response from ${url}: ${err.message}`));
+            });
+        });
     }
 }
 //# sourceMappingURL=web-loader.js.map

package/dist/loaders/web-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"web-loader.js","sourceRoot":"","sources":["../../src/loaders/web-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAEhC,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAE3C,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IAChC,WAAW;IACX,WAAW;IACX,KAAK;IACL,OAAO;IACP,SAAS;IACT,iBAAiB;IACjB,0BAA0B;CAC3B,CAAC,CAAC;AAEH,MAAM,aAAa,GAAG,CAAC,CAAC;AAExB,SAAS,WAAW,CAAC,EAAU;IAC7B,IAAI,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACtC,IAAI,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,OAAO,IAAI,CAAC;IAC3C,IAAI,EAAE,KAAK,SAAS,IAAI,EAAE,KAAK,WAAW,IAAI,EAAE,KAAK,KAAK;QAAE,OAAO,IAAI,CAAC;IACxE,IAAI,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,OAAO,IAAI,CAAC;IAC3C,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IAErF,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC1C,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACzC,IAAI,MAAM,IAAI,EAAE,IAAI,MAAM,IAAI,EAAE;YAAE,OAAO,IAAI,CAAC;IAChD,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,SAAiB;IAC1C,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,2BAA2B,SAAS,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CACb,oCAAoC,GAAG,CAAC,QAAQ,qCAAqC,CACtF,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAC5C,IAAI,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,yBAAyB,QAAQ,6CAA6C,CAAC,CAAC;IAClG,CAAC;IAED,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,oCAAoC,QAAQ,cAAc,CAAC,CAAC;IAC9E,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC3C,IAAI,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,KAAK,CAAC,eAAe,QAAQ,4BAA4B,OAAO,YAAY,CAAC,CAAC;YAC1F,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC;gBAAE,MAAM,GAAG,CAAC;QAC9E,CAAC;IACH,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AASD,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,0BAA0B,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAEpD,MAAM,OAAO,SAAS;IACX,cAAc,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC3B,UAAU,CAAa;IACvB,OAAO,CAA0B;IACjC,SAAS,CAAS;IAClB,gBAAgB,CAAS;IAE1C,YAAY,OAA0B;QACpC,IAAI,CAAC,UAAU,GAAG,IAAI,UAAU,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QAClE,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;QAChC,IAAI,CAAC,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,kBAAkB,CAAC;QAC1D,IAAI,CAAC,gBAAgB,GAAG,OAAO,EAAE,gBAAgB,IAAI,0BAA0B,CAAC;IAClF,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,IAAI,UAAU,GAAG,MAAM,CAAC;QAExB,KAAK,IAAI,aAAa,GAAG,CAAC,EAAE,aAAa,IAAI,aAAa,EAAE,aAAa,EAAE,EAAE,CAAC;YAC5E,MAAM,WAAW,CAAC,UAAU,CAAC,CAAC;YAE9B,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,UAAU,EAAE;gBACvC,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,EAAE;gBAC3B,QAAQ,EAAE,QAAQ;gBAClB,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC;aAC5C,CAAC,CAAC;YAEH,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACpD,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBAClD,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,MAAM,IAAI,KAAK,CACb,uBAAuB,QAAQ,CAAC,MAAM,iCAAiC,UAAU,EAAE,CACpF,CAAC;gBACJ,CAAC;gBACD,UAAU,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC;gBAChD,SAAS;YACX,CAAC;YAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CACb,8BAA8B,UAAU,KAAK,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CACtF,CAAC;YACJ,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;YAChE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;YACjE,OAAO,CAAC,GAAG,CAAC,CAAC;QACf,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,sCAAsC,aAAa,SAAS,MAAM,EAAE,CAAC,CAAC;IACxF,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,QAAkB,EAAE,GAAW;QAC7D,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;QAC3B,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACxC,MAAM,IAAI,KAAK,CACb,4BAA4B,GAAG,YAAY,IAAI,CAAC,gBAAgB,cAAc,CAC/E,CAAC;YACJ,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAChC,MAAM,MAAM,GAAiB,EAAE,CAAC;QAChC,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,IAAI,CAAC;YACH,SAAS,CAAC;gBACR,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,IAAI;oBAAE,MAAM;gBAEhB,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;gBAC/B,IAAI,UAAU,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;oBACvC,MAAM,IAAI,KAAK,CACb,4BAA4B,GAAG,YAAY,IAAI,CAAC,gBAAgB,cAAc,CAC/E,CAAC;gBACJ,CAAC;gBACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,WAAW,EAAE,CAAC;QACvB,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC;QAC7B,CAAC;QAED,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACjD,CAAC;CACF"}
+{"version":3,"file":"web-loader.js","sourceRoot":"","sources":["../../src/loaders/web-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,SAAS,EAA0C,MAAM,UAAU,CAAC;AACvF,OAAO,IAA8B,MAAM,WAAW,CAAC;AACvD,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAE3C,OAAO,EAAE,sBAAsB,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAEhF,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,MAAM,aAAa,GAAG,CAAC,CAAC;AACxB,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,0BAA0B,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAEpD,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,0BAA0B,EAAE,UAAU,CAAC,CAAC,CAAC;AAEzF,MAAM,cAAc,GAAG,IAAI,SAAS,EAAE,CAAC;AACvC,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI;IAC9B,CAAC,SAAS,EAAE,CAAC,CAAC;IACd,CAAC,UAAU,EAAE,CAAC,CAAC;IACf,CAAC,YAAY,EAAE,EAAE,CAAC;IAClB,CAAC,WAAW,EAAE,CAAC,CAAC;IAChB,CAAC,aAAa,EAAE,EAAE,CAAC;IACnB,CAAC,YAAY,EAAE,EAAE,CAAC;IAClB,CAAC,WAAW,EAAE,EAAE,CAAC;IACjB,CAAC,aAAa,EAAE,EAAE,CAAC;IACnB,CAAC,YAAY,EAAE,EAAE,CAAC;IAClB,CAAC,WAAW,EAAE,CAAC,CAAC;IAChB,CAAC,WAAW,EAAE,CAAC,CAAC;CACR,EAAE,CAAC;IACX,cAAc,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AACpD,CAAC;AACD,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI;IAC9B,CAAC,IAAI,EAAE,GAAG,CAAC;IACX,CAAC,KAAK,EAAE,GAAG,CAAC;IACZ,CAAC,QAAQ,EAAE,CAAC,CAAC;IACb,CAAC,QAAQ,EAAE,EAAE,CAAC;IACd,CAAC,QAAQ,EAAE,CAAC,CAAC;CACL,EAAE,CAAC;IACX,cAAc,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AACpD,CAAC;AAED,SAAS,YAAY,CAAC,IAAY;IAChC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC;IAChD,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC;YAAE,SAAS;QACxC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAClC,MAAM,GAAG,GAAG,mCAAmC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3D,IAAI,GAAG,EAAE,CAAC;YACR,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAE,EAAE,EAAE,CAAC,CAAC;YACnC,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAE,EAAE,EAAE,CAAC,CAAC;YAClC,OAAO,CAAC,IAAI,IAAI,CAAC,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAe;IAC9C,MAAM,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IACxB,IAAI,MAAM,KAAK,CAAC;QAAE,OAAO,cAAc,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAC1D,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACjB,MAAM,EAAE,GAAG,YAAY,CAAC,EAAE,CAAC,CAAC;QAC5B,IAAI,EAAE;YAAE,OAAO,cAAc,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QAChD,OAAO,cAAc,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAaD,MAAM,cAAc,GAAa,CAAC,QAAQ,EAAE,QAAQ,EAAE,EAAE;IACtD,SAAS,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,QAAQ,CAAC,CAAC;AAC/D,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,WAAqB,cAAc;IACrE,OAAO,CAAC,QAAgB,EAAE,OAAsB,EAAE,QAAwB,EAAQ,EAAE;QAClF,QAAQ,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE;YACpC,IAAI,GAAG,EAAE,CAAC;gBACR,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;gBAClB,OAAO;YACT,CAAC;YACD,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CACjC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,CAC9D,CAAC;YACF,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YAC5E,IAAI,OAAO,EAAE,CAAC;gBACZ,QAAQ,CACN,MAAM,CAAC,MAAM,CACX,IAAI,KAAK,CACP,eAAe,QAAQ,4BAA4B,OAAO,CAAC,OAAO,YAAY,CAC/E,EACD,EAAE,IAAI,EAAE,UAAU,EAAE,CACrB,EACD,EAAE,CACH,CAAC;gBACF,OAAO;YACT,CAAC;YACD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC5B,QAAQ,CACN,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,sCAAsC,QAAQ,GAAG,CAAC,EAAE;oBAC1E,IAAI,EAAE,WAAW;iBAClB,CAAC,EACF,EAAE,CACH,CAAC;gBACF,OAAO;YACT,CAAC;YACD,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAE,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAC;YAChE,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,SAAiB,EAAE,mBAA4B;IAC/D,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,2BAA2B,SAAS,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CACb,oCAAoC,GAAG,CAAC,QAAQ,qCAAqC,CACtF,CAAC;IACJ,CAAC;IAED,IAAI,mBAAmB;QAAE,OAAO,GAAG,CAAC;IAEpC,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ;SAC1B,WAAW,EAAE;SACb,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACtB,IAAI,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;QACvE,MAAM,IAAI,KAAK,CAAC,yBAAyB,QAAQ,6CAA6C,CAAC,CAAC;IAClG,CAAC;IACD,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,gBAAgB,CAAC,QAAQ,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CAAC,oCAAoC,QAAQ,cAAc,CAAC,CAAC;IAC9E,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,UAAU,CAAC,IAAY,EAAE,WAAmB;IACnD,MAAM,OAAO,GAAG,+BAA+B,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACvE,IAAI,OAAO,EAAE,CAAC;QACZ,IAAI,CAAC;YACH,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC/C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IACD,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,UAAU,CAAC,QAAyB;IAC3C,MAAM,QAAQ,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;IACnF,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,MAAM,CAAC;QACZ,KAAK,QAAQ;YACX,OAAO,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;QACvC,KAAK,SAAS;YACZ,OAAO,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;QACxC,KAAK,IAAI;YACP,OAAO,QAAQ,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC,CAAC;QACjD;YACE,OAAO,QAAQ,CAAC;IACpB,CAAC;AACH,CAAC;AAoBD;;;;GAIG;AACH,MAAM,OAAO,SAAS;IACX,cAAc,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC3B,UAAU,CAAa;IACvB,OAAO,CAA0B;IACjC,SAAS,CAAS;IAClB,gBAAgB,CAAS;IACzB,mBAAmB,CAAU;IAC7B,MAAM,GAAG,mBAAmB,EAAE,CAAC;IAEhD,YAAY,OAA0B;QACpC,IAAI,CAAC,UAAU,GAAG,IAAI,UAAU,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QAClE,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;QAChC,IAAI,CAAC,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,kBAAkB,CAAC;QAC1D,IAAI,CAAC,gBAAgB,GAAG,OAAO,EAAE,gBAAgB,IAAI,0BAA0B,CAAC;QAChF,IAAI,CAAC,mBAAmB,GAAG,OAAO,EAAE,mBAAmB,IAAI,KAAK,CAAC;IACnE,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAElE,IAAI,IAAI,KAAK,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3F,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;YACtE,OAAO;gBACL,EAAE,GAAG,GAAG,EAAE,QAAQ,EAAE,EAAE,GAAG,GAAG,CAAC,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,IAAI,WAAW,EAAE,EAAE;aAC3F,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,IAAI,KAAK,kBAAkB,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YACtF,OAAO;gBACL;oBACE,EAAE,EAAE,MAAM,EAAE;oBACZ,OAAO,EAAE,IAAI,CAAC,IAAI;oBAClB,MAAM;oBACN,UAAU,EAAE,KAAK;oBACjB,QAAQ,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE;iBAC/C;aACF,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,UAAU,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACpF,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,MAAc;QACpC,IAAI,OAAO,GAAG,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAEzD,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,IAAI,aAAa,EAAE,SAAS,EAAE,EAAE,CAAC;YAChE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAC7C,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,IAAI,CAAC,CAAC;YAExC,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,EAAE,CAAC;gBAClC,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAClB,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC;gBAC3C,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,MAAM,IAAI,KAAK,CACb,uBAAuB,MAAM,iCAAiC,OAAO,CAAC,IAAI,EAAE,CAC7E,CAAC;gBACJ,CAAC;gBACD,OAAO,GAAG,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;gBAC9E,SAAS;YACX,CAAC;YAED,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;gBAClC,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CACb,8BAA8B,OAAO,CAAC,IAAI,KAAK,MAAM,IAAI,QAAQ,CAAC,aAAa,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,CAC/F,CAAC;YACJ,CAAC;YAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAC3D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;YACzD,OAAO,EAAE,GAAG,EAAE,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,EAAE,CAAC;QACjF,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,sCAAsC,aAAa,SAAS,MAAM,EAAE,CAAC,CAAC;IACxF,CAAC;IAEO,OAAO,CAAC,GAAQ;QACtB,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;QAC3D,OAAO,IAAI,OAAO,CAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACtD,MAAM,GAAG,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE;gBAC7B,OAAO,EAAE;oBACP,YAAY,EAAE,yBAAyB;oBACvC,MAAM,EAAE,4DAA4D;oBACpE,iBAAiB,EAAE,mBAAmB;oBACtC,GAAG,IAAI,CAAC,OAAO;iBAChB;gBACD,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC;gBAC3C,GAAG,CAAC,CAAC,IAAI,CAAC,mBAAmB,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;aAC1D,CAAC,CAAC;YACH,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YAC9B,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,GAA8B,EAAE,EAAE;gBACnD,IAAI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;oBACzC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACd,CAAC;qBAAM,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;oBACjE,MAAM,CACJ,IAAI,KAAK,CAAC,yBAAyB,GAAG,CAAC,IAAI,oBAAoB,IAAI,CAAC,SAAS,IAAI,CAAC,CACnF,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,KAAK,CAAC,yBAAyB,GAAG,CAAC,IAAI,YAAY,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;gBAChF,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,QAAQ,CAAC,QAAyB,EAAE,GAAW;QACrD,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QACpC,OAAO,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC7C,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;gBAClC,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;gBACtB,IAAI,KAAK,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;oBAClC,QAAQ,CAAC,OAAO,EAAE,CAAC;oBACnB,MAAM,CAAC,OAAO,EAAE,CAAC;oBACjB,MAAM,CACJ,IAAI,KAAK,CACP,4BAA4B,GAAG,YAAY,IAAI,CAAC,gBAAgB,cAAc,CAC/E,CACF,CAAC;oBACF,OAAO;gBACT,CAAC;gBACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC,CAAC,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACzD,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE;gBAClC,MAAM,CAAC,IAAI,KAAK,CAAC,2CAA2C,GAAG,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACtF,CAAC,CAAC,CAAC;YACH,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE;gBACpC,MAAM,CAAC,IAAI,KAAK,CAAC,2CAA2C,GAAG,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACtF,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}

package/dist/rag-builder.d.ts

@@ -1,4 +1,5 @@
-import type { DocumentLoader, Chunker, AsyncChunker, EmbeddingService, EmbeddingAdapter, Retriever, Reranker } from '@cogitator-ai/types';
+import type { DocumentLoader, Chunker, AsyncChunker, EmbeddingService, EmbeddingAdapter, Retriever, Reranker, HybridSearchWeights } from '@cogitator-ai/types';
+import type { HybridSearch } from '@cogitator-ai/memory';
 import { type RAGPipelineConfigInput } from './schema.js';
 import { RAGPipeline } from './rag-pipeline.js';
 export declare class RAGPipelineBuilder {
@@ -9,14 +10,22 @@ export declare class RAGPipelineBuilder {
     private retriever?;
     private reranker?;
     private configInput?;
+    private hybridSearch?;
+    private hybridWeights?;
+    private queryExpander?;
     withLoader(loader: DocumentLoader): this;
     withChunker(chunker: Chunker | AsyncChunker): this;
     withEmbeddingService(service: EmbeddingService): this;
     withEmbeddingAdapter(adapter: EmbeddingAdapter): this;
     withRetriever(retriever: Retriever): this;
     withReranker(reranker: Reranker): this;
+    /** HybridSearch used by the `hybrid` strategy (and as the multi-query base when set). */
+    withHybridSearch(hybridSearch: HybridSearch, weights?: HybridSearchWeights): this;
+    /** Query expansion function used by the `multi-query` strategy. */
+    withQueryExpander(expandQuery: (query: string) => Promise<string[]>): this;
     withConfig(config: RAGPipelineConfigInput): this;
     build(): RAGPipeline;
+    private createRetriever;
     private resolveConfig;
 }
 //# sourceMappingURL=rag-builder.d.ts.map