@cogitator-ai/rag 0.1.10 → 0.3.0

package/dist/loaders/html-loader.js

@@ -9,6 +9,21 @@ async function loadCheerio() {
         throw new Error('cheerio is required for HTMLLoader. Install it: pnpm add cheerio');
     }
 }
+const NON_CONTENT_SELECTOR = 'script, style, noscript, template, svg, iframe, object, head';
+const BLOCK_SELECTOR = 'address, article, aside, blockquote, dd, div, dl, dt, fieldset, figcaption, figure, footer, ' +
+    'form, h1, h2, h3, h4, h5, h6, header, hr, li, main, nav, ol, p, pre, section, table, tr, ul';
+function normalizeWhitespace(text) {
+    return text
+        .replace(/\r\n?/g, '\n')
+        .replace(/[^\S\n]+/g, ' ')
+        .replace(/ *\n */g, '\n')
+        .replace(/\n{3,}/g, '\n\n')
+        .trim();
+}
+/**
+ * Extracts readable text from HTML: scripts, styles and other non-content elements are
+ * dropped, block elements become line breaks and whitespace is normalized.
+ */
 export class HTMLLoader {
     supportedTypes = ['html', 'htm'];
     selector;
@@ -23,8 +38,17 @@ export class HTMLLoader {
     async parseHTML(html, source, sourceType) {
         const cheerio = await loadCheerio();
         const $ = cheerio.load(html);
-        const content = $(this.selector).text().trim();
-        const title = $('title').text().trim() || undefined;
+        const title = $('title').first().text().trim() || undefined;
+        $(NON_CONTENT_SELECTOR).remove();
+        $('br').replaceWith('\n');
+        $(BLOCK_SELECTOR).each((_, element) => {
+            $(element).before('\n').after('\n');
+        });
+        $('td, th').after(' ');
+        const content = normalizeWhitespace($(this.selector)
+            .map((_, element) => $(element).text())
+            .get()
+            .join('\n\n'));
         const metadata = {};
         if (title)
             metadata.title = title;

package/dist/loaders/html-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"html-loader.js","sourceRoot":"","sources":["../../src/loaders/html-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,KAAK,UAAU,WAAW;IACxB,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;IACjC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;IACtF,CAAC;AACH,CAAC;AAMD,MAAM,OAAO,UAAU;IACZ,cAAc,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACjC,QAAQ,CAAS;IAE1B,YAAY,OAA2B;QACrC,IAAI,CAAC,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,MAAM,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC/C,OAAO,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,IAAY,EAAE,MAAc,EAAE,UAA0B;QACtE,MAAM,OAAO,GAAG,MAAM,WAAW,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE7B,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAC/C,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC;QAEpD,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,IAAI,KAAK;YAAE,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;QAElC,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM;YACN,UAAU;YACV,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;SACtD,CAAC;IACJ,CAAC;CACF"}
+{"version":3,"file":"html-loader.js","sourceRoot":"","sources":["../../src/loaders/html-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,KAAK,UAAU,WAAW;IACxB,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;IACjC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;IACtF,CAAC;AACH,CAAC;AAMD,MAAM,oBAAoB,GAAG,8DAA8D,CAAC;AAC5F,MAAM,cAAc,GAClB,8FAA8F;IAC9F,6FAA6F,CAAC;AAEhG,SAAS,mBAAmB,CAAC,IAAY;IACvC,OAAO,IAAI;SACR,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;SACvB,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC;SACzB,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC;SACxB,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;SAC1B,IAAI,EAAE,CAAC;AACZ,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,UAAU;IACZ,cAAc,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACjC,QAAQ,CAAS;IAE1B,YAAY,OAA2B;QACrC,IAAI,CAAC,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,MAAM,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC/C,OAAO,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,IAAY,EAAE,MAAc,EAAE,UAA0B;QACtE,MAAM,OAAO,GAAG,MAAM,WAAW,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE7B,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC;QAE5D,CAAC,CAAC,oBAAoB,CAAC,CAAC,MAAM,EAAE,CAAC;QACjC,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;YACpC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEvB,MAAM,OAAO,GAAG,mBAAmB,CACjC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC;aACb,GAAG,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;aACtC,GAAG,EAAE;aACL,IAAI,CAAC,MAAM,CAAC,CAChB,CAAC;QAEF,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,IAAI,KAAK;YAAE,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;QAElC,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM;YACN,UAAU;YACV,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;SACtD,CAAC;IACJ,CAAC;CACF"}

package/dist/loaders/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/loaders/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,YAAY,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACvD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,YAAY,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,YAAY,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACvD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,YAAY,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,YAAY,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/loaders/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,YAAY,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,YAAY,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,YAAY,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC"}

package/dist/loaders/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/loaders/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAE3C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAE3C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/loaders/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAEtD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC"}

package/dist/loaders/json-loader.js

@@ -19,7 +19,7 @@ export class JSONLoader {
         }
         catch (err) {
             const message = err instanceof Error ? err.message : String(err);
-            throw new Error(`JSONLoader: failed to parse "${filePath}": ${message}`);
+            throw new Error(`JSONLoader: failed to parse "${filePath}": ${message}`, { cause: err });
         }
         const items = Array.isArray(data) ? data : [data];
         return items.map((item) => this.itemToDocument(item, filePath));

package/dist/loaders/json-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"json-loader.js","sourceRoot":"","sources":["../../src/loaders/json-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,cAAc,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAOnD,MAAM,OAAO,UAAU;IACZ,cAAc,GAAG,CAAC,MAAM,CAAC,CAAC;IAClB,YAAY,CAAU;IACtB,cAAc,CAAY;IAE3C,YAAY,OAA2B;QACrC,IAAI,CAAC,YAAY,GAAG,OAAO,EAAE,YAAY,CAAC;QAC1C,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,cAAc,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAE9C,IAAI,IAAa,CAAC;QAClB,IAAI,CAAC;YACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,gCAAgC,QAAQ,MAAM,OAAO,EAAE,CAAC,CAAC;QAC3E,CAAC;QAED,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAClD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;IAClE,CAAC;IAEO,cAAc,CAAC,IAAa,EAAE,MAAc;QAClD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACrE,OAAO;gBACL,EAAE,EAAE,MAAM,EAAE;gBACZ,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC;gBACrB,MAAM;gBACN,UAAU,EAAE,MAAM;aACnB,CAAC;QACJ,CAAC;QACD,MAAM,GAAG,GAAG,IAA+B,CAAC;QAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QAE3C,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM;YACN,UAAU,EAAE,MAAM;YAClB,GAAG,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;SAClE,CAAC;IACJ,CAAC;IAEO,cAAc,CAAC,GAA4B;QACjD,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY,IAAI,GAAG,EAAE,CAAC;YAClD,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;QACpD,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;gBACjB,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACtC,CAAC;IAEO,aAAa,CAAC,KAAc;QAClC,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QAC5C,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACxC,CAAC;IAEO,eAAe,CAAC,GAA4B;QAClD,IAAI,CAAC,IAAI,CAAC,cAAc;YAAE,OAAO,SAAS,CAAC;QAE3C,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxC,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;gBACjB,QAAQ,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF"}
+{"version":3,"file":"json-loader.js","sourceRoot":"","sources":["../../src/loaders/json-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,cAAc,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAOnD,MAAM,OAAO,UAAU;IACZ,cAAc,GAAG,CAAC,MAAM,CAAC,CAAC;IAClB,YAAY,CAAU;IACtB,cAAc,CAAY;IAE3C,YAAY,OAA2B;QACrC,IAAI,CAAC,YAAY,GAAG,OAAO,EAAE,YAAY,CAAC;QAC1C,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,cAAc,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAE9C,IAAI,IAAa,CAAC;QAClB,IAAI,CAAC;YACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,gCAAgC,QAAQ,MAAM,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;QAC3F,CAAC;QAED,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAClD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;IAClE,CAAC;IAEO,cAAc,CAAC,IAAa,EAAE,MAAc;QAClD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACrE,OAAO;gBACL,EAAE,EAAE,MAAM,EAAE;gBACZ,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC;gBACrB,MAAM;gBACN,UAAU,EAAE,MAAM;aACnB,CAAC;QACJ,CAAC;QACD,MAAM,GAAG,GAAG,IAA+B,CAAC;QAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QAE3C,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM;YACN,UAAU,EAAE,MAAM;YAClB,GAAG,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;SAClE,CAAC;IACJ,CAAC;IAEO,cAAc,CAAC,GAA4B;QACjD,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY,IAAI,GAAG,EAAE,CAAC;YAClD,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;QACpD,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;gBACjB,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACtC,CAAC;IAEO,aAAa,CAAC,KAAc;QAClC,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QAC5C,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACxC,CAAC;IAEO,eAAe,CAAC,GAA4B;QAClD,IAAI,CAAC,IAAI,CAAC,cAAc;YAAE,OAAO,SAAS,CAAC;QAE3C,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxC,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;gBACjB,QAAQ,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF"}

package/dist/loaders/markdown-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"markdown-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/markdown-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAMvE,MAAM,WAAW,qBAAqB;IACpC,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,qBAAa,cAAe,YAAW,cAAc;IACnD,QAAQ,CAAC,cAAc,WAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAU;gBAE/B,OAAO,CAAC,EAAE,qBAAqB;IAIrC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YAWpC,QAAQ;IAsBtB,OAAO,CAAC,gBAAgB;YAWV,aAAa;CAiB5B"}
+{"version":3,"file":"markdown-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/markdown-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAWvE,MAAM,WAAW,qBAAqB;IACpC,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,qBAAa,cAAe,YAAW,cAAc;IACnD,QAAQ,CAAC,cAAc,WAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAU;gBAE/B,OAAO,CAAC,EAAE,qBAAqB;IAIrC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YAWpC,QAAQ;IAsBtB,OAAO,CAAC,gBAAgB;YAWV,aAAa;CAiB5B"}

package/dist/loaders/markdown-loader.js

@@ -3,6 +3,10 @@ import { join, resolve } from 'node:path';
 import { nanoid } from 'nanoid';
 const MD_EXTENSIONS = new Set(['md', 'mdx']);
 const FRONTMATTER_RE = /^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/;
+function unquote(value) {
+    const quoted = /^(['"])(.*)\1$/.exec(value);
+    return quoted ? quoted[2] : value;
+}
 export class MarkdownLoader {
     supportedTypes = ['md', 'mdx'];
     stripFrontmatter;
@@ -38,16 +42,16 @@ export class MarkdownLoader {
     }
     parseFrontmatter(raw) {
         const result = {};
-        for (const line of raw.split('\n')) {
+        for (const line of raw.split(/\r?\n/)) {
             const match = /^(\w[\w\s-]*?):\s*(.+)$/.exec(line);
             if (match) {
-                result[match[1].trim()] = match[2].trim();
+                result[match[1].trim()] = unquote(match[2].trim());
             }
         }
         return result;
     }
     async loadDirectory(dirPath) {
-        const entries = await readdir(dirPath);
+        const entries = (await readdir(dirPath)).sort();
         const docs = [];
         for (const entry of entries) {
             const ext = entry.split('.').pop()?.toLowerCase() ?? '';

package/dist/loaders/markdown-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"markdown-loader.js","sourceRoot":"","sources":["../../src/loaders/markdown-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;AAE7C,MAAM,cAAc,GAAG,wCAAwC,CAAC;AAMhE,MAAM,OAAO,cAAc;IAChB,cAAc,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACvB,gBAAgB,CAAU;IAE3C,YAAY,OAA+B;QACzC,IAAI,CAAC,gBAAgB,GAAG,OAAO,EAAE,gBAAgB,IAAI,KAAK,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;QAElC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,QAAgB;QACrC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9C,IAAI,OAAO,GAAG,GAAG,CAAC;QAClB,IAAI,QAA6C,CAAC;QAElD,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvC,IAAI,KAAK,EAAE,CAAC;gBACV,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC3C,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACvC,CAAC;QACH,CAAC;QAED,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,UAAU;YACtB,GAAG,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,CAAC;SAC9B,CAAC;IACJ,CAAC;IAEO,gBAAgB,CAAC,GAAW;QAClC,MAAM,MAAM,GAA4B,EAAE,CAAC;QAC3C,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnD,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC5C,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAe;QACzC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;QACvC,MAAM,IAAI,GAAkB,EAAE,CAAC;QAE/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YACxD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAEtC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClC,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;gBAClB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}
+{"version":3,"file":"markdown-loader.js","sourceRoot":"","sources":["../../src/loaders/markdown-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;AAE7C,MAAM,cAAc,GAAG,wCAAwC,CAAC;AAEhE,SAAS,OAAO,CAAC,KAAa;IAC5B,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC5C,OAAO,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,KAAK,CAAC;AACrC,CAAC;AAMD,MAAM,OAAO,cAAc;IAChB,cAAc,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACvB,gBAAgB,CAAU;IAE3C,YAAY,OAA+B;QACzC,IAAI,CAAC,gBAAgB,GAAG,OAAO,EAAE,gBAAgB,IAAI,KAAK,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;QAElC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,QAAgB;QACrC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9C,IAAI,OAAO,GAAG,GAAG,CAAC;QAClB,IAAI,QAA6C,CAAC;QAElD,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvC,IAAI,KAAK,EAAE,CAAC;gBACV,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC3C,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACvC,CAAC;QACH,CAAC;QAED,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,UAAU;YACtB,GAAG,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,CAAC;SAC9B,CAAC;IACJ,CAAC;IAEO,gBAAgB,CAAC,GAAW;QAClC,MAAM,MAAM,GAA4B,EAAE,CAAC;QAC3C,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YACtC,MAAM,KAAK,GAAG,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnD,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,CAAC;YACvD,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAe;QACzC,MAAM,OAAO,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,MAAM,IAAI,GAAkB,EAAE,CAAC;QAE/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YACxD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAEtC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClC,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;gBAClB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}

package/dist/loaders/pdf-loader.d.ts

@@ -7,9 +7,9 @@ export declare class PDFLoader implements DocumentLoader {
     private readonly splitPages;
     constructor(options?: PDFLoaderOptions);
     load(source: string): Promise<RAGDocument[]>;
-    private loadSingleDocument;
-    private loadSplitPages;
-    private parseWithErrorContext;
+    private parse;
+    private buildSingleDoc;
+    private buildPageDocs;
     private buildPageDoc;
 }
 //# sourceMappingURL=pdf-loader.d.ts.map

package/dist/loaders/pdf-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pdf-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/pdf-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AA4BvE,MAAM,WAAW,gBAAgB;IAC/B,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,qBAAa,SAAU,YAAW,cAAc;IAC9C,QAAQ,CAAC,cAAc,WAAW;IAClC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAU;gBAEzB,OAAO,CAAC,EAAE,gBAAgB;IAIhC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YAYpC,kBAAkB;YAsBlB,cAAc;YA6Bd,qBAAqB;IAcnC,OAAO,CAAC,YAAY;CAkBrB"}
+{"version":3,"file":"pdf-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/pdf-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AA0BvE,MAAM,WAAW,gBAAgB;IAC/B,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,qBAAa,SAAU,YAAW,cAAc;IAC9C,QAAQ,CAAC,cAAc,WAAW;IAClC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAU;gBAEzB,OAAO,CAAC,EAAE,gBAAgB;IAIhC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YAapC,KAAK;IAwBnB,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,aAAa;IAYrB,OAAO,CAAC,YAAY;CAkBrB"}

package/dist/loaders/pdf-loader.js

@@ -4,12 +4,18 @@ import { nanoid } from 'nanoid';
 async function loadPdfParse() {
     try {
         const mod = await import('pdf-parse');
-        return mod.default;
+        return mod.PDFParse;
     }
     catch {
         throw new Error('pdf-parse is required for PDFLoader. Install it: pnpm add pdf-parse');
     }
 }
+function extractTitle(info) {
+    if (typeof info !== 'object' || info === null || !('Title' in info))
+        return undefined;
+    const title = info.Title;
+    return typeof title === 'string' && title.length > 0 ? title : undefined;
+}
 export class PDFLoader {
     supportedTypes = ['pdf'];
     splitPages;
@@ -17,57 +23,54 @@ export class PDFLoader {
         this.splitPages = options?.splitPages ?? false;
     }
     async load(source) {
-        const pdfParse = await loadPdfParse();
+        const PDFParse = await loadPdfParse();
         const filePath = resolve(source);
         const buffer = await readFile(filePath);
+        const parsed = await this.parse(PDFParse, buffer, filePath);
         if (this.splitPages) {
-            return this.loadSplitPages(pdfParse, buffer, filePath);
+            return this.buildPageDocs(parsed, filePath);
         }
-        return this.loadSingleDocument(pdfParse, buffer, filePath);
+        return [this.buildSingleDoc(parsed, filePath)];
     }
-    async loadSingleDocument(pdfParse, buffer, source) {
-        const result = await this.parseWithErrorContext(pdfParse, buffer, source);
-        const title = result.info?.Title;
-        const metadata = { pages: result.numpages };
-        if (title)
-            metadata.title = title;
-        return [
-            {
-                id: nanoid(),
-                content: result.text,
-                source,
-                sourceType: 'pdf',
-                metadata,
-            },
-        ];
-    }
-    async loadSplitPages(pdfParse, buffer, source) {
-        const pageTexts = [];
-        const result = await this.parseWithErrorContext(pdfParse, buffer, source, {
-            pagerender: async (pageData) => {
-                if (typeof pageData.getTextContent !== 'function')
-                    return '';
-                const textContent = await pageData.getTextContent();
-                const text = textContent.items.map((item) => item.str).join('');
-                pageTexts.push(text);
-                return text;
-            },
-        });
-        const title = result.info?.Title;
-        const pages = pageTexts.filter((p) => p.trim().length > 0);
-        if (pages.length === 0) {
-            return [this.buildPageDoc(result.text, source, 1, result.numpages, title)];
-        }
-        return pages.map((text, i) => this.buildPageDoc(text.trim(), source, i + 1, result.numpages, title));
-    }
-    async parseWithErrorContext(pdfParse, buffer, source, options) {
+    async parse(PDFParse, buffer, source) {
+        let parser;
         try {
-            return await pdfParse(buffer, options);
+            parser = new PDFParse({ data: new Uint8Array(buffer) });
+            const info = await parser.getInfo();
+            const result = await parser.getText({ pageJoiner: '' });
+            return {
+                text: result.text,
+                totalPages: result.total,
+                pages: result.pages.map(({ num, text }) => ({ num, text })),
+                title: extractTitle(info.info),
+            };
         }
         catch (err) {
             const message = err instanceof Error ? err.message : String(err);
-            throw new Error(`PDFLoader: failed to parse "${source}": ${message}`);
+            throw new Error(`PDFLoader: failed to parse "${source}": ${message}`, { cause: err });
+        }
+        finally {
+            await parser?.destroy();
+        }
+    }
+    buildSingleDoc(parsed, source) {
+        const metadata = { pages: parsed.totalPages };
+        if (parsed.title)
+            metadata.title = parsed.title;
+        return {
+            id: nanoid(),
+            content: parsed.text,
+            source,
+            sourceType: 'pdf',
+            metadata,
+        };
+    }
+    buildPageDocs(parsed, source) {
+        const pages = parsed.pages.filter((page) => page.text.trim().length > 0);
+        if (pages.length === 0) {
+            return [this.buildPageDoc(parsed.text, source, 1, parsed.totalPages, parsed.title)];
         }
+        return pages.map((page) => this.buildPageDoc(page.text.trim(), source, page.num, parsed.totalPages, parsed.title));
     }
     buildPageDoc(content, source, pageNumber, totalPages, title) {
         const metadata = { pageNumber, totalPages };

package/dist/loaders/pdf-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"pdf-loader.js","sourceRoot":"","sources":["../../src/loaders/pdf-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAoBhC,KAAK,UAAU,YAAY;IACzB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;QACtC,OAAO,GAAG,CAAC,OAAgC,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;IACzF,CAAC;AACH,CAAC;AAMD,MAAM,OAAO,SAAS;IACX,cAAc,GAAG,CAAC,KAAK,CAAC,CAAC;IACjB,UAAU,CAAU;IAErC,YAAY,OAA0B;QACpC,IAAI,CAAC,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,KAAK,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,MAAM,YAAY,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAExC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;QACzD,CAAC;QAED,OAAO,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC7D,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,QAAoB,EACpB,MAAc,EACd,MAAc;QAEd,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QAC1E,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC;QAEjC,MAAM,QAAQ,GAA4B,EAAE,KAAK,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;QACrE,IAAI,KAAK;YAAE,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;QAElC,OAAO;YACL;gBACE,EAAE,EAAE,MAAM,EAAE;gBACZ,OAAO,EAAE,MAAM,CAAC,IAAI;gBACpB,MAAM;gBACN,UAAU,EAAE,KAAK;gBACjB,QAAQ;aACT;SACF,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,QAAoB,EACpB,MAAc,EACd,MAAc;QAEd,MAAM,SAAS,GAAa,EAAE,CAAC;QAE/B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE;YACxE,UAAU,EAAE,KAAK,EAAE,QAAqB,EAAE,EAAE;gBAC1C,IAAI,OAAO,QAAQ,CAAC,cAAc,KAAK,UAAU;oBAAE,OAAO,EAAE,CAAC;gBAC7D,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,cAAc,EAAE,CAAC;gBACpD,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAChE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACrB,OAAO,IAAI,CAAC;YACd,CAAC;SACF,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC;QACjC,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAE3D,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;QAC7E,CAAC;QAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAC3B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,CACtE,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,qBAAqB,CACjC,QAAoB,EACpB,MAAc,EACd,MAAc,EACd,OAAmC;QAEnC,IAAI,CAAC;YACH,OAAO,MAAM,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,+BAA+B,MAAM,MAAM,OAAO,EAAE,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IAEO,YAAY,CAClB,OAAe,EACf,MAAc,EACd,UAAkB,EAClB,UAAkB,EAClB,KAAc;QAEd,MAAM,QAAQ,GAA4B,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;QACrE,IAAI,KAAK;YAAE,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;QAElC,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM;YACN,UAAU,EAAE,KAAK;YACjB,QAAQ;SACT,CAAC;IACJ,CAAC;CACF"}
+{"version":3,"file":"pdf-loader.js","sourceRoot":"","sources":["../../src/loaders/pdf-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAahC,KAAK,UAAU,YAAY;IACzB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;QACtC,OAAO,GAAG,CAAC,QAAQ,CAAC;IACtB,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;IACzF,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,IAAa;IACjC,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACtF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACzB,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AAC3E,CAAC;AAMD,MAAM,OAAO,SAAS;IACX,cAAc,GAAG,CAAC,KAAK,CAAC,CAAC;IACjB,UAAU,CAAU;IAErC,YAAY,OAA0B;QACpC,IAAI,CAAC,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,KAAK,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,MAAM,YAAY,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACxC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;QAE5D,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAC9C,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;IACjD,CAAC;IAEO,KAAK,CAAC,KAAK,CACjB,QAA6B,EAC7B,MAAc,EACd,MAAc;QAEd,IAAI,MAAiC,CAAC;QACtC,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,IAAI,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACxD,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;YACpC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,CAAC;YACxD,OAAO;gBACL,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,UAAU,EAAE,MAAM,CAAC,KAAK;gBACxB,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC3D,KAAK,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;aAC/B,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,+BAA+B,MAAM,MAAM,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;QACxF,CAAC;gBAAS,CAAC;YACT,MAAM,MAAM,EAAE,OAAO,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,MAAiB,EAAE,MAAc;QACtD,MAAM,QAAQ,GAA4B,EAAE,KAAK,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC;QACvE,IAAI,MAAM,CAAC,KAAK;YAAE,QAAQ,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QAEhD,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO,EAAE,MAAM,CAAC,IAAI;YACpB,MAAM;YACN,UAAU,EAAE,KAAK;YACjB,QAAQ;SACT,CAAC;IACJ,CAAC;IAEO,aAAa,CAAC,MAAiB,EAAE,MAAc;QACrD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAEzE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACtF,CAAC;QAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACxB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,CACvF,CAAC;IACJ,CAAC;IAEO,YAAY,CAClB,OAAe,EACf,MAAc,EACd,UAAkB,EAClB,UAAkB,EAClB,KAAyB;QAEzB,MAAM,QAAQ,GAA4B,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;QACrE,IAAI,KAAK;YAAE,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;QAElC,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM;YACN,UAAU,EAAE,KAAK;YACjB,QAAQ;SACT,CAAC;IACJ,CAAC;CACF"}

package/dist/loaders/text-loader.js

@@ -22,7 +22,7 @@ export class TextLoader {
         };
     }
     async loadDirectory(dirPath) {
-        const entries = await readdir(dirPath);
+        const entries = (await readdir(dirPath)).sort();
         const docs = [];
         for (const entry of entries) {
             const ext = entry.split('.').pop()?.toLowerCase() ?? '';

package/dist/loaders/text-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"text-loader.js","sourceRoot":"","sources":["../../src/loaders/text-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;AAEjD,MAAM,OAAO,UAAU;IACZ,cAAc,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAE1C,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;QAElC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,QAAgB;QACrC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAClD,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,MAAM;SACnB,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAe;QACzC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;QACvC,MAAM,IAAI,GAAkB,EAAE,CAAC;QAE/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YACxD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAExC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClC,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;gBAClB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}
+{"version":3,"file":"text-loader.js","sourceRoot":"","sources":["../../src/loaders/text-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;AAEjD,MAAM,OAAO,UAAU;IACZ,cAAc,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAE1C,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;QAElC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,QAAgB;QACrC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAClD,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,MAAM;SACnB,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAe;QACzC,MAAM,OAAO,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,MAAM,IAAI,GAAkB,EAAE,CAAC;QAE/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YACxD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAExC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClC,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;gBAClB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}

package/dist/loaders/web-loader.d.ts

@@ -1,18 +1,46 @@
+import { type LookupAddress, type LookupOptions } from 'node:dns';
 import type { DocumentLoader, RAGDocument } from '@cogitator-ai/types';
+/**
+ * Whether an IP address points to a loopback, private, link-local, CGNAT, multicast or
+ * otherwise non-public network (including IPv4-mapped / NAT64 IPv6 forms).
+ */
+export declare function isBlockedAddress(address: string): boolean;
+type LookupCallback = (err: NodeJS.ErrnoException | null, address: string | LookupAddress[], family?: number) => void;
+type Resolver = (hostname: string, callback: (err: NodeJS.ErrnoException | null, addresses: LookupAddress[]) => void) => void;
+/**
+ * DNS lookup for `http.request` that rejects private addresses at connect time,
+ * which also defeats DNS-rebinding between validation and connection.
+ */
+export declare function createGuardedLookup(resolver?: Resolver): (hostname: string, options: LookupOptions, callback: LookupCallback) => void;
 export interface WebLoaderOptions {
     selector?: string;
     headers?: Record<string, string>;
     timeoutMs?: number;
     maxResponseBytes?: number;
+    /**
+     * Allow fetching loopback / private-network hosts (e.g. an intranet wiki).
+     * Disabled by default to prevent SSRF when URLs come from untrusted input.
+     */
+    allowPrivateNetwork?: boolean;
 }
+/**
+ * Loads a web page over http(s). HTML is converted to text (scripts and styles removed);
+ * plain-text and JSON responses are kept as-is. Redirects are followed (max 5) and every
+ * hop — including DNS resolution at connect time — is checked against private networks.
+ */
 export declare class WebLoader implements DocumentLoader {
     readonly supportedTypes: string[];
     private readonly htmlLoader;
     private readonly headers?;
     private readonly timeoutMs;
     private readonly maxResponseBytes;
+    private readonly allowPrivateNetwork;
+    private readonly lookup;
     constructor(options?: WebLoaderOptions);
     load(source: string): Promise<RAGDocument[]>;
-    private readBodyWithLimit;
+    private fetchPage;
+    private request;
+    private readBody;
 }
+export {};
 //# sourceMappingURL=web-loader.d.ts.map

package/dist/loaders/web-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"web-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/web-loader.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAoEvE,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAKD,qBAAa,SAAU,YAAW,cAAc;IAC9C,QAAQ,CAAC,cAAc,WAAqB;IAC5C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAyB;IAClD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;gBAE9B,OAAO,CAAC,EAAE,gBAAgB;IAOhC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YAqCpC,iBAAiB;CA0ChC"}
+{"version":3,"file":"web-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/web-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAuB,KAAK,aAAa,EAAE,KAAK,aAAa,EAAE,MAAM,UAAU,CAAC;AAMvF,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAqDvE;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAUzD;AAED,KAAK,cAAc,GAAG,CACpB,GAAG,EAAE,MAAM,CAAC,cAAc,GAAG,IAAI,EACjC,OAAO,EAAE,MAAM,GAAG,aAAa,EAAE,EACjC,MAAM,CAAC,EAAE,MAAM,KACZ,IAAI,CAAC;AAEV,KAAK,QAAQ,GAAG,CACd,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,CAAC,GAAG,EAAE,MAAM,CAAC,cAAc,GAAG,IAAI,EAAE,SAAS,EAAE,aAAa,EAAE,KAAK,IAAI,KAC9E,IAAI,CAAC;AAMV;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,GAAE,QAAyB,IAC7D,UAAU,MAAM,EAAE,SAAS,aAAa,EAAE,UAAU,cAAc,KAAG,IAAI,CAsClF;AA2DD,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;OAGG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC/B;AAQD;;;;GAIG;AACH,qBAAa,SAAU,YAAW,cAAc;IAC9C,QAAQ,CAAC,cAAc,WAAqB;IAC5C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAyB;IAClD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAU;IAC9C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAyB;gBAEpC,OAAO,CAAC,EAAE,gBAAgB;IAQhC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YA0BpC,SAAS;IAkCvB,OAAO,CAAC,OAAO;IA4Bf,OAAO,CAAC,QAAQ;CA4BjB"}