@cogitator-ai/rag 0.1.9 → 0.2.0

package/dist/loaders/text-loader.js

@@ -22,7 +22,7 @@ export class TextLoader {
         };
     }
     async loadDirectory(dirPath) {
-        const entries = await readdir(dirPath);
+        const entries = (await readdir(dirPath)).sort();
         const docs = [];
         for (const entry of entries) {
             const ext = entry.split('.').pop()?.toLowerCase() ?? '';

package/dist/loaders/text-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"text-loader.js","sourceRoot":"","sources":["../../src/loaders/text-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;AAEjD,MAAM,OAAO,UAAU;IACZ,cAAc,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAE1C,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;QAElC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,QAAgB;QACrC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAClD,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,MAAM;SACnB,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAe;QACzC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;QACvC,MAAM,IAAI,GAAkB,EAAE,CAAC;QAE/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YACxD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAExC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClC,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;gBAClB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}
+{"version":3,"file":"text-loader.js","sourceRoot":"","sources":["../../src/loaders/text-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;AAEjD,MAAM,OAAO,UAAU;IACZ,cAAc,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAE1C,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;QAElC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,QAAgB;QACrC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAClD,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,MAAM;SACnB,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAe;QACzC,MAAM,OAAO,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,MAAM,IAAI,GAAkB,EAAE,CAAC;QAE/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YACxD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAExC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClC,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;gBAClB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}

package/dist/loaders/web-loader.d.ts

@@ -1,13 +1,46 @@
+import { type LookupAddress, type LookupOptions } from 'node:dns';
 import type { DocumentLoader, RAGDocument } from '@cogitator-ai/types';
+/**
+ * Whether an IP address points to a loopback, private, link-local, CGNAT, multicast or
+ * otherwise non-public network (including IPv4-mapped / NAT64 IPv6 forms).
+ */
+export declare function isBlockedAddress(address: string): boolean;
+type LookupCallback = (err: NodeJS.ErrnoException | null, address: string | LookupAddress[], family?: number) => void;
+type Resolver = (hostname: string, callback: (err: NodeJS.ErrnoException | null, addresses: LookupAddress[]) => void) => void;
+/**
+ * DNS lookup for `http.request` that rejects private addresses at connect time,
+ * which also defeats DNS-rebinding between validation and connection.
+ */
+export declare function createGuardedLookup(resolver?: Resolver): (hostname: string, options: LookupOptions, callback: LookupCallback) => void;
 export interface WebLoaderOptions {
     selector?: string;
     headers?: Record<string, string>;
+    timeoutMs?: number;
+    maxResponseBytes?: number;
+    /**
+     * Allow fetching loopback / private-network hosts (e.g. an intranet wiki).
+     * Disabled by default to prevent SSRF when URLs come from untrusted input.
+     */
+    allowPrivateNetwork?: boolean;
 }
+/**
+ * Loads a web page over http(s). HTML is converted to text (scripts and styles removed);
+ * plain-text and JSON responses are kept as-is. Redirects are followed (max 5) and every
+ * hop — including DNS resolution at connect time — is checked against private networks.
+ */
 export declare class WebLoader implements DocumentLoader {
     readonly supportedTypes: string[];
     private readonly htmlLoader;
     private readonly headers?;
+    private readonly timeoutMs;
+    private readonly maxResponseBytes;
+    private readonly allowPrivateNetwork;
+    private readonly lookup;
     constructor(options?: WebLoaderOptions);
     load(source: string): Promise<RAGDocument[]>;
+    private fetchPage;
+    private request;
+    private readBody;
 }
+export {};
 //# sourceMappingURL=web-loader.d.ts.map

package/dist/loaders/web-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"web-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/web-loader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAGvE,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAED,qBAAa,SAAU,YAAW,cAAc;IAC9C,QAAQ,CAAC,cAAc,WAAqB;IAC5C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAyB;gBAEtC,OAAO,CAAC,EAAE,gBAAgB;IAKhC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;CAcnD"}
+{"version":3,"file":"web-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/web-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAuB,KAAK,aAAa,EAAE,KAAK,aAAa,EAAE,MAAM,UAAU,CAAC;AAMvF,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAqDvE;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAUzD;AAED,KAAK,cAAc,GAAG,CACpB,GAAG,EAAE,MAAM,CAAC,cAAc,GAAG,IAAI,EACjC,OAAO,EAAE,MAAM,GAAG,aAAa,EAAE,EACjC,MAAM,CAAC,EAAE,MAAM,KACZ,IAAI,CAAC;AAEV,KAAK,QAAQ,GAAG,CACd,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,CAAC,GAAG,EAAE,MAAM,CAAC,cAAc,GAAG,IAAI,EAAE,SAAS,EAAE,aAAa,EAAE,KAAK,IAAI,KAC9E,IAAI,CAAC;AAMV;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,GAAE,QAAyB,IAC7D,UAAU,MAAM,EAAE,SAAS,aAAa,EAAE,UAAU,cAAc,KAAG,IAAI,CAsClF;AA2DD,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;OAGG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC/B;AAQD;;;;GAIG;AACH,qBAAa,SAAU,YAAW,cAAc;IAC9C,QAAQ,CAAC,cAAc,WAAqB;IAC5C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAyB;IAClD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAU;IAC9C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAyB;gBAEpC,OAAO,CAAC,EAAE,gBAAgB;IAQhC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YA0BpC,SAAS;IAkCvB,OAAO,CAAC,OAAO;IA4Bf,OAAO,CAAC,QAAQ;CA4BjB"}

package/dist/loaders/web-loader.js

@@ -1,22 +1,276 @@
+import { lookup as dnsLookup } from 'node:dns';
+import http from 'node:http';
+import https from 'node:https';
+import { BlockList, isIP } from 'node:net';
+import { createBrotliDecompress, createGunzip, createInflate } from 'node:zlib';
+import { nanoid } from 'nanoid';
 import { HTMLLoader } from './html-loader.js';
+const MAX_REDIRECTS = 5;
+const DEFAULT_TIMEOUT_MS = 30_000;
+const DEFAULT_MAX_RESPONSE_BYTES = 50 * 1024 * 1024;
+const BLOCKED_HOSTNAMES = new Set(['localhost', 'metadata.google.internal', 'metadata']);
+const PRIVATE_RANGES = new BlockList();
+for (const [network, prefix] of [
+    ['0.0.0.0', 8],
+    ['10.0.0.0', 8],
+    ['100.64.0.0', 10],
+    ['127.0.0.0', 8],
+    ['169.254.0.0', 16],
+    ['172.16.0.0', 12],
+    ['192.0.0.0', 24],
+    ['192.168.0.0', 16],
+    ['198.18.0.0', 15],
+    ['224.0.0.0', 4],
+    ['240.0.0.0', 4],
+]) {
+    PRIVATE_RANGES.addSubnet(network, prefix, 'ipv4');
+}
+for (const [network, prefix] of [
+    ['::', 128],
+    ['::1', 128],
+    ['fc00::', 7],
+    ['fe80::', 10],
+    ['ff00::', 8],
+]) {
+    PRIVATE_RANGES.addSubnet(network, prefix, 'ipv6');
+}
+function embeddedIPv4(ipv6) {
+    const lower = ipv6.toLowerCase();
+    const prefixes = ['::ffff:', '64:ff9b::', '::'];
+    for (const prefix of prefixes) {
+        if (!lower.startsWith(prefix))
+            continue;
+        const rest = lower.slice(prefix.length);
+        if (isIP(rest) === 4)
+            return rest;
+        const hex = /^([0-9a-f]{1,4}):([0-9a-f]{1,4})$/.exec(rest);
+        if (hex) {
+            const high = parseInt(hex[1], 16);
+            const low = parseInt(hex[2], 16);
+            return [high >> 8, high & 0xff, low >> 8, low & 0xff].join('.');
+        }
+    }
+    return null;
+}
+/**
+ * Whether an IP address points to a loopback, private, link-local, CGNAT, multicast or
+ * otherwise non-public network (including IPv4-mapped / NAT64 IPv6 forms).
+ */
+export function isBlockedAddress(address) {
+    const ip = address.replace(/^\[|\]$/g, '');
+    const family = isIP(ip);
+    if (family === 4)
+        return PRIVATE_RANGES.check(ip, 'ipv4');
+    if (family === 6) {
+        const v4 = embeddedIPv4(ip);
+        if (v4)
+            return PRIVATE_RANGES.check(v4, 'ipv4');
+        return PRIVATE_RANGES.check(ip, 'ipv6');
+    }
+    return false;
+}
+const systemResolver = (hostname, callback) => {
+    dnsLookup(hostname, { all: true, verbatim: true }, callback);
+};
+/**
+ * DNS lookup for `http.request` that rejects private addresses at connect time,
+ * which also defeats DNS-rebinding between validation and connection.
+ */
+export function createGuardedLookup(resolver = systemResolver) {
+    return (hostname, options, callback) => {
+        resolver(hostname, (err, addresses) => {
+            if (err) {
+                callback(err, []);
+                return;
+            }
+            const candidates = addresses.filter((entry) => !options.family || entry.family === options.family);
+            const blocked = candidates.find((entry) => isBlockedAddress(entry.address));
+            if (blocked) {
+                callback(Object.assign(new Error(`WebLoader: "${hostname}" resolves to private IP ${blocked.address} — blocked`), { code: 'EBLOCKED' }), []);
+                return;
+            }
+            if (candidates.length === 0) {
+                callback(Object.assign(new Error(`WebLoader: no addresses found for "${hostname}"`), {
+                    code: 'ENOTFOUND',
+                }), []);
+                return;
+            }
+            if (options.all) {
+                callback(null, candidates);
+            }
+            else {
+                callback(null, candidates[0].address, candidates[0].family);
+            }
+        });
+    };
+}
+function parseUrl(urlString, allowPrivateNetwork) {
+    let url;
+    try {
+        url = new URL(urlString);
+    }
+    catch {
+        throw new Error(`WebLoader: invalid URL: ${urlString}`);
+    }
+    if (url.protocol !== 'http:' && url.protocol !== 'https:') {
+        throw new Error(`WebLoader: unsupported protocol "${url.protocol}" — only http and https are allowed`);
+    }
+    if (allowPrivateNetwork)
+        return url;
+    const hostname = url.hostname
+        .toLowerCase()
+        .replace(/^\[|\]$/g, '')
+        .replace(/\.$/, '');
+    if (BLOCKED_HOSTNAMES.has(hostname) || hostname.endsWith('.localhost')) {
+        throw new Error(`WebLoader: access to "${hostname}" is blocked (loopback / metadata endpoint)`);
+    }
+    if (isIP(hostname) && isBlockedAddress(hostname)) {
+        throw new Error(`WebLoader: access to private IP "${hostname}" is blocked`);
+    }
+    return url;
+}
+function decodeBody(body, contentType) {
+    const charset = /charset\s*=\s*"?([\w.:-]+)"?/i.exec(contentType)?.[1];
+    if (charset) {
+        try {
+            return new TextDecoder(charset).decode(body);
+        }
+        catch {
+            return new TextDecoder('utf-8').decode(body);
+        }
+    }
+    return new TextDecoder('utf-8').decode(body);
+}
+function decompress(response) {
+    const encoding = (response.headers['content-encoding'] ?? '').toLowerCase().trim();
+    switch (encoding) {
+        case 'gzip':
+        case 'x-gzip':
+            return response.pipe(createGunzip());
+        case 'deflate':
+            return response.pipe(createInflate());
+        case 'br':
+            return response.pipe(createBrotliDecompress());
+        default:
+            return response;
+    }
+}
+/**
+ * Loads a web page over http(s). HTML is converted to text (scripts and styles removed);
+ * plain-text and JSON responses are kept as-is. Redirects are followed (max 5) and every
+ * hop — including DNS resolution at connect time — is checked against private networks.
+ */
 export class WebLoader {
     supportedTypes = ['http', 'https'];
     htmlLoader;
     headers;
+    timeoutMs;
+    maxResponseBytes;
+    allowPrivateNetwork;
+    lookup = createGuardedLookup();
     constructor(options) {
         this.htmlLoader = new HTMLLoader({ selector: options?.selector });
         this.headers = options?.headers;
+        this.timeoutMs = options?.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+        this.maxResponseBytes = options?.maxResponseBytes ?? DEFAULT_MAX_RESPONSE_BYTES;
+        this.allowPrivateNetwork = options?.allowPrivateNetwork ?? false;
     }
     async load(source) {
-        const response = await fetch(source, {
-            headers: this.headers ?? {},
-        });
-        if (!response.ok) {
-            throw new Error(`Failed to fetch ${source}: ${response.status} ${response.statusText}`);
+        const page = await this.fetchPage(source);
+        const mime = page.contentType.split(';')[0].trim().toLowerCase();
+        if (mime === '' || mime.includes('html') || mime.endsWith('/xml') || mime.endsWith('+xml')) {
+            const doc = await this.htmlLoader.parseHTML(page.body, source, 'web');
+            return [
+                { ...doc, metadata: { ...doc.metadata, url: page.url, contentType: mime || 'text/html' } },
+            ];
         }
-        const html = await response.text();
-        const doc = await this.htmlLoader.parseHTML(html, source, 'web');
-        return [doc];
+        if (mime.startsWith('text/') || mime === 'application/json' || mime.endsWith('+json')) {
+            return [
+                {
+                    id: nanoid(),
+                    content: page.body,
+                    source,
+                    sourceType: 'web',
+                    metadata: { url: page.url, contentType: mime },
+                },
+            ];
+        }
+        throw new Error(`WebLoader: unsupported content type "${mime}" from ${page.url}`);
+    }
+    async fetchPage(source) {
+        let current = parseUrl(source, this.allowPrivateNetwork);
+        for (let redirects = 0; redirects <= MAX_REDIRECTS; redirects++) {
+            const response = await this.request(current);
+            const status = response.statusCode ?? 0;
+            if (status >= 300 && status < 400) {
+                response.resume();
+                const location = response.headers.location;
+                if (!location) {
+                    throw new Error(`WebLoader: redirect ${status} without Location header from ${current.href}`);
+                }
+                current = parseUrl(new URL(location, current).href, this.allowPrivateNetwork);
+                continue;
+            }
+            if (status < 200 || status >= 300) {
+                response.resume();
+                throw new Error(`WebLoader: failed to fetch ${current.href}: ${status} ${response.statusMessage ?? ''}`.trim());
+            }
+            const contentType = response.headers['content-type'] ?? '';
+            const body = await this.readBody(response, current.href);
+            return { url: current.href, contentType, body: decodeBody(body, contentType) };
+        }
+        throw new Error(`WebLoader: too many redirects (max ${MAX_REDIRECTS}) for ${source}`);
+    }
+    request(url) {
+        const transport = url.protocol === 'https:' ? https : http;
+        return new Promise((resolve, reject) => {
+            const req = transport.get(url, {
+                headers: {
+                    'user-agent': 'cogitator-rag-webloader',
+                    accept: 'text/html,application/xhtml+xml,text/plain;q=0.9,*/*;q=0.5',
+                    'accept-encoding': 'gzip, deflate, br',
+                    ...this.headers,
+                },
+                signal: AbortSignal.timeout(this.timeoutMs),
+                ...(!this.allowPrivateNetwork && { lookup: this.lookup }),
+            });
+            req.once('response', resolve);
+            req.once('error', (err) => {
+                if (err.message.startsWith('WebLoader:')) {
+                    reject(err);
+                }
+                else if (err.name === 'AbortError' || err.code === 'ABORT_ERR') {
+                    reject(new Error(`WebLoader: request to ${url.href} timed out after ${this.timeoutMs}ms`));
+                }
+                else {
+                    reject(new Error(`WebLoader: request to ${url.href} failed: ${err.message}`));
+                }
+            });
+        });
+    }
+    readBody(response, url) {
+        const stream = decompress(response);
+        return new Promise((resolve, reject) => {
+            const chunks = [];
+            let total = 0;
+            stream.on('data', (chunk) => {
+                total += chunk.length;
+                if (total > this.maxResponseBytes) {
+                    response.destroy();
+                    stream.destroy();
+                    reject(new Error(`WebLoader: response from ${url} exceeds ${this.maxResponseBytes} bytes limit`));
+                    return;
+                }
+                chunks.push(chunk);
+            });
+            stream.once('end', () => resolve(Buffer.concat(chunks)));
+            stream.once('error', (err) => {
+                reject(new Error(`WebLoader: failed to read response from ${url}: ${err.message}`));
+            });
+            response.once('error', (err) => {
+                reject(new Error(`WebLoader: failed to read response from ${url}: ${err.message}`));
+            });
+        });
     }
 }
 //# sourceMappingURL=web-loader.js.map

package/dist/loaders/web-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"web-loader.js","sourceRoot":"","sources":["../../src/loaders/web-loader.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAO3C,MAAM,OAAO,SAAS;IACX,cAAc,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC3B,UAAU,CAAa;IACvB,OAAO,CAA0B;IAElD,YAAY,OAA0B;QACpC,IAAI,CAAC,UAAU,GAAG,IAAI,UAAU,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QAClE,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE;YACnC,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,EAAE;SAC5B,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,mBAAmB,MAAM,KAAK,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QAC1F,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;QAEjE,OAAO,CAAC,GAAG,CAAC,CAAC;IACf,CAAC;CACF"}
+{"version":3,"file":"web-loader.js","sourceRoot":"","sources":["../../src/loaders/web-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,SAAS,EAA0C,MAAM,UAAU,CAAC;AACvF,OAAO,IAA8B,MAAM,WAAW,CAAC;AACvD,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAE3C,OAAO,EAAE,sBAAsB,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAEhF,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,MAAM,aAAa,GAAG,CAAC,CAAC;AACxB,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,0BAA0B,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAEpD,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,0BAA0B,EAAE,UAAU,CAAC,CAAC,CAAC;AAEzF,MAAM,cAAc,GAAG,IAAI,SAAS,EAAE,CAAC;AACvC,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI;IAC9B,CAAC,SAAS,EAAE,CAAC,CAAC;IACd,CAAC,UAAU,EAAE,CAAC,CAAC;IACf,CAAC,YAAY,EAAE,EAAE,CAAC;IAClB,CAAC,WAAW,EAAE,CAAC,CAAC;IAChB,CAAC,aAAa,EAAE,EAAE,CAAC;IACnB,CAAC,YAAY,EAAE,EAAE,CAAC;IAClB,CAAC,WAAW,EAAE,EAAE,CAAC;IACjB,CAAC,aAAa,EAAE,EAAE,CAAC;IACnB,CAAC,YAAY,EAAE,EAAE,CAAC;IAClB,CAAC,WAAW,EAAE,CAAC,CAAC;IAChB,CAAC,WAAW,EAAE,CAAC,CAAC;CACR,EAAE,CAAC;IACX,cAAc,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AACpD,CAAC;AACD,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI;IAC9B,CAAC,IAAI,EAAE,GAAG,CAAC;IACX,CAAC,KAAK,EAAE,GAAG,CAAC;IACZ,CAAC,QAAQ,EAAE,CAAC,CAAC;IACb,CAAC,QAAQ,EAAE,EAAE,CAAC;IACd,CAAC,QAAQ,EAAE,CAAC,CAAC;CACL,EAAE,CAAC;IACX,cAAc,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AACpD,CAAC;AAED,SAAS,YAAY,CAAC,IAAY;IAChC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC;IAChD,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC;YAAE,SAAS;QACxC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAClC,MAAM,GAAG,GAAG,mCAAmC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3D,IAAI,GAAG,EAAE,CAAC;YACR,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAE,EAAE,EAAE,CAAC,CAAC;YACnC,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAE,EAAE,EAAE,CAAC,CAAC;YAClC,OAAO,CAAC,IAAI,IAAI,CAAC,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAe;IAC9C,MAAM,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IACxB,IAAI,MAAM,KAAK,CAAC;QAAE,OAAO,cAAc,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAC1D,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACjB,MAAM,EAAE,GAAG,YAAY,CAAC,EAAE,CAAC,CAAC;QAC5B,IAAI,EAAE;YAAE,OAAO,cAAc,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QAChD,OAAO,cAAc,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAaD,MAAM,cAAc,GAAa,CAAC,QAAQ,EAAE,QAAQ,EAAE,EAAE;IACtD,SAAS,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,QAAQ,CAAC,CAAC;AAC/D,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,WAAqB,cAAc;IACrE,OAAO,CAAC,QAAgB,EAAE,OAAsB,EAAE,QAAwB,EAAQ,EAAE;QAClF,QAAQ,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE;YACpC,IAAI,GAAG,EAAE,CAAC;gBACR,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;gBAClB,OAAO;YACT,CAAC;YACD,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CACjC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,CAC9D,CAAC;YACF,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YAC5E,IAAI,OAAO,EAAE,CAAC;gBACZ,QAAQ,CACN,MAAM,CAAC,MAAM,CACX,IAAI,KAAK,CACP,eAAe,QAAQ,4BAA4B,OAAO,CAAC,OAAO,YAAY,CAC/E,EACD,EAAE,IAAI,EAAE,UAAU,EAAE,CACrB,EACD,EAAE,CACH,CAAC;gBACF,OAAO;YACT,CAAC;YACD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC5B,QAAQ,CACN,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,sCAAsC,QAAQ,GAAG,CAAC,EAAE;oBAC1E,IAAI,EAAE,WAAW;iBAClB,CAAC,EACF,EAAE,CACH,CAAC;gBACF,OAAO;YACT,CAAC;YACD,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAE,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAC;YAChE,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,SAAiB,EAAE,mBAA4B;IAC/D,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,2BAA2B,SAAS,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CACb,oCAAoC,GAAG,CAAC,QAAQ,qCAAqC,CACtF,CAAC;IACJ,CAAC;IAED,IAAI,mBAAmB;QAAE,OAAO,GAAG,CAAC;IAEpC,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ;SAC1B,WAAW,EAAE;SACb,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACtB,IAAI,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;QACvE,MAAM,IAAI,KAAK,CAAC,yBAAyB,QAAQ,6CAA6C,CAAC,CAAC;IAClG,CAAC;IACD,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,gBAAgB,CAAC,QAAQ,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CAAC,oCAAoC,QAAQ,cAAc,CAAC,CAAC;IAC9E,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,UAAU,CAAC,IAAY,EAAE,WAAmB;IACnD,MAAM,OAAO,GAAG,+BAA+B,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACvE,IAAI,OAAO,EAAE,CAAC;QACZ,IAAI,CAAC;YACH,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC/C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IACD,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,UAAU,CAAC,QAAyB;IAC3C,MAAM,QAAQ,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;IACnF,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,MAAM,CAAC;QACZ,KAAK,QAAQ;YACX,OAAO,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;QACvC,KAAK,SAAS;YACZ,OAAO,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;QACxC,KAAK,IAAI;YACP,OAAO,QAAQ,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC,CAAC;QACjD;YACE,OAAO,QAAQ,CAAC;IACpB,CAAC;AACH,CAAC;AAoBD;;;;GAIG;AACH,MAAM,OAAO,SAAS;IACX,cAAc,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC3B,UAAU,CAAa;IACvB,OAAO,CAA0B;IACjC,SAAS,CAAS;IAClB,gBAAgB,CAAS;IACzB,mBAAmB,CAAU;IAC7B,MAAM,GAAG,mBAAmB,EAAE,CAAC;IAEhD,YAAY,OAA0B;QACpC,IAAI,CAAC,UAAU,GAAG,IAAI,UAAU,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QAClE,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;QAChC,IAAI,CAAC,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,kBAAkB,CAAC;QAC1D,IAAI,CAAC,gBAAgB,GAAG,OAAO,EAAE,gBAAgB,IAAI,0BAA0B,CAAC;QAChF,IAAI,CAAC,mBAAmB,GAAG,OAAO,EAAE,mBAAmB,IAAI,KAAK,CAAC;IACnE,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAElE,IAAI,IAAI,KAAK,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3F,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;YACtE,OAAO;gBACL,EAAE,GAAG,GAAG,EAAE,QAAQ,EAAE,EAAE,GAAG,GAAG,CAAC,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,IAAI,WAAW,EAAE,EAAE;aAC3F,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,IAAI,KAAK,kBAAkB,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YACtF,OAAO;gBACL;oBACE,EAAE,EAAE,MAAM,EAAE;oBACZ,OAAO,EAAE,IAAI,CAAC,IAAI;oBAClB,MAAM;oBACN,UAAU,EAAE,KAAK;oBACjB,QAAQ,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE;iBAC/C;aACF,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,UAAU,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACpF,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,MAAc;QACpC,IAAI,OAAO,GAAG,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAEzD,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,IAAI,aAAa,EAAE,SAAS,EAAE,EAAE,CAAC;YAChE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAC7C,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,IAAI,CAAC,CAAC;YAExC,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,EAAE,CAAC;gBAClC,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAClB,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC;gBAC3C,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,MAAM,IAAI,KAAK,CACb,uBAAuB,MAAM,iCAAiC,OAAO,CAAC,IAAI,EAAE,CAC7E,CAAC;gBACJ,CAAC;gBACD,OAAO,GAAG,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;gBAC9E,SAAS;YACX,CAAC;YAED,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;gBAClC,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CACb,8BAA8B,OAAO,CAAC,IAAI,KAAK,MAAM,IAAI,QAAQ,CAAC,aAAa,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,CAC/F,CAAC;YACJ,CAAC;YAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAC3D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;YACzD,OAAO,EAAE,GAAG,EAAE,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,EAAE,CAAC;QACjF,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,sCAAsC,aAAa,SAAS,MAAM,EAAE,CAAC,CAAC;IACxF,CAAC;IAEO,OAAO,CAAC,GAAQ;QACtB,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;QAC3D,OAAO,IAAI,OAAO,CAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACtD,MAAM,GAAG,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE;gBAC7B,OAAO,EAAE;oBACP,YAAY,EAAE,yBAAyB;oBACvC,MAAM,EAAE,4DAA4D;oBACpE,iBAAiB,EAAE,mBAAmB;oBACtC,GAAG,IAAI,CAAC,OAAO;iBAChB;gBACD,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC;gBAC3C,GAAG,CAAC,CAAC,IAAI,CAAC,mBAAmB,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;aAC1D,CAAC,CAAC;YACH,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YAC9B,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,GAA8B,EAAE,EAAE;gBACnD,IAAI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;oBACzC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACd,CAAC;qBAAM,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;oBACjE,MAAM,CACJ,IAAI,KAAK,CAAC,yBAAyB,GAAG,CAAC,IAAI,oBAAoB,IAAI,CAAC,SAAS,IAAI,CAAC,CACnF,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,KAAK,CAAC,yBAAyB,GAAG,CAAC,IAAI,YAAY,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;gBAChF,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,QAAQ,CAAC,QAAyB,EAAE,GAAW;QACrD,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QACpC,OAAO,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC7C,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;gBAClC,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;gBACtB,IAAI,KAAK,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;oBAClC,QAAQ,CAAC,OAAO,EAAE,CAAC;oBACnB,MAAM,CAAC,OAAO,EAAE,CAAC;oBACjB,MAAM,CACJ,IAAI,KAAK,CACP,4BAA4B,GAAG,YAAY,IAAI,CAAC,gBAAgB,cAAc,CAC/E,CACF,CAAC;oBACF,OAAO;gBACT,CAAC;gBACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC,CAAC,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACzD,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE;gBAClC,MAAM,CAAC,IAAI,KAAK,CAAC,2CAA2C,GAAG,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACtF,CAAC,CAAC,CAAC;YACH,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE;gBACpC,MAAM,CAAC,IAAI,KAAK,CAAC,2CAA2C,GAAG,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACtF,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}

package/dist/rag-builder.d.ts

@@ -1,4 +1,5 @@
-import type { DocumentLoader, Chunker, AsyncChunker, EmbeddingService, EmbeddingAdapter, Retriever, Reranker } from '@cogitator-ai/types';
+import type { DocumentLoader, Chunker, AsyncChunker, EmbeddingService, EmbeddingAdapter, Retriever, Reranker, HybridSearchWeights } from '@cogitator-ai/types';
+import type { HybridSearch } from '@cogitator-ai/memory';
 import { type RAGPipelineConfigInput } from './schema.js';
 import { RAGPipeline } from './rag-pipeline.js';
 export declare class RAGPipelineBuilder {
@@ -9,14 +10,22 @@ export declare class RAGPipelineBuilder {
     private retriever?;
     private reranker?;
     private configInput?;
+    private hybridSearch?;
+    private hybridWeights?;
+    private queryExpander?;
     withLoader(loader: DocumentLoader): this;
     withChunker(chunker: Chunker | AsyncChunker): this;
     withEmbeddingService(service: EmbeddingService): this;
     withEmbeddingAdapter(adapter: EmbeddingAdapter): this;
     withRetriever(retriever: Retriever): this;
     withReranker(reranker: Reranker): this;
+    /** HybridSearch used by the `hybrid` strategy (and as the multi-query base when set). */
+    withHybridSearch(hybridSearch: HybridSearch, weights?: HybridSearchWeights): this;
+    /** Query expansion function used by the `multi-query` strategy. */
+    withQueryExpander(expandQuery: (query: string) => Promise<string[]>): this;
     withConfig(config: RAGPipelineConfigInput): this;
     build(): RAGPipeline;
+    private createRetriever;
     private resolveConfig;
 }
 //# sourceMappingURL=rag-builder.d.ts.map

package/dist/rag-builder.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"rag-builder.d.ts","sourceRoot":"","sources":["../src/rag-builder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,cAAc,EACd,OAAO,EACP,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAChB,SAAS,EACT,QAAQ,EAET,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAA2B,KAAK,sBAAsB,EAAE,MAAM,UAAU,CAAC;AAChF,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAI7C,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,MAAM,CAAC,CAAiB;IAChC,OAAO,CAAC,OAAO,CAAC,CAAyB;IACzC,OAAO,CAAC,gBAAgB,CAAC,CAAmB;IAC5C,OAAO,CAAC,gBAAgB,CAAC,CAAmB;IAC5C,OAAO,CAAC,SAAS,CAAC,CAAY;IAC9B,OAAO,CAAC,QAAQ,CAAC,CAAW;IAC5B,OAAO,CAAC,WAAW,CAAC,CAAyB;IAE7C,UAAU,CAAC,MAAM,EAAE,cAAc,GAAG,IAAI;IAKxC,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,YAAY,GAAG,IAAI;IAKlD,oBAAoB,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAKrD,oBAAoB,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAKrD,aAAa,CAAC,SAAS,EAAE,SAAS,GAAG,IAAI;IAKzC,YAAY,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI;IAKtC,UAAU,CAAC,MAAM,EAAE,sBAAsB,GAAG,IAAI;IAKhD,KAAK,IAAI,WAAW;IA8BpB,OAAO,CAAC,aAAa;CAMtB"}
+{"version":3,"file":"rag-builder.d.ts","sourceRoot":"","sources":["../src/rag-builder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,cAAc,EACd,OAAO,EACP,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAChB,SAAS,EACT,QAAQ,EAER,mBAAmB,EACpB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAA2B,KAAK,sBAAsB,EAAE,MAAM,aAAa,CAAC;AACnF,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAOhD,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,MAAM,CAAC,CAAiB;IAChC,OAAO,CAAC,OAAO,CAAC,CAAyB;IACzC,OAAO,CAAC,gBAAgB,CAAC,CAAmB;IAC5C,OAAO,CAAC,gBAAgB,CAAC,CAAmB;IAC5C,OAAO,CAAC,SAAS,CAAC,CAAY;IAC9B,OAAO,CAAC,QAAQ,CAAC,CAAW;IAC5B,OAAO,CAAC,WAAW,CAAC,CAAyB;IAC7C,OAAO,CAAC,YAAY,CAAC,CAAe;IACpC,OAAO,CAAC,aAAa,CAAC,CAAsB;IAC5C,OAAO,CAAC,aAAa,CAAC,CAAuC;IAE7D,UAAU,CAAC,MAAM,EAAE,cAAc,GAAG,IAAI;IAKxC,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,YAAY,GAAG,IAAI;IAKlD,oBAAoB,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAKrD,oBAAoB,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAKrD,aAAa,CAAC,SAAS,EAAE,SAAS,GAAG,IAAI;IAKzC,YAAY,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI;IAKtC,yFAAyF;IACzF,gBAAgB,CAAC,YAAY,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,IAAI;IAMjF,mEAAmE;IACnE,iBAAiB,CAAC,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,IAAI;IAK1E,UAAU,CAAC,MAAM,EAAE,sBAAsB,GAAG,IAAI;IAKhD,KAAK,IAAI,WAAW;IA8BpB,OAAO,CAAC,eAAe;IAmEvB,OAAO,CAAC,aAAa;CAMtB"}

package/dist/rag-builder.js

@@ -2,6 +2,9 @@ import { RAGPipelineConfigSchema } from './schema.js';
 import { RAGPipeline } from './rag-pipeline.js';
 import { createChunker } from './chunkers/create-chunker.js';
 import { SimilarityRetriever } from './retrievers/similarity-retriever.js';
+import { MMRRetriever } from './retrievers/mmr-retriever.js';
+import { HybridRetriever } from './retrievers/hybrid-retriever.js';
+import { MultiQueryRetriever } from './retrievers/multi-query-retriever.js';
 export class RAGPipelineBuilder {
     loader;
     chunker;
@@ -10,6 +13,9 @@ export class RAGPipelineBuilder {
     retriever;
     reranker;
     configInput;
+    hybridSearch;
+    hybridWeights;
+    queryExpander;
     withLoader(loader) {
         this.loader = loader;
         return this;
@@ -34,6 +40,17 @@ export class RAGPipelineBuilder {
         this.reranker = reranker;
         return this;
     }
+    /** HybridSearch used by the `hybrid` strategy (and as the multi-query base when set). */
+    withHybridSearch(hybridSearch, weights) {
+        this.hybridSearch = hybridSearch;
+        this.hybridWeights = weights;
+        return this;
+    }
+    /** Query expansion function used by the `multi-query` strategy. */
+    withQueryExpander(expandQuery) {
+        this.queryExpander = expandQuery;
+        return this;
+    }
     withConfig(config) {
         this.configInput = config;
         return this;
@@ -50,11 +67,7 @@ export class RAGPipelineBuilder {
         }
         const config = this.resolveConfig();
         const chunker = this.chunker ?? createChunker(config.chunking, this.embeddingService);
-        const retriever = this.retriever ??
-            new SimilarityRetriever({
-                embeddingAdapter: this.embeddingAdapter,
-                embeddingService: this.embeddingService,
-            });
+        const retriever = this.retriever ?? this.createRetriever(config, this.embeddingService, this.embeddingAdapter);
         return new RAGPipeline(config, {
             loader: this.loader,
             chunker,
@@ -62,7 +75,59 @@ export class RAGPipelineBuilder {
             embeddingAdapter: this.embeddingAdapter,
             retriever,
             reranker: this.reranker,
-        });
+        }, true);
+    }
+    createRetriever(config, embeddingService, embeddingAdapter) {
+        const { retrieval } = config;
+        const vectorRetriever = () => this.hybridSearch
+            ? new HybridRetriever({
+                hybridSearch: this.hybridSearch,
+                defaultWeights: this.hybridWeights,
+                defaultTopK: retrieval.topK,
+                defaultThreshold: retrieval.threshold,
+            })
+            : new SimilarityRetriever({
+                embeddingAdapter,
+                embeddingService,
+                defaultTopK: retrieval.topK,
+                defaultThreshold: retrieval.threshold,
+            });
+        switch (retrieval.strategy) {
+            case 'similarity':
+                return new SimilarityRetriever({
+                    embeddingAdapter,
+                    embeddingService,
+                    defaultTopK: retrieval.topK,
+                    defaultThreshold: retrieval.threshold,
+                });
+            case 'mmr':
+                return new MMRRetriever({
+                    embeddingAdapter,
+                    embeddingService,
+                    defaultLambda: retrieval.mmrLambda,
+                    defaultTopK: retrieval.topK,
+                    defaultThreshold: retrieval.threshold,
+                });
+            case 'hybrid':
+                if (!this.hybridSearch) {
+                    throw new Error('retrieval strategy "hybrid" requires withHybridSearch() (or a custom withRetriever())');
+                }
+                return vectorRetriever();
+            case 'multi-query':
+                if (!this.queryExpander) {
+                    throw new Error('retrieval strategy "multi-query" requires withQueryExpander() (or a custom withRetriever())');
+                }
+                return new MultiQueryRetriever({
+                    baseRetriever: vectorRetriever(),
+                    expandQuery: this.queryExpander,
+                    defaultTopK: retrieval.topK,
+                    defaultMaxQueries: retrieval.multiQueryCount,
+                });
+            default: {
+                const exhaustive = retrieval.strategy;
+                throw new Error(`Unknown retrieval strategy: ${String(exhaustive)}`);
+            }
+        }
     }
     resolveConfig() {
         if (!this.configInput) {

package/dist/rag-builder.js.map

@@ -1 +1 @@
-{"version":3,"file":"rag-builder.js","sourceRoot":"","sources":["../src/rag-builder.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,uBAAuB,EAA+B,MAAM,UAAU,CAAC;AAChF,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AAExE,MAAM,OAAO,kBAAkB;IACrB,MAAM,CAAkB;IACxB,OAAO,CAA0B;IACjC,gBAAgB,CAAoB;IACpC,gBAAgB,CAAoB;IACpC,SAAS,CAAa;IACtB,QAAQ,CAAY;IACpB,WAAW,CAA0B;IAE7C,UAAU,CAAC,MAAsB;QAC/B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,WAAW,CAAC,OAA+B;QACzC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,oBAAoB,CAAC,OAAyB;QAC5C,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC;QAChC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,oBAAoB,CAAC,OAAyB;QAC5C,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC;QAChC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,aAAa,CAAC,SAAoB;QAChC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,YAAY,CAAC,QAAkB;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,UAAU,CAAC,MAA8B;QACvC,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;QAC3E,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;QAC/F,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;QAC/F,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACtF,MAAM,SAAS,GACb,IAAI,CAAC,SAAS;YACd,IAAI,mBAAmB,CAAC;gBACtB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;gBACvC,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;aACxC,CAAC,CAAC;QAEL,OAAO,IAAI,WAAW,CAAC,MAAM,EAAE;YAC7B,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO;YACP,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,SAAS;YACT,QAAQ,EAAE,IAAI,CAAC,QAAQ;SACxB,CAAC,CAAC;IACL,CAAC;IAEO,aAAa;QACnB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;QAC3E,CAAC;QACD,OAAO,uBAAuB,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACzD,CAAC;CACF"}
+{"version":3,"file":"rag-builder.js","sourceRoot":"","sources":["../src/rag-builder.ts"],"names":[],"mappings":"AAYA,OAAO,EAAE,uBAAuB,EAA+B,MAAM,aAAa,CAAC;AACnF,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAE,mBAAmB,EAAE,MAAM,sCAAsC,CAAC;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAC7D,OAAO,EAAE,eAAe,EAAE,MAAM,kCAAkC,CAAC;AACnE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AAE5E,MAAM,OAAO,kBAAkB;IACrB,MAAM,CAAkB;IACxB,OAAO,CAA0B;IACjC,gBAAgB,CAAoB;IACpC,gBAAgB,CAAoB;IACpC,SAAS,CAAa;IACtB,QAAQ,CAAY;IACpB,WAAW,CAA0B;IACrC,YAAY,CAAgB;IAC5B,aAAa,CAAuB;IACpC,aAAa,CAAwC;IAE7D,UAAU,CAAC,MAAsB;QAC/B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,WAAW,CAAC,OAA+B;QACzC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,oBAAoB,CAAC,OAAyB;QAC5C,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC;QAChC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,oBAAoB,CAAC,OAAyB;QAC5C,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC;QAChC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,aAAa,CAAC,SAAoB;QAChC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,YAAY,CAAC,QAAkB;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,yFAAyF;IACzF,gBAAgB,CAAC,YAA0B,EAAE,OAA6B;QACxE,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,mEAAmE;IACnE,iBAAiB,CAAC,WAAiD;QACjE,IAAI,CAAC,aAAa,GAAG,WAAW,CAAC;QACjC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,UAAU,CAAC,MAA8B;QACvC,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;QAC3E,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;QAC/F,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;QAC/F,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACtF,MAAM,SAAS,GACb,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,IAAI,CAAC,gBAAgB,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAE/F,OAAO,IAAI,WAAW,CACpB,MAAM,EACN;YACE,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO;YACP,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,SAAS;YACT,QAAQ,EAAE,IAAI,CAAC,QAAQ;SACxB,EACD,IAAI,CACL,CAAC;IACJ,CAAC;IAEO,eAAe,CACrB,MAAyB,EACzB,gBAAkC,EAClC,gBAAkC;QAElC,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC;QAC7B,MAAM,eAAe,GAAG,GAAc,EAAE,CACtC,IAAI,CAAC,YAAY;YACf,CAAC,CAAC,IAAI,eAAe,CAAC;gBAClB,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,cAAc,EAAE,IAAI,CAAC,aAAa;gBAClC,WAAW,EAAE,SAAS,CAAC,IAAI;gBAC3B,gBAAgB,EAAE,SAAS,CAAC,SAAS;aACtC,CAAC;YACJ,CAAC,CAAC,IAAI,mBAAmB,CAAC;gBACtB,gBAAgB;gBAChB,gBAAgB;gBAChB,WAAW,EAAE,SAAS,CAAC,IAAI;gBAC3B,gBAAgB,EAAE,SAAS,CAAC,SAAS;aACtC,CAAC,CAAC;QAET,QAAQ,SAAS,CAAC,QAAQ,EAAE,CAAC;YAC3B,KAAK,YAAY;gBACf,OAAO,IAAI,mBAAmB,CAAC;oBAC7B,gBAAgB;oBAChB,gBAAgB;oBAChB,WAAW,EAAE,SAAS,CAAC,IAAI;oBAC3B,gBAAgB,EAAE,SAAS,CAAC,SAAS;iBACtC,CAAC,CAAC;YAEL,KAAK,KAAK;gBACR,OAAO,IAAI,YAAY,CAAC;oBACtB,gBAAgB;oBAChB,gBAAgB;oBAChB,aAAa,EAAE,SAAS,CAAC,SAAS;oBAClC,WAAW,EAAE,SAAS,CAAC,IAAI;oBAC3B,gBAAgB,EAAE,SAAS,CAAC,SAAS;iBACtC,CAAC,CAAC;YAEL,KAAK,QAAQ;gBACX,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;oBACvB,MAAM,IAAI,KAAK,CACb,uFAAuF,CACxF,CAAC;gBACJ,CAAC;gBACD,OAAO,eAAe,EAAE,CAAC;YAE3B,KAAK,aAAa;gBAChB,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;oBACxB,MAAM,IAAI,KAAK,CACb,6FAA6F,CAC9F,CAAC;gBACJ,CAAC;gBACD,OAAO,IAAI,mBAAmB,CAAC;oBAC7B,aAAa,EAAE,eAAe,EAAE;oBAChC,WAAW,EAAE,IAAI,CAAC,aAAa;oBAC/B,WAAW,EAAE,SAAS,CAAC,IAAI;oBAC3B,iBAAiB,EAAE,SAAS,CAAC,eAAe;iBAC7C,CAAC,CAAC;YAEL,OAAO,CAAC,CAAC,CAAC;gBACR,MAAM,UAAU,GAAU,SAAS,CAAC,QAAQ,CAAC;gBAC7C,MAAM,IAAI,KAAK,CAAC,+BAA+B,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YACvE,CAAC;QACH,CAAC;IACH,CAAC;IAEO,aAAa;QACnB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;QAC3E,CAAC;QACD,OAAO,uBAAuB,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACzD,CAAC;CACF"}

package/dist/rag-pipeline.d.ts

@@ -11,7 +11,7 @@ export declare class RAGPipeline {
     private readonly config;
     private readonly deps;
     private stats;
-    constructor(config: RAGPipelineConfig, deps: RAGPipelineDeps);
+    constructor(config: RAGPipelineConfig, deps: RAGPipelineDeps, alreadyParsed?: boolean);
     ingest(source: string): Promise<{
         documents: number;
         chunks: number;
@@ -23,6 +23,7 @@ export declare class RAGPipeline {
         queriesProcessed: number;
     };
     private chunkDocument;
+    private storeChunks;
     private storeChunk;
 }
 //# sourceMappingURL=rag-pipeline.d.ts.map

package/dist/rag-pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"rag-pipeline.d.ts","sourceRoot":"","sources":["../src/rag-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,cAAc,EACd,OAAO,EACP,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAChB,SAAS,EACT,QAAQ,EACR,iBAAiB,EACjB,eAAe,EACf,eAAe,EAGhB,MAAM,qBAAqB,CAAC;AAG7B,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,cAAc,CAAC;IACvB,OAAO,EAAE,OAAO,GAAG,YAAY,CAAC;IAChC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,SAAS,EAAE,SAAS,CAAC;IACrB,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACrB;AAED,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoB;IAC3C,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAkB;IACvC,OAAO,CAAC,KAAK,CAAkE;gBAEnE,MAAM,EAAE,iBAAiB,EAAE,IAAI,EAAE,eAAe;IAKtD,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IA8BtE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAWzF,QAAQ;;;;;YAIM,aAAa;YAIb,UAAU;CAkBzB"}
+{"version":3,"file":"rag-pipeline.d.ts","sourceRoot":"","sources":["../src/rag-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,cAAc,EACd,OAAO,EACP,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAChB,SAAS,EACT,QAAQ,EACR,iBAAiB,EACjB,eAAe,EACf,eAAe,EAGhB,MAAM,qBAAqB,CAAC;AAI7B,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,cAAc,CAAC;IACvB,OAAO,EAAE,OAAO,GAAG,YAAY,CAAC;IAChC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,SAAS,EAAE,SAAS,CAAC;IACrB,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACrB;AASD,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoB;IAC3C,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAkB;IACvC,OAAO,CAAC,KAAK,CAAkE;gBAEnE,MAAM,EAAE,iBAAiB,EAAE,IAAI,EAAE,eAAe,EAAE,aAAa,UAAQ;IAK7E,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IA2BtE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAezF,QAAQ;;;;;YAIM,aAAa;YAKb,WAAW;YAiBX,UAAU;CAkCzB"}