@hazeljs/rag 0.2.0-alpha.1

package/dist/loaders/text-file.loader.js

@@ -0,0 +1,69 @@
+"use strict";
+/**
+ * TextFileLoader
+ *
+ * Loads one or more plain-text files (.txt, .log, .tsv, etc.) from disk.
+ * Each file becomes a single `Document`.
+ *
+ * @example
+ * ```typescript
+ * const loader = new TextFileLoader({ path: './docs/notes.txt' });
+ * const docs = await loader.load();
+ * // docs[0].content === "...file contents..."
+ * // docs[0].metadata.source === "notes.txt"
+ * ```
+ *
+ * Multiple files:
+ * ```typescript
+ * const loader = new TextFileLoader({ paths: ['./a.txt', './b.txt'] });
+ * ```
+ */
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TextFileLoader = void 0;
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+const base_loader_1 = require("./base.loader");
+let TextFileLoader = class TextFileLoader extends base_loader_1.BaseDocumentLoader {
+    constructor(options) {
+        super();
+        if (!options.path && (!options.paths || options.paths.length === 0)) {
+            throw new Error('TextFileLoader: provide at least one file path via `path` or `paths`.');
+        }
+        this.paths = options.paths ?? (options.path ? [options.path] : []);
+        this.encoding = options.encoding ?? 'utf-8';
+        this.extraMetadata = options.metadata ?? {};
+    }
+    async load() {
+        const results = [];
+        for (const filePath of this.paths) {
+            const content = await (0, promises_1.readFile)(filePath, { encoding: this.encoding });
+            results.push(this.createDocument(content, {
+                source: (0, path_1.basename)(filePath),
+                filePath,
+                loaderType: 'text',
+                ...this.extraMetadata,
+            }));
+        }
+        return results;
+    }
+};
+exports.TextFileLoader = TextFileLoader;
+exports.TextFileLoader = TextFileLoader = __decorate([
+    (0, base_loader_1.Loader)({
+        name: 'TextFileLoader',
+        description: 'Loads plain-text files from the local filesystem.',
+        extensions: ['.txt', '.text', '.log', '.tsv', '.nfo'],
+        mimeTypes: ['text/plain'],
+    }),
+    __metadata("design:paramtypes", [Object])
+], TextFileLoader);
+//# sourceMappingURL=text-file.loader.js.map

package/dist/loaders/text-file.loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"text-file.loader.js","sourceRoot":"","sources":["../../src/loaders/text-file.loader.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;GAkBG;;;;;;;;;;;;AAEH,0CAAuC;AACvC,+BAAgC;AAChC,+CAA2D;AAoBpD,IAAM,cAAc,GAApB,MAAM,cAAe,SAAQ,gCAAkB;IAKpD,YAAY,OAA8B;QACxC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;YACpE,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAC;QAC3F,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACnE,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC;QAC5C,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,OAAO,GAAe,EAAE,CAAC;QAE/B,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAClC,MAAM,OAAO,GAAG,MAAM,IAAA,mBAAQ,EAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;YACtE,OAAO,CAAC,IAAI,CACV,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE;gBAC3B,MAAM,EAAE,IAAA,eAAQ,EAAC,QAAQ,CAAC;gBAC1B,QAAQ;gBACR,UAAU,EAAE,MAAM;gBAClB,GAAG,IAAI,CAAC,aAAa;aACtB,CAAC,CACH,CAAC;QACJ,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF,CAAA;AAhCY,wCAAc;yBAAd,cAAc;IAN1B,IAAA,oBAAM,EAAC;QACN,IAAI,EAAE,gBAAgB;QACtB,WAAW,EAAE,mDAAmD;QAChE,UAAU,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;QACrD,SAAS,EAAE,CAAC,YAAY,CAAC;KAC1B,CAAC;;GACW,cAAc,CAgC1B"}

package/dist/loaders/web.loader.d.ts

@@ -0,0 +1,87 @@
+/**
+ * WebLoader
+ *
+ * Scrapes one or more URLs and converts the page content to plain text
+ * (or optionally HTML) suitable for RAG indexing.
+ *
+ * Two operation modes:
+ *  - **built-in** (default) — uses Node.js `fetch` + a lightweight HTML
+ *    stripper.  No extra dependencies.
+ *  - **cheerio** mode — uses the optional `cheerio` peer dependency for
+ *    CSS-selector-based content extraction (much more precise).
+ *
+ * Install cheerio for the selector mode:
+ * ```bash
+ * npm install cheerio
+ * ```
+ *
+ * @example
+ * ```typescript
+ * // Scrape a single page (no extra deps)
+ * const loader = new WebLoader({ url: 'https://hazeljs.com/docs' });
+ * const docs = await loader.load();
+ *
+ * // Scrape multiple URLs
+ * const loader = new WebLoader({
+ *   urls: ['https://example.com/a', 'https://example.com/b'],
+ * });
+ *
+ * // Extract only the <article> element (requires cheerio)
+ * const loader = new WebLoader({
+ *   url: 'https://blog.example.com/post',
+ *   selector: 'article',
+ * });
+ * ```
+ */
+import { BaseDocumentLoader } from './base.loader';
+import type { Document } from '../types';
+export interface WebLoaderOptions {
+    /** Single URL to scrape. */
+    url?: string;
+    /** Multiple URLs — loaded in parallel (max `concurrency`). */
+    urls?: string[];
+    /**
+     * CSS selector to extract content from.
+     * Requires the optional `cheerio` peer dependency.
+     * @example 'article', 'main', '.post-content', '#readme'
+     */
+    selector?: string;
+    /**
+     * Maximum number of concurrent fetches.
+     * @default 3
+     */
+    concurrency?: number;
+    /**
+     * Request timeout in milliseconds.
+     * @default 15000
+     */
+    timeout?: number;
+    /**
+     * Additional headers sent with every request.
+     * @example { 'User-Agent': 'HazelJSBot/1.0' }
+     */
+    headers?: Record<string, string>;
+    /**
+     * Retry failed requests this many times before giving up.
+     * @default 2
+     */
+    retries?: number;
+    /** Extra metadata merged into every document. */
+    metadata?: Record<string, unknown>;
+}
+export declare class WebLoader extends BaseDocumentLoader {
+    private readonly urls;
+    private readonly selector?;
+    private readonly concurrency;
+    private readonly timeout;
+    private readonly headers;
+    private readonly retries;
+    private readonly extraMetadata;
+    constructor(options: WebLoaderOptions);
+    load(): Promise<Document[]>;
+    private scrape;
+    private fetchWithRetry;
+    private extractTitle;
+    private stripTags;
+}
+//# sourceMappingURL=web.loader.d.ts.map

package/dist/loaders/web.loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"web.loader.d.ts","sourceRoot":"","sources":["../../src/loaders/web.loader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,OAAO,EAAE,kBAAkB,EAAU,MAAM,eAAe,CAAC;AAC3D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEzC,MAAM,WAAW,gBAAgB;IAC/B,4BAA4B;IAC5B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,8DAA8D;IAC9D,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,qBAKa,SAAU,SAAQ,kBAAkB;IAC/C,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAW;IAChC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAyB;IACjD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA0B;gBAE5C,OAAO,EAAE,gBAAgB;IAiB/B,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YAuBnB,MAAM;YAgDN,cAAc;IA0C5B,OAAO,CAAC,YAAY;IAKpB,OAAO,CAAC,SAAS;CAoBlB"}

package/dist/loaders/web.loader.js

@@ -0,0 +1,194 @@
+"use strict";
+/**
+ * WebLoader
+ *
+ * Scrapes one or more URLs and converts the page content to plain text
+ * (or optionally HTML) suitable for RAG indexing.
+ *
+ * Two operation modes:
+ *  - **built-in** (default) — uses Node.js `fetch` + a lightweight HTML
+ *    stripper.  No extra dependencies.
+ *  - **cheerio** mode — uses the optional `cheerio` peer dependency for
+ *    CSS-selector-based content extraction (much more precise).
+ *
+ * Install cheerio for the selector mode:
+ * ```bash
+ * npm install cheerio
+ * ```
+ *
+ * @example
+ * ```typescript
+ * // Scrape a single page (no extra deps)
+ * const loader = new WebLoader({ url: 'https://hazeljs.com/docs' });
+ * const docs = await loader.load();
+ *
+ * // Scrape multiple URLs
+ * const loader = new WebLoader({
+ *   urls: ['https://example.com/a', 'https://example.com/b'],
+ * });
+ *
+ * // Extract only the <article> element (requires cheerio)
+ * const loader = new WebLoader({
+ *   url: 'https://blog.example.com/post',
+ *   selector: 'article',
+ * });
+ * ```
+ */
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WebLoader = void 0;
+const base_loader_1 = require("./base.loader");
+let WebLoader = class WebLoader extends base_loader_1.BaseDocumentLoader {
+    constructor(options) {
+        super();
+        if (!options.url && (!options.urls || options.urls.length === 0)) {
+            throw new Error('WebLoader: provide at least one URL via `url` or `urls`.');
+        }
+        this.urls = options.urls ?? (options.url ? [options.url] : []);
+        this.selector = options.selector;
+        this.concurrency = options.concurrency ?? 3;
+        this.timeout = options.timeout ?? 15000;
+        this.headers = {
+            'User-Agent': 'Mozilla/5.0 (compatible; HazelJSBot/1.0)',
+            ...options.headers,
+        };
+        this.retries = options.retries ?? 2;
+        this.extraMetadata = options.metadata ?? {};
+    }
+    async load() {
+        const allDocs = [];
+        // Process in chunks of `concurrency`
+        for (let i = 0; i < this.urls.length; i += this.concurrency) {
+            const batch = this.urls.slice(i, i + this.concurrency);
+            const results = await Promise.allSettled(batch.map((url) => this.scrape(url)));
+            for (const result of results) {
+                if (result.status === 'fulfilled' && result.value) {
+                    allDocs.push(result.value);
+                }
+                else if (result.status === 'rejected') {
+                    // eslint-disable-next-line no-console
+                    console.warn('[WebLoader] Failed to scrape URL:', result.reason);
+                }
+            }
+        }
+        return allDocs;
+    }
+    // ── Private helpers ──────────────────────────────────────────────────────
+    async scrape(url) {
+        const html = await this.fetchWithRetry(url);
+        if (!html)
+            return null;
+        let text;
+        let title = '';
+        if (this.selector) {
+            try {
+                // eslint-disable-next-line @typescript-eslint/no-require-imports
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-require-imports
+                const cheerio = require('cheerio');
+                const $ = cheerio.load(html);
+                title = $('title').first().text().trim();
+                text = $(this.selector).text().trim();
+                if (!text) {
+                    // eslint-disable-next-line no-console
+                    console.warn(`[WebLoader] Selector "${this.selector}" found no content at ${url}. Falling back to full page.`);
+                    text = this.stripTags(html);
+                }
+            }
+            catch {
+                // cheerio not installed
+                if (this.selector) {
+                    throw new Error(`[WebLoader] The \`selector\` option requires the optional peer dependency "cheerio". ` +
+                        `Run: npm install cheerio\n` +
+                        `Or remove the \`selector\` option to use built-in HTML stripping.`);
+                }
+                text = this.stripTags(html);
+            }
+        }
+        else {
+            title = this.extractTitle(html);
+            text = this.stripTags(html);
+        }
+        return this.createDocument(text, {
+            source: url,
+            url,
+            loaderType: 'web',
+            ...(title && { title }),
+            scrapedAt: new Date().toISOString(),
+            ...this.extraMetadata,
+        });
+    }
+    async fetchWithRetry(url) {
+        let lastError;
+        for (let attempt = 0; attempt <= this.retries; attempt++) {
+            try {
+                const controller = new AbortController();
+                const timer = setTimeout(() => controller.abort(), this.timeout);
+                const response = await fetch(url, {
+                    headers: this.headers,
+                    signal: controller.signal,
+                });
+                clearTimeout(timer);
+                if (!response.ok) {
+                    throw new Error(`HTTP ${response.status} ${response.statusText}`);
+                }
+                const contentType = response.headers.get('content-type') ?? '';
+                if (!contentType.includes('text/')) {
+                    // eslint-disable-next-line no-console
+                    console.warn(`[WebLoader] Skipping non-text content at ${url}: ${contentType}`);
+                    return null;
+                }
+                return await response.text();
+            }
+            catch (err) {
+                lastError = err;
+                if (attempt < this.retries) {
+                    // Exponential backoff: 500ms, 1000ms, 2000ms
+                    await new Promise((r) => setTimeout(r, 500 * Math.pow(2, attempt)));
+                }
+            }
+        }
+        throw new Error(`[WebLoader] Failed to fetch ${url} after ${this.retries + 1} attempts: ${lastError instanceof Error ? lastError.message : String(lastError)}`);
+    }
+    extractTitle(html) {
+        const match = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+        return match ? match[1].trim() : '';
+    }
+    stripTags(html) {
+        return html
+            .replace(/<script[\s\S]*?<\/script>/gi, ' ')
+            .replace(/<style[\s\S]*?<\/style>/gi, ' ')
+            .replace(/<nav[\s\S]*?<\/nav>/gi, ' ')
+            .replace(/<header[\s\S]*?<\/header>/gi, ' ')
+            .replace(/<footer[\s\S]*?<\/footer>/gi, ' ')
+            .replace(/<!--[\s\S]*?-->/g, ' ')
+            .replace(/<\/(p|div|h[1-6]|li|tr|br)[^>]*>/gi, '\n')
+            .replace(/<[^>]+>/g, ' ')
+            .replace(/&amp;/g, '&')
+            .replace(/&lt;/g, '<')
+            .replace(/&gt;/g, '>')
+            .replace(/&quot;/g, '"')
+            .replace(/&nbsp;/g, ' ')
+            .replace(/&#(\d+);/g, (_, code) => String.fromCharCode(parseInt(code, 10)))
+            .replace(/[ \t]+/g, ' ')
+            .replace(/\n{3,}/g, '\n\n')
+            .trim();
+    }
+};
+exports.WebLoader = WebLoader;
+exports.WebLoader = WebLoader = __decorate([
+    (0, base_loader_1.Loader)({
+        name: 'WebLoader',
+        description: 'Scrapes web pages and converts them to plain text documents.',
+        mimeTypes: ['text/html'],
+    }),
+    __metadata("design:paramtypes", [Object])
+], WebLoader);
+//# sourceMappingURL=web.loader.js.map

package/dist/loaders/web.loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"web.loader.js","sourceRoot":"","sources":["../../src/loaders/web.loader.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;;;;;;;;;;;;AAEH,+CAA2D;AA2CpD,IAAM,SAAS,GAAf,MAAM,SAAU,SAAQ,gCAAkB;IAS/C,YAAY,OAAyB;QACnC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;QAC9E,CAAC;QACD,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC/D,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QACjC,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;QAC5C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,KAAM,CAAC;QACzC,IAAI,CAAC,OAAO,GAAG;YACb,YAAY,EAAE,0CAA0C;YACxD,GAAG,OAAO,CAAC,OAAO;SACnB,CAAC;QACF,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;QACpC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,OAAO,GAAe,EAAE,CAAC;QAE/B,qCAAqC;QACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YAC5D,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC;YACvD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAE/E,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;oBAClD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC7B,CAAC;qBAAM,IAAI,MAAM,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;oBACxC,sCAAsC;oBACtC,OAAO,CAAC,IAAI,CAAC,mCAAmC,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;gBACnE,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,4EAA4E;IAEpE,KAAK,CAAC,MAAM,CAAC,GAAW;QAC9B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC5C,IAAI,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC;QAEvB,IAAI,IAAY,CAAC;QACjB,IAAI,KAAK,GAAG,EAAE,CAAC;QAEf,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,IAAI,CAAC;gBACH,iEAAiE;gBACjE,qGAAqG;gBACrG,MAAM,OAAO,GAAG,OAAO,CAAC,SAAS,CAAQ,CAAC;gBAC1C,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC7B,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBACzC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBACtC,IAAI,CAAC,IAAI,EAAE,CAAC;oBACV,sCAAsC;oBACtC,OAAO,CAAC,IAAI,CACV,yBAAyB,IAAI,CAAC,QAAQ,yBAAyB,GAAG,8BAA8B,CACjG,CAAC;oBACF,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;gBAC9B,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,wBAAwB;gBACxB,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAClB,MAAM,IAAI,KAAK,CACb,uFAAuF;wBACrF,4BAA4B;wBAC5B,mEAAmE,CACtE,CAAC;gBACJ,CAAC;gBACD,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;aAAM,CAAC;YACN,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAChC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;QAED,OAAO,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE;YAC/B,MAAM,EAAE,GAAG;YACX,GAAG;YACH,UAAU,EAAE,KAAK;YACjB,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;YACvB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,GAAG,IAAI,CAAC,aAAa;SACtB,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,GAAW;QACtC,IAAI,SAAkB,CAAC;QAEvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YACzD,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;gBACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;gBAEjE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;oBAChC,OAAO,EAAE,IAAI,CAAC,OAAO;oBACrB,MAAM,EAAE,UAAU,CAAC,MAAM;iBAC1B,CAAC,CAAC;gBACH,YAAY,CAAC,KAAK,CAAC,CAAC;gBAEpB,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;gBACpE,CAAC;gBAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;gBAC/D,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;oBACnC,sCAAsC;oBACtC,OAAO,CAAC,IAAI,CAAC,4CAA4C,GAAG,KAAK,WAAW,EAAE,CAAC,CAAC;oBAChF,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAC/B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,SAAS,GAAG,GAAG,CAAC;gBAChB,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;oBAC3B,6CAA6C;oBAC7C,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;gBACtE,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,IAAI,KAAK,CACb,+BAA+B,GAAG,UAAU,IAAI,CAAC,OAAO,GAAG,CAAC,cAC1D,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CACnE,EAAE,CACH,CAAC;IACJ,CAAC;IAEO,YAAY,CAAC,IAAY;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;QAC7D,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACtC,CAAC;IAEO,SAAS,CAAC,IAAY;QAC5B,OAAO,IAAI;aACR,OAAO,CAAC,6BAA6B,EAAE,GAAG,CAAC;aAC3C,OAAO,CAAC,2BAA2B,EAAE,GAAG,CAAC;aACzC,OAAO,CAAC,uBAAuB,EAAE,GAAG,CAAC;aACrC,OAAO,CAAC,6BAA6B,EAAE,GAAG,CAAC;aAC3C,OAAO,CAAC,6BAA6B,EAAE,GAAG,CAAC;aAC3C,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC;aAChC,OAAO,CAAC,oCAAoC,EAAE,IAAI,CAAC;aACnD,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;aACxB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;aACtB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;aACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;aACrB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aACvB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aACvB,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;aAC1E,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aACvB,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;aAC1B,IAAI,EAAE,CAAC;IACZ,CAAC;CACF,CAAA;AApKY,8BAAS;oBAAT,SAAS;IALrB,IAAA,oBAAM,EAAC;QACN,IAAI,EAAE,WAAW;QACjB,WAAW,EAAE,8DAA8D;QAC3E,SAAS,EAAE,CAAC,WAAW,CAAC;KACzB,CAAC;;GACW,SAAS,CAoKrB"}

package/dist/loaders/youtube-transcript.loader.d.ts

@@ -0,0 +1,92 @@
+/**
+ * YouTubeTranscriptLoader
+ *
+ * Downloads the caption/transcript for one or more YouTube videos and
+ * converts them to `Document` objects — no extra npm dependency required.
+ * Uses Node.js built-in `fetch` and parses the YouTube InnerTube API response.
+ *
+ * How it works:
+ *  1. Fetches the YouTube watch page for the video.
+ *  2. Extracts the `ytInitialPlayerResponse` JSON blob from the page HTML.
+ *  3. Locates the caption tracks list and selects the preferred language.
+ *  4. Fetches the caption XML file.
+ *  5. Parses `<text>` elements and decodes HTML entities.
+ *  6. Returns the transcript as one document (or split into segments).
+ *
+ * Accepts video IDs, full watch URLs, and youtu.be short URLs.
+ *
+ * @example
+ * ```typescript
+ * const loader = new YouTubeTranscriptLoader({
+ *   videoId: 'dQw4w9WgXcQ',
+ * });
+ * const docs = await loader.load();
+ * // docs[0].content === "Never gonna give you up, never gonna let you..."
+ * // docs[0].metadata.title === "Rick Astley - Never Gonna Give You Up"
+ * // docs[0].metadata.videoId === "dQw4w9WgXcQ"
+ * ```
+ *
+ * Multiple videos:
+ * ```typescript
+ * const loader = new YouTubeTranscriptLoader({
+ *   videoIds: ['VIDEO_ID_1', 'VIDEO_ID_2'],
+ * });
+ * ```
+ *
+ * Split into timed segments (useful for timestamped RAG retrieval):
+ * ```typescript
+ * const loader = new YouTubeTranscriptLoader({
+ *   videoId: 'VIDEO_ID',
+ *   segmentDuration: 120, // one document per 2-minute window
+ * });
+ * ```
+ */
+import { BaseDocumentLoader } from './base.loader';
+import type { Document } from '../types';
+export interface YouTubeTranscriptLoaderOptions {
+    /** Single YouTube video ID or URL. */
+    videoId?: string;
+    /** Multiple YouTube video IDs or URLs. */
+    videoIds?: string[];
+    /**
+     * Preferred caption language code.
+     * Falls back to the first available track if not found.
+     * @default 'en'
+     */
+    language?: string;
+    /**
+     * If set, the transcript is split into segments of approximately this
+     * many seconds.  Each segment becomes a separate `Document` with
+     * `metadata.startTime` and `metadata.endTime`.
+     * @default undefined (single document per video)
+     */
+    segmentDuration?: number;
+    /**
+     * Request timeout in milliseconds.
+     * @default 15000
+     */
+    timeout?: number;
+    /** Extra metadata merged into every document. */
+    metadata?: Record<string, unknown>;
+}
+export declare class YouTubeTranscriptLoader extends BaseDocumentLoader {
+    private readonly videoIds;
+    private readonly language;
+    private readonly segmentDuration?;
+    private readonly timeout;
+    private readonly extraMetadata;
+    private static readonly YOUTUBE_BASE;
+    private static readonly RE_PLAYER_RESPONSE;
+    private static readonly RE_CAPTION_TEXT;
+    constructor(options: YouTubeTranscriptLoaderOptions);
+    load(): Promise<Document[]>;
+    private loadVideo;
+    private extractCaptionInfo;
+    private parseCaptionXml;
+    private buildSegments;
+    private normaliseId;
+    private fetch;
+    private decodeEntities;
+    private formatTime;
+}
+//# sourceMappingURL=youtube-transcript.loader.d.ts.map

package/dist/loaders/youtube-transcript.loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"youtube-transcript.loader.d.ts","sourceRoot":"","sources":["../../src/loaders/youtube-transcript.loader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AAEH,OAAO,EAAE,kBAAkB,EAAU,MAAM,eAAe,CAAC;AAC3D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEzC,MAAM,WAAW,8BAA8B;IAC7C,sCAAsC;IACtC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,0CAA0C;IAC1C,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;OAKG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAQD,qBAIa,uBAAwB,SAAQ,kBAAkB;IAC7D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA0B;IAExD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAA6B;IACjE,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,kBAAkB,CAA+C;IACzF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CACkC;gBAE7D,OAAO,EAAE,8BAA8B;IAa7C,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YAmBnB,SAAS;IA+BvB,OAAO,CAAC,kBAAkB;IAqD1B,OAAO,CAAC,eAAe;IAgBvB,OAAO,CAAC,aAAa;IAqDrB,OAAO,CAAC,WAAW;YAYL,KAAK;IAsBnB,OAAO,CAAC,cAAc;IAUtB,OAAO,CAAC,UAAU;CAMnB"}