@hazeljs/rag 0.2.0-beta.59 → 0.2.0-beta.61

package/dist/loaders/youtube-transcript.loader.js

@@ -0,0 +1,254 @@
+"use strict";
+/**
+ * YouTubeTranscriptLoader
+ *
+ * Downloads the caption/transcript for one or more YouTube videos and
+ * converts them to `Document` objects — no extra npm dependency required.
+ * Uses Node.js built-in `fetch` and parses the YouTube InnerTube API response.
+ *
+ * How it works:
+ *  1. Fetches the YouTube watch page for the video.
+ *  2. Extracts the `ytInitialPlayerResponse` JSON blob from the page HTML.
+ *  3. Locates the caption tracks list and selects the preferred language.
+ *  4. Fetches the caption XML file.
+ *  5. Parses `<text>` elements and decodes HTML entities.
+ *  6. Returns the transcript as one document (or split into segments).
+ *
+ * Accepts video IDs, full watch URLs, and youtu.be short URLs.
+ *
+ * @example
+ * ```typescript
+ * const loader = new YouTubeTranscriptLoader({
+ *   videoId: 'dQw4w9WgXcQ',
+ * });
+ * const docs = await loader.load();
+ * // docs[0].content === "Never gonna give you up, never gonna let you..."
+ * // docs[0].metadata.title === "Rick Astley - Never Gonna Give You Up"
+ * // docs[0].metadata.videoId === "dQw4w9WgXcQ"
+ * ```
+ *
+ * Multiple videos:
+ * ```typescript
+ * const loader = new YouTubeTranscriptLoader({
+ *   videoIds: ['VIDEO_ID_1', 'VIDEO_ID_2'],
+ * });
+ * ```
+ *
+ * Split into timed segments (useful for timestamped RAG retrieval):
+ * ```typescript
+ * const loader = new YouTubeTranscriptLoader({
+ *   videoId: 'VIDEO_ID',
+ *   segmentDuration: 120, // one document per 2-minute window
+ * });
+ * ```
+ */
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var YouTubeTranscriptLoader_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.YouTubeTranscriptLoader = void 0;
+const base_loader_1 = require("./base.loader");
+let YouTubeTranscriptLoader = YouTubeTranscriptLoader_1 = class YouTubeTranscriptLoader extends base_loader_1.BaseDocumentLoader {
+    constructor(options) {
+        super();
+        if (!options.videoId && (!options.videoIds || options.videoIds.length === 0)) {
+            throw new Error('YouTubeTranscriptLoader: provide at least one videoId.');
+        }
+        const raw = options.videoIds ?? (options.videoId ? [options.videoId] : []);
+        this.videoIds = raw.map((id) => this.normaliseId(id));
+        this.language = options.language ?? 'en';
+        this.segmentDuration = options.segmentDuration;
+        this.timeout = options.timeout ?? 15000;
+        this.extraMetadata = options.metadata ?? {};
+    }
+    async load() {
+        const allDocs = [];
+        for (const videoId of this.videoIds) {
+            try {
+                const docs = await this.loadVideo(videoId);
+                allDocs.push(...docs);
+            }
+            catch (err) {
+                const message = err instanceof Error ? err.message : String(err);
+                // eslint-disable-next-line no-console
+                console.warn(`[YouTubeTranscriptLoader] Skipping ${videoId}: ${message}`);
+            }
+        }
+        return allDocs;
+    }
+    // ── Private: per-video pipeline ──────────────────────────────────────────
+    async loadVideo(videoId) {
+        const watchUrl = `${YouTubeTranscriptLoader_1.YOUTUBE_BASE}/watch?v=${videoId}`;
+        const html = await this.fetch(watchUrl);
+        const { captionUrl, title, channelName } = this.extractCaptionInfo(html, videoId);
+        const captionXml = await this.fetch(captionUrl);
+        const lines = this.parseCaptionXml(captionXml);
+        if (lines.length === 0) {
+            throw new Error(`No transcript lines found for video ${videoId}`);
+        }
+        const baseMetadata = {
+            source: watchUrl,
+            videoId,
+            loaderType: 'youtube-transcript',
+            language: this.language,
+            ...(title && { title }),
+            ...(channelName && { channelName }),
+            ...this.extraMetadata,
+        };
+        if (!this.segmentDuration) {
+            const fullText = lines.map((l) => l.text).join(' ');
+            return [this.createDocument(fullText, { ...baseMetadata, totalLines: lines.length })];
+        }
+        return this.buildSegments(lines, videoId, baseMetadata);
+    }
+    extractCaptionInfo(html, videoId) {
+        // Extract ytInitialPlayerResponse
+        const match = YouTubeTranscriptLoader_1.RE_PLAYER_RESPONSE.exec(html);
+        if (!match) {
+            throw new Error(`Could not find ytInitialPlayerResponse for video ${videoId}. ` +
+                `The video may be unavailable, age-restricted, or have no captions.`);
+        }
+        // Minimal safe JSON parse — the JSON is often very large
+        let playerResponse;
+        try {
+            playerResponse = JSON.parse(match[1]);
+        }
+        catch {
+            throw new Error(`Failed to parse ytInitialPlayerResponse for video ${videoId}`);
+        }
+        const title = playerResponse.videoDetails?.title;
+        const channelName = playerResponse.videoDetails
+            ?.author;
+        // Navigate: captions → playerCaptionsTracklistRenderer → captionTracks[]
+        const captionTracks = playerResponse.captions?.playerCaptionsTracklistRenderer?.captionTracks;
+        if (!captionTracks || captionTracks.length === 0) {
+            throw new Error(`No caption tracks available for video ${videoId}. ` +
+                `The video may not have captions/subtitles.`);
+        }
+        // Prefer the requested language; fall back to the first available track
+        const track = captionTracks.find((t) => t.languageCode === this.language) ?? captionTracks[0];
+        return {
+            captionUrl: `${track.baseUrl}&fmt=xml`,
+            title,
+            channelName,
+        };
+    }
+    parseCaptionXml(xml) {
+        const lines = [];
+        const re = new RegExp(YouTubeTranscriptLoader_1.RE_CAPTION_TEXT.source, 'g');
+        let match;
+        while ((match = re.exec(xml)) !== null) {
+            const start = parseFloat(match[1]);
+            const duration = parseFloat(match[2]);
+            const raw = match[3];
+            const text = this.decodeEntities(raw.replace(/<[^>]+>/g, '')).trim();
+            if (text)
+                lines.push({ start, duration, text });
+        }
+        return lines;
+    }
+    buildSegments(lines, videoId, baseMetadata) {
+        const segments = [];
+        const duration = this.segmentDuration;
+        let segStart = 0;
+        let buffer = [];
+        let windowEnd = segStart + duration;
+        for (const line of lines) {
+            if (line.start >= windowEnd) {
+                if (buffer.length > 0) {
+                    const text = buffer.join(' ');
+                    const startTime = this.formatTime(segStart);
+                    const endTime = this.formatTime(windowEnd);
+                    segments.push(this.createDocument(text, {
+                        ...baseMetadata,
+                        startTime,
+                        endTime,
+                        startSeconds: segStart,
+                        youtubeUrl: `https://youtu.be/${videoId}?t=${Math.floor(segStart)}`,
+                    }));
+                    buffer = [];
+                }
+                segStart = Math.floor(line.start / duration) * duration;
+                windowEnd = segStart + duration;
+            }
+            buffer.push(line.text);
+        }
+        // Final segment
+        if (buffer.length > 0) {
+            segments.push(this.createDocument(buffer.join(' '), {
+                ...baseMetadata,
+                startTime: this.formatTime(segStart),
+                endTime: this.formatTime(windowEnd),
+                startSeconds: segStart,
+                youtubeUrl: `https://youtu.be/${videoId}?t=${Math.floor(segStart)}`,
+            }));
+        }
+        return segments;
+    }
+    // ── Utilities ────────────────────────────────────────────────────────────
+    normaliseId(input) {
+        // youtu.be/ID
+        const shortMatch = input.match(/youtu\.be\/([a-zA-Z0-9_-]{11})/);
+        if (shortMatch)
+            return shortMatch[1];
+        // youtube.com/watch?v=ID
+        const watchMatch = input.match(/[?&]v=([a-zA-Z0-9_-]{11})/);
+        if (watchMatch)
+            return watchMatch[1];
+        // Already an 11-char ID
+        if (/^[a-zA-Z0-9_-]{11}$/.test(input))
+            return input;
+        throw new Error(`YouTubeTranscriptLoader: cannot parse video ID from "${input}"`);
+    }
+    async fetch(url) {
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), this.timeout);
+        const response = await fetch(url, {
+            headers: {
+                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' +
+                    '(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+                'Accept-Language': 'en-US,en;q=0.9',
+            },
+            signal: controller.signal,
+        });
+        clearTimeout(timer);
+        if (!response.ok) {
+            throw new Error(`HTTP ${response.status} ${response.statusText} fetching ${url}`);
+        }
+        return response.text();
+    }
+    decodeEntities(text) {
+        return text
+            .replace(/&amp;/g, '&')
+            .replace(/&lt;/g, '<')
+            .replace(/&gt;/g, '>')
+            .replace(/&quot;/g, '"')
+            .replace(/&#39;/g, "'")
+            .replace(/&#(\d+);/g, (_, code) => String.fromCharCode(parseInt(code, 10)));
+    }
+    formatTime(seconds) {
+        const h = Math.floor(seconds / 3600);
+        const m = Math.floor((seconds % 3600) / 60);
+        const s = Math.floor(seconds % 60);
+        return [h, m, s].map((v) => String(v).padStart(2, '0')).join(':');
+    }
+};
+exports.YouTubeTranscriptLoader = YouTubeTranscriptLoader;
+YouTubeTranscriptLoader.YOUTUBE_BASE = 'https://www.youtube.com';
+YouTubeTranscriptLoader.RE_PLAYER_RESPONSE = /ytInitialPlayerResponse\s*=\s*({.+?})\s*;/;
+YouTubeTranscriptLoader.RE_CAPTION_TEXT = /<text[^>]*start="([^"]*)"[^>]*dur="([^"]*)"[^>]*>([\s\S]*?)<\/text>/g;
+exports.YouTubeTranscriptLoader = YouTubeTranscriptLoader = YouTubeTranscriptLoader_1 = __decorate([
+    (0, base_loader_1.Loader)({
+        name: 'YouTubeTranscriptLoader',
+        description: 'Downloads YouTube video transcripts/captions — no API key needed.',
+    }),
+    __metadata("design:paramtypes", [Object])
+], YouTubeTranscriptLoader);
+//# sourceMappingURL=youtube-transcript.loader.js.map

package/dist/loaders/youtube-transcript.loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"youtube-transcript.loader.js","sourceRoot":"","sources":["../../src/loaders/youtube-transcript.loader.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;;;;;;;;;;;;;AAEH,+CAA2D;AAwCpD,IAAM,uBAAuB,+BAA7B,MAAM,uBAAwB,SAAQ,gCAAkB;IAY7D,YAAY,OAAuC;QACjD,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;YAC7E,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC5E,CAAC;QACD,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC3E,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC;QACtD,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC;QACzC,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;QAC/C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,KAAM,CAAC;QACzC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,OAAO,GAAe,EAAE,CAAC;QAE/B,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpC,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;gBAC3C,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;YACxB,CAAC;YAAC,OAAO,GAAY,EAAE,CAAC;gBACtB,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACjE,sCAAsC;gBACtC,OAAO,CAAC,IAAI,CAAC,sCAAsC,OAAO,KAAK,OAAO,EAAE,CAAC,CAAC;YAC5E,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,4EAA4E;IAEpE,KAAK,CAAC,SAAS,CAAC,OAAe;QACrC,MAAM,QAAQ,GAAG,GAAG,yBAAuB,CAAC,YAAY,YAAY,OAAO,EAAE,CAAC;QAE9E,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACxC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAElF,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;QAE/C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,uCAAuC,OAAO,EAAE,CAAC,CAAC;QACpE,CAAC;QAED,MAAM,YAAY,GAA4B;YAC5C,MAAM,EAAE,QAAQ;YAChB,OAAO;YACP,UAAU,EAAE,oBAAoB;YAChC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;YACvB,GAAG,CAAC,WAAW,IAAI,EAAE,WAAW,EAAE,CAAC;YACnC,GAAG,IAAI,CAAC,aAAa;SACtB,CAAC;QAEF,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpD,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,EAAE,GAAG,YAAY,EAAE,UAAU,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACxF,CAAC;QAED,OAAO,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC;IAC1D,CAAC;IAEO,kBAAkB,CACxB,IAAY,EACZ,OAAe;QAEf,kCAAkC;QAClC,MAAM,KAAK,GAAG,yBAAuB,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpE,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CACb,oDAAoD,OAAO,IAAI;gBAC7D,oEAAoE,CACvE,CAAC;QACJ,CAAC;QAED,yDAAyD;QACzD,IAAI,cAAuC,CAAC;QAC5C,IAAI,CAAC;YACH,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAA4B,CAAC;QACnE,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,qDAAqD,OAAO,EAAE,CAAC,CAAC;QAClF,CAAC;QAED,MAAM,KAAK,GAAI,cAAwD,CAAC,YAAY,EAAE,KAAK,CAAC;QAC5F,MAAM,WAAW,GAAI,cAAyD,CAAC,YAAY;YACzF,EAAE,MAAM,CAAC;QAEX,yEAAyE;QACzE,MAAM,aAAa,GACjB,cAOD,CAAC,QAAQ,EAAE,+BAA+B,EAAE,aAAa,CAAC;QAE3D,IAAI,CAAC,aAAa,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjD,MAAM,IAAI,KAAK,CACb,yCAAyC,OAAO,IAAI;gBAClD,4CAA4C,CAC/C,CAAC;QACJ,CAAC;QAED,wEAAwE;QACxE,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,IAAI,CAAC,QAAQ,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC,CAAC;QAE9F,OAAO;YACL,UAAU,EAAE,GAAG,KAAK,CAAC,OAAO,UAAU;YACtC,KAAK;YACL,WAAW;SACZ,CAAC;IACJ,CAAC;IAEO,eAAe,CAAC,GAAW;QACjC,MAAM,KAAK,GAAqB,EAAE,CAAC;QACnC,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,yBAAuB,CAAC,eAAe,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC3E,IAAI,KAA6B,CAAC;QAElC,OAAO,CAAC,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACvC,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACnC,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACtC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACrB,MAAM,IAAI,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACrE,IAAI,IAAI;gBAAE,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAClD,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,aAAa,CACnB,KAAuB,EACvB,OAAe,EACf,YAAqC;QAErC,MAAM,QAAQ,GAAe,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAgB,CAAC;QAEvC,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,MAAM,GAAa,EAAE,CAAC;QAC1B,IAAI,SAAS,GAAG,QAAQ,GAAG,QAAQ,CAAC;QAEpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,KAAK,IAAI,SAAS,EAAE,CAAC;gBAC5B,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACtB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;oBAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;oBAC3C,QAAQ,CAAC,IAAI,CACX,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE;wBACxB,GAAG,YAAY;wBACf,SAAS;wBACT,OAAO;wBACP,YAAY,EAAE,QAAQ;wBACtB,UAAU,EAAE,oBAAoB,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE;qBACpE,CAAC,CACH,CAAC;oBACF,MAAM,GAAG,EAAE,CAAC;gBACd,CAAC;gBACD,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAC,GAAG,QAAQ,CAAC;gBACxD,SAAS,GAAG,QAAQ,GAAG,QAAQ,CAAC;YAClC,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzB,CAAC;QAED,gBAAgB;QAChB,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,QAAQ,CAAC,IAAI,CACX,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;gBACpC,GAAG,YAAY;gBACf,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;gBACpC,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;gBACnC,YAAY,EAAE,QAAQ;gBACtB,UAAU,EAAE,oBAAoB,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE;aACpE,CAAC,CACH,CAAC;QACJ,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,4EAA4E;IAEpE,WAAW,CAAC,KAAa;QAC/B,cAAc;QACd,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;QACjE,IAAI,UAAU;YAAE,OAAO,UAAU,CAAC,CAAC,CAAC,CAAC;QACrC,yBAAyB;QACzB,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC5D,IAAI,UAAU;YAAE,OAAO,UAAU,CAAC,CAAC,CAAC,CAAC;QACrC,wBAAwB;QACxB,IAAI,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,wDAAwD,KAAK,GAAG,CAAC,CAAC;IACpF,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,GAAW;QAC7B,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAEjE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAChC,OAAO,EAAE;gBACP,YAAY,EACV,+DAA+D;oBAC/D,oDAAoD;gBACtD,iBAAiB,EAAE,gBAAgB;aACpC;YACD,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;QACH,YAAY,CAAC,KAAK,CAAC,CAAC;QAEpB,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,aAAa,GAAG,EAAE,CAAC,CAAC;QACpF,CAAC;QAED,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAEO,cAAc,CAAC,IAAY;QACjC,OAAO,IAAI;aACR,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;aACtB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;aACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;aACrB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;aACtB,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IAChF,CAAC;IAEO,UAAU,CAAC,OAAe;QAChC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;QACrC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAC5C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;QACnC,OAAO,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACpE,CAAC;;AAtPU,0DAAuB;AAOV,oCAAY,GAAG,yBAAyB,AAA5B,CAA6B;AACzC,0CAAkB,GAAG,2CAA2C,AAA9C,CAA+C;AACjE,uCAAe,GACrC,sEAAsE,AADjC,CACkC;kCAV9D,uBAAuB;IAJnC,IAAA,oBAAM,EAAC;QACN,IAAI,EAAE,yBAAyB;QAC/B,WAAW,EAAE,mEAAmE;KACjF,CAAC;;GACW,uBAAuB,CAuPnC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hazeljs/rag",
-  "version": "0.2.0-beta.59",
+  "version": "0.2.0-beta.61",
   "description": "Retrieval-Augmented Generation (RAG) and vector search for HazelJS framework",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -23,9 +23,12 @@
     "@huggingface/inference": "^2.0.0",
     "@pinecone-database/pinecone": "^3.0.0",
     "@qdrant/js-client-rest": "^1.9.0",
+    "cheerio": "^1.0.0",
     "chromadb": "^1.8.0",
     "cohere-ai": "^7.0.0",
+    "mammoth": "^1.11.0",
     "openai": "^4.0.0",
+    "pdf-parse": "^1.1.1",
     "weaviate-ts-client": "^2.0.0"
   },
   "peerDependenciesMeta": {
@@ -49,6 +52,51 @@
     },
     "@huggingface/inference": {
       "optional": true
+    },
+    "pdf-parse": {
+      "optional": true
+    },
+    "mammoth": {
+      "optional": true
+    },
+    "cheerio": {
+      "optional": true
+    }
+  },
+  "jest": {
+    "preset": "ts-jest",
+    "testEnvironment": "node",
+    "testMatch": [
+      "**/src/__tests__/**/*.test.ts"
+    ],
+    "collectCoverageFrom": [
+      "src/**/*.ts",
+      "!src/**/*.d.ts",
+      "!src/index.ts",
+      "!src/rag.module.ts",
+      "!src/rag.service.ts",
+      "!src/rag-pipeline-with-memory.ts",
+      "!src/vector-stores/**",
+      "!src/embeddings/**",
+      "!src/agentic/**",
+      "!src/decorators/**",
+      "!src/memory/**",
+      "!src/loaders/pdf.loader.ts",
+      "!src/loaders/docx.loader.ts",
+      "!src/loaders/web.loader.ts",
+      "!src/loaders/github.loader.ts",
+      "!src/loaders/youtube-transcript.loader.ts",
+      "!src/loaders/index.ts",
+      "!src/graph/index.ts",
+      "!src/types/**"
+    ],
+    "coverageThreshold": {
+      "global": {
+        "statements": 85,
+        "branches": 75,
+        "functions": 85,
+        "lines": 85
+      }
     }
   },
   "devDependencies": {
@@ -89,5 +137,5 @@
     "url": "https://github.com/hazeljs/hazel-js/issues"
   },
   "homepage": "https://hazeljs.com",
-  "gitHead": "f6d8ee8162a40e2298ccce46d843269838bbe6ff"
+  "gitHead": "2205447dd2f88f83a7748b0ffdee0417be0f3970"
 }