@hazeljs/rag 0.2.0-alpha.1

package/dist/loaders/youtube-transcript.loader.js

@@ -0,0 +1,254 @@
+"use strict";
+/**
+ * YouTubeTranscriptLoader
+ *
+ * Downloads the caption/transcript for one or more YouTube videos and
+ * converts them to `Document` objects — no extra npm dependency required.
+ * Uses Node.js built-in `fetch` and parses the YouTube InnerTube API response.
+ *
+ * How it works:
+ *  1. Fetches the YouTube watch page for the video.
+ *  2. Extracts the `ytInitialPlayerResponse` JSON blob from the page HTML.
+ *  3. Locates the caption tracks list and selects the preferred language.
+ *  4. Fetches the caption XML file.
+ *  5. Parses `<text>` elements and decodes HTML entities.
+ *  6. Returns the transcript as one document (or split into segments).
+ *
+ * Accepts video IDs, full watch URLs, and youtu.be short URLs.
+ *
+ * @example
+ * ```typescript
+ * const loader = new YouTubeTranscriptLoader({
+ *   videoId: 'dQw4w9WgXcQ',
+ * });
+ * const docs = await loader.load();
+ * // docs[0].content === "Never gonna give you up, never gonna let you..."
+ * // docs[0].metadata.title === "Rick Astley - Never Gonna Give You Up"
+ * // docs[0].metadata.videoId === "dQw4w9WgXcQ"
+ * ```
+ *
+ * Multiple videos:
+ * ```typescript
+ * const loader = new YouTubeTranscriptLoader({
+ *   videoIds: ['VIDEO_ID_1', 'VIDEO_ID_2'],
+ * });
+ * ```
+ *
+ * Split into timed segments (useful for timestamped RAG retrieval):
+ * ```typescript
+ * const loader = new YouTubeTranscriptLoader({
+ *   videoId: 'VIDEO_ID',
+ *   segmentDuration: 120, // one document per 2-minute window
+ * });
+ * ```
+ */
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var YouTubeTranscriptLoader_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.YouTubeTranscriptLoader = void 0;
+const base_loader_1 = require("./base.loader");
+let YouTubeTranscriptLoader = YouTubeTranscriptLoader_1 = class YouTubeTranscriptLoader extends base_loader_1.BaseDocumentLoader {
+    constructor(options) {
+        super();
+        if (!options.videoId && (!options.videoIds || options.videoIds.length === 0)) {
+            throw new Error('YouTubeTranscriptLoader: provide at least one videoId.');
+        }
+        const raw = options.videoIds ?? (options.videoId ? [options.videoId] : []);
+        this.videoIds = raw.map((id) => this.normaliseId(id));
+        this.language = options.language ?? 'en';
+        this.segmentDuration = options.segmentDuration;
+        this.timeout = options.timeout ?? 15000;
+        this.extraMetadata = options.metadata ?? {};
+    }
+    async load() {
+        const allDocs = [];
+        for (const videoId of this.videoIds) {
+            try {
+                const docs = await this.loadVideo(videoId);
+                allDocs.push(...docs);
+            }
+            catch (err) {
+                const message = err instanceof Error ? err.message : String(err);
+                // eslint-disable-next-line no-console
+                console.warn(`[YouTubeTranscriptLoader] Skipping ${videoId}: ${message}`);
+            }
+        }
+        return allDocs;
+    }
+    // ── Private: per-video pipeline ──────────────────────────────────────────
+    async loadVideo(videoId) {
+        const watchUrl = `${YouTubeTranscriptLoader_1.YOUTUBE_BASE}/watch?v=${videoId}`;
+        const html = await this.fetch(watchUrl);
+        const { captionUrl, title, channelName } = this.extractCaptionInfo(html, videoId);
+        const captionXml = await this.fetch(captionUrl);
+        const lines = this.parseCaptionXml(captionXml);
+        if (lines.length === 0) {
+            throw new Error(`No transcript lines found for video ${videoId}`);
+        }
+        const baseMetadata = {
+            source: watchUrl,
+            videoId,
+            loaderType: 'youtube-transcript',
+            language: this.language,
+            ...(title && { title }),
+            ...(channelName && { channelName }),
+            ...this.extraMetadata,
+        };
+        if (!this.segmentDuration) {
+            const fullText = lines.map((l) => l.text).join(' ');
+            return [this.createDocument(fullText, { ...baseMetadata, totalLines: lines.length })];
+        }
+        return this.buildSegments(lines, videoId, baseMetadata);
+    }
+    extractCaptionInfo(html, videoId) {
+        // Extract ytInitialPlayerResponse
+        const match = YouTubeTranscriptLoader_1.RE_PLAYER_RESPONSE.exec(html);
+        if (!match) {
+            throw new Error(`Could not find ytInitialPlayerResponse for video ${videoId}. ` +
+                `The video may be unavailable, age-restricted, or have no captions.`);
+        }
+        // Minimal safe JSON parse — the JSON is often very large
+        let playerResponse;
+        try {
+            playerResponse = JSON.parse(match[1]);
+        }
+        catch {
+            throw new Error(`Failed to parse ytInitialPlayerResponse for video ${videoId}`);
+        }
+        const title = playerResponse.videoDetails?.title;
+        const channelName = playerResponse.videoDetails
+            ?.author;
+        // Navigate: captions → playerCaptionsTracklistRenderer → captionTracks[]
+        const captionTracks = playerResponse.captions?.playerCaptionsTracklistRenderer?.captionTracks;
+        if (!captionTracks || captionTracks.length === 0) {
+            throw new Error(`No caption tracks available for video ${videoId}. ` +
+                `The video may not have captions/subtitles.`);
+        }
+        // Prefer the requested language; fall back to the first available track
+        const track = captionTracks.find((t) => t.languageCode === this.language) ?? captionTracks[0];
+        return {
+            captionUrl: `${track.baseUrl}&fmt=xml`,
+            title,
+            channelName,
+        };
+    }
+    parseCaptionXml(xml) {
+        const lines = [];
+        const re = new RegExp(YouTubeTranscriptLoader_1.RE_CAPTION_TEXT.source, 'g');
+        let match;
+        while ((match = re.exec(xml)) !== null) {
+            const start = parseFloat(match[1]);
+            const duration = parseFloat(match[2]);
+            const raw = match[3];
+            const text = this.decodeEntities(raw.replace(/<[^>]+>/g, '')).trim();
+            if (text)
+                lines.push({ start, duration, text });
+        }
+        return lines;
+    }
+    buildSegments(lines, videoId, baseMetadata) {
+        const segments = [];
+        const duration = this.segmentDuration;
+        let segStart = 0;
+        let buffer = [];
+        let windowEnd = segStart + duration;
+        for (const line of lines) {
+            if (line.start >= windowEnd) {
+                if (buffer.length > 0) {
+                    const text = buffer.join(' ');
+                    const startTime = this.formatTime(segStart);
+                    const endTime = this.formatTime(windowEnd);
+                    segments.push(this.createDocument(text, {
+                        ...baseMetadata,
+                        startTime,
+                        endTime,
+                        startSeconds: segStart,
+                        youtubeUrl: `https://youtu.be/${videoId}?t=${Math.floor(segStart)}`,
+                    }));
+                    buffer = [];
+                }
+                segStart = Math.floor(line.start / duration) * duration;
+                windowEnd = segStart + duration;
+            }
+            buffer.push(line.text);
+        }
+        // Final segment
+        if (buffer.length > 0) {
+            segments.push(this.createDocument(buffer.join(' '), {
+                ...baseMetadata,
+                startTime: this.formatTime(segStart),
+                endTime: this.formatTime(windowEnd),
+                startSeconds: segStart,
+                youtubeUrl: `https://youtu.be/${videoId}?t=${Math.floor(segStart)}`,
+            }));
+        }
+        return segments;
+    }
+    // ── Utilities ────────────────────────────────────────────────────────────
+    normaliseId(input) {
+        // youtu.be/ID
+        const shortMatch = input.match(/youtu\.be\/([a-zA-Z0-9_-]{11})/);
+        if (shortMatch)
+            return shortMatch[1];
+        // youtube.com/watch?v=ID
+        const watchMatch = input.match(/[?&]v=([a-zA-Z0-9_-]{11})/);
+        if (watchMatch)
+            return watchMatch[1];
+        // Already an 11-char ID
+        if (/^[a-zA-Z0-9_-]{11}$/.test(input))
+            return input;
+        throw new Error(`YouTubeTranscriptLoader: cannot parse video ID from "${input}"`);
+    }
+    async fetch(url) {
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), this.timeout);
+        const response = await fetch(url, {
+            headers: {
+                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' +
+                    '(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+                'Accept-Language': 'en-US,en;q=0.9',
+            },
+            signal: controller.signal,
+        });
+        clearTimeout(timer);
+        if (!response.ok) {
+            throw new Error(`HTTP ${response.status} ${response.statusText} fetching ${url}`);
+        }
+        return response.text();
+    }
+    decodeEntities(text) {
+        return text
+            .replace(/&amp;/g, '&')
+            .replace(/&lt;/g, '<')
+            .replace(/&gt;/g, '>')
+            .replace(/&quot;/g, '"')
+            .replace(/&#39;/g, "'")
+            .replace(/&#(\d+);/g, (_, code) => String.fromCharCode(parseInt(code, 10)));
+    }
+    formatTime(seconds) {
+        const h = Math.floor(seconds / 3600);
+        const m = Math.floor((seconds % 3600) / 60);
+        const s = Math.floor(seconds % 60);
+        return [h, m, s].map((v) => String(v).padStart(2, '0')).join(':');
+    }
+};
+exports.YouTubeTranscriptLoader = YouTubeTranscriptLoader;
+YouTubeTranscriptLoader.YOUTUBE_BASE = 'https://www.youtube.com';
+YouTubeTranscriptLoader.RE_PLAYER_RESPONSE = /ytInitialPlayerResponse\s*=\s*({.+?})\s*;/;
+YouTubeTranscriptLoader.RE_CAPTION_TEXT = /<text[^>]*start="([^"]*)"[^>]*dur="([^"]*)"[^>]*>([\s\S]*?)<\/text>/g;
+exports.YouTubeTranscriptLoader = YouTubeTranscriptLoader = YouTubeTranscriptLoader_1 = __decorate([
+    (0, base_loader_1.Loader)({
+        name: 'YouTubeTranscriptLoader',
+        description: 'Downloads YouTube video transcripts/captions — no API key needed.',
+    }),
+    __metadata("design:paramtypes", [Object])
+], YouTubeTranscriptLoader);
+//# sourceMappingURL=youtube-transcript.loader.js.map

package/dist/loaders/youtube-transcript.loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"youtube-transcript.loader.js","sourceRoot":"","sources":["../../src/loaders/youtube-transcript.loader.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;;;;;;;;;;;;;AAEH,+CAA2D;AAwCpD,IAAM,uBAAuB,+BAA7B,MAAM,uBAAwB,SAAQ,gCAAkB;IAY7D,YAAY,OAAuC;QACjD,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;YAC7E,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC5E,CAAC;QACD,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC3E,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC;QACtD,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC;QACzC,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;QAC/C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,KAAM,CAAC;QACzC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,OAAO,GAAe,EAAE,CAAC;QAE/B,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpC,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;gBAC3C,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;YACxB,CAAC;YAAC,OAAO,GAAY,EAAE,CAAC;gBACtB,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACjE,sCAAsC;gBACtC,OAAO,CAAC,IAAI,CAAC,sCAAsC,OAAO,KAAK,OAAO,EAAE,CAAC,CAAC;YAC5E,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,4EAA4E;IAEpE,KAAK,CAAC,SAAS,CAAC,OAAe;QACrC,MAAM,QAAQ,GAAG,GAAG,yBAAuB,CAAC,YAAY,YAAY,OAAO,EAAE,CAAC;QAE9E,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACxC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAElF,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;QAE/C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,uCAAuC,OAAO,EAAE,CAAC,CAAC;QACpE,CAAC;QAED,MAAM,YAAY,GAA4B;YAC5C,MAAM,EAAE,QAAQ;YAChB,OAAO;YACP,UAAU,EAAE,oBAAoB;YAChC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;YACvB,GAAG,CAAC,WAAW,IAAI,EAAE,WAAW,EAAE,CAAC;YACnC,GAAG,IAAI,CAAC,aAAa;SACtB,CAAC;QAEF,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpD,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,EAAE,GAAG,YAAY,EAAE,UAAU,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACxF,CAAC;QAED,OAAO,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC;IAC1D,CAAC;IAEO,kBAAkB,CACxB,IAAY,EACZ,OAAe;QAEf,kCAAkC;QAClC,MAAM,KAAK,GAAG,yBAAuB,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpE,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CACb,oDAAoD,OAAO,IAAI;gBAC7D,oEAAoE,CACvE,CAAC;QACJ,CAAC;QAED,yDAAyD;QACzD,IAAI,cAAuC,CAAC;QAC5C,IAAI,CAAC;YACH,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAA4B,CAAC;QACnE,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,qDAAqD,OAAO,EAAE,CAAC,CAAC;QAClF,CAAC;QAED,MAAM,KAAK,GAAI,cAAwD,CAAC,YAAY,EAAE,KAAK,CAAC;QAC5F,MAAM,WAAW,GAAI,cAAyD,CAAC,YAAY;YACzF,EAAE,MAAM,CAAC;QAEX,yEAAyE;QACzE,MAAM,aAAa,GACjB,cAOD,CAAC,QAAQ,EAAE,+BAA+B,EAAE,aAAa,CAAC;QAE3D,IAAI,CAAC,aAAa,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjD,MAAM,IAAI,KAAK,CACb,yCAAyC,OAAO,IAAI;gBAClD,4CAA4C,CAC/C,CAAC;QACJ,CAAC;QAED,wEAAwE;QACxE,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,IAAI,CAAC,QAAQ,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC,CAAC;QAE9F,OAAO;YACL,UAAU,EAAE,GAAG,KAAK,CAAC,OAAO,UAAU;YACtC,KAAK;YACL,WAAW;SACZ,CAAC;IACJ,CAAC;IAEO,eAAe,CAAC,GAAW;QACjC,MAAM,KAAK,GAAqB,EAAE,CAAC;QACnC,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,yBAAuB,CAAC,eAAe,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC3E,IAAI,KAA6B,CAAC;QAElC,OAAO,CAAC,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACvC,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACnC,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACtC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACrB,MAAM,IAAI,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACrE,IAAI,IAAI;gBAAE,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAClD,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,aAAa,CACnB,KAAuB,EACvB,OAAe,EACf,YAAqC;QAErC,MAAM,QAAQ,GAAe,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAgB,CAAC;QAEvC,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,MAAM,GAAa,EAAE,CAAC;QAC1B,IAAI,SAAS,GAAG,QAAQ,GAAG,QAAQ,CAAC;QAEpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,KAAK,IAAI,SAAS,EAAE,CAAC;gBAC5B,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACtB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;oBAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;oBAC3C,QAAQ,CAAC,IAAI,CACX,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE;wBACxB,GAAG,YAAY;wBACf,SAAS;wBACT,OAAO;wBACP,YAAY,EAAE,QAAQ;wBACtB,UAAU,EAAE,oBAAoB,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE;qBACpE,CAAC,CACH,CAAC;oBACF,MAAM,GAAG,EAAE,CAAC;gBACd,CAAC;gBACD,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAC,GAAG,QAAQ,CAAC;gBACxD,SAAS,GAAG,QAAQ,GAAG,QAAQ,CAAC;YAClC,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzB,CAAC;QAED,gBAAgB;QAChB,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,QAAQ,CAAC,IAAI,CACX,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;gBACpC,GAAG,YAAY;gBACf,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;gBACpC,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;gBACnC,YAAY,EAAE,QAAQ;gBACtB,UAAU,EAAE,oBAAoB,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE;aACpE,CAAC,CACH,CAAC;QACJ,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,4EAA4E;IAEpE,WAAW,CAAC,KAAa;QAC/B,cAAc;QACd,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;QACjE,IAAI,UAAU;YAAE,OAAO,UAAU,CAAC,CAAC,CAAC,CAAC;QACrC,yBAAyB;QACzB,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC5D,IAAI,UAAU;YAAE,OAAO,UAAU,CAAC,CAAC,CAAC,CAAC;QACrC,wBAAwB;QACxB,IAAI,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,wDAAwD,KAAK,GAAG,CAAC,CAAC;IACpF,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,GAAW;QAC7B,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAEjE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAChC,OAAO,EAAE;gBACP,YAAY,EACV,+DAA+D;oBAC/D,oDAAoD;gBACtD,iBAAiB,EAAE,gBAAgB;aACpC;YACD,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;QACH,YAAY,CAAC,KAAK,CAAC,CAAC;QAEpB,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,aAAa,GAAG,EAAE,CAAC,CAAC;QACpF,CAAC;QAED,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAEO,cAAc,CAAC,IAAY;QACjC,OAAO,IAAI;aACR,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;aACtB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;aACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;aACrB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;aACtB,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IAChF,CAAC;IAEO,UAAU,CAAC,OAAe;QAChC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;QACrC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAC5C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;QACnC,OAAO,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACpE,CAAC;;AAtPU,0DAAuB;AAOV,oCAAY,GAAG,yBAAyB,AAA5B,CAA6B;AACzC,0CAAkB,GAAG,2CAA2C,AAA9C,CAA+C;AACjE,uCAAe,GACrC,sEAAsE,AADjC,CACkC;kCAV9D,uBAAuB;IAJnC,IAAA,oBAAM,EAAC;QACN,IAAI,EAAE,yBAAyB;QAC/B,WAAW,EAAE,mEAAmE;KACjF,CAAC;;GACW,uBAAuB,CAuPnC"}

package/dist/memory/index.d.ts

@@ -0,0 +1,11 @@
+/**
+ * @hazeljs/rag - Memory Module
+ * Persistent context and conversation management
+ */
+export * from './types';
+export * from './memory-store.interface';
+export * from './stores/buffer-memory';
+export * from './stores/vector-memory';
+export * from './stores/hybrid-memory';
+export * from './memory-manager';
+//# sourceMappingURL=index.d.ts.map

package/dist/memory/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/memory/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,cAAc,SAAS,CAAC;AAGxB,cAAc,0BAA0B,CAAC;AAGzC,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AAGvC,cAAc,kBAAkB,CAAC"}

package/dist/memory/index.js

@@ -0,0 +1,31 @@
+"use strict";
+/**
+ * @hazeljs/rag - Memory Module
+ * Persistent context and conversation management
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+// Types
+__exportStar(require("./types"), exports);
+// Core interfaces
+__exportStar(require("./memory-store.interface"), exports);
+// Memory stores
+__exportStar(require("./stores/buffer-memory"), exports);
+__exportStar(require("./stores/vector-memory"), exports);
+__exportStar(require("./stores/hybrid-memory"), exports);
+// Memory manager
+__exportStar(require("./memory-manager"), exports);
+//# sourceMappingURL=index.js.map

package/dist/memory/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/memory/index.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;;;;;;;;;;;AAEH,QAAQ;AACR,0CAAwB;AAExB,kBAAkB;AAClB,2DAAyC;AAEzC,gBAAgB;AAChB,yDAAuC;AACvC,yDAAuC;AACvC,yDAAuC;AAEvC,iBAAiB;AACjB,mDAAiC"}

package/dist/memory/memory-manager.d.ts

@@ -0,0 +1,96 @@
+/**
+ * Memory Manager - High-level interface for managing different types of memory
+ */
+import { MemoryStore } from './memory-store.interface';
+import { Memory, MemoryConfig, Message, Entity, MemoryQuery, MemorySearchOptions } from './types';
+/**
+ * Central manager for all memory operations
+ * Provides high-level APIs for conversation, entity, fact, and working memory
+ */
+export declare class MemoryManager {
+    private store;
+    private config;
+    constructor(store: MemoryStore, config?: MemoryConfig);
+    initialize(): Promise<void>;
+    /**
+     * Add a message to conversation history
+     */
+    addMessage(message: Message, sessionId: string, userId?: string): Promise<string>;
+    /**
+     * Get conversation history for a session
+     */
+    getConversationHistory(sessionId: string, limit?: number): Promise<Message[]>;
+    /**
+     * Summarize conversation history
+     */
+    summarizeConversation(sessionId: string): Promise<string>;
+    /**
+     * Clear conversation history for a session
+     */
+    clearConversation(sessionId: string): Promise<void>;
+    /**
+     * Track an entity mentioned in conversation
+     */
+    trackEntity(entity: Entity, sessionId?: string): Promise<void>;
+    /**
+     * Get entity by name
+     */
+    getEntity(name: string): Promise<Entity | null>;
+    /**
+     * Update entity information
+     */
+    updateEntity(name: string, updates: Partial<Entity>): Promise<void>;
+    /**
+     * Get all tracked entities
+     */
+    getAllEntities(sessionId?: string): Promise<Entity[]>;
+    /**
+     * Store a fact or piece of knowledge
+     */
+    storeFact(fact: string, metadata?: Record<string, unknown>): Promise<string>;
+    /**
+     * Recall facts related to a query
+     */
+    recallFacts(query: string, options?: MemorySearchOptions): Promise<string[]>;
+    /**
+     * Update a fact
+     */
+    updateFact(id: string, newContent: string): Promise<void>;
+    /**
+     * Set a value in working memory (temporary context)
+     */
+    setContext(key: string, value: unknown, sessionId: string): Promise<void>;
+    /**
+     * Get a value from working memory
+     */
+    getContext(key: string, sessionId: string): Promise<unknown>;
+    /**
+     * Clear working memory for a session
+     */
+    clearContext(sessionId: string): Promise<void>;
+    /**
+     * Get relevant memories for a query
+     */
+    relevantMemories(query: string, options: MemorySearchOptions): Promise<Memory[]>;
+    /**
+     * Get memories by query parameters
+     */
+    getMemories(query: MemoryQuery): Promise<Memory[]>;
+    /**
+     * Get memory statistics
+     */
+    getStats(sessionId?: string): Promise<import('./types').MemoryStats>;
+    /**
+     * Calculate importance score for content
+     */
+    private calculateImportance;
+    /**
+     * Check if conversation needs summarization
+     */
+    private checkAndSummarize;
+    /**
+     * Prune working memory to max size
+     */
+    private pruneWorkingMemory;
+}
+//# sourceMappingURL=memory-manager.d.ts.map

package/dist/memory/memory-manager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"memory-manager.d.ts","sourceRoot":"","sources":["../../src/memory/memory-manager.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,EACL,MAAM,EAEN,YAAY,EACZ,OAAO,EACP,MAAM,EACN,WAAW,EACX,mBAAmB,EACpB,MAAM,SAAS,CAAC;AAGjB;;;GAGG;AACH,qBAAa,aAAa;IACxB,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,MAAM,CAAe;gBAEjB,KAAK,EAAE,WAAW,EAAE,MAAM,GAAE,YAAiB;IAcnD,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAMjC;;OAEG;IACG,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAuBvF;;OAEG;IACG,sBAAsB,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAenF;;OAEG;IACG,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAQ/D;;OAEG;IACG,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAczD;;OAEG;IACG,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBpE;;OAEG;IACG,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAgBrD;;OAEG;IACG,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA8BzE;;OAEG;IACG,cAAc,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAoB3D;;OAEG;IACG,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAelF;;OAEG;IACG,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAUlF;;OAEG;IACG,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAY/D;;OAEG;IACG,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAwC/E;;OAEG;IACG,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAiBlE;;OAEG;IACG,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAcpD;;OAEG;IACG,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAItF;;OAEG;IACG,WAAW,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAIxD;;OAEG;IACG,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,SAAS,EAAE,WAAW,CAAC;IAM1E;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAa3B;;OAEG;YACW,iBAAiB;IAuB/B;;OAEG;YACW,kBAAkB;CAiBjC"}