@hazeljs/rag 0.2.0-alpha.1

package/dist/loaders/directory.loader.js

@@ -0,0 +1,163 @@
+"use strict";
+/**
+ * DirectoryLoader
+ *
+ * Walks a directory (optionally recursively) and loads every file whose
+ * extension is matched by a registered loader or the `loaders` map.
+ *
+ * Supported patterns:
+ *  1. **Extension map** (recommended) — pass a `loaders` map of
+ *     `{ '.pdf': (path) => new PdfLoader({ path }) }` to customise which
+ *     loader handles which extension.
+ *  2. **Auto-detect** — falls back to the built-in file loaders for the
+ *     common types (.txt, .md, .html, .json, .csv) with default options.
+ *  3. **Glob filter** — pass `glob` to only load files matching a pattern.
+ *
+ * @example
+ * ```typescript
+ * const loader = new DirectoryLoader({
+ *   path: './docs',
+ *   recursive: true,
+ *   loaders: {
+ *     '.txt':  (p) => new TextFileLoader({ path: p }),
+ *     '.md':   (p) => new MarkdownFileLoader({ path: p, splitByHeading: 'h2' }),
+ *     '.pdf':  (p) => new PdfLoader({ path: p }),
+ *   },
+ * });
+ * const docs = await loader.load();
+ * ```
+ */
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DirectoryLoader = void 0;
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+const base_loader_1 = require("./base.loader");
+const text_file_loader_1 = require("./text-file.loader");
+const markdown_file_loader_1 = require("./markdown-file.loader");
+const json_file_loader_1 = require("./json-file.loader");
+const csv_file_loader_1 = require("./csv-file.loader");
+const html_file_loader_1 = require("./html-file.loader");
+let DirectoryLoader = class DirectoryLoader extends base_loader_1.BaseDocumentLoader {
+    constructor(options) {
+        super();
+        this.opts = {
+            path: options.path,
+            loaders: options.loaders ?? {},
+            recursive: options.recursive ?? true,
+            includeExtensions: options.includeExtensions ?? [],
+            excludeExtensions: options.excludeExtensions ?? [],
+            maxFiles: options.maxFiles ?? 1000,
+            metadata: options.metadata ?? {},
+        };
+    }
+    async load() {
+        const filePaths = await this.collectFiles(this.opts.path, 0);
+        const allDocs = [];
+        for (const filePath of filePaths) {
+            const loader = this.resolveLoader(filePath);
+            if (!loader)
+                continue;
+            try {
+                const docs = await loader.load();
+                // Merge directory-level metadata into each document
+                for (const doc of docs) {
+                    doc.metadata = {
+                        ...doc.metadata,
+                        directoryPath: this.opts.path,
+                        ...this.opts.metadata,
+                    };
+                    allDocs.push(doc);
+                }
+            }
+            catch (err) {
+                const message = err instanceof Error ? err.message : String(err);
+                // eslint-disable-next-line no-console
+                console.warn(`[DirectoryLoader] Skipping ${(0, path_1.basename)(filePath)}: ${message}`);
+            }
+            if (allDocs.length >= this.opts.maxFiles) {
+                // eslint-disable-next-line no-console
+                console.warn(`[DirectoryLoader] maxFiles limit (${this.opts.maxFiles}) reached. ` +
+                    `Stopping early. Loaded ${allDocs.length} documents.`);
+                break;
+            }
+        }
+        return allDocs;
+    }
+    // ── Private helpers ──────────────────────────────────────────────────────
+    async collectFiles(dir, depth) {
+        const entries = await (0, promises_1.readdir)(dir);
+        const files = [];
+        for (const entry of entries) {
+            // Skip hidden files/directories
+            if (entry.startsWith('.'))
+                continue;
+            const fullPath = (0, path_1.join)(dir, entry);
+            const stats = await (0, promises_1.stat)(fullPath);
+            if (stats.isDirectory()) {
+                if (this.opts.recursive) {
+                    const subFiles = await this.collectFiles(fullPath, depth + 1);
+                    files.push(...subFiles);
+                }
+            }
+            else {
+                const ext = (0, path_1.extname)(entry).toLowerCase();
+                if (this.opts.includeExtensions.length > 0 && !this.opts.includeExtensions.includes(ext)) {
+                    continue;
+                }
+                if (this.opts.excludeExtensions.includes(ext)) {
+                    continue;
+                }
+                files.push(fullPath);
+            }
+        }
+        return files;
+    }
+    resolveLoader(filePath) {
+        const ext = (0, path_1.extname)(filePath).toLowerCase();
+        // 1. Check explicitly configured loaders first
+        const factory = this.opts.loaders[ext];
+        if (factory)
+            return factory(filePath);
+        // 2. Auto-detect using built-in loaders for common types
+        return this.autoDetectLoader(filePath, ext);
+    }
+    autoDetectLoader(filePath, ext) {
+        switch (ext) {
+            case '.txt':
+            case '.text':
+            case '.log':
+                return new text_file_loader_1.TextFileLoader({ path: filePath });
+            case '.md':
+            case '.markdown':
+            case '.mdx':
+                return new markdown_file_loader_1.MarkdownFileLoader({ path: filePath });
+            case '.json':
+                return new json_file_loader_1.JSONFileLoader({ path: filePath });
+            case '.csv':
+                return new csv_file_loader_1.CSVFileLoader({ path: filePath });
+            case '.html':
+            case '.htm':
+                return new html_file_loader_1.HTMLFileLoader({ path: filePath });
+            default:
+                return undefined;
+        }
+    }
+};
+exports.DirectoryLoader = DirectoryLoader;
+exports.DirectoryLoader = DirectoryLoader = __decorate([
+    (0, base_loader_1.Loader)({
+        name: 'DirectoryLoader',
+        description: 'Walks a directory and loads all supported files using registered loaders.',
+    }),
+    __metadata("design:paramtypes", [Object])
+], DirectoryLoader);
+//# sourceMappingURL=directory.loader.js.map

package/dist/loaders/directory.loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"directory.loader.js","sourceRoot":"","sources":["../../src/loaders/directory.loader.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;;;;;;;;;;;;AAEH,0CAA4C;AAC5C,+BAA+C;AAC/C,+CAA2D;AAG3D,yDAAoD;AACpD,iEAA4D;AAC5D,yDAAoD;AACpD,uDAAkD;AAClD,yDAAoD;AAmC7C,IAAM,eAAe,GAArB,MAAM,eAAgB,SAAQ,gCAAkB;IAGrD,YAAY,OAA+B;QACzC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,IAAI,GAAG;YACV,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,EAAE;YAC9B,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,IAAI;YACpC,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,EAAE;YAClD,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,EAAE;YAClD,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI;YAClC,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,EAAE;SACjC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC7D,MAAM,OAAO,GAAe,EAAE,CAAC;QAE/B,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;YAC5C,IAAI,CAAC,MAAM;gBAAE,SAAS;YAEtB,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBACjC,oDAAoD;gBACpD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;oBACvB,GAAG,CAAC,QAAQ,GAAG;wBACb,GAAG,GAAG,CAAC,QAAQ;wBACf,aAAa,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI;wBAC7B,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ;qBACtB,CAAC;oBACF,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACpB,CAAC;YACH,CAAC;YAAC,OAAO,GAAY,EAAE,CAAC;gBACtB,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACjE,sCAAsC;gBACtC,OAAO,CAAC,IAAI,CAAC,8BAA8B,IAAA,eAAQ,EAAC,QAAQ,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC;YAC/E,CAAC;YAED,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACzC,sCAAsC;gBACtC,OAAO,CAAC,IAAI,CACV,qCAAqC,IAAI,CAAC,IAAI,CAAC,QAAQ,aAAa;oBAClE,0BAA0B,OAAO,CAAC,MAAM,aAAa,CACxD,CAAC;gBACF,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,4EAA4E;IAEpE,KAAK,CAAC,YAAY,CAAC,GAAW,EAAE,KAAa;QACnD,MAAM,OAAO,GAAG,MAAM,IAAA,kBAAO,EAAC,GAAG,CAAC,CAAC;QACnC,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,gCAAgC;YAChC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,SAAS;YAEpC,MAAM,QAAQ,GAAG,IAAA,WAAI,EAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAClC,MAAM,KAAK,GAAG,MAAM,IAAA,eAAI,EAAC,QAAQ,CAAC,CAAC;YAEnC,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACxB,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;oBACxB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;oBAC9D,KAAK,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;gBAC1B,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,GAAG,IAAA,cAAO,EAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;gBAEzC,IAAI,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBACzF,SAAS;gBACX,CAAC;gBACD,IAAI,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC9C,SAAS;gBACX,CAAC;gBAED,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,aAAa,CAAC,QAAgB;QACpC,MAAM,GAAG,GAAG,IAAA,cAAO,EAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;QAE5C,+CAA+C;QAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACvC,IAAI,OAAO;YAAE,OAAO,OAAO,CAAC,QAAQ,CAAC,CAAC;QAEtC,yDAAyD;QACzD,OAAO,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAC9C,CAAC;IAEO,gBAAgB,CAAC,QAAgB,EAAE,GAAW;QACpD,QAAQ,GAAG,EAAE,CAAC;YACZ,KAAK,MAAM,CAAC;YACZ,KAAK,OAAO,CAAC;YACb,KAAK,MAAM;gBACT,OAAO,IAAI,iCAAc,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;YAChD,KAAK,KAAK,CAAC;YACX,KAAK,WAAW,CAAC;YACjB,KAAK,MAAM;gBACT,OAAO,IAAI,yCAAkB,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;YACpD,KAAK,OAAO;gBACV,OAAO,IAAI,iCAAc,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;YAChD,KAAK,MAAM;gBACT,OAAO,IAAI,+BAAa,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;YAC/C,KAAK,OAAO,CAAC;YACb,KAAK,MAAM;gBACT,OAAO,IAAI,iCAAc,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;YAChD;gBACE,OAAO,SAAS,CAAC;QACrB,CAAC;IACH,CAAC;CACF,CAAA;AAzHY,0CAAe;0BAAf,eAAe;IAJ3B,IAAA,oBAAM,EAAC;QACN,IAAI,EAAE,iBAAiB;QACvB,WAAW,EAAE,2EAA2E;KACzF,CAAC;;GACW,eAAe,CAyH3B"}

package/dist/loaders/docx.loader.d.ts

@@ -0,0 +1,52 @@
+/**
+ * DocxLoader
+ *
+ * Loads Microsoft Word (.docx) files using the optional `mammoth` peer
+ * dependency.  Converts rich-text document content to plain text or HTML.
+ *
+ * Install the peer dependency:
+ * ```bash
+ * npm install mammoth
+ * ```
+ *
+ * @example
+ * ```typescript
+ * const loader = new DocxLoader({ path: './proposal.docx' });
+ * const docs = await loader.load();
+ * // docs[0].content === extracted plain text
+ * // docs[0].metadata.source === 'proposal.docx'
+ * ```
+ *
+ * Extract HTML instead of plain text (useful for preserving table structure):
+ * ```typescript
+ * const loader = new DocxLoader({ path: './report.docx', outputFormat: 'html' });
+ * ```
+ *
+ * Legacy .doc files are NOT supported — mammoth only handles the modern OOXML
+ * format (.docx).  Use LibreOffice to convert .doc → .docx first.
+ */
+import { BaseDocumentLoader } from './base.loader';
+import type { Document } from '../types';
+export interface DocxLoaderOptions {
+    /** Path to the .docx file. */
+    path: string;
+    /**
+     * Output format.
+     * - `'text'` (default) — plain text, stripping all formatting
+     * - `'html'` — HTML string preserving basic structure
+     */
+    outputFormat?: 'text' | 'html';
+    /**
+     * Custom style map passed to mammoth for HTML output.
+     * @see https://github.com/mwilliamson/mammoth.js#writing-style-maps
+     */
+    styleMap?: string[];
+    /** Extra metadata merged into every document. */
+    metadata?: Record<string, unknown>;
+}
+export declare class DocxLoader extends BaseDocumentLoader {
+    private readonly opts;
+    constructor(options: DocxLoaderOptions);
+    load(): Promise<Document[]>;
+}
+//# sourceMappingURL=docx.loader.d.ts.map

package/dist/loaders/docx.loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"docx.loader.d.ts","sourceRoot":"","sources":["../../src/loaders/docx.loader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAIH,OAAO,EAAE,kBAAkB,EAAU,MAAM,eAAe,CAAC;AAC3D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEzC,MAAM,WAAW,iBAAiB;IAChC,8BAA8B;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC/B;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,iDAAiD;IACjD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,qBAMa,UAAW,SAAQ,kBAAkB;IAChD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA8B;gBAEvC,OAAO,EAAE,iBAAiB;IAUhC,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;CAyDlC"}

package/dist/loaders/docx.loader.js

@@ -0,0 +1,110 @@
+"use strict";
+/**
+ * DocxLoader
+ *
+ * Loads Microsoft Word (.docx) files using the optional `mammoth` peer
+ * dependency.  Converts rich-text document content to plain text or HTML.
+ *
+ * Install the peer dependency:
+ * ```bash
+ * npm install mammoth
+ * ```
+ *
+ * @example
+ * ```typescript
+ * const loader = new DocxLoader({ path: './proposal.docx' });
+ * const docs = await loader.load();
+ * // docs[0].content === extracted plain text
+ * // docs[0].metadata.source === 'proposal.docx'
+ * ```
+ *
+ * Extract HTML instead of plain text (useful for preserving table structure):
+ * ```typescript
+ * const loader = new DocxLoader({ path: './report.docx', outputFormat: 'html' });
+ * ```
+ *
+ * Legacy .doc files are NOT supported — mammoth only handles the modern OOXML
+ * format (.docx).  Use LibreOffice to convert .doc → .docx first.
+ */
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DocxLoader = void 0;
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+const base_loader_1 = require("./base.loader");
+let DocxLoader = class DocxLoader extends base_loader_1.BaseDocumentLoader {
+    constructor(options) {
+        super();
+        this.opts = {
+            path: options.path,
+            outputFormat: options.outputFormat ?? 'text',
+            styleMap: options.styleMap ?? [],
+            metadata: options.metadata ?? {},
+        };
+    }
+    async load() {
+        // Dynamic import — mammoth is only required when DocxLoader is actually used
+        let mammoth;
+        try {
+            // eslint-disable-next-line @typescript-eslint/no-require-imports
+            const mod = require('mammoth');
+            mammoth = mod.default ?? mod;
+        }
+        catch {
+            throw new Error('[DocxLoader] mammoth is not installed. Run: npm install mammoth');
+        }
+        const buffer = await (0, promises_1.readFile)(this.opts.path);
+        const fileName = (0, path_1.basename)(this.opts.path);
+        let content;
+        let messages;
+        if (this.opts.outputFormat === 'html') {
+            const result = await mammoth.convertToHtml({
+                buffer,
+                styleMap: this.opts.styleMap.length > 0 ? this.opts.styleMap : undefined,
+            });
+            content = result.value;
+            messages = result.messages;
+        }
+        else {
+            const result = await mammoth.extractRawText({ buffer });
+            content = result.value;
+            messages = result.messages;
+        }
+        // Log any conversion warnings (e.g. unsupported features in the DOCX)
+        const warnings = messages
+            .filter((m) => m.type === 'warning')
+            .map((m) => m.message);
+        if (warnings.length > 0) {
+            // eslint-disable-next-line no-console
+            console.warn(`[DocxLoader] Conversion warnings for ${fileName}:`, warnings);
+        }
+        return [
+            this.createDocument(content, {
+                source: fileName,
+                filePath: this.opts.path,
+                loaderType: 'docx',
+                outputFormat: this.opts.outputFormat,
+                ...this.opts.metadata,
+            }),
+        ];
+    }
+};
+exports.DocxLoader = DocxLoader;
+exports.DocxLoader = DocxLoader = __decorate([
+    (0, base_loader_1.Loader)({
+        name: 'DocxLoader',
+        description: 'Loads Microsoft Word .docx files using mammoth (optional peer dependency).',
+        extensions: ['.docx'],
+        mimeTypes: ['application/vnd.openxmlformats-officedocument.wordprocessingml.document'],
+    }),
+    __metadata("design:paramtypes", [Object])
+], DocxLoader);
+//# sourceMappingURL=docx.loader.js.map

package/dist/loaders/docx.loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"docx.loader.js","sourceRoot":"","sources":["../../src/loaders/docx.loader.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;;;;;;;;;;;;AAEH,0CAAuC;AACvC,+BAAgC;AAChC,+CAA2D;AA2BpD,IAAM,UAAU,GAAhB,MAAM,UAAW,SAAQ,gCAAkB;IAGhD,YAAY,OAA0B;QACpC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,IAAI,GAAG;YACV,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,MAAM;YAC5C,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,EAAE;YAChC,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,EAAE;SACjC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAI;QACR,6EAA6E;QAC7E,IAAI,OAMH,CAAC;QAEF,IAAI,CAAC;YACH,iEAAiE;YACjE,MAAM,GAAG,GAAG,OAAO,CAAC,SAAS,CAAkD,CAAC;YAChF,OAAO,GAAG,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,iEAAiE,CAAC,CAAC;QACrF,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAA,mBAAQ,EAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9C,MAAM,QAAQ,GAAG,IAAA,eAAQ,EAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE1C,IAAI,OAAe,CAAC;QACpB,IAAI,QAAmB,CAAC;QAExB,IAAI,IAAI,CAAC,IAAI,CAAC,YAAY,KAAK,MAAM,EAAE,CAAC;YACtC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC;gBACzC,MAAM;gBACN,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;aACzE,CAAC,CAAC;YACH,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACvB,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QAC7B,CAAC;aAAM,CAAC;YACN,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;YACxD,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACvB,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QAC7B,CAAC;QAED,sEAAsE;QACtE,MAAM,QAAQ,GAAI,QAAqD;aACpE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC;aACnC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAEzB,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,sCAAsC;YACtC,OAAO,CAAC,IAAI,CAAC,wCAAwC,QAAQ,GAAG,EAAE,QAAQ,CAAC,CAAC;QAC9E,CAAC;QAED,OAAO;YACL,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE;gBAC3B,MAAM,EAAE,QAAQ;gBAChB,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI;gBACxB,UAAU,EAAE,MAAM;gBAClB,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY;gBACpC,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ;aACtB,CAAC;SACH,CAAC;IACJ,CAAC;CACF,CAAA;AAtEY,gCAAU;qBAAV,UAAU;IANtB,IAAA,oBAAM,EAAC;QACN,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,4EAA4E;QACzF,UAAU,EAAE,CAAC,OAAO,CAAC;QACrB,SAAS,EAAE,CAAC,yEAAyE,CAAC;KACvF,CAAC;;GACW,UAAU,CAsEtB"}

package/dist/loaders/github.loader.d.ts

@@ -0,0 +1,114 @@
+/**
+ * GitHubLoader
+ *
+ * Loads files from a GitHub repository using the GitHub REST API.
+ * No extra npm dependency required — uses Node.js built-in `fetch`.
+ *
+ * Supports:
+ *  - Public repositories (no token needed, but rate-limited to 60 req/h)
+ *  - Private repositories (requires `token`)
+ *  - Specific branches, tags, or commits via `ref`
+ *  - Recursive directory traversal with depth limit
+ *  - Glob-style extension filtering (`includeExtensions`)
+ *  - Path prefix filtering (`includePaths`)
+ *
+ * Rate limits:
+ *  - Unauthenticated: 60 requests per hour per IP
+ *  - Authenticated:   5,000 requests per hour per token
+ *  For large repos, always provide a `token`.
+ *
+ * @example
+ * ```typescript
+ * // Load all Markdown files from a public repo
+ * const loader = new GitHubLoader({
+ *   owner: 'hazeljs',
+ *   repo: 'hazeljs',
+ *   ref: 'main',
+ *   includeExtensions: ['.md'],
+ *   token: process.env.GITHUB_TOKEN,
+ * });
+ * const docs = await loader.load();
+ * // docs[0].metadata.path === 'README.md'
+ * // docs[0].metadata.url  === 'https://github.com/hazeljs/hazeljs/blob/main/README.md'
+ * ```
+ *
+ * Load a specific directory only:
+ * ```typescript
+ * const loader = new GitHubLoader({
+ *   owner: 'facebook',
+ *   repo: 'react',
+ *   ref: 'main',
+ *   directory: 'packages/react/src',
+ *   includeExtensions: ['.js', '.ts'],
+ * });
+ * ```
+ */
+import { BaseDocumentLoader } from './base.loader';
+import type { Document } from '../types';
+export interface GitHubLoaderOptions {
+    /** GitHub organisation or user name. */
+    owner: string;
+    /** Repository name. */
+    repo: string;
+    /**
+     * Branch, tag, or commit SHA.
+     * @default 'main'
+     */
+    ref?: string;
+    /**
+     * Sub-directory inside the repository to load.
+     * Defaults to the repository root (`''`).
+     */
+    directory?: string;
+    /**
+     * Only load files with these extensions.
+     * @example ['.ts', '.md']
+     */
+    includeExtensions?: string[];
+    /**
+     * Only load files under these path prefixes (relative to `directory`).
+     * @example ['src/', 'docs/']
+     */
+    includePaths?: string[];
+    /**
+     * Skip files under these path prefixes.
+     * @example ['node_modules/', 'dist/', '__tests__/']
+     */
+    excludePaths?: string[];
+    /**
+     * Maximum number of files to load (safety limit).
+     * @default 500
+     */
+    maxFiles?: number;
+    /**
+     * Maximum file size in bytes to load.
+     * Files larger than this are skipped with a warning.
+     * @default 102400 (100 KB)
+     */
+    maxFileSize?: number;
+    /**
+     * GitHub personal access token (PAT) or fine-grained token.
+     * Required for private repos; recommended for public repos to avoid rate limits.
+     */
+    token?: string;
+    /**
+     * Request timeout in milliseconds.
+     * @default 15000
+     */
+    timeout?: number;
+    /** Extra metadata merged into every document. */
+    metadata?: Record<string, unknown>;
+}
+export declare class GitHubLoader extends BaseDocumentLoader {
+    private readonly opts;
+    private readonly apiBase;
+    constructor(options: GitHubLoaderOptions);
+    load(): Promise<Document[]>;
+    private getTree;
+    private filterItems;
+    private getFileContent;
+    private apiGet;
+    private httpGet;
+    private isBinary;
+}
+//# sourceMappingURL=github.loader.d.ts.map

package/dist/loaders/github.loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"github.loader.d.ts","sourceRoot":"","sources":["../../src/loaders/github.loader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AAEH,OAAO,EAAE,kBAAkB,EAAU,MAAM,eAAe,CAAC;AAC3D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEzC,MAAM,WAAW,mBAAmB;IAClC,wCAAwC;IACxC,KAAK,EAAE,MAAM,CAAC;IACd,uBAAuB;IACvB,IAAI,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAUD,qBAIa,YAAa,SAAQ,kBAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAgC;IACrD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA4B;gBAExC,OAAO,EAAE,mBAAmB;IAkBlC,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YA6CnB,OAAO;IA8BrB,OAAO,CAAC,WAAW;YAoCL,cAAc;YAgBd,MAAM;YAKN,OAAO;IA6BrB,OAAO,CAAC,QAAQ;CAYjB"}