@hazeljs/rag 0.2.0-alpha.1

package/dist/loaders/github.loader.js

@@ -0,0 +1,217 @@
+"use strict";
+/**
+ * GitHubLoader
+ *
+ * Loads files from a GitHub repository using the GitHub REST API.
+ * No extra npm dependency required — uses Node.js built-in `fetch`.
+ *
+ * Supports:
+ *  - Public repositories (no token needed, but rate-limited to 60 req/h)
+ *  - Private repositories (requires `token`)
+ *  - Specific branches, tags, or commits via `ref`
+ *  - Recursive directory traversal with depth limit
+ *  - Glob-style extension filtering (`includeExtensions`)
+ *  - Path prefix filtering (`includePaths`)
+ *
+ * Rate limits:
+ *  - Unauthenticated: 60 requests per hour per IP
+ *  - Authenticated:   5,000 requests per hour per token
+ *  For large repos, always provide a `token`.
+ *
+ * @example
+ * ```typescript
+ * // Load all Markdown files from a public repo
+ * const loader = new GitHubLoader({
+ *   owner: 'hazeljs',
+ *   repo: 'hazeljs',
+ *   ref: 'main',
+ *   includeExtensions: ['.md'],
+ *   token: process.env.GITHUB_TOKEN,
+ * });
+ * const docs = await loader.load();
+ * // docs[0].metadata.path === 'README.md'
+ * // docs[0].metadata.url  === 'https://github.com/hazeljs/hazeljs/blob/main/README.md'
+ * ```
+ *
+ * Load a specific directory only:
+ * ```typescript
+ * const loader = new GitHubLoader({
+ *   owner: 'facebook',
+ *   repo: 'react',
+ *   ref: 'main',
+ *   directory: 'packages/react/src',
+ *   includeExtensions: ['.js', '.ts'],
+ * });
+ * ```
+ */
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GitHubLoader = void 0;
+const base_loader_1 = require("./base.loader");
+let GitHubLoader = class GitHubLoader extends base_loader_1.BaseDocumentLoader {
+    constructor(options) {
+        super();
+        this.apiBase = 'https://api.github.com';
+        this.opts = {
+            owner: options.owner,
+            repo: options.repo,
+            ref: options.ref ?? 'main',
+            directory: options.directory ?? '',
+            includeExtensions: options.includeExtensions ?? [],
+            includePaths: options.includePaths ?? [],
+            excludePaths: options.excludePaths ?? ['node_modules/', 'dist/', '.git/'],
+            maxFiles: options.maxFiles ?? 500,
+            maxFileSize: options.maxFileSize ?? 102400,
+            token: options.token ?? '',
+            timeout: options.timeout ?? 15000,
+            metadata: options.metadata ?? {},
+        };
+    }
+    async load() {
+        const treeItems = await this.getTree();
+        const filtered = this.filterItems(treeItems);
+        if (filtered.length === 0) {
+            // eslint-disable-next-line no-console
+            console.warn(`[GitHubLoader] No files matched the filter criteria in ${this.opts.owner}/${this.opts.repo}`);
+            return [];
+        }
+        const docs = [];
+        for (let i = 0; i < filtered.length && docs.length < this.opts.maxFiles; i++) {
+            const item = filtered[i];
+            try {
+                const content = await this.getFileContent(item);
+                if (content !== null) {
+                    docs.push(this.createDocument(content, {
+                        source: item.path,
+                        path: item.path,
+                        repo: `${this.opts.owner}/${this.opts.repo}`,
+                        ref: this.opts.ref,
+                        url: `https://github.com/${this.opts.owner}/${this.opts.repo}/blob/${this.opts.ref}/${item.path}`,
+                        rawUrl: `https://raw.githubusercontent.com/${this.opts.owner}/${this.opts.repo}/${this.opts.ref}/${item.path}`,
+                        sha: item.sha,
+                        loaderType: 'github',
+                        ...this.opts.metadata,
+                    }));
+                }
+            }
+            catch (err) {
+                const message = err instanceof Error ? err.message : String(err);
+                // eslint-disable-next-line no-console
+                console.warn(`[GitHubLoader] Skipping ${item.path}: ${message}`);
+            }
+        }
+        return docs;
+    }
+    // ── Private: GitHub API helpers ──────────────────────────────────────────
+    async getTree() {
+        // Use the recursive git tree endpoint — one API call for the whole repo
+        const url = `${this.apiBase}/repos/${this.opts.owner}/${this.opts.repo}` +
+            `/git/trees/${this.opts.ref}?recursive=1`;
+        const data = await this.apiGet(url);
+        if (data.truncated) {
+            // eslint-disable-next-line no-console
+            console.warn(`[GitHubLoader] Repository tree was truncated by the GitHub API ` +
+                `(too many files). Use \`directory\` or \`includePaths\` to narrow the scope.`);
+        }
+        let items = data.tree.filter((item) => item.type === 'blob');
+        // Restrict to `directory` prefix if set
+        if (this.opts.directory) {
+            const prefix = this.opts.directory.replace(/\/$/, '') + '/';
+            items = items.filter((item) => item.path.startsWith(prefix));
+        }
+        return items;
+    }
+    filterItems(items) {
+        return items.filter((item) => {
+            const path = item.path;
+            const ext = '.' + path.split('.').pop().toLowerCase();
+            // Extension filter
+            if (this.opts.includeExtensions.length > 0 && !this.opts.includeExtensions.includes(ext)) {
+                return false;
+            }
+            // Include path filter
+            if (this.opts.includePaths.length > 0 &&
+                !this.opts.includePaths.some((prefix) => path.startsWith(prefix))) {
+                return false;
+            }
+            // Exclude path filter
+            if (this.opts.excludePaths.some((prefix) => path.startsWith(prefix))) {
+                return false;
+            }
+            // File size filter
+            if (item.size !== undefined && item.size > this.opts.maxFileSize) {
+                // eslint-disable-next-line no-console
+                console.warn(`[GitHubLoader] Skipping ${path} — size ${item.size} bytes exceeds maxFileSize ${this.opts.maxFileSize}`);
+                return false;
+            }
+            return true;
+        });
+    }
+    async getFileContent(item) {
+        // Use raw content URL to avoid base64 decoding and extra API calls
+        const rawUrl = `https://raw.githubusercontent.com/${this.opts.owner}/${this.opts.repo}` +
+            `/${this.opts.ref}/${item.path}`;
+        const response = await this.httpGet(rawUrl);
+        // Skip binary files
+        if (this.isBinary(response)) {
+            return null;
+        }
+        return response;
+    }
+    async apiGet(url) {
+        const text = await this.httpGet(url);
+        return JSON.parse(text);
+    }
+    async httpGet(url) {
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), this.opts.timeout);
+        const headers = {
+            Accept: 'application/vnd.github.v3+json',
+            'User-Agent': 'HazelJS-RAG/1.0',
+        };
+        if (this.opts.token) {
+            headers['Authorization'] = `Bearer ${this.opts.token}`;
+        }
+        const response = await fetch(url, { headers, signal: controller.signal });
+        clearTimeout(timer);
+        if (!response.ok) {
+            const body = await response.text().catch(() => '');
+            const hint = response.status === 403
+                ? ' (rate limited — provide a GitHub token with the `token` option)'
+                : response.status === 404
+                    ? ' (repo or path not found — check owner/repo/ref/directory)'
+                    : '';
+            throw new Error(`GitHub API error ${response.status}${hint}: ${body.slice(0, 200)}`);
+        }
+        return response.text();
+    }
+    isBinary(text) {
+        // Heuristic: if > 10% of the first 512 bytes are non-text codepoints, treat as binary
+        const sample = text.slice(0, 512);
+        let nonText = 0;
+        for (let i = 0; i < sample.length; i++) {
+            const code = sample.charCodeAt(i);
+            if (code < 9 || (code > 13 && code < 32 && code !== 27)) {
+                nonText++;
+            }
+        }
+        return nonText / sample.length > 0.1;
+    }
+};
+exports.GitHubLoader = GitHubLoader;
+exports.GitHubLoader = GitHubLoader = __decorate([
+    (0, base_loader_1.Loader)({
+        name: 'GitHubLoader',
+        description: 'Loads files from a GitHub repository using the GitHub REST API.',
+    }),
+    __metadata("design:paramtypes", [Object])
+], GitHubLoader);
+//# sourceMappingURL=github.loader.js.map

package/dist/loaders/github.loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"github.loader.js","sourceRoot":"","sources":["../../src/loaders/github.loader.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;;;;;;;;;;;;AAEH,+CAA2D;AAsEpD,IAAM,YAAY,GAAlB,MAAM,YAAa,SAAQ,gCAAkB;IAIlD,YAAY,OAA4B;QACtC,KAAK,EAAE,CAAC;QAHO,YAAO,GAAG,wBAAwB,CAAC;QAIlD,IAAI,CAAC,IAAI,GAAG;YACV,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,GAAG,EAAE,OAAO,CAAC,GAAG,IAAI,MAAM;YAC1B,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,EAAE;YAClC,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,EAAE;YAClD,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,EAAE;YACxC,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,CAAC,eAAe,EAAE,OAAO,EAAE,OAAO,CAAC;YACzE,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,GAAG;YACjC,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,MAAO;YAC3C,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,EAAE;YAC1B,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,KAAM;YAClC,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,EAAE;SACjC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QAE7C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,sCAAsC;YACtC,OAAO,CAAC,IAAI,CACV,0DAA0D,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAC9F,CAAC;YACF,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,IAAI,GAAe,EAAE,CAAC;QAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7E,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;gBAChD,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;oBACrB,IAAI,CAAC,IAAI,CACP,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE;wBAC3B,MAAM,EAAE,IAAI,CAAC,IAAI;wBACjB,IAAI,EAAE,IAAI,CAAC,IAAI;wBACf,IAAI,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;wBAC5C,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG;wBAClB,GAAG,EAAE,sBAAsB,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE;wBACjG,MAAM,EAAE,qCAAqC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE;wBAC9G,GAAG,EAAE,IAAI,CAAC,GAAG;wBACb,UAAU,EAAE,QAAQ;wBACpB,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ;qBACtB,CAAC,CACH,CAAC;gBACJ,CAAC;YACH,CAAC;YAAC,OAAO,GAAY,EAAE,CAAC;gBACtB,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACjE,sCAAsC;gBACtC,OAAO,CAAC,IAAI,CAAC,2BAA2B,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC,CAAC;YACnE,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED,4EAA4E;IAEpE,KAAK,CAAC,OAAO;QACnB,wEAAwE;QACxE,MAAM,GAAG,GACP,GAAG,IAAI,CAAC,OAAO,UAAU,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAC5D,cAAc,IAAI,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC;QAE5C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAG3B,GAAG,CAAC,CAAC;QAER,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,sCAAsC;YACtC,OAAO,CAAC,IAAI,CACV,iEAAiE;gBAC/D,8EAA8E,CACjF,CAAC;QACJ,CAAC;QAED,IAAI,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;QAE7D,wCAAwC;QACxC,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACxB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC;YAC5D,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;QAC/D,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,WAAW,CAAC,KAAuB;QACzC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;YAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;YACvB,MAAM,GAAG,GAAG,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAG,CAAC,WAAW,EAAE,CAAC;YAEvD,mBAAmB;YACnB,IAAI,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBACzF,OAAO,KAAK,CAAC;YACf,CAAC;YAED,sBAAsB;YACtB,IACE,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC;gBACjC,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,EACjE,CAAC;gBACD,OAAO,KAAK,CAAC;YACf,CAAC;YAED,sBAAsB;YACtB,IAAI,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;gBACrE,OAAO,KAAK,CAAC;YACf,CAAC;YAED,mBAAmB;YACnB,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjE,sCAAsC;gBACtC,OAAO,CAAC,IAAI,CACV,2BAA2B,IAAI,WAAW,IAAI,CAAC,IAAI,8BAA8B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CACzG,CAAC;gBACF,OAAO,KAAK,CAAC;YACf,CAAC;YAED,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,IAAoB;QAC/C,mEAAmE;QACnE,MAAM,MAAM,GACV,qCAAqC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YACxE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QAEnC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAE5C,oBAAoB;QACpB,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,MAAM,CAAI,GAAW;QACjC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACrC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAM,CAAC;IAC/B,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,GAAW;QAC/B,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEtE,MAAM,OAAO,GAA2B;YACtC,MAAM,EAAE,gCAAgC;YACxC,YAAY,EAAE,iBAAiB;SAChC,CAAC;QACF,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACpB,OAAO,CAAC,eAAe,CAAC,GAAG,UAAU,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QACzD,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1E,YAAY,CAAC,KAAK,CAAC,CAAC;QAEpB,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YACnD,MAAM,IAAI,GACR,QAAQ,CAAC,MAAM,KAAK,GAAG;gBACrB,CAAC,CAAC,kEAAkE;gBACpE,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,GAAG;oBACvB,CAAC,CAAC,4DAA4D;oBAC9D,CAAC,CAAC,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CAAC,oBAAoB,QAAQ,CAAC,MAAM,GAAG,IAAI,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACvF,CAAC;QAED,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAEO,QAAQ,CAAC,IAAY;QAC3B,sFAAsF;QACtF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAClC,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAClC,IAAI,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,GAAG,EAAE,IAAI,IAAI,GAAG,EAAE,IAAI,IAAI,KAAK,EAAE,CAAC,EAAE,CAAC;gBACxD,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC;QACD,OAAO,OAAO,GAAG,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC;IACvC,CAAC;CACF,CAAA;AAnMY,oCAAY;uBAAZ,YAAY;IAJxB,IAAA,oBAAM,EAAC;QACN,IAAI,EAAE,cAAc;QACpB,WAAW,EAAE,iEAAiE;KAC/E,CAAC;;GACW,YAAY,CAmMxB"}

package/dist/loaders/html-file.loader.d.ts

@@ -0,0 +1,55 @@
+/**
+ * HTMLFileLoader
+ *
+ * Loads HTML files from disk and converts them to plain text.
+ *
+ * Uses a built-in tag stripper with no external dependencies.
+ * For advanced HTML parsing (CSS selectors, JavaScript rendering), use
+ * `WebLoader` with the optional `cheerio` dependency instead.
+ *
+ * Optionally strips:
+ *  - `<script>` and `<style>` blocks (content + tags)
+ *  - HTML comments
+ *  - All remaining HTML tags (leaving only text nodes)
+ *
+ * @example
+ * ```typescript
+ * const loader = new HTMLFileLoader({ path: './page.html' });
+ * const docs = await loader.load();
+ * // docs[0].content === stripped plain text
+ * // docs[0].metadata.title === page <title>
+ * ```
+ */
+import { BaseDocumentLoader } from './base.loader';
+import type { Document } from '../types';
+export interface HTMLFileLoaderOptions {
+    path?: string;
+    paths?: string[];
+    /**
+     * CSS selector string used to limit extraction to a specific element.
+     * Requires the optional `cheerio` peer dependency to be installed.
+     * If cheerio is not installed, the full stripped text is returned.
+     * @example 'article', 'main', '#content'
+     */
+    selector?: string;
+    /** Strip <script> and <style> blocks entirely. @default true */
+    stripScripts?: boolean;
+    /** Normalise multiple blank lines into a single blank line. @default true */
+    collapseWhitespace?: boolean;
+    /** Extra metadata merged into every document. */
+    metadata?: Record<string, unknown>;
+}
+export declare class HTMLFileLoader extends BaseDocumentLoader {
+    private readonly paths;
+    private readonly selector?;
+    private readonly stripScripts;
+    private readonly collapseWhitespace;
+    private readonly extraMetadata;
+    constructor(options: HTMLFileLoaderOptions);
+    load(): Promise<Document[]>;
+    private extractTitle;
+    private stripTags;
+    private decodeEntities;
+    private clean;
+}
+//# sourceMappingURL=html-file.loader.d.ts.map

package/dist/loaders/html-file.loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-file.loader.d.ts","sourceRoot":"","sources":["../../src/loaders/html-file.loader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAIH,OAAO,EAAE,kBAAkB,EAAU,MAAM,eAAe,CAAC;AAC3D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEzC,MAAM,WAAW,qBAAqB;IACpC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gEAAgE;IAChE,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,6EAA6E;IAC7E,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,iDAAiD;IACjD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,qBAMa,cAAe,SAAQ,kBAAkB;IACpD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAW;IACjC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAU;IACvC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAU;IAC7C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA0B;gBAE5C,OAAO,EAAE,qBAAqB;IAYpC,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;IA4CjC,OAAO,CAAC,YAAY;IAKpB,OAAO,CAAC,SAAS;IAsBjB,OAAO,CAAC,cAAc;IAYtB,OAAO,CAAC,KAAK;CAOd"}

package/dist/loaders/html-file.loader.js

@@ -0,0 +1,170 @@
+"use strict";
+/**
+ * HTMLFileLoader
+ *
+ * Loads HTML files from disk and converts them to plain text.
+ *
+ * Uses a built-in tag stripper with no external dependencies.
+ * For advanced HTML parsing (CSS selectors, JavaScript rendering), use
+ * `WebLoader` with the optional `cheerio` dependency instead.
+ *
+ * Optionally strips:
+ *  - `<script>` and `<style>` blocks (content + tags)
+ *  - HTML comments
+ *  - All remaining HTML tags (leaving only text nodes)
+ *
+ * @example
+ * ```typescript
+ * const loader = new HTMLFileLoader({ path: './page.html' });
+ * const docs = await loader.load();
+ * // docs[0].content === stripped plain text
+ * // docs[0].metadata.title === page <title>
+ * ```
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HTMLFileLoader = void 0;
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+const base_loader_1 = require("./base.loader");
+let HTMLFileLoader = class HTMLFileLoader extends base_loader_1.BaseDocumentLoader {
+    constructor(options) {
+        super();
+        if (!options.path && (!options.paths || options.paths.length === 0)) {
+            throw new Error('HTMLFileLoader: provide at least one path via `path` or `paths`.');
+        }
+        this.paths = options.paths ?? (options.path ? [options.path] : []);
+        this.selector = options.selector;
+        this.stripScripts = options.stripScripts ?? true;
+        this.collapseWhitespace = options.collapseWhitespace ?? true;
+        this.extraMetadata = options.metadata ?? {};
+    }
+    async load() {
+        const docs = [];
+        for (const filePath of this.paths) {
+            const html = await (0, promises_1.readFile)(filePath, { encoding: 'utf-8' });
+            // Try cheerio for selector support, fall back to built-in stripper
+            let text;
+            let title = '';
+            if (this.selector) {
+                try {
+                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                    const cheerio = await Promise.resolve(`${'cheerio'}`).then(s => __importStar(require(s))).then((m) => m);
+                    const $ = (cheerio.load ?? cheerio.default)(html);
+                    title = $('title').first().text().trim();
+                    text = this.clean($(this.selector).text());
+                }
+                catch {
+                    // cheerio not installed — fall back
+                    text = this.stripTags(html);
+                }
+            }
+            else {
+                title = this.extractTitle(html);
+                text = this.stripTags(html);
+            }
+            docs.push(this.createDocument(text, {
+                source: (0, path_1.basename)(filePath),
+                filePath,
+                loaderType: 'html',
+                ...(title && { title }),
+                ...this.extraMetadata,
+            }));
+        }
+        return docs;
+    }
+    // ── Private helpers ──────────────────────────────────────────────────────
+    extractTitle(html) {
+        const match = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+        return match ? match[1].trim() : '';
+    }
+    stripTags(html) {
+        let text = html;
+        if (this.stripScripts) {
+            // Remove <script> blocks
+            text = text.replace(/<script[\s\S]*?<\/script>/gi, ' ');
+            // Remove <style> blocks
+            text = text.replace(/<style[\s\S]*?<\/style>/gi, ' ');
+        }
+        // Remove HTML comments
+        text = text.replace(/<!--[\s\S]*?-->/g, ' ');
+        // Replace block-level tags with newlines for readability
+        text = text.replace(/<\/(p|div|h[1-6]|li|tr|br)[^>]*>/gi, '\n');
+        // Remove all remaining tags
+        text = text.replace(/<[^>]+>/g, ' ');
+        // Decode common HTML entities
+        text = this.decodeEntities(text);
+        return this.clean(text);
+    }
+    decodeEntities(text) {
+        return text
+            .replace(/&amp;/g, '&')
+            .replace(/&lt;/g, '<')
+            .replace(/&gt;/g, '>')
+            .replace(/&quot;/g, '"')
+            .replace(/&#039;/g, "'")
+            .replace(/&nbsp;/g, ' ')
+            .replace(/&hellip;/g, '...')
+            .replace(/&#(\d+);/g, (_, code) => String.fromCharCode(parseInt(code, 10)));
+    }
+    clean(text) {
+        if (!this.collapseWhitespace)
+            return text.trim();
+        return text
+            .replace(/[ \t]+/g, ' ') // collapse inline spaces
+            .replace(/\n{3,}/g, '\n\n') // collapse multiple blank lines
+            .trim();
+    }
+};
+exports.HTMLFileLoader = HTMLFileLoader;
+exports.HTMLFileLoader = HTMLFileLoader = __decorate([
+    (0, base_loader_1.Loader)({
+        name: 'HTMLFileLoader',
+        description: 'Loads local HTML files and strips tags to produce plain text.',
+        extensions: ['.html', '.htm', '.xhtml'],
+        mimeTypes: ['text/html', 'application/xhtml+xml'],
+    }),
+    __metadata("design:paramtypes", [Object])
+], HTMLFileLoader);
+//# sourceMappingURL=html-file.loader.js.map

package/dist/loaders/html-file.loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-file.loader.js","sourceRoot":"","sources":["../../src/loaders/html-file.loader.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,0CAAuC;AACvC,+BAAgC;AAChC,+CAA2D;AA2BpD,IAAM,cAAc,GAApB,MAAM,cAAe,SAAQ,gCAAkB;IAOpD,YAAY,OAA8B;QACxC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;YACpE,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;QACtF,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACnE,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QACjC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC;QACjD,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,IAAI,CAAC;QAC7D,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,IAAI,GAAe,EAAE,CAAC;QAE5B,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,MAAM,IAAA,mBAAQ,EAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;YAE7D,mEAAmE;YACnE,IAAI,IAAY,CAAC;YACjB,IAAI,KAAK,GAAG,EAAE,CAAC;YAEf,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,IAAI,CAAC;oBACH,8DAA8D;oBAC9D,MAAM,OAAO,GAAG,MAAM,mBAAO,SAAmB,wCAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAQ,CAAC,CAAC;oBACxE,MAAM,CAAC,GAAG,CACR,OAAO,CAAC,IAAI,IAAK,OAAuD,CAAC,OAAO,CACjF,CAAC,IAAI,CAAC,CAAC;oBACR,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;oBACzC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC7C,CAAC;gBAAC,MAAM,CAAC;oBACP,oCAAoC;oBACpC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;gBAC9B,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;gBAChC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAC9B,CAAC;YAED,IAAI,CAAC,IAAI,CACP,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE;gBACxB,MAAM,EAAE,IAAA,eAAQ,EAAC,QAAQ,CAAC;gBAC1B,QAAQ;gBACR,UAAU,EAAE,MAAM;gBAClB,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;gBACvB,GAAG,IAAI,CAAC,aAAa;aACtB,CAAC,CACH,CAAC;QACJ,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED,4EAA4E;IAEpE,YAAY,CAAC,IAAY;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;QAC7D,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACtC,CAAC;IAEO,SAAS,CAAC,IAAY;QAC5B,IAAI,IAAI,GAAG,IAAI,CAAC;QAEhB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,yBAAyB;YACzB,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,6BAA6B,EAAE,GAAG,CAAC,CAAC;YACxD,wBAAwB;YACxB,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,2BAA2B,EAAE,GAAG,CAAC,CAAC;QACxD,CAAC;QAED,uBAAuB;QACvB,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;QAC7C,yDAAyD;QACzD,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,oCAAoC,EAAE,IAAI,CAAC,CAAC;QAChE,4BAA4B;QAC5B,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;QACrC,8BAA8B;QAC9B,IAAI,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAEjC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAEO,cAAc,CAAC,IAAY;QACjC,OAAO,IAAI;aACR,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;aACtB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;aACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;aACrB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aACvB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aACvB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aACvB,OAAO,CAAC,WAAW,EAAE,KAAK,CAAC;aAC3B,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IAChF,CAAC;IAEO,KAAK,CAAC,IAAY;QACxB,IAAI,CAAC,IAAI,CAAC,kBAAkB;YAAE,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;QACjD,OAAO,IAAI;aACR,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,yBAAyB;aACjD,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,gCAAgC;aAC3D,IAAI,EAAE,CAAC;IACZ,CAAC;CACF,CAAA;AA7GY,wCAAc;yBAAd,cAAc;IAN1B,IAAA,oBAAM,EAAC;QACN,IAAI,EAAE,gBAAgB;QACtB,WAAW,EAAE,+DAA+D;QAC5E,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC;QACvC,SAAS,EAAE,CAAC,WAAW,EAAE,uBAAuB,CAAC;KAClD,CAAC;;GACW,cAAc,CA6G1B"}

package/dist/loaders/index.d.ts

@@ -0,0 +1,52 @@
+/**
+ * @hazeljs/rag — Document Loaders
+ *
+ * All document loaders that ship with @hazeljs/rag.
+ *
+ * Built-in loaders (no extra dependencies):
+ *  - TextFileLoader          .txt, .log
+ *  - JSONFileLoader          .json
+ *  - CSVFileLoader           .csv
+ *  - MarkdownFileLoader      .md, .mdx
+ *  - HTMLFileLoader          .html, .htm
+ *  - DirectoryLoader         walks a directory and delegates to the above
+ *  - WebLoader               fetches web pages (Node.js fetch)
+ *  - YouTubeTranscriptLoader YouTube captions via ytInitialPlayerResponse
+ *  - GitHubLoader            GitHub REST API (no extra dep)
+ *
+ * Optional-dependency loaders (graceful error if dep not installed):
+ *  - PdfLoader               requires: npm install pdf-parse
+ *  - DocxLoader              requires: npm install mammoth
+ *  - WebLoader (selector)    requires: npm install cheerio  (only for the `selector` option)
+ *
+ * Base class and decorator (extend to build your own):
+ *  - BaseDocumentLoader
+ *  - @Loader
+ *  - DocumentLoaderRegistry
+ *  - LoaderConfig
+ */
+export { BaseDocumentLoader, Loader, getLoaderConfig, DocumentLoaderRegistry } from './base.loader';
+export type { LoaderConfig } from './base.loader';
+export { TextFileLoader } from './text-file.loader';
+export { JSONFileLoader } from './json-file.loader';
+export { CSVFileLoader } from './csv-file.loader';
+export { MarkdownFileLoader } from './markdown-file.loader';
+export { HTMLFileLoader } from './html-file.loader';
+export { DirectoryLoader } from './directory.loader';
+export { PdfLoader } from './pdf.loader';
+export { DocxLoader } from './docx.loader';
+export { WebLoader } from './web.loader';
+export { YouTubeTranscriptLoader } from './youtube-transcript.loader';
+export { GitHubLoader } from './github.loader';
+export type { TextFileLoaderOptions } from './text-file.loader';
+export type { JSONFileLoaderOptions } from './json-file.loader';
+export type { CSVFileLoaderOptions } from './csv-file.loader';
+export type { MarkdownFileLoaderOptions } from './markdown-file.loader';
+export type { HTMLFileLoaderOptions } from './html-file.loader';
+export type { DirectoryLoaderOptions, LoaderFactory } from './directory.loader';
+export type { PdfLoaderOptions } from './pdf.loader';
+export type { DocxLoaderOptions } from './docx.loader';
+export type { WebLoaderOptions } from './web.loader';
+export type { YouTubeTranscriptLoaderOptions } from './youtube-transcript.loader';
+export type { GitHubLoaderOptions } from './github.loader';
+//# sourceMappingURL=index.d.ts.map

package/dist/loaders/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/loaders/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,EAAE,kBAAkB,EAAE,MAAM,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AACpG,YAAY,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAGlD,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAGrD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAG3C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,uBAAuB,EAAE,MAAM,6BAA6B,CAAC;AACtE,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAG/C,YAAY,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAChE,YAAY,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAChE,YAAY,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAC9D,YAAY,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC;AACxE,YAAY,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAChE,YAAY,EAAE,sBAAsB,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAChF,YAAY,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,YAAY,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACvD,YAAY,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,YAAY,EAAE,8BAA8B,EAAE,MAAM,6BAA6B,CAAC;AAClF,YAAY,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC"}

package/dist/loaders/index.js

@@ -0,0 +1,61 @@
+"use strict";
+/**
+ * @hazeljs/rag — Document Loaders
+ *
+ * All document loaders that ship with @hazeljs/rag.
+ *
+ * Built-in loaders (no extra dependencies):
+ *  - TextFileLoader          .txt, .log
+ *  - JSONFileLoader          .json
+ *  - CSVFileLoader           .csv
+ *  - MarkdownFileLoader      .md, .mdx
+ *  - HTMLFileLoader          .html, .htm
+ *  - DirectoryLoader         walks a directory and delegates to the above
+ *  - WebLoader               fetches web pages (Node.js fetch)
+ *  - YouTubeTranscriptLoader YouTube captions via ytInitialPlayerResponse
+ *  - GitHubLoader            GitHub REST API (no extra dep)
+ *
+ * Optional-dependency loaders (graceful error if dep not installed):
+ *  - PdfLoader               requires: npm install pdf-parse
+ *  - DocxLoader              requires: npm install mammoth
+ *  - WebLoader (selector)    requires: npm install cheerio  (only for the `selector` option)
+ *
+ * Base class and decorator (extend to build your own):
+ *  - BaseDocumentLoader
+ *  - @Loader
+ *  - DocumentLoaderRegistry
+ *  - LoaderConfig
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GitHubLoader = exports.YouTubeTranscriptLoader = exports.WebLoader = exports.DocxLoader = exports.PdfLoader = exports.DirectoryLoader = exports.HTMLFileLoader = exports.MarkdownFileLoader = exports.CSVFileLoader = exports.JSONFileLoader = exports.TextFileLoader = exports.DocumentLoaderRegistry = exports.getLoaderConfig = exports.Loader = exports.BaseDocumentLoader = void 0;
+var base_loader_1 = require("./base.loader");
+Object.defineProperty(exports, "BaseDocumentLoader", { enumerable: true, get: function () { return base_loader_1.BaseDocumentLoader; } });
+Object.defineProperty(exports, "Loader", { enumerable: true, get: function () { return base_loader_1.Loader; } });
+Object.defineProperty(exports, "getLoaderConfig", { enumerable: true, get: function () { return base_loader_1.getLoaderConfig; } });
+Object.defineProperty(exports, "DocumentLoaderRegistry", { enumerable: true, get: function () { return base_loader_1.DocumentLoaderRegistry; } });
+// ── File-system loaders ───────────────────────────────────────────────────
+var text_file_loader_1 = require("./text-file.loader");
+Object.defineProperty(exports, "TextFileLoader", { enumerable: true, get: function () { return text_file_loader_1.TextFileLoader; } });
+var json_file_loader_1 = require("./json-file.loader");
+Object.defineProperty(exports, "JSONFileLoader", { enumerable: true, get: function () { return json_file_loader_1.JSONFileLoader; } });
+var csv_file_loader_1 = require("./csv-file.loader");
+Object.defineProperty(exports, "CSVFileLoader", { enumerable: true, get: function () { return csv_file_loader_1.CSVFileLoader; } });
+var markdown_file_loader_1 = require("./markdown-file.loader");
+Object.defineProperty(exports, "MarkdownFileLoader", { enumerable: true, get: function () { return markdown_file_loader_1.MarkdownFileLoader; } });
+var html_file_loader_1 = require("./html-file.loader");
+Object.defineProperty(exports, "HTMLFileLoader", { enumerable: true, get: function () { return html_file_loader_1.HTMLFileLoader; } });
+var directory_loader_1 = require("./directory.loader");
+Object.defineProperty(exports, "DirectoryLoader", { enumerable: true, get: function () { return directory_loader_1.DirectoryLoader; } });
+// ── Format loaders (optional deps) ───────────────────────────────────────
+var pdf_loader_1 = require("./pdf.loader");
+Object.defineProperty(exports, "PdfLoader", { enumerable: true, get: function () { return pdf_loader_1.PdfLoader; } });
+var docx_loader_1 = require("./docx.loader");
+Object.defineProperty(exports, "DocxLoader", { enumerable: true, get: function () { return docx_loader_1.DocxLoader; } });
+// ── Remote loaders ────────────────────────────────────────────────────────
+var web_loader_1 = require("./web.loader");
+Object.defineProperty(exports, "WebLoader", { enumerable: true, get: function () { return web_loader_1.WebLoader; } });
+var youtube_transcript_loader_1 = require("./youtube-transcript.loader");
+Object.defineProperty(exports, "YouTubeTranscriptLoader", { enumerable: true, get: function () { return youtube_transcript_loader_1.YouTubeTranscriptLoader; } });
+var github_loader_1 = require("./github.loader");
+Object.defineProperty(exports, "GitHubLoader", { enumerable: true, get: function () { return github_loader_1.GitHubLoader; } });
+//# sourceMappingURL=index.js.map

package/dist/loaders/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/loaders/index.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;;;AAEH,6CAAoG;AAA3F,iHAAA,kBAAkB,OAAA;AAAE,qGAAA,MAAM,OAAA;AAAE,8GAAA,eAAe,OAAA;AAAE,qHAAA,sBAAsB,OAAA;AAG5E,6EAA6E;AAC7E,uDAAoD;AAA3C,kHAAA,cAAc,OAAA;AACvB,uDAAoD;AAA3C,kHAAA,cAAc,OAAA;AACvB,qDAAkD;AAAzC,gHAAA,aAAa,OAAA;AACtB,+DAA4D;AAAnD,0HAAA,kBAAkB,OAAA;AAC3B,uDAAoD;AAA3C,kHAAA,cAAc,OAAA;AACvB,uDAAqD;AAA5C,mHAAA,eAAe,OAAA;AAExB,4EAA4E;AAC5E,2CAAyC;AAAhC,uGAAA,SAAS,OAAA;AAClB,6CAA2C;AAAlC,yGAAA,UAAU,OAAA;AAEnB,6EAA6E;AAC7E,2CAAyC;AAAhC,uGAAA,SAAS,OAAA;AAClB,yEAAsE;AAA7D,oIAAA,uBAAuB,OAAA;AAChC,iDAA+C;AAAtC,6GAAA,YAAY,OAAA"}

package/dist/loaders/json-file.loader.d.ts

@@ -0,0 +1,51 @@
+/**
+ * JSONFileLoader
+ *
+ * Loads a JSON file and converts it to documents.
+ *
+ * Supports two shapes:
+ *  - **Array mode** (default) — if the root value is an array, each element
+ *    becomes a document.  The element's text is either the value of `textKey`
+ *    (a specific field) or the full JSON stringification of the element.
+ *  - **Object mode** — a single root object becomes one document.
+ *
+ * @example
+ * ```typescript
+ * // articles.json = [{ "title": "...", "body": "..." }, ...]
+ * const loader = new JSONFileLoader({
+ *   path: './articles.json',
+ *   textKey: 'body',       // use "body" field as content
+ *   metadataKeys: ['title'], // include "title" in metadata
+ * });
+ * const docs = await loader.load();
+ * ```
+ */
+import { BaseDocumentLoader } from './base.loader';
+import type { Document } from '../types';
+export interface JSONFileLoaderOptions {
+    /** Path to the JSON file. */
+    path: string;
+    /**
+     * Key in each array element (or root object) whose value becomes the
+     * document content.  If omitted, the entire element is JSON-stringified.
+     */
+    textKey?: string;
+    /**
+     * Keys to extract from each element into the document metadata.
+     * All other keys are omitted from metadata.
+     */
+    metadataKeys?: string[];
+    /** Extra metadata merged into every document. */
+    metadata?: Record<string, unknown>;
+    /**
+     * JSON pointer (dot-separated path) to the array within the JSON file.
+     * @example 'data.results'  →  reads `json.data.results`
+     */
+    jsonPointer?: string;
+}
+export declare class JSONFileLoader extends BaseDocumentLoader {
+    private readonly opts;
+    constructor(options: JSONFileLoaderOptions);
+    load(): Promise<Document[]>;
+}
+//# sourceMappingURL=json-file.loader.d.ts.map

package/dist/loaders/json-file.loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"json-file.loader.d.ts","sourceRoot":"","sources":["../../src/loaders/json-file.loader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAIH,OAAO,EAAE,kBAAkB,EAAU,MAAM,eAAe,CAAC;AAC3D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEzC,MAAM,WAAW,qBAAqB;IACpC,6BAA6B;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,iDAAiD;IACjD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,qBAMa,cAAe,SAAQ,kBAAkB;IACpD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAGnB;gBAEU,OAAO,EAAE,qBAAqB;IAWpC,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;CAkDlC"}