@hazeljs/rag 0.2.0-alpha.1

package/dist/loaders/base.loader.d.ts

@@ -0,0 +1,108 @@
+/**
+ * Base Document Loader
+ *
+ * Abstract base class for all document loaders in @hazeljs/rag.
+ * Every loader:
+ *  1. Extends `BaseDocumentLoader`
+ *  2. Implements `load(): Promise<Document[]>`
+ *  3. May optionally be decorated with `@Loader` for metadata reflection
+ *
+ * The `@Loader` decorator stores descriptive metadata on the class so that
+ * a DocumentLoaderRegistry or DI container can enumerate available loaders,
+ * display them in a developer UI, and auto-select the right one by MIME type
+ * or file extension.
+ *
+ * @example
+ * ```typescript
+ * @Loader({
+ *   name: 'MyCustomLoader',
+ *   description: 'Loads .xyz files from a proprietary database',
+ *   extensions: ['.xyz'],
+ * })
+ * export class MyCustomLoader extends BaseDocumentLoader {
+ *   async load(): Promise<Document[]> {
+ *     return [this.createDocument('content', { source: 'custom' })];
+ *   }
+ * }
+ * ```
+ */
+import 'reflect-metadata';
+import type { Document, DocumentLoader } from '../types';
+export interface LoaderConfig {
+    /** Human-readable name used in logs and the registry. */
+    name: string;
+    /** Short description of what this loader handles. */
+    description?: string;
+    /**
+     * File extensions this loader handles (with leading dot).
+     * @example ['.pdf', '.PDF']
+     */
+    extensions?: string[];
+    /**
+     * MIME types this loader handles.
+     * @example ['application/pdf']
+     */
+    mimeTypes?: string[];
+}
+/**
+ * Mark a class as a document loader.
+ *
+ * Stores `LoaderConfig` metadata on the class that can be read at runtime
+ * via `getLoaderConfig()`.  Applying `@Loader` is optional — any class that
+ * extends `BaseDocumentLoader` works without it.
+ */
+export declare function Loader(config: LoaderConfig): ClassDecorator;
+/**
+ * Read the `@Loader` config from a class.
+ * Returns `undefined` if the decorator was not applied.
+ */
+export declare function getLoaderConfig(target: object): LoaderConfig | undefined;
+/**
+ * Abstract base class for all document loaders.
+ *
+ * Provides convenience helpers for creating well-formed `Document` objects
+ * and normalising metadata.  All loaders must implement `load()`.
+ */
+export declare abstract class BaseDocumentLoader implements DocumentLoader {
+    /** Load documents from the source. */
+    abstract load(): Promise<Document[]>;
+    /**
+     * Convenience factory for creating a `Document` with normalised whitespace.
+     *
+     * @param content  The raw text content.
+     * @param metadata Optional metadata attached to the document.
+     */
+    protected createDocument(content: string, metadata?: Record<string, unknown>): Document;
+    /**
+     * Split a large string into multiple `Document` objects, one per page or
+     * logical section.  Useful for loaders that produce multi-page content
+     * (PDF, DOCX) where you want one document per page.
+     */
+    protected createDocuments(pages: string[], baseMetadata?: Record<string, unknown>): Document[];
+}
+/**
+ * Simple in-process registry that maps file extensions and MIME types to
+ * loader factories.  Loaders self-register by calling `registerLoader()`.
+ *
+ * @example
+ * ```typescript
+ * DocumentLoaderRegistry.register(PdfLoader, () => new PdfLoader({ path: '...' }));
+ * const loader = DocumentLoaderRegistry.forExtension('.pdf', { path: 'doc.pdf' });
+ * const docs = await loader.load();
+ * ```
+ */
+type LoaderFactory<T extends BaseDocumentLoader> = (opts: unknown) => T;
+export declare class DocumentLoaderRegistry {
+    private static byExtension;
+    private static byMimeType;
+    /** Register a loader class and factory for all its declared extensions / MIME types. */
+    static register<T extends BaseDocumentLoader>(loaderClass: new (...args: unknown[]) => T, factory: LoaderFactory<T>): void;
+    /** Get a loader factory by file extension (e.g. `".pdf"`). */
+    static forExtension(ext: string, opts?: unknown): BaseDocumentLoader | undefined;
+    /** Get a loader factory by MIME type (e.g. `"application/pdf"`). */
+    static forMimeType(mimeType: string, opts?: unknown): BaseDocumentLoader | undefined;
+    /** List all registered extensions. */
+    static registeredExtensions(): string[];
+}
+export {};
+//# sourceMappingURL=base.loader.d.ts.map

package/dist/loaders/base.loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base.loader.d.ts","sourceRoot":"","sources":["../../src/loaders/base.loader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAQzD,MAAM,WAAW,YAAY;IAC3B,yDAAyD;IACzD,IAAI,EAAE,MAAM,CAAC;IACb,qDAAqD;IACrD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AAED;;;;;;GAMG;AACH,wBAAgB,MAAM,CAAC,MAAM,EAAE,YAAY,GAAG,cAAc,CAI3D;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,GAAG,SAAS,CAExE;AAID;;;;;GAKG;AACH,8BAAsB,kBAAmB,YAAW,cAAc;IAChE,sCAAsC;IACtC,QAAQ,CAAC,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;IAEpC;;;;;OAKG;IACH,SAAS,CAAC,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,QAAQ;IAOvF;;;;OAIG;IACH,SAAS,CAAC,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,QAAQ,EAAE;CAW/F;AAID;;;;;;;;;;GAUG;AACH,KAAK,aAAa,CAAC,CAAC,SAAS,kBAAkB,IAAI,CAAC,IAAI,EAAE,OAAO,KAAK,CAAC,CAAC;AAExE,qBAAa,sBAAsB;IACjC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAwD;IAClF,OAAO,CAAC,MAAM,CAAC,UAAU,CAAwD;IAEjF,wFAAwF;IACxF,MAAM,CAAC,QAAQ,CAAC,CAAC,SAAS,kBAAkB,EAC1C,WAAW,EAAE,KAAK,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,EAC1C,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC,GACxB,IAAI;IAkBP,8DAA8D;IAC9D,MAAM,CAAC,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,OAAO,GAAG,kBAAkB,GAAG,SAAS;IAKhF,oEAAoE;IACpE,MAAM,CAAC,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,OAAO,GAAG,kBAAkB,GAAG,SAAS;IAKpF,sCAAsC;IACtC,MAAM,CAAC,oBAAoB,IAAI,MAAM,EAAE;CAGxC"}

package/dist/loaders/base.loader.js

@@ -0,0 +1,123 @@
+"use strict";
+/**
+ * Base Document Loader
+ *
+ * Abstract base class for all document loaders in @hazeljs/rag.
+ * Every loader:
+ *  1. Extends `BaseDocumentLoader`
+ *  2. Implements `load(): Promise<Document[]>`
+ *  3. May optionally be decorated with `@Loader` for metadata reflection
+ *
+ * The `@Loader` decorator stores descriptive metadata on the class so that
+ * a DocumentLoaderRegistry or DI container can enumerate available loaders,
+ * display them in a developer UI, and auto-select the right one by MIME type
+ * or file extension.
+ *
+ * @example
+ * ```typescript
+ * @Loader({
+ *   name: 'MyCustomLoader',
+ *   description: 'Loads .xyz files from a proprietary database',
+ *   extensions: ['.xyz'],
+ * })
+ * export class MyCustomLoader extends BaseDocumentLoader {
+ *   async load(): Promise<Document[]> {
+ *     return [this.createDocument('content', { source: 'custom' })];
+ *   }
+ * }
+ * ```
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DocumentLoaderRegistry = exports.BaseDocumentLoader = void 0;
+exports.Loader = Loader;
+exports.getLoaderConfig = getLoaderConfig;
+require("reflect-metadata");
+// ── Metadata keys ────────────────────────────────────────────────────────────
+const LOADER_METADATA_KEY = Symbol('hazel:rag:loader');
+/**
+ * Mark a class as a document loader.
+ *
+ * Stores `LoaderConfig` metadata on the class that can be read at runtime
+ * via `getLoaderConfig()`.  Applying `@Loader` is optional — any class that
+ * extends `BaseDocumentLoader` works without it.
+ */
+function Loader(config) {
+    return (target) => {
+        Reflect.defineMetadata(LOADER_METADATA_KEY, config, target);
+    };
+}
+/**
+ * Read the `@Loader` config from a class.
+ * Returns `undefined` if the decorator was not applied.
+ */
+function getLoaderConfig(target) {
+    return Reflect.getMetadata(LOADER_METADATA_KEY, target);
+}
+// ── Base class ───────────────────────────────────────────────────────────────
+/**
+ * Abstract base class for all document loaders.
+ *
+ * Provides convenience helpers for creating well-formed `Document` objects
+ * and normalising metadata.  All loaders must implement `load()`.
+ */
+class BaseDocumentLoader {
+    /**
+     * Convenience factory for creating a `Document` with normalised whitespace.
+     *
+     * @param content  The raw text content.
+     * @param metadata Optional metadata attached to the document.
+     */
+    createDocument(content, metadata) {
+        return {
+            content: content.trim(),
+            metadata: metadata ?? {},
+        };
+    }
+    /**
+     * Split a large string into multiple `Document` objects, one per page or
+     * logical section.  Useful for loaders that produce multi-page content
+     * (PDF, DOCX) where you want one document per page.
+     */
+    createDocuments(pages, baseMetadata) {
+        return pages
+            .map((page, idx) => this.createDocument(page, {
+            ...baseMetadata,
+            pageNumber: idx + 1,
+            totalPages: pages.length,
+        }))
+            .filter((doc) => doc.content.length > 0);
+    }
+}
+exports.BaseDocumentLoader = BaseDocumentLoader;
+class DocumentLoaderRegistry {
+    /** Register a loader class and factory for all its declared extensions / MIME types. */
+    static register(loaderClass, factory) {
+        const config = getLoaderConfig(loaderClass);
+        if (!config)
+            return;
+        for (const ext of config.extensions ?? []) {
+            DocumentLoaderRegistry.byExtension.set(ext.toLowerCase(), factory);
+        }
+        for (const mime of config.mimeTypes ?? []) {
+            DocumentLoaderRegistry.byMimeType.set(mime.toLowerCase(), factory);
+        }
+    }
+    /** Get a loader factory by file extension (e.g. `".pdf"`). */
+    static forExtension(ext, opts) {
+        const factory = DocumentLoaderRegistry.byExtension.get(ext.toLowerCase());
+        return factory ? factory(opts) : undefined;
+    }
+    /** Get a loader factory by MIME type (e.g. `"application/pdf"`). */
+    static forMimeType(mimeType, opts) {
+        const factory = DocumentLoaderRegistry.byMimeType.get(mimeType.toLowerCase());
+        return factory ? factory(opts) : undefined;
+    }
+    /** List all registered extensions. */
+    static registeredExtensions() {
+        return [...DocumentLoaderRegistry.byExtension.keys()];
+    }
+}
+exports.DocumentLoaderRegistry = DocumentLoaderRegistry;
+DocumentLoaderRegistry.byExtension = new Map();
+DocumentLoaderRegistry.byMimeType = new Map();
+//# sourceMappingURL=base.loader.js.map

package/dist/loaders/base.loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"base.loader.js","sourceRoot":"","sources":["../../src/loaders/base.loader.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;;;AAmCH,wBAIC;AAMD,0CAEC;AA7CD,4BAA0B;AAG1B,gFAAgF;AAEhF,MAAM,mBAAmB,GAAG,MAAM,CAAC,kBAAkB,CAAC,CAAC;AAqBvD;;;;;;GAMG;AACH,SAAgB,MAAM,CAAC,MAAoB;IACzC,OAAO,CAAC,MAAM,EAAE,EAAE;QAChB,OAAO,CAAC,cAAc,CAAC,mBAAmB,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9D,CAAC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAgB,eAAe,CAAC,MAAc;IAC5C,OAAO,OAAO,CAAC,WAAW,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAAC;AAC1D,CAAC;AAED,gFAAgF;AAEhF;;;;;GAKG;AACH,MAAsB,kBAAkB;IAItC;;;;;OAKG;IACO,cAAc,CAAC,OAAe,EAAE,QAAkC;QAC1E,OAAO;YACL,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE;YACvB,QAAQ,EAAE,QAAQ,IAAI,EAAE;SACzB,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACO,eAAe,CAAC,KAAe,EAAE,YAAsC;QAC/E,OAAO,KAAK;aACT,GAAG,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CACjB,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE;YACxB,GAAG,YAAY;YACf,UAAU,EAAE,GAAG,GAAG,CAAC;YACnB,UAAU,EAAE,KAAK,CAAC,MAAM;SACzB,CAAC,CACH;aACA,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC7C,CAAC;CACF;AAjCD,gDAiCC;AAiBD,MAAa,sBAAsB;IAIjC,wFAAwF;IACxF,MAAM,CAAC,QAAQ,CACb,WAA0C,EAC1C,OAAyB;QAEzB,MAAM,MAAM,GAAG,eAAe,CAAC,WAAW,CAAC,CAAC;QAC5C,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,UAAU,IAAI,EAAE,EAAE,CAAC;YAC1C,sBAAsB,CAAC,WAAW,CAAC,GAAG,CACpC,GAAG,CAAC,WAAW,EAAE,EACjB,OAA4C,CAC7C,CAAC;QACJ,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,SAAS,IAAI,EAAE,EAAE,CAAC;YAC1C,sBAAsB,CAAC,UAAU,CAAC,GAAG,CACnC,IAAI,CAAC,WAAW,EAAE,EAClB,OAA4C,CAC7C,CAAC;QACJ,CAAC;IACH,CAAC;IAED,8DAA8D;IAC9D,MAAM,CAAC,YAAY,CAAC,GAAW,EAAE,IAAc;QAC7C,MAAM,OAAO,GAAG,sBAAsB,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC;QAC1E,OAAO,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC7C,CAAC;IAED,oEAAoE;IACpE,MAAM,CAAC,WAAW,CAAC,QAAgB,EAAE,IAAc;QACjD,MAAM,OAAO,GAAG,sBAAsB,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QAC9E,OAAO,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC7C,CAAC;IAED,sCAAsC;IACtC,MAAM,CAAC,oBAAoB;QACzB,OAAO,CAAC,GAAG,sBAAsB,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;IACxD,CAAC;;AAzCH,wDA0CC;AAzCgB,kCAAW,GAAG,IAAI,GAAG,EAA6C,CAAC;AACnE,iCAAU,GAAG,IAAI,GAAG,EAA6C,CAAC"}

package/dist/loaders/csv-file.loader.d.ts

@@ -0,0 +1,61 @@
+/**
+ * CSVFileLoader
+ *
+ * Loads a CSV file and converts each row to a `Document`.
+ * No external dependency — uses a built-in parser that handles quoted fields
+ * and escaped commas.
+ *
+ * Each row's content is built from either:
+ *  - A specific `contentColumns` list (concatenated with a separator), or
+ *  - All columns joined as `key: value` pairs.
+ *
+ * @example
+ * ```typescript
+ * // products.csv has columns: id, name, description, price
+ * const loader = new CSVFileLoader({
+ *   path: './products.csv',
+ *   contentColumns: ['name', 'description'],
+ *   metadataColumns: ['id', 'price'],
+ * });
+ * const docs = await loader.load();
+ * // docs[0].content  === "Product Name\nProduct description here"
+ * // docs[0].metadata === { id: '1', price: '9.99', row: 0, source: 'products.csv' }
+ * ```
+ */
+import { BaseDocumentLoader } from './base.loader';
+import type { Document } from '../types';
+export interface CSVFileLoaderOptions {
+    path: string;
+    /** Column delimiter. @default ',' */
+    delimiter?: string;
+    /** Quote character for fields containing delimiters. @default '"' */
+    quote?: string;
+    /** If true, the first row is a header row. @default true */
+    hasHeader?: boolean;
+    /**
+     * Columns whose values are concatenated to form the document content.
+     * If omitted, all columns are used in `key: value` format.
+     */
+    contentColumns?: string[];
+    /** Separator used between content column values. @default '\n' */
+    contentSeparator?: string;
+    /**
+     * Columns included in the document metadata.
+     * If omitted, all non-content columns are included.
+     */
+    metadataColumns?: string[];
+    /** Extra metadata merged into every document. */
+    metadata?: Record<string, unknown>;
+}
+export declare class CSVFileLoader extends BaseDocumentLoader {
+    private readonly opts;
+    constructor(options: CSVFileLoaderOptions);
+    load(): Promise<Document[]>;
+    /**
+     * Minimal RFC-4180-compliant CSV parser.
+     * Handles quoted fields with embedded commas and escaped quotes.
+     */
+    private parseCSV;
+    private parseLine;
+}
+//# sourceMappingURL=csv-file.loader.d.ts.map

package/dist/loaders/csv-file.loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv-file.loader.d.ts","sourceRoot":"","sources":["../../src/loaders/csv-file.loader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAIH,OAAO,EAAE,kBAAkB,EAAU,MAAM,eAAe,CAAC;AAC3D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEzC,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,qCAAqC;IACrC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,qEAAqE;IACrE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,4DAA4D;IAC5D,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,kEAAkE;IAClE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,iDAAiD;IACjD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,qBAMa,aAAc,SAAQ,kBAAkB;IACnD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiC;gBAE1C,OAAO,EAAE,oBAAoB;IAcnC,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;IA4DjC;;;OAGG;IACH,OAAO,CAAC,QAAQ;IAYhB,OAAO,CAAC,SAAS;CAgClB"}

package/dist/loaders/csv-file.loader.js

@@ -0,0 +1,162 @@
+"use strict";
+/**
+ * CSVFileLoader
+ *
+ * Loads a CSV file and converts each row to a `Document`.
+ * No external dependency — uses a built-in parser that handles quoted fields
+ * and escaped commas.
+ *
+ * Each row's content is built from either:
+ *  - A specific `contentColumns` list (concatenated with a separator), or
+ *  - All columns joined as `key: value` pairs.
+ *
+ * @example
+ * ```typescript
+ * // products.csv has columns: id, name, description, price
+ * const loader = new CSVFileLoader({
+ *   path: './products.csv',
+ *   contentColumns: ['name', 'description'],
+ *   metadataColumns: ['id', 'price'],
+ * });
+ * const docs = await loader.load();
+ * // docs[0].content  === "Product Name\nProduct description here"
+ * // docs[0].metadata === { id: '1', price: '9.99', row: 0, source: 'products.csv' }
+ * ```
+ */
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CSVFileLoader = void 0;
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+const base_loader_1 = require("./base.loader");
+let CSVFileLoader = class CSVFileLoader extends base_loader_1.BaseDocumentLoader {
+    constructor(options) {
+        super();
+        this.opts = {
+            path: options.path,
+            delimiter: options.delimiter ?? ',',
+            quote: options.quote ?? '"',
+            hasHeader: options.hasHeader ?? true,
+            contentColumns: options.contentColumns ?? [],
+            contentSeparator: options.contentSeparator ?? '\n',
+            metadataColumns: options.metadataColumns ?? [],
+            metadata: options.metadata ?? {},
+        };
+    }
+    async load() {
+        const raw = await (0, promises_1.readFile)(this.opts.path, { encoding: 'utf-8' });
+        const rows = this.parseCSV(raw);
+        if (rows.length === 0)
+            return [];
+        let headers;
+        let dataRows;
+        if (this.opts.hasHeader) {
+            headers = rows[0];
+            dataRows = rows.slice(1);
+        }
+        else {
+            // Generate column names col0, col1, ...
+            headers = rows[0].map((_, i) => `col${i}`);
+            dataRows = rows;
+        }
+        const fileName = (0, path_1.basename)(this.opts.path);
+        const contentCols = this.opts.contentColumns.length > 0 ? this.opts.contentColumns : headers;
+        const metaCols = this.opts.metadataColumns.length > 0
+            ? this.opts.metadataColumns
+            : headers.filter((h) => !contentCols.includes(h));
+        return dataRows
+            .map((row, rowIdx) => {
+            const obj = {};
+            headers.forEach((h, i) => {
+                obj[h] = row[i] ?? '';
+            });
+            // Build content
+            const contentParts = contentCols
+                .filter((col) => col in obj && obj[col].trim().length > 0)
+                .map((col) => contentCols.length === 1
+                ? obj[col] // single column: raw value
+                : `${col}: ${obj[col]}` // multiple: key: value
+            );
+            const content = contentParts.join(this.opts.contentSeparator);
+            // Build metadata
+            const metadata = {
+                source: fileName,
+                filePath: this.opts.path,
+                row: rowIdx,
+                loaderType: 'csv',
+                ...this.opts.metadata,
+            };
+            for (const col of metaCols) {
+                if (col in obj)
+                    metadata[col] = obj[col];
+            }
+            return this.createDocument(content, metadata);
+        })
+            .filter((doc) => doc.content.length > 0);
+    }
+    /**
+     * Minimal RFC-4180-compliant CSV parser.
+     * Handles quoted fields with embedded commas and escaped quotes.
+     */
+    parseCSV(text) {
+        const rows = [];
+        const lines = text.split(/\r?\n/);
+        for (const line of lines) {
+            if (line.trim() === '')
+                continue;
+            rows.push(this.parseLine(line));
+        }
+        return rows;
+    }
+    parseLine(line) {
+        const fields = [];
+        const { delimiter, quote } = this.opts;
+        let field = '';
+        let inQuotes = false;
+        let i = 0;
+        while (i < line.length) {
+            const char = line[i];
+            if (char === quote) {
+                if (inQuotes && line[i + 1] === quote) {
+                    // Escaped quote inside quoted field
+                    field += quote;
+                    i += 2;
+                }
+                else {
+                    inQuotes = !inQuotes;
+                    i++;
+                }
+            }
+            else if (char === delimiter && !inQuotes) {
+                fields.push(field);
+                field = '';
+                i++;
+            }
+            else {
+                field += char;
+                i++;
+            }
+        }
+        fields.push(field);
+        return fields;
+    }
+};
+exports.CSVFileLoader = CSVFileLoader;
+exports.CSVFileLoader = CSVFileLoader = __decorate([
+    (0, base_loader_1.Loader)({
+        name: 'CSVFileLoader',
+        description: 'Loads rows from a CSV file, one document per row.',
+        extensions: ['.csv'],
+        mimeTypes: ['text/csv'],
+    }),
+    __metadata("design:paramtypes", [Object])
+], CSVFileLoader);
+//# sourceMappingURL=csv-file.loader.js.map

package/dist/loaders/csv-file.loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv-file.loader.js","sourceRoot":"","sources":["../../src/loaders/csv-file.loader.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;;;;;;;;;;;;AAEH,0CAAuC;AACvC,+BAAgC;AAChC,+CAA2D;AAiCpD,IAAM,aAAa,GAAnB,MAAM,aAAc,SAAQ,gCAAkB;IAGnD,YAAY,OAA6B;QACvC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,IAAI,GAAG;YACV,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,GAAG;YACnC,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,GAAG;YAC3B,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,IAAI;YACpC,cAAc,EAAE,OAAO,CAAC,cAAc,IAAI,EAAE;YAC5C,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,IAAI,IAAI;YAClD,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,EAAE;YAC9C,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,EAAE;SACjC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,GAAG,GAAG,MAAM,IAAA,mBAAQ,EAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;QAClE,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAEhC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjC,IAAI,OAAiB,CAAC;QACtB,IAAI,QAAoB,CAAC;QAEzB,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACxB,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC3B,CAAC;aAAM,CAAC;YACN,wCAAwC;YACxC,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAC3C,QAAQ,GAAG,IAAI,CAAC;QAClB,CAAC;QAED,MAAM,QAAQ,GAAG,IAAA,eAAQ,EAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC;QAC7F,MAAM,QAAQ,GACZ,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC;YAClC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe;YAC3B,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAEtD,OAAO,QAAQ;aACZ,GAAG,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE;YACnB,MAAM,GAAG,GAA2B,EAAE,CAAC;YACvC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;gBACvB,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACxB,CAAC,CAAC,CAAC;YAEH,gBAAgB;YAChB,MAAM,YAAY,GAAG,WAAW;iBAC7B,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;iBACzD,GAAG,CACF,CAAC,GAAG,EAAE,EAAE,CACN,WAAW,CAAC,MAAM,KAAK,CAAC;gBACtB,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,2BAA2B;gBACtC,CAAC,CAAC,GAAG,GAAG,KAAK,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,uBAAuB;aACpD,CAAC;YACJ,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YAE9D,iBAAiB;YACjB,MAAM,QAAQ,GAA4B;gBACxC,MAAM,EAAE,QAAQ;gBAChB,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI;gBACxB,GAAG,EAAE,MAAM;gBACX,UAAU,EAAE,KAAK;gBACjB,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ;aACtB,CAAC;YACF,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;gBAC3B,IAAI,GAAG,IAAI,GAAG;oBAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3C,CAAC;YAED,OAAO,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QAChD,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACK,QAAQ,CAAC,IAAY;QAC3B,MAAM,IAAI,GAAe,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAElC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;gBAAE,SAAS;YACjC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QAClC,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,SAAS,CAAC,IAAY;QAC5B,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC;QACvC,IAAI,KAAK,GAAG,EAAE,CAAC;QACf,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,CAAC,GAAG,CAAC,CAAC;QAEV,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;YAErB,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;gBACnB,IAAI,QAAQ,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,KAAK,EAAE,CAAC;oBACtC,oCAAoC;oBACpC,KAAK,IAAI,KAAK,CAAC;oBACf,CAAC,IAAI,CAAC,CAAC;gBACT,CAAC;qBAAM,CAAC;oBACN,QAAQ,GAAG,CAAC,QAAQ,CAAC;oBACrB,CAAC,EAAE,CAAC;gBACN,CAAC;YACH,CAAC;iBAAM,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAC3C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACnB,KAAK,GAAG,EAAE,CAAC;gBACX,CAAC,EAAE,CAAC;YACN,CAAC;iBAAM,CAAC;gBACN,KAAK,IAAI,IAAI,CAAC;gBACd,CAAC,EAAE,CAAC;YACN,CAAC;QACH,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,OAAO,MAAM,CAAC;IAChB,CAAC;CACF,CAAA;AA7HY,sCAAa;wBAAb,aAAa;IANzB,IAAA,oBAAM,EAAC;QACN,IAAI,EAAE,eAAe;QACrB,WAAW,EAAE,mDAAmD;QAChE,UAAU,EAAE,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,CAAC,UAAU,CAAC;KACxB,CAAC;;GACW,aAAa,CA6HzB"}

package/dist/loaders/directory.loader.d.ts

@@ -0,0 +1,67 @@
+/**
+ * DirectoryLoader
+ *
+ * Walks a directory (optionally recursively) and loads every file whose
+ * extension is matched by a registered loader or the `loaders` map.
+ *
+ * Supported patterns:
+ *  1. **Extension map** (recommended) — pass a `loaders` map of
+ *     `{ '.pdf': (path) => new PdfLoader({ path }) }` to customise which
+ *     loader handles which extension.
+ *  2. **Auto-detect** — falls back to the built-in file loaders for the
+ *     common types (.txt, .md, .html, .json, .csv) with default options.
+ *  3. **Glob filter** — pass `glob` to only load files matching a pattern.
+ *
+ * @example
+ * ```typescript
+ * const loader = new DirectoryLoader({
+ *   path: './docs',
+ *   recursive: true,
+ *   loaders: {
+ *     '.txt':  (p) => new TextFileLoader({ path: p }),
+ *     '.md':   (p) => new MarkdownFileLoader({ path: p, splitByHeading: 'h2' }),
+ *     '.pdf':  (p) => new PdfLoader({ path: p }),
+ *   },
+ * });
+ * const docs = await loader.load();
+ * ```
+ */
+import { BaseDocumentLoader } from './base.loader';
+import type { Document } from '../types';
+import type { BaseDocumentLoader as IBaseDocumentLoader } from './base.loader';
+export type LoaderFactory = (filePath: string) => IBaseDocumentLoader;
+export interface DirectoryLoaderOptions {
+    /** Root directory path to scan. */
+    path: string;
+    /**
+     * Map of file extension → loader factory.
+     * @example { '.txt': (p) => new TextFileLoader({ path: p }) }
+     */
+    loaders?: Record<string, LoaderFactory>;
+    /** Recurse into sub-directories. @default true */
+    recursive?: boolean;
+    /**
+     * Extensions to include.  If provided, only files with these extensions are
+     * loaded (even if a loader is registered for other types).
+     * @example ['.txt', '.md', '.pdf']
+     */
+    includeExtensions?: string[];
+    /**
+     * Extensions to exclude.
+     * @example ['.DS_Store', '.gitkeep']
+     */
+    excludeExtensions?: string[];
+    /** Maximum number of files to load (safety limit). @default 1000 */
+    maxFiles?: number;
+    /** Extra metadata merged into every document produced by this loader. */
+    metadata?: Record<string, unknown>;
+}
+export declare class DirectoryLoader extends BaseDocumentLoader {
+    private readonly opts;
+    constructor(options: DirectoryLoaderOptions);
+    load(): Promise<Document[]>;
+    private collectFiles;
+    private resolveLoader;
+    private autoDetectLoader;
+}
+//# sourceMappingURL=directory.loader.d.ts.map

package/dist/loaders/directory.loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"directory.loader.d.ts","sourceRoot":"","sources":["../../src/loaders/directory.loader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAIH,OAAO,EAAE,kBAAkB,EAAU,MAAM,eAAe,CAAC;AAC3D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,KAAK,EAAE,kBAAkB,IAAI,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAO/E,MAAM,MAAM,aAAa,GAAG,CAAC,QAAQ,EAAE,MAAM,KAAK,mBAAmB,CAAC;AAEtE,MAAM,WAAW,sBAAsB;IACrC,mCAAmC;IACnC,IAAI,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IACxC,kDAAkD;IAClD,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,oEAAoE;IACpE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,qBAIa,eAAgB,SAAQ,kBAAkB;IACrD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAmC;gBAE5C,OAAO,EAAE,sBAAsB;IAarC,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YAwCnB,YAAY;IAiC1B,OAAO,CAAC,aAAa;IAWrB,OAAO,CAAC,gBAAgB;CAqBzB"}