deeplake 0.1.0 → 0.2.0

package/dist/node/node/normalizers/binary.js

@@ -0,0 +1,51 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalizeBinary = normalizeBinary;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const crypto = __importStar(require("crypto"));
+function normalizeBinary(filePath) {
+    const fileId = crypto.randomUUID();
+    const data = fs.readFileSync(filePath);
+    const filename = path.basename(filePath);
+    return {
+        id: [`${fileId}_0`],
+        file_id: [fileId],
+        data: [data],
+        filename: [filename],
+    };
+}
+//# sourceMappingURL=binary.js.map

package/dist/node/node/normalizers/binary.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"binary.js","sourceRoot":"","sources":["../../../../src/node/normalizers/binary.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAKA,0CAWC;AAhBD,uCAAyB;AACzB,2CAA6B;AAC7B,+CAAiC;AAGjC,SAAgB,eAAe,CAAC,QAAgB;IAC9C,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAEzC,OAAO;QACL,EAAE,EAAE,CAAC,GAAG,MAAM,IAAI,CAAC;QACnB,OAAO,EAAE,CAAC,MAAM,CAAC;QACjB,IAAI,EAAE,CAAC,IAAI,CAAC;QACZ,QAAQ,EAAE,CAAC,QAAQ,CAAC;KACrB,CAAC;AACJ,CAAC"}

package/dist/node/node/normalizers/dict.d.ts

@@ -0,0 +1,3 @@
+import type { Batch } from '../../shared/types';
+export declare function normalizeDict(data: Record<string, unknown[]>, batchSize?: number): Generator<Batch>;
+//# sourceMappingURL=dict.d.ts.map

package/dist/node/node/normalizers/dict.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dict.d.ts","sourceRoot":"","sources":["../../../../src/node/normalizers/dict.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAEhD,wBAAiB,aAAa,CAC5B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,EAC/B,SAAS,GAAE,MAAa,GACvB,SAAS,CAAC,KAAK,CAAC,CAalB"}

package/dist/node/node/normalizers/dict.js

@@ -0,0 +1,17 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalizeDict = normalizeDict;
+function* normalizeDict(data, batchSize = 1000) {
+    const firstCol = Object.keys(data)[0];
+    if (!firstCol)
+        return;
+    const rowCount = data[firstCol].length;
+    for (let i = 0; i < rowCount; i += batchSize) {
+        const batch = {};
+        for (const [col, values] of Object.entries(data)) {
+            batch[col] = values.slice(i, i + batchSize);
+        }
+        yield batch;
+    }
+}
+//# sourceMappingURL=dict.js.map

package/dist/node/node/normalizers/dict.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dict.js","sourceRoot":"","sources":["../../../../src/node/normalizers/dict.ts"],"names":[],"mappings":";;AAEA,sCAgBC;AAhBD,QAAe,CAAC,CAAC,aAAa,CAC5B,IAA+B,EAC/B,YAAoB,IAAI;IAExB,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACtC,IAAI,CAAC,QAAQ;QAAE,OAAO;IAEtB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IAEvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QAC7C,MAAM,KAAK,GAAU,EAAE,CAAC;QACxB,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACjD,KAAK,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;QAC9C,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC"}

package/dist/node/node/normalizers/files.d.ts

@@ -0,0 +1,3 @@
+import type { Batch } from '../../shared/types';
+export declare function normalizeFiles(files: string[], chunkSize: number, chunkOverlap: number, concurrency?: number): AsyncGenerator<Batch>;
+//# sourceMappingURL=files.d.ts.map

package/dist/node/node/normalizers/files.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"files.d.ts","sourceRoot":"","sources":["../../../../src/node/normalizers/files.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAiDhD,wBAAuB,cAAc,CACnC,KAAK,EAAE,MAAM,EAAE,EACf,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,MAAM,EACpB,WAAW,GAAE,MAAU,GACtB,cAAc,CAAC,KAAK,CAAC,CA6CvB"}

package/dist/node/node/normalizers/files.js

@@ -0,0 +1,109 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalizeFiles = normalizeFiles;
+const path = __importStar(require("path"));
+const fs = __importStar(require("fs"));
+const errors_1 = require("../../shared/errors");
+const image_1 = require("./image");
+const text_1 = require("./text");
+const binary_1 = require("./binary");
+const video_1 = require("./video");
+const pdf_1 = require("./pdf");
+const VIDEO_EXTS = new Set(['.mp4', '.mov', '.avi', '.mkv', '.webm']);
+const IMAGE_EXTS = new Set(['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.webp']);
+const TEXT_EXTS = new Set(['.txt', '.md', '.json', '.csv', '.xml', '.html']);
+async function normalizeOneFile(filePath, chunkSize, chunkOverlap) {
+    if (!fs.existsSync(filePath)) {
+        throw new errors_1.IngestError(`File not found: ${filePath}`);
+    }
+    const ext = path.extname(filePath).toLowerCase();
+    if (VIDEO_EXTS.has(ext)) {
+        const batches = [];
+        for await (const batch of (0, video_1.normalizeVideo)(filePath)) {
+            batches.push(batch);
+        }
+        return batches;
+    }
+    if (IMAGE_EXTS.has(ext)) {
+        return [(0, image_1.normalizeImage)(filePath)];
+    }
+    if (ext === '.pdf') {
+        const batches = [];
+        for await (const batch of (0, pdf_1.normalizePdf)(filePath)) {
+            batches.push(batch);
+        }
+        return batches;
+    }
+    if (TEXT_EXTS.has(ext)) {
+        return [(0, text_1.normalizeText)(filePath, chunkSize, chunkOverlap)];
+    }
+    return [(0, binary_1.normalizeBinary)(filePath)];
+}
+async function* normalizeFiles(files, chunkSize, chunkOverlap, concurrency = 4) {
+    const maxWorkers = Math.min(concurrency, files.length);
+    if (maxWorkers <= 1) {
+        for (const filePath of files) {
+            const batches = await normalizeOneFile(filePath, chunkSize, chunkOverlap);
+            for (const batch of batches) {
+                yield batch;
+            }
+        }
+        return;
+    }
+    // Process files with limited concurrency
+    let index = 0;
+    const pending = [];
+    function enqueue() {
+        while (pending.length < maxWorkers && index < files.length) {
+            const filePath = files[index++];
+            pending.push(normalizeOneFile(filePath, chunkSize, chunkOverlap)
+                .then((batches) => ({ filePath, batches }))
+                .catch((e) => {
+                throw new errors_1.IngestError(`Failed to process ${filePath}: ${e instanceof Error ? e.message : e}`);
+            }));
+        }
+    }
+    enqueue();
+    while (pending.length > 0) {
+        const result = await Promise.race(pending.map((p, i) => p.then((r) => ({ ...r, idx: i }))));
+        pending.splice(result.idx, 1);
+        enqueue();
+        for (const batch of result.batches) {
+            yield batch;
+        }
+    }
+}
+//# sourceMappingURL=files.js.map

package/dist/node/node/normalizers/files.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"files.js","sourceRoot":"","sources":["../../../../src/node/normalizers/files.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoDA,wCAkDC;AAtGD,2CAA6B;AAC7B,uCAAyB;AACzB,gDAAkD;AAElD,mCAAyC;AACzC,iCAAuC;AACvC,qCAA2C;AAC3C,mCAAyC;AACzC,+BAAqC;AAErC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;AACtE,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;AAC/E,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;AAE7E,KAAK,UAAU,gBAAgB,CAC7B,QAAgB,EAChB,SAAiB,EACjB,YAAoB;IAEpB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,oBAAW,CAAC,mBAAmB,QAAQ,EAAE,CAAC,CAAC;IACvD,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;IAEjD,IAAI,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,OAAO,GAAY,EAAE,CAAC;QAC5B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAA,sBAAc,EAAC,QAAQ,CAAC,EAAE,CAAC;YACnD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,IAAI,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,IAAA,sBAAc,EAAC,QAAQ,CAAC,CAAC,CAAC;IACpC,CAAC;IAED,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;QACnB,MAAM,OAAO,GAAY,EAAE,CAAC;QAC5B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAA,kBAAY,EAAC,QAAQ,CAAC,EAAE,CAAC;YACjD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,IAAA,oBAAa,EAAC,QAAQ,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC,CAAC;IAC5D,CAAC;IAED,OAAO,CAAC,IAAA,wBAAe,EAAC,QAAQ,CAAC,CAAC,CAAC;AACrC,CAAC;AAEM,KAAK,SAAS,CAAC,CAAC,cAAc,CACnC,KAAe,EACf,SAAiB,EACjB,YAAoB,EACpB,cAAsB,CAAC;IAEvB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAEvD,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;QACpB,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,QAAQ,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;YAC1E,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC5B,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;QACD,OAAO;IACT,CAAC;IAED,yCAAyC;IACzC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,MAAM,OAAO,GAAsD,EAAE,CAAC;IAEtE,SAAS,OAAO;QACd,OAAO,OAAO,CAAC,MAAM,GAAG,UAAU,IAAI,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC3D,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;YAChC,OAAO,CAAC,IAAI,CACV,gBAAgB,CAAC,QAAQ,EAAE,SAAS,EAAE,YAAY,CAAC;iBAChD,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;iBAC1C,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;gBACX,MAAM,IAAI,oBAAW,CACnB,qBAAqB,QAAQ,KAAK,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CACvE,CAAC;YACJ,CAAC,CAAC,CACL,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,EAAE,CAAC;IAEV,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAC/B,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CACzD,CAAC;QACF,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAC9B,OAAO,EAAE,CAAC;QAEV,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnC,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/node/node/normalizers/huggingface.d.ts

@@ -0,0 +1,3 @@
+import type { Batch } from '../../shared/types';
+export declare function normalizeHuggingface(datasetName: string, batchSize?: number, split?: string): AsyncGenerator<Batch>;
+//# sourceMappingURL=huggingface.d.ts.map

package/dist/node/node/normalizers/huggingface.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"huggingface.d.ts","sourceRoot":"","sources":["../../../../src/node/normalizers/huggingface.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAEhD,wBAAuB,oBAAoB,CACzC,WAAW,EAAE,MAAM,EACnB,SAAS,GAAE,MAAY,EACvB,KAAK,GAAE,MAAgB,GACtB,cAAc,CAAC,KAAK,CAAC,CA4CvB"}

package/dist/node/node/normalizers/huggingface.js

@@ -0,0 +1,158 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalizeHuggingface = normalizeHuggingface;
+async function* normalizeHuggingface(datasetName, batchSize = 100, split = 'train') {
+    let rows;
+    try {
+        const hub = require('@huggingface/hub');
+        if (typeof hub.listDatasetFiles === 'function') {
+            rows = streamViaHub(hub, datasetName, split);
+        }
+        else {
+            throw new Error('Incompatible @huggingface/hub version');
+        }
+    }
+    catch {
+        try {
+            rows = streamViaPython(datasetName, split);
+        }
+        catch {
+            throw new Error(`No HuggingFace dataset library available. Install one of:\n` +
+                `  npm install @huggingface/hub\n` +
+                `Or ensure Python is available with: pip install datasets`);
+        }
+    }
+    const batch = [];
+    for await (const item of rows) {
+        const row = {};
+        for (const [col, val] of Object.entries(item)) {
+            if (val instanceof Uint8Array && !Buffer.isBuffer(val)) {
+                row[col] = Buffer.from(val);
+            }
+            else {
+                row[col] = val;
+            }
+        }
+        batch.push(row);
+        if (batch.length >= batchSize) {
+            yield rowsToBatch(batch);
+            batch.length = 0;
+        }
+    }
+    if (batch.length > 0) {
+        yield rowsToBatch(batch);
+    }
+}
+async function* streamViaHub(hub, datasetName, split) {
+    const baseUrl = `https://datasets-server.huggingface.co/rows?dataset=${encodeURIComponent(datasetName)}&config=default&split=${encodeURIComponent(split)}`;
+    let offset = 0;
+    const length = 100;
+    while (true) {
+        const url = `${baseUrl}&offset=${offset}&length=${length}`;
+        const resp = await fetch(url);
+        if (!resp.ok)
+            break;
+        const data = (await resp.json());
+        const rows = data.rows;
+        if (!rows || rows.length === 0)
+            break;
+        for (const rowWrapper of rows) {
+            yield rowWrapper.row;
+        }
+        if (rows.length < length)
+            break;
+        offset += rows.length;
+    }
+}
+async function* streamViaPython(datasetName, split) {
+    const { spawn } = await Promise.resolve().then(() => __importStar(require('child_process')));
+    const readline = await Promise.resolve().then(() => __importStar(require('readline')));
+    const pyScript = `
+import json, sys
+from datasets import load_dataset
+ds = load_dataset("${datasetName.replace(/"/g, '\\"')}", split="${split.replace(/"/g, '\\"')}", streaming=True)
+for item in ds:
+    row = {}
+    for k, v in item.items():
+        if hasattr(v, 'tobytes'):
+            import base64
+            row[k] = {"__bytes__": base64.b64encode(v.tobytes()).decode()}
+        elif hasattr(v, 'tolist'):
+            row[k] = v.tolist()
+        else:
+            row[k] = v
+    print(json.dumps(row))
+`;
+    const proc = spawn('python3', ['-c', pyScript], {
+        stdio: ['pipe', 'pipe', 'pipe'],
+    });
+    const rl = readline.createInterface({ input: proc.stdout });
+    for await (const line of rl) {
+        const trimmed = line.trim();
+        if (!trimmed)
+            continue;
+        const row = JSON.parse(trimmed);
+        for (const [key, val] of Object.entries(row)) {
+            if (val &&
+                typeof val === 'object' &&
+                '__bytes__' in val) {
+                row[key] = Buffer.from(val.__bytes__, 'base64');
+            }
+        }
+        yield row;
+    }
+    await new Promise((resolve, reject) => {
+        proc.on('close', (code) => {
+            if (code !== 0) {
+                reject(new Error(`Python process exited with code ${code}`));
+            }
+            else {
+                resolve();
+            }
+        });
+    });
+}
+function rowsToBatch(rows) {
+    if (rows.length === 0)
+        return {};
+    const keys = Object.keys(rows[0]);
+    const result = {};
+    for (const key of keys) {
+        result[key] = rows.map((r) => r[key]);
+    }
+    return result;
+}
+//# sourceMappingURL=huggingface.js.map

package/dist/node/node/normalizers/huggingface.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"huggingface.js","sourceRoot":"","sources":["../../../../src/node/normalizers/huggingface.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,oDAgDC;AAhDM,KAAK,SAAS,CAAC,CAAC,oBAAoB,CACzC,WAAmB,EACnB,YAAoB,GAAG,EACvB,QAAgB,OAAO;IAEvB,IAAI,IAA4C,CAAC;IAEjD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,OAAO,CAAC,kBAAkB,CAAQ,CAAC;QAC/C,IAAI,OAAO,GAAG,CAAC,gBAAgB,KAAK,UAAU,EAAE,CAAC;YAC/C,IAAI,GAAG,YAAY,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;QAC/C,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,IAAI,CAAC;YACH,IAAI,GAAG,eAAe,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QAC7C,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CACb,6DAA6D;gBAC7D,kCAAkC;gBAClC,0DAA0D,CAC3D,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAA8B,EAAE,CAAC;IAE5C,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QAC9B,MAAM,GAAG,GAA4B,EAAE,CAAC;QACxC,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9C,IAAI,GAAG,YAAY,UAAU,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBACvD,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC9B,CAAC;iBAAM,CAAC;gBACN,GAAG,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;YACjB,CAAC;QACH,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEhB,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;YAC9B,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC;YACzB,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC;AACH,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,YAAY,CAC1B,GAAQ,EACR,WAAmB,EACnB,KAAa;IAEb,MAAM,OAAO,GAAG,uDAAuD,kBAAkB,CAAC,WAAW,CAAC,yBAAyB,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;IAE3J,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,MAAM,GAAG,GAAG,CAAC;IAEnB,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,GAAG,GAAG,GAAG,OAAO,WAAW,MAAM,WAAW,MAAM,EAAE,CAAC;QAC3D,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,MAAM;QAEpB,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAQ,CAAC;QACxC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAa,CAAC;QAChC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM;QAEtC,KAAK,MAAM,UAAU,IAAI,IAAI,EAAE,CAAC;YAC9B,MAAM,UAAU,CAAC,GAA8B,CAAC;QAClD,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM;YAAE,MAAM;QAChC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC;IACxB,CAAC;AACH,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,eAAe,CAC7B,WAAmB,EACnB,KAAa;IAEb,MAAM,EAAE,KAAK,EAAE,GAAG,wDAAa,eAAe,GAAC,CAAC;IAChD,MAAM,QAAQ,GAAG,wDAAa,UAAU,GAAC,CAAC;IAE1C,MAAM,QAAQ,GAAG;;;qBAGE,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,aAAa,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;;;;;;;;;;;;CAY3F,CAAC;IAEA,MAAM,IAAI,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAE;QAC9C,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;KAChC,CAAC,CAAC;IAEH,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,MAAO,EAAE,CAAC,CAAC;IAE7D,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;QAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAA4B,CAAC;QAE3D,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7C,IACE,GAAG;gBACH,OAAO,GAAG,KAAK,QAAQ;gBACvB,WAAW,IAAK,GAA+B,EAC/C,CAAC;gBACD,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,CACnB,GAA6B,CAAC,SAAS,EACxC,QAAQ,CACT,CAAC;YACJ,CAAC;QACH,CAAC;QAED,MAAM,GAAG,CAAC;IACZ,CAAC;IAED,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC1C,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;YACxB,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBACf,MAAM,CAAC,IAAI,KAAK,CAAC,mCAAmC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC/D,CAAC;iBAAM,CAAC;gBACN,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,WAAW,CAAC,IAA+B;IAClD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACjC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAClC,MAAM,MAAM,GAAU,EAAE,CAAC;IACzB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/node/node/normalizers/image.d.ts

@@ -0,0 +1,3 @@
+import type { Batch } from '../../shared/types';
+export declare function normalizeImage(filePath: string): Batch;
+//# sourceMappingURL=image.d.ts.map

package/dist/node/node/normalizers/image.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"image.d.ts","sourceRoot":"","sources":["../../../../src/node/normalizers/image.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAEhD,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,KAAK,CAYtD"}

package/dist/node/node/normalizers/image.js

@@ -0,0 +1,52 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalizeImage = normalizeImage;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const crypto = __importStar(require("crypto"));
+function normalizeImage(filePath) {
+    const fileId = crypto.randomUUID();
+    const data = fs.readFileSync(filePath);
+    const filename = path.basename(filePath);
+    return {
+        id: [`${fileId}_0`],
+        file_id: [fileId],
+        image: [data],
+        filename: [filename],
+        text: [`Image: ${filename}`],
+    };
+}
+//# sourceMappingURL=image.js.map

package/dist/node/node/normalizers/image.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"image.js","sourceRoot":"","sources":["../../../../src/node/normalizers/image.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAKA,wCAYC;AAjBD,uCAAyB;AACzB,2CAA6B;AAC7B,+CAAiC;AAGjC,SAAgB,cAAc,CAAC,QAAgB;IAC7C,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAEzC,OAAO;QACL,EAAE,EAAE,CAAC,GAAG,MAAM,IAAI,CAAC;QACnB,OAAO,EAAE,CAAC,MAAM,CAAC;QACjB,KAAK,EAAE,CAAC,IAAI,CAAC;QACb,QAAQ,EAAE,CAAC,QAAQ,CAAC;QACpB,IAAI,EAAE,CAAC,UAAU,QAAQ,EAAE,CAAC;KAC7B,CAAC;AACJ,CAAC"}

package/dist/node/node/normalizers/index.d.ts

@@ -0,0 +1,9 @@
+export { normalizeDict } from './dict';
+export { chunkText, normalizeText } from './text';
+export { normalizeBinary } from './binary';
+export { normalizeImage } from './image';
+export { normalizeVideo } from './video';
+export { normalizePdf } from './pdf';
+export { normalizeFiles } from './files';
+export { normalizeHuggingface } from './huggingface';
+//# sourceMappingURL=index.d.ts.map

package/dist/node/node/normalizers/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/node/normalizers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,OAAO,CAAC;AACrC,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC"}

package/dist/node/node/normalizers/index.js

@@ -0,0 +1,21 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalizeHuggingface = exports.normalizeFiles = exports.normalizePdf = exports.normalizeVideo = exports.normalizeImage = exports.normalizeBinary = exports.normalizeText = exports.chunkText = exports.normalizeDict = void 0;
+var dict_1 = require("./dict");
+Object.defineProperty(exports, "normalizeDict", { enumerable: true, get: function () { return dict_1.normalizeDict; } });
+var text_1 = require("./text");
+Object.defineProperty(exports, "chunkText", { enumerable: true, get: function () { return text_1.chunkText; } });
+Object.defineProperty(exports, "normalizeText", { enumerable: true, get: function () { return text_1.normalizeText; } });
+var binary_1 = require("./binary");
+Object.defineProperty(exports, "normalizeBinary", { enumerable: true, get: function () { return binary_1.normalizeBinary; } });
+var image_1 = require("./image");
+Object.defineProperty(exports, "normalizeImage", { enumerable: true, get: function () { return image_1.normalizeImage; } });
+var video_1 = require("./video");
+Object.defineProperty(exports, "normalizeVideo", { enumerable: true, get: function () { return video_1.normalizeVideo; } });
+var pdf_1 = require("./pdf");
+Object.defineProperty(exports, "normalizePdf", { enumerable: true, get: function () { return pdf_1.normalizePdf; } });
+var files_1 = require("./files");
+Object.defineProperty(exports, "normalizeFiles", { enumerable: true, get: function () { return files_1.normalizeFiles; } });
+var huggingface_1 = require("./huggingface");
+Object.defineProperty(exports, "normalizeHuggingface", { enumerable: true, get: function () { return huggingface_1.normalizeHuggingface; } });
+//# sourceMappingURL=index.js.map

package/dist/node/node/normalizers/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/node/normalizers/index.ts"],"names":[],"mappings":";;;AAAA,+BAAuC;AAA9B,qGAAA,aAAa,OAAA;AACtB,+BAAkD;AAAzC,iGAAA,SAAS,OAAA;AAAE,qGAAA,aAAa,OAAA;AACjC,mCAA2C;AAAlC,yGAAA,eAAe,OAAA;AACxB,iCAAyC;AAAhC,uGAAA,cAAc,OAAA;AACvB,iCAAyC;AAAhC,uGAAA,cAAc,OAAA;AACvB,6BAAqC;AAA5B,mGAAA,YAAY,OAAA;AACrB,iCAAyC;AAAhC,uGAAA,cAAc,OAAA;AACvB,6CAAqD;AAA5C,mHAAA,oBAAoB,OAAA"}

package/dist/node/node/normalizers/pdf.d.ts

@@ -0,0 +1,3 @@
+import type { Batch } from '../../shared/types';
+export declare function normalizePdf(filePath: string): AsyncGenerator<Batch>;
+//# sourceMappingURL=pdf.d.ts.map

package/dist/node/node/normalizers/pdf.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf.d.ts","sourceRoot":"","sources":["../../../../src/node/normalizers/pdf.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAEhD,wBAAuB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,CAsC3E"}

package/dist/node/node/normalizers/pdf.js

@@ -0,0 +1,75 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalizePdf = normalizePdf;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const crypto = __importStar(require("crypto"));
+async function* normalizePdf(filePath) {
+    const fileId = crypto.randomUUID();
+    const filename = path.basename(filePath);
+    let pdfjs;
+    try {
+        pdfjs = require('pdfjs-dist');
+    }
+    catch {
+        // Fallback: store whole PDF as binary
+        yield {
+            id: [`${fileId}_0`],
+            file_id: [fileId],
+            page_index: [0],
+            data: [fs.readFileSync(filePath)],
+            text: [`PDF: ${filename}`],
+        };
+        return;
+    }
+    const data = new Uint8Array(fs.readFileSync(filePath));
+    const doc = await pdfjs.getDocument({ data }).promise;
+    for (let pageIdx = 0; pageIdx < doc.numPages; pageIdx++) {
+        const page = await doc.getPage(pageIdx + 1);
+        const textContent = await page.getTextContent();
+        const text = textContent.items
+            .map((item) => item.str)
+            .join(' ');
+        yield {
+            id: [`${fileId}_${pageIdx}`],
+            file_id: [fileId],
+            page_index: [pageIdx],
+            text: [text || `PDF page ${pageIdx + 1}`],
+        };
+    }
+    doc.destroy();
+}
+//# sourceMappingURL=pdf.js.map

package/dist/node/node/normalizers/pdf.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf.js","sourceRoot":"","sources":["../../../../src/node/normalizers/pdf.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAKA,oCAsCC;AA3CD,uCAAyB;AACzB,2CAA6B;AAC7B,+CAAiC;AAG1B,KAAK,SAAS,CAAC,CAAC,YAAY,CAAC,QAAgB;IAClD,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAEzC,IAAI,KAAU,CAAC;IACf,IAAI,CAAC;QACH,KAAK,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,sCAAsC;QACtC,MAAM;YACJ,EAAE,EAAE,CAAC,GAAG,MAAM,IAAI,CAAC;YACnB,OAAO,EAAE,CAAC,MAAM,CAAC;YACjB,UAAU,EAAE,CAAC,CAAC,CAAC;YACf,IAAI,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;YACjC,IAAI,EAAE,CAAC,QAAQ,QAAQ,EAAE,CAAC;SAC3B,CAAC;QACF,OAAO;IACT,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC;IACvD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC;IAEtD,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,GAAG,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,CAAC;QACxD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;QAC5C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAChD,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK;aAC3B,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC;aAC5B,IAAI,CAAC,GAAG,CAAC,CAAC;QAEb,MAAM;YACJ,EAAE,EAAE,CAAC,GAAG,MAAM,IAAI,OAAO,EAAE,CAAC;YAC5B,OAAO,EAAE,CAAC,MAAM,CAAC;YACjB,UAAU,EAAE,CAAC,OAAO,CAAC;YACrB,IAAI,EAAE,CAAC,IAAI,IAAI,YAAY,OAAO,GAAG,CAAC,EAAE,CAAC;SAC1C,CAAC;IACJ,CAAC;IAED,GAAG,CAAC,OAAO,EAAE,CAAC;AAChB,CAAC"}

package/dist/node/node/normalizers/text.d.ts

@@ -0,0 +1,4 @@
+import type { Batch } from '../../shared/types';
+export declare function chunkText(text: string, chunkSize: number, overlap: number): string[];
+export declare function normalizeText(filePath: string, chunkSize: number, chunkOverlap: number): Batch;
+//# sourceMappingURL=text.d.ts.map

package/dist/node/node/normalizers/text.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"text.d.ts","sourceRoot":"","sources":["../../../../src/node/normalizers/text.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAEhD,wBAAgB,SAAS,CACvB,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,GACd,MAAM,EAAE,CAoCV;AAED,wBAAgB,aAAa,CAC3B,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,MAAM,GACnB,KAAK,CAYP"}