deeplake 0.1.0 → 0.2.0

package/dist/node/node/normalizers/text.js

@@ -0,0 +1,83 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.chunkText = chunkText;
+exports.normalizeText = normalizeText;
+const fs = __importStar(require("fs"));
+const crypto = __importStar(require("crypto"));
+function chunkText(text, chunkSize, overlap) {
+    if (text.length <= chunkSize) {
+        return text.trim() ? [text] : [];
+    }
+    const chunks = [];
+    let start = 0;
+    while (start < text.length) {
+        let end = start + chunkSize;
+        if (end >= text.length) {
+            const tail = text.slice(start).trim();
+            if (tail)
+                chunks.push(tail);
+            break;
+        }
+        const searchStart = Math.max(start + chunkSize - 100, start);
+        let bestBreak = end;
+        for (const sep of ['. ', '? ', '! ', '\n\n', '\n']) {
+            const pos = text.lastIndexOf(sep, end + 50);
+            if (pos >= searchStart && pos > start) {
+                bestBreak = pos + sep.length;
+                break;
+            }
+        }
+        const chunk = text.slice(start, bestBreak).trim();
+        if (chunk)
+            chunks.push(chunk);
+        const newStart = bestBreak - overlap;
+        start = newStart <= start ? bestBreak : newStart;
+    }
+    return chunks;
+}
+function normalizeText(filePath, chunkSize, chunkOverlap) {
+    const fileId = crypto.randomUUID();
+    const content = fs.readFileSync(filePath, 'utf-8');
+    const chunks = chunkText(content, chunkSize, chunkOverlap);
+    const name = filePath.split('/').pop() ?? filePath;
+    return {
+        id: chunks.map((_, i) => `${fileId}_${i}`),
+        file_id: Array(chunks.length).fill(fileId),
+        chunk_index: chunks.map((_, i) => i),
+        text: chunks.length > 0 ? chunks : [`Text: ${name}`],
+    };
+}
+//# sourceMappingURL=text.js.map

package/dist/node/node/normalizers/text.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"text.js","sourceRoot":"","sources":["../../../../src/node/normalizers/text.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIA,8BAwCC;AAED,sCAgBC;AA9DD,uCAAyB;AACzB,+CAAiC;AAGjC,SAAgB,SAAS,CACvB,IAAY,EACZ,SAAiB,EACjB,OAAe;IAEf,IAAI,IAAI,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACnC,CAAC;IAED,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC3B,IAAI,GAAG,GAAG,KAAK,GAAG,SAAS,CAAC;QAE5B,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;YACtC,IAAI,IAAI;gBAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5B,MAAM;QACR,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,SAAS,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;QAC7D,IAAI,SAAS,GAAG,GAAG,CAAC;QAEpB,KAAK,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;YACnD,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,EAAE,CAAC,CAAC;YAC5C,IAAI,GAAG,IAAI,WAAW,IAAI,GAAG,GAAG,KAAK,EAAE,CAAC;gBACtC,SAAS,GAAG,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC;gBAC7B,MAAM;YACR,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;QAClD,IAAI,KAAK;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;QACrC,KAAK,GAAG,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;IACnD,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAgB,aAAa,CAC3B,QAAgB,EAChB,SAAiB,EACjB,YAAoB;IAEpB,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IACnC,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,QAAQ,CAAC;IAEnD,OAAO;QACL,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC;QAC1C,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;QAC1C,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QACpC,IAAI,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,CAAC;KACrD,CAAC;AACJ,CAAC"}

package/dist/node/node/normalizers/video.d.ts

@@ -0,0 +1,3 @@
+import type { Batch } from '../../shared/types';
+export declare function normalizeVideo(filePath: string, chunkInterval?: number): AsyncGenerator<Batch>;
+//# sourceMappingURL=video.d.ts.map

package/dist/node/node/normalizers/video.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"video.d.ts","sourceRoot":"","sources":["../../../../src/node/normalizers/video.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAmBhD,wBAAuB,cAAc,CACnC,QAAQ,EAAE,MAAM,EAChB,aAAa,GAAE,MAAW,GACzB,cAAc,CAAC,KAAK,CAAC,CAwEvB"}

package/dist/node/node/normalizers/video.js

@@ -0,0 +1,119 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalizeVideo = normalizeVideo;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const crypto = __importStar(require("crypto"));
+const child_process_1 = require("child_process");
+const util_1 = require("util");
+const execFileAsync = (0, util_1.promisify)(child_process_1.execFile);
+async function probeDuration(filePath) {
+    try {
+        const { stdout } = await execFileAsync('ffprobe', [
+            '-v', 'quiet',
+            '-print_format', 'json',
+            '-show_format',
+            filePath,
+        ]);
+        const data = JSON.parse(stdout);
+        return parseFloat(data?.format?.duration ?? '0');
+    }
+    catch {
+        return 0;
+    }
+}
+async function* normalizeVideo(filePath, chunkInterval = 10) {
+    const fileId = crypto.randomUUID();
+    const filename = path.basename(filePath);
+    const duration = await probeDuration(filePath);
+    if (duration <= 0) {
+        yield {
+            id: [`${fileId}_0`],
+            file_id: [fileId],
+            chunk_index: [0],
+            start_time: [0.0],
+            end_time: [0.0],
+            video_data: [fs.readFileSync(filePath)],
+            thumbnail: [null],
+            text: [`Video: ${filename}`],
+        };
+        return;
+    }
+    const numChunks = Math.ceil(duration / chunkInterval);
+    const os = await Promise.resolve().then(() => __importStar(require('os')));
+    const tmpBase = os.tmpdir();
+    for (let idx = 0; idx < numChunks; idx++) {
+        const start = idx * chunkInterval;
+        const segDuration = Math.min(chunkInterval, duration - start);
+        const end = start + segDuration;
+        const tmpDir = fs.mkdtempSync(path.join(tmpBase, 'dl-video-'));
+        try {
+            const segPath = path.join(tmpDir, `seg_${idx}.mp4`);
+            try {
+                await execFileAsync('ffmpeg', ['-y', '-ss', String(start), '-t', String(segDuration), '-i', filePath, '-c', 'copy', segPath], { timeout: 300_000 });
+            }
+            catch {
+                continue;
+            }
+            let thumbnail = null;
+            const thumbPath = path.join(tmpDir, `thumb_${idx}.jpg`);
+            try {
+                await execFileAsync('ffmpeg', ['-y', '-ss', String((start + end) / 2), '-i', filePath, '-vframes', '1', '-s', '320x180', thumbPath], { timeout: 30_000 });
+                if (fs.existsSync(thumbPath)) {
+                    thumbnail = fs.readFileSync(thumbPath);
+                }
+            }
+            catch {
+                thumbnail = null;
+            }
+            const videoData = fs.existsSync(segPath) ? fs.readFileSync(segPath) : null;
+            yield {
+                id: [`${fileId}_${idx}`],
+                file_id: [fileId],
+                chunk_index: [idx],
+                start_time: [Math.round(start * 1000) / 1000],
+                end_time: [Math.round(end * 1000) / 1000],
+                video_data: [videoData],
+                thumbnail: [thumbnail],
+                text: [`Video segment ${idx}: ${start.toFixed(1)}s - ${end.toFixed(1)}s`],
+            };
+        }
+        finally {
+            fs.rmSync(tmpDir, { recursive: true, force: true });
+        }
+    }
+}
+//# sourceMappingURL=video.js.map

package/dist/node/node/normalizers/video.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"video.js","sourceRoot":"","sources":["../../../../src/node/normalizers/video.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwBA,wCA2EC;AAnGD,uCAAyB;AACzB,2CAA6B;AAC7B,+CAAiC;AACjC,iDAAyC;AACzC,+BAAiC;AAGjC,MAAM,aAAa,GAAG,IAAA,gBAAS,EAAC,wBAAQ,CAAC,CAAC;AAE1C,KAAK,UAAU,aAAa,CAAC,QAAgB;IAC3C,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,SAAS,EAAE;YAChD,IAAI,EAAE,OAAO;YACb,eAAe,EAAE,MAAM;YACvB,cAAc;YACd,QAAQ;SACT,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAChC,OAAO,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,IAAI,GAAG,CAAC,CAAC;IACnD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,CAAC;IACX,CAAC;AACH,CAAC;AAEM,KAAK,SAAS,CAAC,CAAC,cAAc,CACnC,QAAgB,EAChB,gBAAwB,EAAE;IAE1B,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,CAAC;IAE/C,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;QAClB,MAAM;YACJ,EAAE,EAAE,CAAC,GAAG,MAAM,IAAI,CAAC;YACnB,OAAO,EAAE,CAAC,MAAM,CAAC;YACjB,WAAW,EAAE,CAAC,CAAC,CAAC;YAChB,UAAU,EAAE,CAAC,GAAG,CAAC;YACjB,QAAQ,EAAE,CAAC,GAAG,CAAC;YACf,UAAU,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;YACvC,SAAS,EAAE,CAAC,IAAI,CAAC;YACjB,IAAI,EAAE,CAAC,UAAU,QAAQ,EAAE,CAAC;SAC7B,CAAC;QACF,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,aAAa,CAAC,CAAC;IACtD,MAAM,EAAE,GAAG,wDAAa,IAAI,GAAC,CAAC;IAC9B,MAAM,OAAO,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC;IAE5B,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,SAAS,EAAE,GAAG,EAAE,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,GAAG,GAAG,aAAa,CAAC;QAClC,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,QAAQ,GAAG,KAAK,CAAC,CAAC;QAC9D,MAAM,GAAG,GAAG,KAAK,GAAG,WAAW,CAAC;QAEhC,MAAM,MAAM,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC;QAC/D,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,CAAC,CAAC;YACpD,IAAI,CAAC;gBACH,MAAM,aAAa,CACjB,QAAQ,EACR,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAC9F,EAAE,OAAO,EAAE,OAAO,EAAE,CACrB,CAAC;YACJ,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;YAED,IAAI,SAAS,GAAkB,IAAI,CAAC;YACpC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,CAAC,CAAC;YACxD,IAAI,CAAC;gBACH,MAAM,aAAa,CACjB,QAAQ,EACR,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,EACrG,EAAE,OAAO,EAAE,MAAM,EAAE,CACpB,CAAC;gBACF,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;oBAC7B,SAAS,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;gBACzC,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS,GAAG,IAAI,CAAC;YACnB,CAAC;YAED,MAAM,SAAS,GAAG,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAE3E,MAAM;gBACJ,EAAE,EAAE,CAAC,GAAG,MAAM,IAAI,GAAG,EAAE,CAAC;gBACxB,OAAO,EAAE,CAAC,MAAM,CAAC;gBACjB,WAAW,EAAE,CAAC,GAAG,CAAC;gBAClB,UAAU,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;gBAC7C,QAAQ,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;gBACzC,UAAU,EAAE,CAAC,SAAS,CAAC;gBACvB,SAAS,EAAE,CAAC,SAAS,CAAC;gBACtB,IAAI,EAAE,CAAC,iBAAiB,GAAG,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;aAC1E,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/node/node/storage.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Node.js S3 storage client factory for the WASM deeplake engine.
+ *
+ * The C++ WASM code calls js_storage::client_factory()(type, config, mode?)
+ * to create a JS client object with methods like download, exists, list, etc.
+ * This module provides the factory that creates those client objects using
+ * the AWS SDK v3.
+ *
+ * Credential rotation: STS credentials are refreshed automatically before
+ * they expire (5 minutes before expiry, matching the native C++ behavior).
+ * The C++ side passes refresh metadata (orgId, dsName, token) through the
+ * config object so the JS side can call the backend API for fresh creds.
+ */
+export declare function createStorageFactory(wasmModule: any): (type: string, config: any, mode?: string) => any;
+//# sourceMappingURL=storage.d.ts.map

package/dist/node/node/storage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage.d.ts","sourceRoot":"","sources":["../../../src/node/storage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAsKH,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,GAAG,IAEhD,MAAM,MAAM,EACZ,QAAQ,GAAG,EACX,OAAO,MAAM,KACZ,GAAG,CA8FP"}