deeplake 0.1.0 → 0.2.0

package/dist/node/node/formats/lerobot.d.ts

@@ -0,0 +1,42 @@
+import type { Batch, Format, SchemaMap } from '../../shared/types';
+export interface LeRobotTasksOptions {
+    datasetDir: string;
+}
+export declare class LeRobotTasks implements Format {
+    private tasksFile;
+    constructor(options: LeRobotTasksOptions);
+    schema(): SchemaMap;
+    normalize(): Generator<Batch>;
+}
+export interface LeRobotFramesOptions {
+    datasetDir: string;
+    chunkStart?: number;
+    chunkEnd?: number;
+}
+export declare class LeRobotFrames implements Format {
+    private datasetDir;
+    private chunkStart;
+    private chunkEnd;
+    constructor(options: LeRobotFramesOptions);
+    schema(): SchemaMap;
+    normalize(): Generator<Batch>;
+}
+export interface LeRobotEpisodesOptions {
+    datasetDir: string;
+    chunkStart?: number;
+    chunkEnd?: number;
+}
+export declare class LeRobotEpisodes implements Format {
+    private datasetDir;
+    private chunkStart;
+    private chunkEnd;
+    private _tasks;
+    private _episodes;
+    constructor(options: LeRobotEpisodesOptions);
+    schema(): SchemaMap;
+    pg_schema(): Record<string, string>;
+    normalize(): Generator<Batch>;
+    private _pullChunkVideos;
+    private _restoreChunkPointers;
+}
+//# sourceMappingURL=lerobot.d.ts.map

package/dist/node/node/formats/lerobot.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lerobot.d.ts","sourceRoot":"","sources":["../../../../src/node/formats/lerobot.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAMnE,MAAM,WAAW,mBAAmB;IAClC,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,YAAa,YAAW,MAAM;IACzC,OAAO,CAAC,SAAS,CAAS;gBAEd,OAAO,EAAE,mBAAmB;IAIxC,MAAM,IAAI,SAAS;IAIlB,SAAS,IAAI,SAAS,CAAC,KAAK,CAAC;CA0B/B;AAkBD,MAAM,WAAW,oBAAoB;IACnC,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,qBAAa,aAAc,YAAW,MAAM;IAC1C,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,QAAQ,CAAS;gBAEb,OAAO,EAAE,oBAAoB;IAMzC,MAAM,IAAI,SAAS;IAIlB,SAAS,IAAI,SAAS,CAAC,KAAK,CAAC;CA8D/B;AAeD,MAAM,WAAW,sBAAsB;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,qBAAa,eAAgB,YAAW,MAAM;IAC5C,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,MAAM,CAAsB;IACpC,OAAO,CAAC,SAAS,CAAuC;gBAE5C,OAAO,EAAE,sBAAsB;IA4B3C,MAAM,IAAI,SAAS;IAQnB,SAAS,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IAQlC,SAAS,IAAI,SAAS,CAAC,KAAK,CAAC;IAiF9B,OAAO,CAAC,gBAAgB;IAiBxB,OAAO,CAAC,qBAAqB;CAmC9B"}

package/dist/node/node/formats/lerobot.js

@@ -0,0 +1,351 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LeRobotEpisodes = exports.LeRobotFrames = exports.LeRobotTasks = void 0;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const child_process_1 = require("child_process");
+class LeRobotTasks {
+    tasksFile;
+    constructor(options) {
+        this.tasksFile = path.join(options.datasetDir, 'meta', 'tasks.jsonl');
+    }
+    schema() {
+        return {};
+    }
+    *normalize() {
+        if (!fs.existsSync(this.tasksFile)) {
+            throw new Error(`Tasks file not found: ${this.tasksFile}`);
+        }
+        const taskIndices = [];
+        const tasks = [];
+        const lines = fs.readFileSync(this.tasksFile, 'utf-8').split('\n');
+        for (const line of lines) {
+            const trimmed = line.trim();
+            if (!trimmed)
+                continue;
+            const rec = JSON.parse(trimmed);
+            taskIndices.push(rec.task_index);
+            tasks.push(rec.task);
+            if (taskIndices.length >= 1000) {
+                yield { task_index: taskIndices.splice(0), task: tasks.splice(0) };
+            }
+        }
+        if (taskIndices.length > 0) {
+            yield { task_index: taskIndices, task: tasks };
+        }
+    }
+}
+exports.LeRobotTasks = LeRobotTasks;
+// =====================================================================
+// LeRobotFrames
+// =====================================================================
+const STATE_NAMES = [
+    'state_x', 'state_y', 'state_z',
+    'state_roll', 'state_pitch', 'state_yaw',
+    'state_pad', 'state_gripper',
+];
+const ACTION_NAMES = [
+    'action_x', 'action_y', 'action_z',
+    'action_roll', 'action_pitch', 'action_yaw',
+    'action_gripper',
+];
+class LeRobotFrames {
+    datasetDir;
+    chunkStart;
+    chunkEnd;
+    constructor(options) {
+        this.datasetDir = options.datasetDir;
+        this.chunkStart = options.chunkStart ?? 0;
+        this.chunkEnd = options.chunkEnd ?? 10;
+    }
+    schema() {
+        return {};
+    }
+    *normalize() {
+        const allColumns = [
+            'index', 'episode_index', 'frame_index', 'task_index', 'timestamp',
+            ...STATE_NAMES, ...ACTION_NAMES,
+        ];
+        let batch = {};
+        for (const col of allColumns) {
+            batch[col] = [];
+        }
+        let batchSize = 0;
+        for (let chunkIdx = this.chunkStart; chunkIdx <= this.chunkEnd; chunkIdx++) {
+            const chunkDir = path.join(this.datasetDir, 'data', `chunk-${String(chunkIdx).padStart(3, '0')}`);
+            if (!fs.existsSync(chunkDir))
+                continue;
+            const parquetFiles = fs.readdirSync(chunkDir)
+                .filter((f) => f.startsWith('episode_') && f.endsWith('.parquet'))
+                .sort()
+                .map((f) => path.join(chunkDir, f));
+            for (const pqPath of parquetFiles) {
+                const rows = readParquetFile(pqPath);
+                if (!rows || rows.length === 0)
+                    continue;
+                for (const row of rows) {
+                    batch['index'].push(row.index);
+                    batch['episode_index'].push(row.episode_index);
+                    batch['frame_index'].push(row.frame_index);
+                    batch['task_index'].push(row.task_index);
+                    batch['timestamp'].push(row.timestamp);
+                    const state = row['observation.state'] ?? [];
+                    for (let i = 0; i < STATE_NAMES.length; i++) {
+                        batch[STATE_NAMES[i]].push(state[i] ?? 0);
+                    }
+                    const action = row['action'] ?? [];
+                    for (let i = 0; i < ACTION_NAMES.length; i++) {
+                        batch[ACTION_NAMES[i]].push(action[i] ?? 0);
+                    }
+                    batchSize++;
+                }
+                if (batchSize >= 5000) {
+                    yield batch;
+                    batch = {};
+                    for (const col of allColumns) {
+                        batch[col] = [];
+                    }
+                    batchSize = 0;
+                }
+            }
+        }
+        if (batchSize > 0) {
+            yield batch;
+        }
+    }
+}
+exports.LeRobotFrames = LeRobotFrames;
+// =====================================================================
+// LeRobotEpisodes
+// =====================================================================
+const VIDEO_KEYS = [
+    'observation.images.exterior_image_1_left',
+    'observation.images.exterior_image_2_left',
+    'observation.images.wrist_image_left',
+];
+const COL_NAMES = ['exterior_1_video', 'exterior_2_video', 'wrist_video'];
+const EPISODE_COLUMNS = ['episode_index', 'task', 'length', ...COL_NAMES];
+class LeRobotEpisodes {
+    datasetDir;
+    chunkStart;
+    chunkEnd;
+    _tasks;
+    _episodes;
+    constructor(options) {
+        this.datasetDir = options.datasetDir;
+        this.chunkStart = options.chunkStart ?? 0;
+        this.chunkEnd = options.chunkEnd ?? 10;
+        const tasksFile = path.join(this.datasetDir, 'meta', 'tasks.jsonl');
+        this._tasks = new Map();
+        if (fs.existsSync(tasksFile)) {
+            for (const line of fs.readFileSync(tasksFile, 'utf-8').split('\n')) {
+                const trimmed = line.trim();
+                if (!trimmed)
+                    continue;
+                const rec = JSON.parse(trimmed);
+                this._tasks.set(rec.task_index, rec.task);
+            }
+        }
+        const episodesFile = path.join(this.datasetDir, 'meta', 'episodes.jsonl');
+        this._episodes = new Map();
+        if (fs.existsSync(episodesFile)) {
+            for (const line of fs.readFileSync(episodesFile, 'utf-8').split('\n')) {
+                const trimmed = line.trim();
+                if (!trimmed)
+                    continue;
+                const rec = JSON.parse(trimmed);
+                this._episodes.set(rec.episode_index, rec);
+            }
+        }
+    }
+    schema() {
+        return {
+            exterior_1_video: 'BINARY',
+            exterior_2_video: 'BINARY',
+            wrist_video: 'BINARY',
+        };
+    }
+    pg_schema() {
+        return {
+            exterior_1_video: 'VIDEO',
+            exterior_2_video: 'VIDEO',
+            wrist_video: 'VIDEO',
+        };
+    }
+    *normalize() {
+        for (let chunkIdx = this.chunkStart; chunkIdx <= this.chunkEnd; chunkIdx++) {
+            const chunk = `chunk-${String(chunkIdx).padStart(3, '0')}`;
+            this._pullChunkVideos(chunkIdx);
+            const chunkDir = path.join(this.datasetDir, 'videos', chunk);
+            if (!fs.existsSync(chunkDir))
+                continue;
+            const firstCam = path.join(chunkDir, VIDEO_KEYS[0]);
+            if (!fs.existsSync(firstCam))
+                continue;
+            const episodeFiles = fs
+                .readdirSync(firstCam)
+                .filter((f) => f.startsWith('episode_') && f.endsWith('.mp4'))
+                .sort();
+            const batch = {
+                episode_index: [],
+                task: [],
+                length: [],
+                exterior_1_video: [],
+                exterior_2_video: [],
+                wrist_video: [],
+            };
+            for (const epFile of episodeFiles) {
+                const epName = path.basename(epFile, '.mp4');
+                const epIdx = parseInt(epName.split('_')[1], 10);
+                const epMeta = this._episodes.get(epIdx);
+                if (!epMeta)
+                    continue;
+                const taskList = epMeta.tasks ?? [];
+                const taskText = taskList.filter(Boolean).join('; ');
+                const videos = [];
+                let skip = false;
+                for (const vkey of VIDEO_KEYS) {
+                    const vpath = path.join(chunkDir, vkey, epFile);
+                    if (!fs.existsSync(vpath)) {
+                        skip = true;
+                        break;
+                    }
+                    const stat = fs.statSync(vpath);
+                    if (stat.size < 200) {
+                        skip = true;
+                        break;
+                    }
+                    videos.push(fs.readFileSync(vpath));
+                }
+                if (skip)
+                    continue;
+                batch.episode_index.push(epIdx);
+                batch.task.push(taskText);
+                batch.length.push(epMeta.length ?? 0);
+                for (let i = 0; i < COL_NAMES.length; i++) {
+                    batch[COL_NAMES[i]].push(videos[i]);
+                }
+                if (batch.episode_index.length >= 50) {
+                    yield batch;
+                    for (const key of EPISODE_COLUMNS) {
+                        batch[key] = [];
+                    }
+                }
+            }
+            if (batch.episode_index.length > 0) {
+                yield batch;
+            }
+            this._restoreChunkPointers(chunkIdx);
+        }
+    }
+    _pullChunkVideos(chunkIdx) {
+        const chunk = `chunk-${String(chunkIdx).padStart(3, '0')}`;
+        try {
+            (0, child_process_1.execFileSync)('git', ['lfs', 'pull', `--include=videos/${chunk}/**`], {
+                cwd: this.datasetDir,
+                timeout: 600_000,
+                stdio: 'pipe',
+            });
+        }
+        catch {
+            // Non-fatal
+        }
+    }
+    _restoreChunkPointers(chunkIdx) {
+        const chunk = `chunk-${String(chunkIdx).padStart(3, '0')}`;
+        const videosDir = path.join(this.datasetDir, 'videos', chunk);
+        if (!fs.existsSync(videosDir))
+            return;
+        let count = 0;
+        for (const camDir of fs.readdirSync(videosDir)) {
+            const camPath = path.join(videosDir, camDir);
+            if (!fs.statSync(camPath).isDirectory())
+                continue;
+            for (const file of fs.readdirSync(camPath)) {
+                if (!file.endsWith('.mp4'))
+                    continue;
+                const filePath = path.join(camPath, file);
+                if (fs.statSync(filePath).size > 200) {
+                    fs.unlinkSync(filePath);
+                    count++;
+                }
+            }
+        }
+        if (count > 0) {
+            try {
+                (0, child_process_1.execFileSync)('git', ['checkout', '--', `videos/${chunk}/`], {
+                    cwd: this.datasetDir,
+                    timeout: 60_000,
+                    stdio: 'pipe',
+                });
+            }
+            catch {
+                // Non-fatal
+            }
+        }
+    }
+}
+exports.LeRobotEpisodes = LeRobotEpisodes;
+// =====================================================================
+// Parquet reading helper
+// =====================================================================
+function readParquetFile(filePath) {
+    try {
+        const { ParquetReader } = require('parquetjs-lite');
+        throw new Error('parquetjs requires async reading — use LeRobotFrames with async normalize');
+    }
+    catch {
+        // No parquet reader available
+    }
+    try {
+        const result = (0, child_process_1.execFileSync)('duckdb', ['-json', '-c', `SELECT * FROM read_parquet('${filePath}')`], { timeout: 30_000, stdio: ['pipe', 'pipe', 'pipe'], maxBuffer: 100 * 1024 * 1024 });
+        return JSON.parse(result.toString('utf-8'));
+    }
+    catch {
+        // duckdb not available
+    }
+    try {
+        const pyScript = `import pandas,json,sys;df=pandas.read_parquet("${filePath}");print(json.dumps([{k:v if not hasattr(v,'tolist') else v.tolist() for k,v in r.items()} for _,r in df.iterrows()]))`;
+        const result = (0, child_process_1.execFileSync)('python3', ['-c', pyScript], { timeout: 30_000, stdio: ['pipe', 'pipe', 'pipe'], maxBuffer: 100 * 1024 * 1024 });
+        return JSON.parse(result.toString('utf-8'));
+    }
+    catch {
+        // python not available
+    }
+    throw new Error(`No parquet reader available. Install one of: duckdb CLI, or Python with pandas. ` +
+        `File: ${filePath}`);
+}
+//# sourceMappingURL=lerobot.js.map

package/dist/node/node/formats/lerobot.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"lerobot.js","sourceRoot":"","sources":["../../../../src/node/formats/lerobot.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAC7B,iDAA6C;AAW7C,MAAa,YAAY;IACf,SAAS,CAAS;IAE1B,YAAY,OAA4B;QACtC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,aAAa,CAAC,CAAC;IACxE,CAAC;IAED,MAAM;QACJ,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,CAAC,SAAS;QACR,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,yBAAyB,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QAC7D,CAAC;QAED,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,MAAM,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACnE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO;gBAAE,SAAS;YAEvB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAChC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YACjC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAErB,IAAI,WAAW,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;gBAC/B,MAAM,EAAE,UAAU,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YACrE,CAAC;QACH,CAAC;QAED,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;QACjD,CAAC;IACH,CAAC;CACF;AArCD,oCAqCC;AAED,wEAAwE;AACxE,gBAAgB;AAChB,wEAAwE;AAExE,MAAM,WAAW,GAAG;IAClB,SAAS,EAAE,SAAS,EAAE,SAAS;IAC/B,YAAY,EAAE,aAAa,EAAE,WAAW;IACxC,WAAW,EAAE,eAAe;CAC7B,CAAC;AAEF,MAAM,YAAY,GAAG;IACnB,UAAU,EAAE,UAAU,EAAE,UAAU;IAClC,aAAa,EAAE,cAAc,EAAE,YAAY;IAC3C,gBAAgB;CACjB,CAAC;AAQF,MAAa,aAAa;IAChB,UAAU,CAAS;IACnB,UAAU,CAAS;IACnB,QAAQ,CAAS;IAEzB,YAAY,OAA6B;QACvC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACrC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,CAAC,CAAC;QAC1C,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;IACzC,CAAC;IAED,MAAM;QACJ,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,CAAC,SAAS;QACR,MAAM,UAAU,GAAG;YACjB,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,YAAY,EAAE,WAAW;YAClE,GAAG,WAAW,EAAE,GAAG,YAAY;SAChC,CAAC;QAEF,IAAI,KAAK,GAA8B,EAAE,CAAC;QAC1C,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;YAC7B,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;QAClB,CAAC;QACD,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,KAAK,IAAI,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,QAAQ,IAAI,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,EAAE,CAAC;YAC3E,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CACxB,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,MAAM,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CACtE,CAAC;YACF,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC;gBAAE,SAAS;YAEvC,MAAM,YAAY,GAAG,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC;iBAC1C,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;iBACjE,IAAI,EAAE;iBACN,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;YAEtC,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;gBAClC,MAAM,IAAI,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;gBACrC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;oBAAE,SAAS;gBAEzC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;oBACvB,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;oBAC/B,KAAK,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;oBAC/C,KAAK,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;oBAC3C,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;oBACzC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;oBAEvC,MAAM,KAAK,GAAG,GAAG,CAAC,mBAAmB,CAAa,IAAI,EAAE,CAAC;oBACzD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;wBAC5C,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;oBAC5C,CAAC;oBAED,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAa,IAAI,EAAE,CAAC;oBAC/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;wBAC7C,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;oBAC9C,CAAC;oBAED,SAAS,EAAE,CAAC;gBACd,CAAC;gBAED,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;oBACtB,MAAM,KAAK,CAAC;oBACZ,KAAK,GAAG,EAAE,CAAC;oBACX,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;wBAC7B,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;oBAClB,CAAC;oBACD,SAAS,GAAG,CAAC,CAAC;gBAChB,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YAClB,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;CACF;AA7ED,sCA6EC;AAED,wEAAwE;AACxE,kBAAkB;AAClB,wEAAwE;AAExE,MAAM,UAAU,GAAG;IACjB,0CAA0C;IAC1C,0CAA0C;IAC1C,qCAAqC;CACtC,CAAC;AAEF,MAAM,SAAS,GAAG,CAAC,kBAAkB,EAAE,kBAAkB,EAAE,aAAa,CAAC,CAAC;AAC1E,MAAM,eAAe,GAAG,CAAC,eAAe,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,CAAC;AAQ1E,MAAa,eAAe;IAClB,UAAU,CAAS;IACnB,UAAU,CAAS;IACnB,QAAQ,CAAS;IACjB,MAAM,CAAsB;IAC5B,SAAS,CAAuC;IAExD,YAAY,OAA+B;QACzC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACrC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,CAAC,CAAC;QAC1C,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;QAEvC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,aAAa,CAAC,CAAC;QACpE,IAAI,CAAC,MAAM,GAAG,IAAI,GAAG,EAAE,CAAC;QACxB,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC7B,KAAK,MAAM,IAAI,IAAI,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnE,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;gBAC5B,IAAI,CAAC,OAAO;oBAAE,SAAS;gBACvB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBAChC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;YAC5C,CAAC;QACH,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,gBAAgB,CAAC,CAAC;QAC1E,IAAI,CAAC,SAAS,GAAG,IAAI,GAAG,EAAE,CAAC;QAC3B,IAAI,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;YAChC,KAAK,MAAM,IAAI,IAAI,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBACtE,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;gBAC5B,IAAI,CAAC,OAAO;oBAAE,SAAS;gBACvB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBAChC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM;QACJ,OAAO;YACL,gBAAgB,EAAE,QAAQ;YAC1B,gBAAgB,EAAE,QAAQ;YAC1B,WAAW,EAAE,QAAQ;SACtB,CAAC;IACJ,CAAC;IAED,SAAS;QACP,OAAO;YACL,gBAAgB,EAAE,OAAO;YACzB,gBAAgB,EAAE,OAAO;YACzB,WAAW,EAAE,OAAO;SACrB,CAAC;IACJ,CAAC;IAED,CAAC,SAAS;QACR,KACE,IAAI,QAAQ,GAAG,IAAI,CAAC,UAAU,EAC9B,QAAQ,IAAI,IAAI,CAAC,QAAQ,EACzB,QAAQ,EAAE,EACV,CAAC;YACD,MAAM,KAAK,GAAG,SAAS,MAAM,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;YAE3D,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;YAEhC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;YAC7D,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC;gBAAE,SAAS;YAEvC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;YACpD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC;gBAAE,SAAS;YAEvC,MAAM,YAAY,GAAG,EAAE;iBACpB,WAAW,CAAC,QAAQ,CAAC;iBACrB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;iBAC7D,IAAI,EAAE,CAAC;YAEV,MAAM,KAAK,GAA8B;gBACvC,aAAa,EAAE,EAAE;gBACjB,IAAI,EAAE,EAAE;gBACR,MAAM,EAAE,EAAE;gBACV,gBAAgB,EAAE,EAAE;gBACpB,gBAAgB,EAAE,EAAE;gBACpB,WAAW,EAAE,EAAE;aAChB,CAAC;YAEF,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;gBAClC,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBAC7C,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAEjD,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACzC,IAAI,CAAC,MAAM;oBAAE,SAAS;gBAEtB,MAAM,QAAQ,GAAI,MAAM,CAAC,KAAkB,IAAI,EAAE,CAAC;gBAClD,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAErD,MAAM,MAAM,GAAa,EAAE,CAAC;gBAC5B,IAAI,IAAI,GAAG,KAAK,CAAC;gBACjB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;oBAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;oBAChD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;wBAC1B,IAAI,GAAG,IAAI,CAAC;wBACZ,MAAM;oBACR,CAAC;oBACD,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;oBAChC,IAAI,IAAI,CAAC,IAAI,GAAG,GAAG,EAAE,CAAC;wBACpB,IAAI,GAAG,IAAI,CAAC;wBACZ,MAAM;oBACR,CAAC;oBACD,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC;gBACtC,CAAC;gBAED,IAAI,IAAI;oBAAE,SAAS;gBAEnB,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAChC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC1B,KAAK,CAAC,MAAM,CAAC,IAAI,CAAE,MAAM,CAAC,MAAiB,IAAI,CAAC,CAAC,CAAC;gBAClD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC1C,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBACtC,CAAC;gBAED,IAAI,KAAK,CAAC,aAAa,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;oBACrC,MAAM,KAAK,CAAC;oBACZ,KAAK,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;wBAClC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;oBAClB,CAAC;gBACH,CAAC;YACH,CAAC;YAED,IAAI,KAAK,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACnC,MAAM,KAAK,CAAC;YACd,CAAC;YAED,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAEO,gBAAgB,CAAC,QAAgB;QACvC,MAAM,KAAK,GAAG,SAAS,MAAM,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;QAC3D,IAAI,CAAC;YACH,IAAA,4BAAY,EACV,KAAK,EACL,CAAC,KAAK,EAAE,MAAM,EAAE,oBAAoB,KAAK,KAAK,CAAC,EAC/C;gBACE,GAAG,EAAE,IAAI,CAAC,UAAU;gBACpB,OAAO,EAAE,OAAO;gBAChB,KAAK,EAAE,MAAM;aACd,CACF,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,YAAY;QACd,CAAC;IACH,CAAC;IAEO,qBAAqB,CAAC,QAAgB;QAC5C,MAAM,KAAK,GAAG,SAAS,MAAM,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;QAC3D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;QAC9D,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO;QAEtC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,MAAM,IAAI,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,EAAE,CAAC;YAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YAC7C,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE;gBAAE,SAAS;YAClD,KAAK,MAAM,IAAI,IAAI,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC3C,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;oBAAE,SAAS;gBACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;gBAC1C,IAAI,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,IAAI,GAAG,GAAG,EAAE,CAAC;oBACrC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;oBACxB,KAAK,EAAE,CAAC;gBACV,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,IAAI,CAAC;gBACH,IAAA,4BAAY,EACV,KAAK,EACL,CAAC,UAAU,EAAE,IAAI,EAAE,UAAU,KAAK,GAAG,CAAC,EACtC;oBACE,GAAG,EAAE,IAAI,CAAC,UAAU;oBACpB,OAAO,EAAE,MAAM;oBACf,KAAK,EAAE,MAAM;iBACd,CACF,CAAC;YACJ,CAAC;YAAC,MAAM,CAAC;gBACP,YAAY;YACd,CAAC;QACH,CAAC;IACH,CAAC;CACF;AAxLD,0CAwLC;AAED,wEAAwE;AACxE,yBAAyB;AACzB,wEAAwE;AAExE,SAAS,eAAe,CAAC,QAAgB;IACvC,IAAI,CAAC;QACH,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC,gBAAgB,CAAQ,CAAC;QAC3D,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;IAC/F,CAAC;IAAC,MAAM,CAAC;QACP,8BAA8B;IAChC,CAAC;IAED,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAA,4BAAY,EACzB,QAAQ,EACR,CAAC,OAAO,EAAE,IAAI,EAAE,+BAA+B,QAAQ,IAAI,CAAC,EAC5D,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,SAAS,EAAE,GAAG,GAAG,IAAI,GAAG,IAAI,EAAE,CACnF,CAAC;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,uBAAuB;IACzB,CAAC;IAED,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,kDAAkD,QAAQ,wHAAwH,CAAC;QACpM,MAAM,MAAM,GAAG,IAAA,4BAAY,EACzB,SAAS,EACT,CAAC,IAAI,EAAE,QAAQ,CAAC,EAChB,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,SAAS,EAAE,GAAG,GAAG,IAAI,GAAG,IAAI,EAAE,CACnF,CAAC;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,uBAAuB;IACzB,CAAC;IAED,MAAM,IAAI,KAAK,CACb,kFAAkF;QAClF,SAAS,QAAQ,EAAE,CACpB,CAAC;AACJ,CAAC"}

package/dist/node/node/index.d.ts

@@ -0,0 +1,19 @@
+export { ManagedClient } from './client';
+export { ManagedQueryBuilder } from '../shared/database';
+export type { QueryExecutor } from '../shared/database';
+export { ManagedServiceError, AuthError, TokenError, CredentialsError, IngestError, TableError, WorkspaceError, } from '../shared/errors';
+export { extractOrgId, tryExtractOrgId, fetchOrgIdFromApi } from '../shared/token';
+export { apiRequest } from '../shared/api';
+export type { ApiRequestOptions } from '../shared/api';
+export { getStorageCreds, applyStorageCreds, } from '../shared/credentials';
+export { SCHEMA_TO_PG, SCHEMA_TO_DEEPLAKE, inferType, buildPgSchema, ensureColumns, } from '../shared/schema';
+export { isDataRef, parseDataRef } from '../shared/dlref';
+export type { DataRef } from '../shared/dlref';
+export { mimeForFile } from './mime';
+export { initializeWasm, deinitializeWasm, } from './wasm';
+export { deeplakeOpen, deeplakeCreate, deeplakeExists, deeplakeSetEndpointAndOpen, deeplakeNumRows, deeplakeGetField, deeplakeGetRow, deeplakeGetColumnData, deeplakeSetField, deeplakeSetRow, deeplakeCommit, deeplakeAppend, deeplakeAddColumn, deeplakeDeleteDataset, deeplakeGetSampleByRef, deeplakeRelease, getWasmModule, unwrapNdArray, } from '../shared/wasm-common';
+export { normalizeDict, chunkText, normalizeText, normalizeBinary, normalizeImage, normalizeVideo, normalizePdf, normalizeFiles, normalizeHuggingface, } from './normalizers';
+export { Coco, CocoPanoptic, LeRobotTasks, LeRobotFrames, LeRobotEpisodes } from './formats';
+export type { CocoOptions, CocoPanopticOptions, LeRobotTasksOptions, LeRobotFramesOptions, LeRobotEpisodesOptions, } from './formats';
+export type { ManagedClientOptions, IngestResult, IngestOptions, Format, Batch, SchemaMap, QueryRow, StorageCreds, } from '../shared/types';
+//# sourceMappingURL=index.d.ts.map

package/dist/node/node/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/node/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,YAAY,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EACL,mBAAmB,EACnB,SAAS,EACT,UAAU,EACV,gBAAgB,EAChB,WAAW,EACX,UAAU,EACV,cAAc,GACf,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACnF,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,YAAY,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACvD,OAAO,EACL,eAAe,EACf,iBAAiB,GAClB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,YAAY,EACZ,kBAAkB,EAClB,SAAS,EACT,aAAa,EACb,aAAa,GACd,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC1D,YAAY,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AACrC,OAAO,EACL,cAAc,EACd,gBAAgB,GACjB,MAAM,QAAQ,CAAC;AAChB,OAAO,EACL,YAAY,EACZ,cAAc,EACd,cAAc,EACd,0BAA0B,EAC1B,eAAe,EACf,gBAAgB,EAChB,cAAc,EACd,qBAAqB,EACrB,gBAAgB,EAChB,cAAc,EACd,cAAc,EACd,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,eAAe,EACf,aAAa,EACb,aAAa,GACd,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,aAAa,EACb,SAAS,EACT,aAAa,EACb,eAAe,EACf,cAAc,EACd,cAAc,EACd,YAAY,EACZ,cAAc,EACd,oBAAoB,GACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC7F,YAAY,EACV,WAAW,EACX,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EACpB,sBAAsB,GACvB,MAAM,WAAW,CAAC;AACnB,YAAY,EACV,oBAAoB,EACpB,YAAY,EACZ,aAAa,EACb,MAAM,EACN,KAAK,EACL,SAAS,EACT,QAAQ,EACR,YAAY,GACb,MAAM,iBAAiB,CAAC"}

package/dist/node/node/index.js

@@ -0,0 +1,75 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalizePdf = exports.normalizeVideo = exports.normalizeImage = exports.normalizeBinary = exports.normalizeText = exports.chunkText = exports.normalizeDict = exports.unwrapNdArray = exports.getWasmModule = exports.deeplakeRelease = exports.deeplakeGetSampleByRef = exports.deeplakeDeleteDataset = exports.deeplakeAddColumn = exports.deeplakeAppend = exports.deeplakeCommit = exports.deeplakeSetRow = exports.deeplakeSetField = exports.deeplakeGetColumnData = exports.deeplakeGetRow = exports.deeplakeGetField = exports.deeplakeNumRows = exports.deeplakeSetEndpointAndOpen = exports.deeplakeExists = exports.deeplakeCreate = exports.deeplakeOpen = exports.deinitializeWasm = exports.initializeWasm = exports.mimeForFile = exports.parseDataRef = exports.isDataRef = exports.ensureColumns = exports.buildPgSchema = exports.inferType = exports.SCHEMA_TO_DEEPLAKE = exports.SCHEMA_TO_PG = exports.applyStorageCreds = exports.getStorageCreds = exports.apiRequest = exports.fetchOrgIdFromApi = exports.tryExtractOrgId = exports.extractOrgId = exports.WorkspaceError = exports.TableError = exports.IngestError = exports.CredentialsError = exports.TokenError = exports.AuthError = exports.ManagedServiceError = exports.ManagedQueryBuilder = exports.ManagedClient = void 0;
+exports.LeRobotEpisodes = exports.LeRobotFrames = exports.LeRobotTasks = exports.CocoPanoptic = exports.Coco = exports.normalizeHuggingface = exports.normalizeFiles = void 0;
+var client_1 = require("./client");
+Object.defineProperty(exports, "ManagedClient", { enumerable: true, get: function () { return client_1.ManagedClient; } });
+var database_1 = require("../shared/database");
+Object.defineProperty(exports, "ManagedQueryBuilder", { enumerable: true, get: function () { return database_1.ManagedQueryBuilder; } });
+var errors_1 = require("../shared/errors");
+Object.defineProperty(exports, "ManagedServiceError", { enumerable: true, get: function () { return errors_1.ManagedServiceError; } });
+Object.defineProperty(exports, "AuthError", { enumerable: true, get: function () { return errors_1.AuthError; } });
+Object.defineProperty(exports, "TokenError", { enumerable: true, get: function () { return errors_1.TokenError; } });
+Object.defineProperty(exports, "CredentialsError", { enumerable: true, get: function () { return errors_1.CredentialsError; } });
+Object.defineProperty(exports, "IngestError", { enumerable: true, get: function () { return errors_1.IngestError; } });
+Object.defineProperty(exports, "TableError", { enumerable: true, get: function () { return errors_1.TableError; } });
+Object.defineProperty(exports, "WorkspaceError", { enumerable: true, get: function () { return errors_1.WorkspaceError; } });
+var token_1 = require("../shared/token");
+Object.defineProperty(exports, "extractOrgId", { enumerable: true, get: function () { return token_1.extractOrgId; } });
+Object.defineProperty(exports, "tryExtractOrgId", { enumerable: true, get: function () { return token_1.tryExtractOrgId; } });
+Object.defineProperty(exports, "fetchOrgIdFromApi", { enumerable: true, get: function () { return token_1.fetchOrgIdFromApi; } });
+var api_1 = require("../shared/api");
+Object.defineProperty(exports, "apiRequest", { enumerable: true, get: function () { return api_1.apiRequest; } });
+var credentials_1 = require("../shared/credentials");
+Object.defineProperty(exports, "getStorageCreds", { enumerable: true, get: function () { return credentials_1.getStorageCreds; } });
+Object.defineProperty(exports, "applyStorageCreds", { enumerable: true, get: function () { return credentials_1.applyStorageCreds; } });
+var schema_1 = require("../shared/schema");
+Object.defineProperty(exports, "SCHEMA_TO_PG", { enumerable: true, get: function () { return schema_1.SCHEMA_TO_PG; } });
+Object.defineProperty(exports, "SCHEMA_TO_DEEPLAKE", { enumerable: true, get: function () { return schema_1.SCHEMA_TO_DEEPLAKE; } });
+Object.defineProperty(exports, "inferType", { enumerable: true, get: function () { return schema_1.inferType; } });
+Object.defineProperty(exports, "buildPgSchema", { enumerable: true, get: function () { return schema_1.buildPgSchema; } });
+Object.defineProperty(exports, "ensureColumns", { enumerable: true, get: function () { return schema_1.ensureColumns; } });
+var dlref_1 = require("../shared/dlref");
+Object.defineProperty(exports, "isDataRef", { enumerable: true, get: function () { return dlref_1.isDataRef; } });
+Object.defineProperty(exports, "parseDataRef", { enumerable: true, get: function () { return dlref_1.parseDataRef; } });
+var mime_1 = require("./mime");
+Object.defineProperty(exports, "mimeForFile", { enumerable: true, get: function () { return mime_1.mimeForFile; } });
+var wasm_1 = require("./wasm");
+Object.defineProperty(exports, "initializeWasm", { enumerable: true, get: function () { return wasm_1.initializeWasm; } });
+Object.defineProperty(exports, "deinitializeWasm", { enumerable: true, get: function () { return wasm_1.deinitializeWasm; } });
+var wasm_common_1 = require("../shared/wasm-common");
+Object.defineProperty(exports, "deeplakeOpen", { enumerable: true, get: function () { return wasm_common_1.deeplakeOpen; } });
+Object.defineProperty(exports, "deeplakeCreate", { enumerable: true, get: function () { return wasm_common_1.deeplakeCreate; } });
+Object.defineProperty(exports, "deeplakeExists", { enumerable: true, get: function () { return wasm_common_1.deeplakeExists; } });
+Object.defineProperty(exports, "deeplakeSetEndpointAndOpen", { enumerable: true, get: function () { return wasm_common_1.deeplakeSetEndpointAndOpen; } });
+Object.defineProperty(exports, "deeplakeNumRows", { enumerable: true, get: function () { return wasm_common_1.deeplakeNumRows; } });
+Object.defineProperty(exports, "deeplakeGetField", { enumerable: true, get: function () { return wasm_common_1.deeplakeGetField; } });
+Object.defineProperty(exports, "deeplakeGetRow", { enumerable: true, get: function () { return wasm_common_1.deeplakeGetRow; } });
+Object.defineProperty(exports, "deeplakeGetColumnData", { enumerable: true, get: function () { return wasm_common_1.deeplakeGetColumnData; } });
+Object.defineProperty(exports, "deeplakeSetField", { enumerable: true, get: function () { return wasm_common_1.deeplakeSetField; } });
+Object.defineProperty(exports, "deeplakeSetRow", { enumerable: true, get: function () { return wasm_common_1.deeplakeSetRow; } });
+Object.defineProperty(exports, "deeplakeCommit", { enumerable: true, get: function () { return wasm_common_1.deeplakeCommit; } });
+Object.defineProperty(exports, "deeplakeAppend", { enumerable: true, get: function () { return wasm_common_1.deeplakeAppend; } });
+Object.defineProperty(exports, "deeplakeAddColumn", { enumerable: true, get: function () { return wasm_common_1.deeplakeAddColumn; } });
+Object.defineProperty(exports, "deeplakeDeleteDataset", { enumerable: true, get: function () { return wasm_common_1.deeplakeDeleteDataset; } });
+Object.defineProperty(exports, "deeplakeGetSampleByRef", { enumerable: true, get: function () { return wasm_common_1.deeplakeGetSampleByRef; } });
+Object.defineProperty(exports, "deeplakeRelease", { enumerable: true, get: function () { return wasm_common_1.deeplakeRelease; } });
+Object.defineProperty(exports, "getWasmModule", { enumerable: true, get: function () { return wasm_common_1.getWasmModule; } });
+Object.defineProperty(exports, "unwrapNdArray", { enumerable: true, get: function () { return wasm_common_1.unwrapNdArray; } });
+var normalizers_1 = require("./normalizers");
+Object.defineProperty(exports, "normalizeDict", { enumerable: true, get: function () { return normalizers_1.normalizeDict; } });
+Object.defineProperty(exports, "chunkText", { enumerable: true, get: function () { return normalizers_1.chunkText; } });
+Object.defineProperty(exports, "normalizeText", { enumerable: true, get: function () { return normalizers_1.normalizeText; } });
+Object.defineProperty(exports, "normalizeBinary", { enumerable: true, get: function () { return normalizers_1.normalizeBinary; } });
+Object.defineProperty(exports, "normalizeImage", { enumerable: true, get: function () { return normalizers_1.normalizeImage; } });
+Object.defineProperty(exports, "normalizeVideo", { enumerable: true, get: function () { return normalizers_1.normalizeVideo; } });
+Object.defineProperty(exports, "normalizePdf", { enumerable: true, get: function () { return normalizers_1.normalizePdf; } });
+Object.defineProperty(exports, "normalizeFiles", { enumerable: true, get: function () { return normalizers_1.normalizeFiles; } });
+Object.defineProperty(exports, "normalizeHuggingface", { enumerable: true, get: function () { return normalizers_1.normalizeHuggingface; } });
+var formats_1 = require("./formats");
+Object.defineProperty(exports, "Coco", { enumerable: true, get: function () { return formats_1.Coco; } });
+Object.defineProperty(exports, "CocoPanoptic", { enumerable: true, get: function () { return formats_1.CocoPanoptic; } });
+Object.defineProperty(exports, "LeRobotTasks", { enumerable: true, get: function () { return formats_1.LeRobotTasks; } });
+Object.defineProperty(exports, "LeRobotFrames", { enumerable: true, get: function () { return formats_1.LeRobotFrames; } });
+Object.defineProperty(exports, "LeRobotEpisodes", { enumerable: true, get: function () { return formats_1.LeRobotEpisodes; } });
+//# sourceMappingURL=index.js.map

package/dist/node/node/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/node/index.ts"],"names":[],"mappings":";;;;AAAA,mCAAyC;AAAhC,uGAAA,aAAa,OAAA;AACtB,+CAAyD;AAAhD,+GAAA,mBAAmB,OAAA;AAE5B,2CAQ0B;AAPxB,6GAAA,mBAAmB,OAAA;AACnB,mGAAA,SAAS,OAAA;AACT,oGAAA,UAAU,OAAA;AACV,0GAAA,gBAAgB,OAAA;AAChB,qGAAA,WAAW,OAAA;AACX,oGAAA,UAAU,OAAA;AACV,wGAAA,cAAc,OAAA;AAEhB,yCAAmF;AAA1E,qGAAA,YAAY,OAAA;AAAE,wGAAA,eAAe,OAAA;AAAE,0GAAA,iBAAiB,OAAA;AACzD,qCAA2C;AAAlC,iGAAA,UAAU,OAAA;AAEnB,qDAG+B;AAF7B,8GAAA,eAAe,OAAA;AACf,gHAAA,iBAAiB,OAAA;AAEnB,2CAM0B;AALxB,sGAAA,YAAY,OAAA;AACZ,4GAAA,kBAAkB,OAAA;AAClB,mGAAA,SAAS,OAAA;AACT,uGAAA,aAAa,OAAA;AACb,uGAAA,aAAa,OAAA;AAEf,yCAA0D;AAAjD,kGAAA,SAAS,OAAA;AAAE,qGAAA,YAAY,OAAA;AAEhC,+BAAqC;AAA5B,mGAAA,WAAW,OAAA;AACpB,+BAGgB;AAFd,sGAAA,cAAc,OAAA;AACd,wGAAA,gBAAgB,OAAA;AAElB,qDAmB+B;AAlB7B,2GAAA,YAAY,OAAA;AACZ,6GAAA,cAAc,OAAA;AACd,6GAAA,cAAc,OAAA;AACd,yHAAA,0BAA0B,OAAA;AAC1B,8GAAA,eAAe,OAAA;AACf,+GAAA,gBAAgB,OAAA;AAChB,6GAAA,cAAc,OAAA;AACd,oHAAA,qBAAqB,OAAA;AACrB,+GAAA,gBAAgB,OAAA;AAChB,6GAAA,cAAc,OAAA;AACd,6GAAA,cAAc,OAAA;AACd,6GAAA,cAAc,OAAA;AACd,gHAAA,iBAAiB,OAAA;AACjB,oHAAA,qBAAqB,OAAA;AACrB,qHAAA,sBAAsB,OAAA;AACtB,8GAAA,eAAe,OAAA;AACf,4GAAA,aAAa,OAAA;AACb,4GAAA,aAAa,OAAA;AAEf,6CAUuB;AATrB,4GAAA,aAAa,OAAA;AACb,wGAAA,SAAS,OAAA;AACT,4GAAA,aAAa,OAAA;AACb,8GAAA,eAAe,OAAA;AACf,6GAAA,cAAc,OAAA;AACd,6GAAA,cAAc,OAAA;AACd,2GAAA,YAAY,OAAA;AACZ,6GAAA,cAAc,OAAA;AACd,mHAAA,oBAAoB,OAAA;AAEtB,qCAA6F;AAApF,+FAAA,IAAI,OAAA;AAAE,uGAAA,YAAY,OAAA;AAAE,uGAAA,YAAY,OAAA;AAAE,wGAAA,aAAa,OAAA;AAAE,0GAAA,eAAe,OAAA"}

package/dist/node/node/mime.d.ts

@@ -0,0 +1,2 @@
+export declare function mimeForFile(filename: string, domainType: string): string;
+//# sourceMappingURL=mime.d.ts.map

package/dist/node/node/mime.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mime.d.ts","sourceRoot":"","sources":["../../../src/node/mime.ts"],"names":[],"mappings":"AAmCA,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,CAKxE"}

package/dist/node/node/mime.js

@@ -0,0 +1,76 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.mimeForFile = mimeForFile;
+const path = __importStar(require("path"));
+const DOMAIN_MIME_DEFAULTS = {
+    IMAGE: 'image/jpeg',
+    SEGMENT_MASK: 'image/png',
+    BINARY_MASK: 'image/png',
+    VIDEO: 'video/mp4',
+    MESH: 'application/octet-stream',
+    MEDICAL: 'application/dicom',
+    BINARY: 'application/octet-stream',
+};
+const EXT_TO_MIME = {
+    '.jpg': 'image/jpeg',
+    '.jpeg': 'image/jpeg',
+    '.png': 'image/png',
+    '.gif': 'image/gif',
+    '.bmp': 'image/bmp',
+    '.webp': 'image/webp',
+    '.svg': 'image/svg+xml',
+    '.mp4': 'video/mp4',
+    '.mov': 'video/quicktime',
+    '.avi': 'video/x-msvideo',
+    '.mkv': 'video/x-matroska',
+    '.webm': 'video/webm',
+    '.pdf': 'application/pdf',
+    '.txt': 'text/plain',
+    '.html': 'text/html',
+    '.json': 'application/json',
+    '.csv': 'text/csv',
+    '.xml': 'application/xml',
+    '.md': 'text/markdown',
+    '.dcm': 'application/dicom',
+};
+function mimeForFile(filename, domainType) {
+    const ext = path.extname(filename).toLowerCase();
+    const fromExt = EXT_TO_MIME[ext];
+    if (fromExt)
+        return fromExt;
+    return DOMAIN_MIME_DEFAULTS[domainType] ?? 'application/octet-stream';
+}
+//# sourceMappingURL=mime.js.map

package/dist/node/node/mime.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mime.js","sourceRoot":"","sources":["../../../src/node/mime.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmCA,kCAKC;AAxCD,2CAA6B;AAE7B,MAAM,oBAAoB,GAA2B;IACnD,KAAK,EAAE,YAAY;IACnB,YAAY,EAAE,WAAW;IACzB,WAAW,EAAE,WAAW;IACxB,KAAK,EAAE,WAAW;IAClB,IAAI,EAAE,0BAA0B;IAChC,OAAO,EAAE,mBAAmB;IAC5B,MAAM,EAAE,0BAA0B;CACnC,CAAC;AAEF,MAAM,WAAW,GAA2B;IAC1C,MAAM,EAAE,YAAY;IACpB,OAAO,EAAE,YAAY;IACrB,MAAM,EAAE,WAAW;IACnB,MAAM,EAAE,WAAW;IACnB,MAAM,EAAE,WAAW;IACnB,OAAO,EAAE,YAAY;IACrB,MAAM,EAAE,eAAe;IACvB,MAAM,EAAE,WAAW;IACnB,MAAM,EAAE,iBAAiB;IACzB,MAAM,EAAE,iBAAiB;IACzB,MAAM,EAAE,kBAAkB;IAC1B,OAAO,EAAE,YAAY;IACrB,MAAM,EAAE,iBAAiB;IACzB,MAAM,EAAE,YAAY;IACpB,OAAO,EAAE,WAAW;IACpB,OAAO,EAAE,kBAAkB;IAC3B,MAAM,EAAE,UAAU;IAClB,MAAM,EAAE,iBAAiB;IACzB,KAAK,EAAE,eAAe;IACtB,MAAM,EAAE,mBAAmB;CAC5B,CAAC;AAEF,SAAgB,WAAW,CAAC,QAAgB,EAAE,UAAkB;IAC9D,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;IACjD,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;IACjC,IAAI,OAAO;QAAE,OAAO,OAAO,CAAC;IAC5B,OAAO,oBAAoB,CAAC,UAAU,CAAC,IAAI,0BAA0B,CAAC;AACxE,CAAC"}

package/dist/node/node/normalizers/binary.d.ts

@@ -0,0 +1,3 @@
+import type { Batch } from '../../shared/types';
+export declare function normalizeBinary(filePath: string): Batch;
+//# sourceMappingURL=binary.d.ts.map

package/dist/node/node/normalizers/binary.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"binary.d.ts","sourceRoot":"","sources":["../../../../src/node/normalizers/binary.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAEhD,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,KAAK,CAWvD"}