@dreamlake/dreamlake-cli 0.1.0

package/dist/cli/upload/multipart.js

@@ -0,0 +1,211 @@
+// BSS S3 multipart upload with pause/resume, then registration in BSS and
+// dreamlake-server. Faithful port of dreamlake-py's cli/commands/upload.py
+// `_upload_file` (the 7-phase flow).
+//
+// Resume state lives at ~/.dreamlake/uploads/<hash>.json; on re-run we ask
+// BSS which parts are already done (`.../parts-done`) and only upload the
+// rest.
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync, } from "node:fs";
+import { homedir } from "node:os";
+import path from "node:path";
+import { requestJson } from "../client.js";
+import { humanSize } from "../helpers.js";
+import { PartsProgress } from "../progress.js";
+import { MIME_MAP, extOf } from "./kinds.js";
+const CHUNK_SIZE = 10 * 1024 * 1024; // 10 MB per part (S3 min is 5 MB except last)
+const MAX_WORKERS = 4; // parallel part uploads
+// ─── resume state on disk ────────────────────────────────────────────
+function statePath(rawHash) {
+    const dir = path.join(homedir(), ".dreamlake", "uploads");
+    mkdirSync(dir, { recursive: true });
+    return path.join(dir, `${rawHash}.json`);
+}
+function saveState(rawHash, state) {
+    writeFileSync(statePath(rawHash), JSON.stringify(state));
+}
+function loadState(rawHash) {
+    const p = statePath(rawHash);
+    if (!existsSync(p))
+        return null;
+    try {
+        return JSON.parse(readFileSync(p, "utf8"));
+    }
+    catch {
+        return null;
+    }
+}
+function clearState(rawHash) {
+    const p = statePath(rawHash);
+    if (existsSync(p))
+        rmSync(p);
+}
+// ─── simple bounded-concurrency pool ─────────────────────────────────
+async function pool(items, limit, worker) {
+    let idx = 0;
+    const runners = Array.from({ length: Math.min(limit, items.length) }, async () => {
+        while (idx < items.length) {
+            const current = items[idx++];
+            await worker(current);
+        }
+    });
+    await Promise.all(runners);
+}
+// ─── the upload ──────────────────────────────────────────────────────
+export async function uploadFile(filePath, t, destPath, kind, token, bss, remote) {
+    // Unified BSS route — all kinds go through /files (the old per-category
+    // routes /videos, /audio, ... no longer exist server-side).
+    const route = "files";
+    const filename = path.basename(filePath);
+    const contentType = MIME_MAP[extOf(filename)] ?? "application/octet-stream";
+    // Phase 1: hash + compute parts.
+    const content = readFileSync(filePath);
+    const rawHash = createHash("sha256").update(content).digest("hex").slice(0, 16);
+    const fileSize = content.length;
+    const totalParts = Math.max(1, Math.ceil(fileSize / CHUNK_SIZE));
+    process.stdout.write(`  size:   ${humanSize(fileSize)}\n`);
+    process.stdout.write(`  parts:  ${totalParts} x ${CHUNK_SIZE / 1024 / 1024} MB\n`);
+    let uploadId = null;
+    let key = null;
+    const completedMap = new Map();
+    // Resume from prior state if BSS still has the parts.
+    const prior = loadState(rawHash);
+    if (prior) {
+        try {
+            const done = await requestJson(bss, `/${route}/upload/multipart/parts-done`, { query: { uploadId: prior.uploadId, key: prior.key }, token });
+            if (!done.expired) {
+                uploadId = prior.uploadId;
+                key = prior.key;
+                for (const p of done.parts ?? [])
+                    completedMap.set(p.partNumber, p);
+                const remaining = totalParts - completedMap.size;
+                process.stdout.write(`  resuming: ${completedMap.size}/${totalParts} done, ${remaining} remaining\n`);
+            }
+        }
+        catch {
+            // ignore — start fresh below
+        }
+    }
+    // Phase 2: init (unless resuming).
+    if (!uploadId) {
+        const init = await requestJson(bss, `/${route}/upload/multipart/init`, {
+            method: "POST",
+            token,
+            json: {
+                owner: t.namespace,
+                project: t.project,
+                hash: rawHash,
+                contentType,
+            },
+        });
+        uploadId = init.uploadId;
+        key = init.key;
+        saveState(rawHash, { uploadId, key, totalParts, completedParts: [] });
+    }
+    // Phase 3: presigned URLs for remaining parts.
+    const remainingParts = [];
+    for (let n = 1; n <= totalParts; n++)
+        if (!completedMap.has(n))
+            remainingParts.push(n);
+    let partUrls = {};
+    if (remainingParts.length > 0) {
+        const resp = await requestJson(bss, `/${route}/upload/multipart/parts`, {
+            method: "POST",
+            token,
+            json: { uploadId, key, partNumbers: remainingParts },
+        });
+        partUrls = resp.parts;
+    }
+    // Phase 4: parallel chunk upload to S3 (presigned PUT, no auth header).
+    if (remainingParts.length > 0) {
+        const progress = new PartsProgress(totalParts, "uploading");
+        progress.advance(completedMap.size);
+        let failure = null;
+        await pool(remainingParts, MAX_WORKERS, async (partNumber) => {
+            if (failure)
+                return;
+            const start = (partNumber - 1) * CHUNK_SIZE;
+            const chunk = content.subarray(start, start + CHUNK_SIZE);
+            const url = partUrls[String(partNumber)];
+            try {
+                const res = await fetch(url, {
+                    method: "PUT",
+                    headers: { "Content-Type": contentType },
+                    body: chunk,
+                    signal: AbortSignal.timeout(300000),
+                });
+                if (!res.ok) {
+                    throw new Error(`part ${partNumber} PUT failed (${res.status})`);
+                }
+                const etag = (res.headers.get("etag") ?? "").replace(/"/g, "");
+                completedMap.set(partNumber, { partNumber, etag });
+                saveState(rawHash, {
+                    uploadId: uploadId,
+                    key: key,
+                    totalParts,
+                    completedParts: [...completedMap.values()],
+                });
+                progress.advance();
+            }
+            catch (err) {
+                failure = err;
+            }
+        });
+        progress.done_();
+        if (failure) {
+            throw new Error(`${failure.message} — upload paused, re-run to resume`);
+        }
+    }
+    // Phase 5: complete.
+    const completedParts = [...completedMap.values()].sort((a, b) => a.partNumber - b.partNumber);
+    await requestJson(bss, `/${route}/upload/multipart/complete`, {
+        method: "POST",
+        token,
+        timeoutMs: 60000,
+        json: { uploadId, key, parts: completedParts },
+    });
+    clearState(rawHash);
+    // Phase 6: register in BSS (unified POST /files — kind passes through).
+    const fullName = `/${destPath}/${filename}`.replace(/\/+/g, "/");
+    const bssBody = {
+        name: fullName,
+        owner: t.namespace,
+        project: t.project,
+        stagingHash: rawHash,
+        kind,
+        contentType,
+        size: fileSize,
+        originalName: filename,
+    };
+    if (kind === "text-track") {
+        const fmt = { ".vtt": "vtt", ".srt": "srt", ".jsonl": "jsonl" };
+        bssBody.metadata = { format: fmt[extOf(filename)] ?? "jsonl" };
+    }
+    const bssResult = await requestJson(bss, `/${route}`, {
+        method: "POST",
+        token,
+        json: bssBody,
+    });
+    const bssId = bssResult.id ?? null;
+    // Phase 7: register in dreamlake-server (POST /nodes).
+    const nodeBody = {
+        namespace: t.namespace,
+        kind,
+        name: fullName,
+        project: t.project,
+        metadata: {
+            bssId,
+            hash: rawHash,
+            size: fileSize,
+            contentType,
+        },
+    };
+    if (t.episode)
+        nodeBody.episode = t.episode;
+    const dlResult = await requestJson(remote, "/nodes", {
+        method: "POST",
+        token,
+        json: nodeBody,
+    });
+    return { nodeId: dlResult.id ?? null, bssId };
+}

package/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "@dreamlake/dreamlake-cli",
+  "version": "0.1.0",
+  "description": "dreamlake — data-warehouse CLI. The `dreamlake` command uploads/downloads assets and manages episodes, bindrs, and datasets against a DreamLake server + BSS.",
+  "private": false,
+  "license": "MIT",
+  "type": "module",
+  "bin": {
+    "dreamlake": "bin/dreamlake.js"
+  },
+  "files": [
+    "bin",
+    "dist",
+    "!dist/**/*.map"
+  ],
+  "engines": {
+    "node": ">=20"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/dreamlake-ai/dreamlake-cli.git"
+  },
+  "homepage": "https://github.com/dreamlake-ai/dreamlake-cli#readme",
+  "bugs": {
+    "url": "https://github.com/dreamlake-ai/dreamlake-cli/issues"
+  },
+  "keywords": [
+    "dreamlake",
+    "cli",
+    "data-warehouse",
+    "dataset",
+    "robotics"
+  ],
+  "author": "DreamLake AI",
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsc",
+    "cli": "tsx src/cli/index.ts",
+    "test": "tsc -p tsconfig.json --noEmit && node --import tsx --test src/cli/__tests__/*.test.ts",
+    "prepublishOnly": "npm test",
+    "prepack": "npm run build",
+    "docs:dev": "pnpm -C docs dev",
+    "docs:build": "pnpm -C docs build",
+    "docs:preview": "pnpm -C docs preview"
+  },
+  "devDependencies": {
+    "@types/node": "^22.10.0",
+    "tsx": "^4.19.0",
+    "typescript": "^5.7.0"
+  },
+  "dependencies": {
+    "@inquirer/prompts": "^8.4.3",
+    "commander": "^12.1.0",
+    "yaml": "^2.6.1"
+  }
+}