@coderook/cli 0.1.0

package/dist/desktop-app/src/main/upload.js

@@ -0,0 +1,433 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Uploader = exports.UploadCancelled = void 0;
+/**
+ * Sending a version to CodeRook.
+ *
+ * Every file becomes a content-addressed object, then one version record
+ * names them all. Objects are addressed by the digest of their contents, so
+ * a file that has not changed since the last version costs nothing to send
+ * again — the server already has it.
+ */
+const node_crypto_1 = require("node:crypto");
+const node_fs_1 = require("node:fs");
+const promises_1 = require("node:fs/promises");
+const node_path_1 = __importDefault(require("node:path"));
+const worktree_js_1 = require("./worktree.js");
+/** Above this the API insists on a multipart session. */
+const DIRECT_LIMIT = 95 * 1024 * 1024;
+const PART_SIZE = 32 * 1024 * 1024;
+const MEDIA_TYPES = {
+    ".css": "text/css",
+    ".csv": "text/csv",
+    ".gif": "image/gif",
+    ".html": "text/html",
+    ".jpeg": "image/jpeg",
+    ".jpg": "image/jpeg",
+    ".js": "text/javascript",
+    ".json": "application/json",
+    ".md": "text/markdown",
+    ".mjs": "text/javascript",
+    ".pdf": "application/pdf",
+    ".png": "image/png",
+    ".py": "text/x-python",
+    ".svg": "image/svg+xml",
+    ".ts": "text/typescript",
+    ".tsx": "text/typescript",
+    ".txt": "text/plain",
+    ".wasm": "application/wasm",
+    ".webp": "image/webp",
+    ".xml": "application/xml",
+    ".yaml": "application/yaml",
+    ".yml": "application/yaml",
+    ".zip": "application/zip",
+};
+function mediaTypeOf(file) {
+    return MEDIA_TYPES[node_path_1.default.extname(file).toLowerCase()] ?? "application/octet-stream";
+}
+/** Hash without holding the file in memory; some of these are large. */
+async function digestOf(full) {
+    const hash = (0, node_crypto_1.createHash)("sha256");
+    for await (const chunk of (0, node_fs_1.createReadStream)(full))
+        hash.update(chunk);
+    return hash.digest("hex");
+}
+class UploadCancelled extends Error {
+    constructor() {
+        super("Upload cancelled");
+    }
+}
+exports.UploadCancelled = UploadCancelled;
+class Uploader {
+    credentials;
+    aborted = false;
+    controller = new AbortController();
+    constructor(credentials) {
+        this.credentials = credentials;
+    }
+    cancel() {
+        this.aborted = true;
+        this.controller.abort();
+    }
+    check() {
+        if (this.aborted)
+            throw new UploadCancelled();
+    }
+    async call(route, init) {
+        const token = await this.credentials.token();
+        if (!token)
+            throw new Error("Sign in again before uploading");
+        const response = await fetch(`${this.credentials.origin()}${route}`, {
+            method: init.method,
+            headers: {
+                accept: "application/json",
+                authorization: `Bearer ${token}`,
+                "user-agent": "CodeRook/0.1",
+                ...(init.contentType ? { "content-type": init.contentType } : {}),
+            },
+            body: init.body,
+            signal: this.controller.signal,
+        });
+        const text = await response.text();
+        const body = text ? JSON.parse(text) : {};
+        if (!response.ok) {
+            const message = body?.error?.message ?? `${route} failed (${response.status})`;
+            throw new Error(message);
+        }
+        return body;
+    }
+    async run(request, report) {
+        // A version is a snapshot, not a delta, so it has to name every file in
+        // the project — not merely the ones being sent this time. Anything
+        // unchanged keeps the object the previous version already pointed at.
+        const rules = await (0, worktree_js_1.readRules)(request.localPath);
+        const everything = await (0, worktree_js_1.changedFiles)(request.localPath, rules, null);
+        const ticked = new Set(request.include);
+        if (!ticked.size)
+            throw new Error("Nothing is selected to upload");
+        const prior = request.repositoryId
+            ? await this.latestVersionFiles(request.repositoryId)
+            : new Map();
+        // Ticked files are sent; everything else the project still has keeps the
+        // copy already stored, and a file that is new but unticked is left out.
+        const sending = everything.filter((file) => ticked.has(file.path));
+        const reused = everything.filter((file) => !ticked.has(file.path) && prior.has(file.path));
+        /*
+          A ticked path that the rescan cannot see is either a deletion or a file
+          that has gone since the changes list was drawn. Deletions are meant to
+          disappear; anything else silently missing from a version is data loss,
+          so it is collected and reported rather than quietly skipped.
+        */
+        const onDisk = new Set(everything.map((file) => file.path));
+        const vanished = [...ticked].filter((chosen) => !onDisk.has(chosen) && !prior.has(chosen));
+        // 1. Measure and hash. This is what makes an unchanged file free.
+        const declarations = [];
+        let totalBytes = 0;
+        for (const [index, file] of sending.entries()) {
+            this.check();
+            const full = node_path_1.default.join(request.localPath, file.path);
+            let size;
+            try {
+                size = (await (0, promises_1.stat)(full)).size;
+            }
+            catch {
+                // Unreadable now, though it was listed a moment ago. Skipping it
+                // would publish a version quietly missing a file the person chose.
+                vanished.push(file.path);
+                continue;
+            }
+            report({
+                stage: "hash",
+                files: index,
+                totalFiles: sending.length,
+                bytes: totalBytes,
+                totalBytes: 0,
+                path: file.path,
+                percent: Math.round((index / sending.length) * 20),
+                bytesPerSecond: 0,
+            });
+            declarations.push({
+                path: file.path,
+                full,
+                size,
+                sha256: await digestOf(full),
+                mediaType: mediaTypeOf(file.path),
+            });
+            totalBytes += size;
+        }
+        if (!declarations.length)
+            throw new Error("Nothing is selected to upload");
+        // 2. The project needs somewhere on the account to live.
+        this.check();
+        let repositoryId = request.repositoryId;
+        if (!repositoryId) {
+            const slug = request.projectName
+                .trim()
+                .toLowerCase()
+                .replace(/[^a-z0-9]+/g, "-")
+                .replace(/^-|-$/g, "") || "project";
+            try {
+                const created = await this.call("/v1/repositories", {
+                    method: "POST",
+                    contentType: "application/json",
+                    body: JSON.stringify({
+                        slug,
+                        displayName: request.projectName.trim() || slug,
+                        description: "",
+                        visibility: "private",
+                    }),
+                });
+                repositoryId = created.id;
+            }
+            catch (error) {
+                // The account already has a project of this name — which happens
+                // after a reinstall, or when the folder has moved. Saving into it is
+                // what was meant; failing would strand the folder.
+                const existing = await this.findBySlug(slug);
+                if (!existing)
+                    throw error;
+                repositoryId = existing;
+            }
+        }
+        // 3. Send the objects.
+        const uploaded = [];
+        let sentBytes = 0;
+        const startedAt = Date.now();
+        const rate = () => {
+            const seconds = (Date.now() - startedAt) / 1000;
+            return seconds > 0.2 ? Math.round(sentBytes / seconds) : 0;
+        };
+        for (const [index, declaration] of declarations.entries()) {
+            this.check();
+            report({
+                stage: "upload",
+                files: index,
+                totalFiles: declarations.length,
+                bytes: sentBytes,
+                totalBytes,
+                path: declaration.path,
+                percent: 20 + Math.round((sentBytes / Math.max(totalBytes, 1)) * 72),
+                bytesPerSecond: rate(),
+            });
+            const result = declaration.size <= DIRECT_LIMIT
+                ? await this.putDirect(repositoryId, declaration).catch((error) => {
+                    // Naming the file turns "it failed" into something actionable.
+                    const reason = error instanceof Error ? error.message : String(error);
+                    throw new Error(`${declaration.path}: ${reason}`);
+                })
+                : await this.putMultipart(repositoryId, declaration, (offset) => {
+                    report({
+                        stage: "upload",
+                        files: index,
+                        totalFiles: declarations.length,
+                        bytes: sentBytes + offset,
+                        totalBytes,
+                        path: declaration.path,
+                        percent: 20 +
+                            Math.round(((sentBytes + offset) / Math.max(totalBytes, 1)) * 72),
+                        bytesPerSecond: rate(),
+                    });
+                });
+            uploaded.push({
+                path: declaration.path,
+                objectId: result.objectId,
+                sourceSize: declaration.size,
+                storedSize: result.size,
+                mediaType: declaration.mediaType,
+            });
+            sentBytes += declaration.size;
+        }
+        // 4. Name the version, which is what makes the upload visible.
+        this.check();
+        report({
+            stage: "publish",
+            files: declarations.length,
+            totalFiles: declarations.length,
+            bytes: sentBytes,
+            totalBytes,
+            path: "Recording the version",
+            percent: 96,
+            bytesPerSecond: rate(),
+        });
+        // The version names everything the project holds: what was just sent,
+        // plus every file that kept the object the last version pointed at.
+        const contents = [
+            ...uploaded,
+            ...reused.map((file) => {
+                const held = prior.get(file.path);
+                return {
+                    path: file.path,
+                    objectId: held.objectId,
+                    sourceSize: held.sourceSize,
+                    storedSize: held.storedSize,
+                    mediaType: held.mediaType,
+                };
+            }),
+        ].sort((left, right) => left.path.localeCompare(right.path));
+        /*
+          Last check before the version is written: everything the person ticked
+          has to be in it. Reaching this with something missing would publish a
+          snapshot that silently lacked a file, which is the one failure a backup
+          tool must never have.
+        */
+        const named = new Set(contents.map((item) => item.path));
+        const dropped = [...new Set([...vanished, ...[...ticked].filter((chosen) => !named.has(chosen))])];
+        if (dropped.length) {
+            const shown = dropped.slice(0, 5).join(", ");
+            const rest = dropped.length > 5 ? ` and ${dropped.length - 5} more` : "";
+            throw new Error(`${dropped.length} selected file${dropped.length === 1 ? "" : "s"} could not be read, ` +
+                `so nothing was saved: ${shown}${rest}. Nothing was changed on your account.`);
+        }
+        const sourceBytes = contents.reduce((total, item) => total + item.sourceSize, 0);
+        const storedBytes = contents.reduce((total, item) => total + item.storedSize, 0);
+        const completed = await this.call(`/v1/repositories/${repositoryId}/versions`, {
+            method: "POST",
+            contentType: "application/json",
+            body: JSON.stringify({
+                message: request.message.trim() || "Saved from the CodeRook desktop app",
+                sourceSize: sourceBytes,
+                storedSize: storedBytes,
+                files: contents.map((item) => ({
+                    path: item.path,
+                    objectId: item.objectId,
+                    sourceSize: item.sourceSize,
+                    storedSize: item.storedSize,
+                    mediaType: item.mediaType,
+                    executable: false,
+                })),
+            }),
+        });
+        report({
+            stage: "done",
+            files: declarations.length,
+            totalFiles: declarations.length,
+            bytes: sentBytes,
+            totalBytes,
+            path: `Version ${completed.version.sequence} saved`,
+            percent: 100,
+            bytesPerSecond: rate(),
+        });
+        return {
+            repositoryId,
+            versionId: completed.version.id,
+            sequence: completed.version.sequence,
+            sourceBytes,
+            storedBytes,
+            sentBytes,
+            sentFiles: uploaded.length,
+            reusedFiles: reused.length,
+            // A file that kept its old object records the digest of *that* copy,
+            // not of the file on disk, so an unticked edit is still pending next
+            // time rather than looking as though it had been saved.
+            manifest: {
+                ...Object.fromEntries(reused.map((file) => [file.path, prior.get(file.path).sha256])),
+                ...Object.fromEntries(declarations.map((declaration) => [declaration.path, declaration.sha256])),
+            },
+        };
+    }
+    /** What the newest version holds, so unchanged files can point at it. */
+    async latestVersionFiles(repositoryId) {
+        const held = new Map();
+        try {
+            const versions = await this.call(`/v1/repositories/${repositoryId}/versions`, { method: "GET" });
+            // The API returns them newest first.
+            const latest = (versions.versions ?? [])[0];
+            if (!latest?.id)
+                return held;
+            const body = await this.call(`/v1/repositories/${repositoryId}/versions/${latest.id}/files`, {
+                method: "GET",
+            });
+            for (const row of body.files ?? []) {
+                held.set(String(row.path ?? ""), {
+                    objectId: String(row.objectId ?? ""),
+                    sha256: String(row.sha256 ?? ""),
+                    sourceSize: Number(row.sourceSize ?? 0),
+                    storedSize: Number(row.storedSize ?? 0),
+                    mediaType: String(row.mediaType ?? "application/octet-stream"),
+                });
+            }
+        }
+        catch {
+            // Without the previous list the version can still be written; it will
+            // simply contain only what is being sent now.
+        }
+        return held;
+    }
+    /** The account's repository with this slug, if it has one. */
+    async findBySlug(slug) {
+        try {
+            const body = await this.call("/v1/repositories", { method: "GET" });
+            const found = (body.repositories ?? []).find((row) => row.slug === slug);
+            return found?.id ?? null;
+        }
+        catch {
+            return null;
+        }
+    }
+    async putDirect(repositoryId, declaration) {
+        const body = await (0, promises_1.readFile)(declaration.full);
+        // The file was hashed earlier, and a live file — a database, a log — can
+        // change in between. The bytes about to be sent are what must be
+        // described, so the digest is taken from them rather than trusted.
+        const digest = (0, node_crypto_1.createHash)("sha256").update(body).digest("hex");
+        if (digest !== declaration.sha256) {
+            declaration.sha256 = digest;
+            declaration.size = body.length;
+        }
+        return this.call(`/v1/repositories/${repositoryId}/objects/${digest}?kind=chunk&role=chunk`, {
+            method: "PUT",
+            contentType: declaration.mediaType,
+            body: new Uint8Array(body),
+        });
+    }
+    /** Files past the direct limit go up in parts under an upload session. */
+    async putMultipart(repositoryId, declaration, onOffset) {
+        const session = await this.call(`/v1/repositories/${repositoryId}/uploads`, {
+            method: "POST",
+            contentType: "application/json",
+            body: JSON.stringify({
+                sha256: declaration.sha256,
+                size: declaration.size,
+                mediaType: declaration.mediaType,
+                kind: "chunk",
+                repositoryRole: "chunk",
+            }),
+        });
+        let offset = 0;
+        let partNumber = 1;
+        try {
+            while (offset < declaration.size) {
+                this.check();
+                const end = Math.min(declaration.size, offset + PART_SIZE);
+                const part = await this.readSlice(declaration.full, offset, end);
+                await this.call(`/v1/uploads/${session.uploadSessionId}/parts/${partNumber}`, {
+                    method: "PUT",
+                    contentType: "application/octet-stream",
+                    body: part,
+                });
+                offset = end;
+                partNumber += 1;
+                onOffset(offset);
+            }
+            return await this.call(`/v1/uploads/${session.uploadSessionId}/complete`, { method: "POST", contentType: "application/json", body: "{}" });
+        }
+        catch (error) {
+            // A half-finished session would hold storage forever.
+            await this.call(`/v1/uploads/${session.uploadSessionId}`, {
+                method: "DELETE",
+            }).catch(() => undefined);
+            throw error;
+        }
+    }
+    async readSlice(full, start, end) {
+        const chunks = [];
+        for await (const chunk of (0, node_fs_1.createReadStream)(full, { start, end: end - 1 })) {
+            chunks.push(chunk);
+        }
+        return new Uint8Array(Buffer.concat(chunks));
+    }
+}
+exports.Uploader = Uploader;