@filefreak.io/sdk 0.0.0-stage → 0.2.0

package/dist/uploads.js

@@ -0,0 +1,259 @@
+// Uploading a file: begin, send the parts straight to storage, complete.
+//
+// The three steps the API defines (POST /v1/uploads, a PUT for each part to
+// the address given for it, POST /v1/uploads/:id/complete). No byte of the
+// file goes to the API. The rules are the website's (web/lib/uploads.ts):
+//
+//   - Each part goes with a Content-MD5, so storage refuses a part that
+//     changed on the way, and that part is sent again.
+//   - A part that fails for a passing reason (no answer, storage busy, its
+//     address run out) is sent again, a little later each time, six times at
+//     most. An address that is about to run out is replaced before it is used.
+//   - Storage itself says which parts it has. Carrying on with `resume` sends
+//     only the rest, from this process or another; so does a complete that
+//     finds a part missing.
+//   - A very large file can take storage a while to put together. Complete
+//     answers 202 until it has, and is asked again when it says.
+//
+// A part is read into memory whole to be hashed and sent, so the memory an
+// upload needs is about partBytes times the parts sent at once.
+import { base64, Md5 } from "./md5.js";
+import { FileFreakError, sleep } from "./transport.js";
+const MIB = 1024 * 1024;
+/** The API's bounds on a part. */
+const LEAST_PART = 8 * MIB;
+const MOST_PART = 64 * MIB;
+/** Parts sent at once, unless the caller says. Four of 32 MiB went at about 700 Mbps from a fast line (plan 04, B1.1). */
+const AT_ONCE = 4;
+/** How many times one part is tried before the upload stops. */
+const PART_ATTEMPTS = 6;
+/** An address this close to running out is not used; a fresh one is asked for instead. */
+const ADDRESS_MARGIN_MS = 60_000;
+/** How long to go on asking while storage puts a very large file together. */
+const ASSEMBLY_PATIENCE_MS = 30 * 60_000;
+/** An upload that stopped part way. What arrived stays in storage until the upload expires: carry on with `resume: uploadId`, or cancel it. */
+export class UploadStopped extends Error {
+    uploadId;
+    constructor(uploadId, cause) {
+        const why = (cause instanceof Error ? cause.message : String(cause)).replace(/[.\s]*$/, ".");
+        super(`The upload stopped. ${why} Carry on with resume: "${uploadId}", or cancel it.`, { cause });
+        this.uploadId = uploadId;
+        this.name = "UploadStopped";
+    }
+}
+/** The part size to ask for: 32 MiB, halved while that would leave fewer parts than are sent at once. */
+export function partSizeFor(fileBytes, atOnce) {
+    let size = 32 * MIB;
+    while (size > LEAST_PART && size > fileBytes / atOnce)
+        size /= 2;
+    return size;
+}
+function sizeOf(source) {
+    return source instanceof Blob ? source.size : source.byteLength;
+}
+async function bytesOf(source, from, to) {
+    if (source instanceof Blob)
+        return new Uint8Array(await source.slice(from, to).arrayBuffer());
+    if (source instanceof ArrayBuffer)
+        return new Uint8Array(source.slice(from, to));
+    return source.slice(from, to);
+}
+export async function upload(calls, fetcher, source, options = {}) {
+    const signal = options.signal;
+    const atOnce = Math.max(1, Math.min(16, options.concurrency ?? AT_ONCE));
+    const size = sizeOf(source);
+    const call = { signal };
+    let job;
+    if (options.resume) {
+        const state = await calls.get(options.resume, call);
+        if (state.upload.status === "completed" && state.upload.fileId)
+            return (await calls.file(state.upload.fileId, call)).item;
+        if (state.upload.status !== "pending") {
+            throw new FileFreakError(410, `upload_${state.upload.status}`, `That upload is ${state.upload.status}, so it cannot be carried on. Begin it again.`);
+        }
+        if (state.upload.sizeBytes !== size) {
+            throw new FileFreakError(400, "size_mismatch", `That upload is for ${state.upload.sizeBytes} bytes, and this source has ${size}. Carry it on with the same file it began with.`);
+        }
+        // Begun by a program that cut the file by its content, as filefreak does: its
+        // parts are blocks of different sizes, and this reads part n at n times
+        // partBytes, so every part would be sent from the wrong place and the
+        // upload could never complete.
+        if (state.blocks) {
+            throw new FileFreakError(400, "cut_by_content", "That upload was begun with its file cut by content, which this SDK does not do. Carry it on with the program that began it, or cancel it and begin again.");
+        }
+        job = { source, size, upload: state.upload, addresses: new Map(), fetching: null, done: new Set(state.uploadedParts), sentBytes: 0 };
+    }
+    else {
+        const name = options.name ?? (typeof File !== "undefined" && source instanceof File ? source.name : undefined);
+        if (!name)
+            throw new TypeError("Give the upload a name: { name: \"photo.jpg\" }.");
+        const type = options.contentType ?? (source instanceof Blob && source.type ? source.type : undefined);
+        const body = {
+            name,
+            sizeBytes: size,
+            parentId: options.parentId ?? null,
+            partBytes: Math.max(LEAST_PART, Math.min(MOST_PART, options.partBytes ?? partSizeFor(size, atOnce))),
+        };
+        if (options.onConflict)
+            body.onConflict = options.onConflict;
+        if (options.replacing)
+            body.replacing = options.replacing;
+        if (type)
+            body.contentType = type;
+        if (options.imageSizes)
+            body.imageSizes = options.imageSizes;
+        const begun = await calls.begin(body, { signal, ...(options.idempotencyKey ? { idempotencyKey: options.idempotencyKey } : {}) });
+        job = { source, size, upload: begun.upload, addresses: new Map(begun.parts.map((p) => [p.number, p])), fetching: null, done: new Set(), sentBytes: 0 };
+    }
+    options.onBegun?.(job.upload);
+    try {
+        tally(job, options);
+        await sendParts(calls, fetcher, job, atOnce, options);
+        return await finish(calls, fetcher, job, atOnce, options);
+    }
+    catch (e) {
+        if (signal?.aborted)
+            throw signal.reason;
+        throw new UploadStopped(job.upload.id, e);
+    }
+}
+function tally(job, options) {
+    let sent = 0;
+    for (const n of job.done)
+        sent += Math.min(job.size, (n + 1) * job.upload.partBytes) - n * job.upload.partBytes;
+    job.sentBytes = sent;
+    options.onProgress?.(sent, job.size);
+}
+async function sendParts(calls, fetcher, job, atOnce, options) {
+    const todo = [];
+    for (let n = 0; n < job.upload.partCount; n++)
+        if (!job.done.has(n))
+            todo.push(n);
+    let next = 0;
+    // The first part that cannot be sent stops the others: the file cannot be finished without it.
+    const stop = new AbortController();
+    const outer = options.signal;
+    const passOn = () => stop.abort(outer?.reason);
+    if (outer?.aborted)
+        passOn();
+    outer?.addEventListener("abort", passOn);
+    const worker = async () => {
+        for (;;) {
+            if (stop.signal.aborted)
+                throw stop.signal.reason;
+            const n = todo[next++];
+            if (n === undefined)
+                return;
+            await sendPart(calls, fetcher, job, n, stop.signal);
+            job.done.add(n);
+            tally(job, options);
+        }
+    };
+    try {
+        await Promise.all(Array.from({ length: Math.min(atOnce, todo.length) }, () => worker().catch((e) => {
+            stop.abort(e);
+            throw e;
+        })));
+    }
+    finally {
+        outer?.removeEventListener("abort", passOn);
+    }
+}
+async function sendPart(calls, fetcher, job, n, signal) {
+    const from = n * job.upload.partBytes;
+    let bytes;
+    try {
+        bytes = await bytesOf(job.source, from, Math.min(job.size, from + job.upload.partBytes));
+    }
+    catch (e) {
+        // Browsers and Node both refuse to read a file that changed after it was
+        // opened, which is what keeps one from going up half old and half new.
+        throw Object.assign(new FileFreakError(0, "unreadable", "The file could not be read. It may have been changed, moved or deleted since it was opened: open it again, unchanged, to carry on."), { cause: e });
+    }
+    const md5 = base64(new Md5().update(bytes).digest());
+    for (let attempt = 1;; attempt++) {
+        const part = await address(calls, job, n, signal);
+        let status = 0;
+        let damaged = false;
+        try {
+            const res = await fetcher(part.url, { method: part.method, headers: { ...part.headers, "Content-MD5": md5 }, body: bytes, signal });
+            if (res.ok) {
+                await res.body?.cancel().catch(() => undefined);
+                return;
+            }
+            status = res.status;
+            // Storage hashed what arrived, and it was not what was hashed here.
+            damaged = status === 400 && /Md5Mismatch/.test(await res.text().catch(() => ""));
+        }
+        catch (e) {
+            if (signal.aborted)
+                throw signal.reason;
+            // No answer: the connection dropped. Sent again below.
+            status = 0;
+            if (attempt >= PART_ATTEMPTS)
+                throw new FileFreakError(0, "network_error", `Storage could not be reached: ${e instanceof Error ? e.message : String(e)}.`);
+        }
+        // 403: the address ran out, or was for an upload that has since ended. A fresh address tells which.
+        if (status === 403)
+            job.addresses.delete(n);
+        const passing = damaged || status === 0 || status === 403 || status === 408 || status === 429 || status >= 500;
+        if (!passing)
+            throw new FileFreakError(status, "storage_refused", `Storage refused part ${n} of the file with ${status}.`);
+        if (attempt >= PART_ATTEMPTS) {
+            throw new FileFreakError(status, damaged ? "part_damaged" : "storage_unreachable", `Part ${n} could not be sent after ${PART_ATTEMPTS} tries.`);
+        }
+        // 1, 2, 4, 8, 16 seconds.
+        await sleep(Math.min(16_000, 1000 * 2 ** (attempt - 1)), signal);
+    }
+}
+async function address(calls, job, n, signal) {
+    for (;;) {
+        const known = job.addresses.get(n);
+        // Addresses last half an hour and a long upload outlives them. One that is
+        // about to run out is replaced before it is used, not after it has failed.
+        if (known && Date.parse(known.expiresAt) - Date.now() > ADDRESS_MARGIN_MS)
+            return known;
+        if (job.fetching) {
+            await job.fetching;
+            if (job.addresses.has(n) && Date.parse(job.addresses.get(n).expiresAt) - Date.now() > ADDRESS_MARGIN_MS)
+                continue;
+        }
+        const fetching = calls.parts(job.upload.id, { from: n, count: 100 }, { signal }).then(({ parts }) => {
+            for (const p of parts)
+                job.addresses.set(p.number, p);
+        });
+        job.fetching = fetching.finally(() => (job.fetching = null)).catch(() => undefined);
+        await fetching;
+        const part = job.addresses.get(n);
+        if (!part)
+            throw new FileFreakError(0, "no_address", `The API gave no address for part ${n}.`);
+        return part;
+    }
+}
+async function finish(calls, fetcher, job, atOnce, options) {
+    const started = Date.now();
+    const body = options.sha256 ? { sha256: options.sha256 } : {};
+    for (let round = 0;;) {
+        try {
+            const result = await calls.complete(job.upload.id, body, { signal: options.signal });
+            if ("item" in result)
+                return result.item;
+            // Storage is still putting a very large file together. Ask again.
+            if (Date.now() - started > ASSEMBLY_PATIENCE_MS) {
+                throw new FileFreakError(202, "still_assembling", "Storage is taking a long time to put this file together. Nothing is lost: complete it again in a few minutes.");
+            }
+            await sleep(Math.max(1, result.retryAfterSeconds) * 1000, options.signal);
+        }
+        catch (e) {
+            // Storage does not have every part after all. Send what is missing.
+            if (e instanceof FileFreakError && e.code === "upload_incomplete" && round++ < 2) {
+                const have = Array.isArray(e.body?.uploadedParts) ? e.body.uploadedParts.filter((n) => typeof n === "number") : [];
+                job.done = new Set(have);
+                tally(job, options);
+                await sendParts(calls, fetcher, job, atOnce, options);
+                continue;
+            }
+            throw e;
+        }
+    }
+}

package/package.json

@@ -1,6 +1,29 @@
 {
   "name": "@filefreak.io/sdk",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.2.0",
+  "description": "FileFreak's developer API for Node and the browser: every endpoint, uploads straight to storage in parts, downloads in ranges, and the changes to follow.",
+  "license": "Apache-2.0",
+  "homepage": "https://filefreak.io/developers/reference",
+  "type": "module",
+  "sideEffects": false,
+  "engines": {
+    "node": ">=20"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./node": {
+      "types": "./dist/node.d.ts",
+      "default": "./dist/node.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "publishConfig": {
+    "access": "public"
+  }
+}