@filefreak.io/sdk 0.0.0-stage → 0.1.0

package/dist/transport.js

@@ -0,0 +1,173 @@
+// How every call reaches the API: the address, the key or token, and what is
+// done when an answer is not the one wanted.
+//
+// Sending a request again is where a client makes something twice, so the
+// rule is narrow. A request is sent again only when that cannot do anything
+// twice: a read; a request carrying an Idempotency-Key, which the API answers
+// once (every endpoint that makes something takes one, and one is made here
+// when the caller gave none); or an answer that says the API did nothing,
+// 429 (too many) and 503 waking_up (the database is starting). Retry-After is
+// honoured.
+/** A refusal from the API, or a request that could not be answered at all. Branch on `code`, which does not change; `message` is for a person. */
+export class FileFreakError extends Error {
+    status;
+    code;
+    body;
+    retryAfterSeconds;
+    constructor(status, code, message, body = null, retryAfterSeconds = null) {
+        super(message);
+        this.status = status;
+        this.code = code;
+        this.body = body;
+        this.retryAfterSeconds = retryAfterSeconds;
+        this.name = "FileFreakError";
+    }
+}
+export const DEFAULT_BASE_URL = "https://api.filefreak.io";
+const DEFAULT_RETRIES = 4;
+/** The longest wait between tries, whatever Retry-After says. */
+const MOST_WAIT_MS = 60_000;
+/** A sleeping database takes about a minute to start. Five minutes of asking is plenty. */
+const WAKING_TRIES = 60;
+/** Waits, or stops waiting when the signal is aborted. */
+export function sleep(ms, signal) {
+    return new Promise((resolve, reject) => {
+        if (signal?.aborted)
+            return reject(signal.reason);
+        const timer = setTimeout(done, ms);
+        function done() {
+            signal?.removeEventListener("abort", stop);
+            resolve();
+        }
+        function stop() {
+            clearTimeout(timer);
+            reject(signal.reason);
+        }
+        signal?.addEventListener("abort", stop, { once: true });
+    });
+}
+/** A wait that doubles with each try, with some spread so that many clients do not come back at once. */
+export function backoff(attempt) {
+    return Math.min(MOST_WAIT_MS, 500 * 2 ** attempt) * (0.5 + Math.random() / 2);
+}
+function randomKey() {
+    const bytes = new Uint8Array(16);
+    crypto.getRandomValues(bytes);
+    return Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("");
+}
+function search(query) {
+    if (!query)
+        return "";
+    const params = new URLSearchParams();
+    for (const [name, value] of Object.entries(query)) {
+        if (value === undefined || value === null)
+            continue;
+        params.set(name, String(value));
+    }
+    const text = params.toString();
+    return text ? `?${text}` : "";
+}
+export class Transport {
+    baseUrl;
+    fetcher;
+    retries;
+    apiKey;
+    tokenSource;
+    token = null;
+    constructor(options) {
+        if (!options.apiKey === !options.token)
+            throw new Error("Give the client an apiKey (on a server) or a token (in a web page), and not both.");
+        this.apiKey = options.apiKey ?? null;
+        this.tokenSource = options.token ?? null;
+        this.baseUrl = (options.baseUrl || DEFAULT_BASE_URL).replace(/\/+$/, "");
+        // Bound, because a fetch taken off globalThis and called on its own loses `this` in some browsers.
+        this.fetcher = options.fetch ?? fetch.bind(globalThis);
+        this.retries = options.retries ?? DEFAULT_RETRIES;
+    }
+    /** Another fetch, for reaching storage with the same retries and runtime. */
+    get fetch() {
+        return this.fetcher;
+    }
+    async credential(fresh) {
+        if (this.apiKey)
+            return this.apiKey;
+        const source = this.tokenSource;
+        if (typeof source === "string")
+            return source;
+        if (fresh || !this.token) {
+            this.token = source();
+            // A failed fetch of a token is not kept, so the next call asks again.
+            this.token.catch(() => (this.token = null));
+        }
+        return await this.token;
+    }
+    async send(request) {
+        const options = request.options ?? {};
+        const key = request.idempotent ? options.idempotencyKey || randomKey() : undefined;
+        const safe = request.method === "GET" || key !== undefined || request.repeatable === true;
+        const url = `${this.baseUrl}${request.path}${search(request.query)}`;
+        let refreshed = false;
+        let waking = 0;
+        for (let attempt = 0;; attempt++) {
+            const headers = { Authorization: `Bearer ${await this.credential(false)}`, Accept: "application/json" };
+            if (key)
+                headers["Idempotency-Key"] = key;
+            if (request.body !== undefined)
+                headers["Content-Type"] = "application/json";
+            let res;
+            try {
+                res = await this.fetcher(url, {
+                    method: request.method,
+                    headers,
+                    body: request.body === undefined ? undefined : JSON.stringify(request.body),
+                    signal: options.signal,
+                });
+            }
+            catch (e) {
+                if (options.signal?.aborted)
+                    throw options.signal.reason;
+                // Nothing came back. Whether the API did anything is unknown, so only
+                // a request that cannot do anything twice is sent again.
+                if (!safe || attempt >= this.retries)
+                    throw new FileFreakError(0, "network_error", `The API could not be reached: ${e instanceof Error ? e.message : String(e)}`);
+                await sleep(backoff(attempt), options.signal);
+                continue;
+            }
+            const text = await res.text();
+            let body = null;
+            try {
+                body = text ? JSON.parse(text) : null;
+            }
+            catch {
+                body = text;
+            }
+            if (res.ok)
+                return { status: res.status, body: body, headers: res.headers };
+            const error = body && typeof body === "object" ? body : null;
+            const code = typeof error?.error === "string" ? error.error : `http_${res.status}`;
+            const message = typeof error?.message === "string" ? error.message : `The API answered ${res.status}.`;
+            // Not Number(header) alone: a missing header is null, and Number(null) is 0.
+            const header = res.headers.get("retry-after");
+            const after = header === null ? NaN : Number(header);
+            const retryAfter = Number.isFinite(after) && after >= 0 ? after : null;
+            const refusal = new FileFreakError(res.status, code, message, error, retryAfter);
+            if (res.status === 401 && code === "token_expired" && typeof this.tokenSource === "function" && !refreshed) {
+                refreshed = true;
+                await this.credential(true);
+                attempt--;
+                continue;
+            }
+            if (res.status === 503 && code === "waking_up" && waking < WAKING_TRIES) {
+                waking++;
+                attempt--;
+                await sleep(Math.min(MOST_WAIT_MS, (retryAfter ?? 5) * 1000), options.signal);
+                continue;
+            }
+            // Refused before anything was done, or a server between here and the API that failed.
+            const passing = res.status === 429 || (safe && (res.status === 502 || res.status === 503 || res.status === 504));
+            if (!passing || attempt >= this.retries)
+                throw refusal;
+            await sleep(retryAfter !== null ? Math.min(MOST_WAIT_MS, retryAfter * 1000) : backoff(attempt), options.signal);
+        }
+    }
+}

package/dist/uploads.d.ts

@@ -0,0 +1,53 @@
+import type { FilesGetAnswer, Item, Upload, UploadsBeginAnswer, UploadsBeginBody, UploadsCompleteAnswer, UploadsCompleteBody, UploadsGetAnswer, UploadsPartsAnswer, UploadsPartsQuery } from "./generated.js";
+import { type CallOptions } from "./transport.js";
+/** What can be uploaded. A File's name and type are used when none are given. */
+export type Source = Blob | ArrayBuffer | Uint8Array;
+export interface UploadOptions {
+    /** The name to store it under. Left out, a File's own name. */
+    name?: string;
+    /** The folder to put it in. Null, or left out, for the top. */
+    parentId?: string | null;
+    /** What to do if the folder already has a file of that name. */
+    onConflict?: "rename" | "replace" | "fail";
+    /** With onConflict "replace": the file as it was read. Anything else there, or this file at another revision, refuses the upload with 409 revision_changed, so a copy kept in step never writes over a change it has not seen. */
+    replacing?: {
+        id: string;
+        revision: number;
+    };
+    /** Left out, a Blob's own type, else worked out from the name. */
+    contentType?: string;
+    /** Smaller copies to make of an image as soon as it is stored. */
+    imageSizes?: number[];
+    /** Bytes in each part, from 8 MiB to 64 MiB. Left out, 32 MiB, or less for a small file. */
+    partBytes?: number;
+    /** Parts sent at once. */
+    concurrency?: number;
+    /** The file's SHA-256 in lower-case hex, if it is known. Kept with the file as clientSha256. */
+    sha256?: string;
+    /** The id of an upload that stopped part way: only what storage does not have is sent. */
+    resume?: string;
+    /** Called once the upload exists, before any part is sent. Keep its id to carry on after a crash. */
+    onBegun?: (upload: Upload) => void;
+    /** Called as parts arrive, with the bytes storage has. */
+    onProgress?: (sentBytes: number, totalBytes: number) => void;
+    /** Stops the upload. What reached storage is kept, to carry on with `resume`. */
+    signal?: AbortSignal;
+    /** For beginning: the caller's own key, so that a begin sent again makes one upload. */
+    idempotencyKey?: string;
+}
+/** An upload that stopped part way. What arrived stays in storage until the upload expires: carry on with `resume: uploadId`, or cancel it. */
+export declare class UploadStopped extends Error {
+    readonly uploadId: string;
+    constructor(uploadId: string, cause: unknown);
+}
+/** The calls an upload makes: the client's own, so that they are retried as any other call is. */
+export interface UploadCalls {
+    begin(body: UploadsBeginBody, options?: CallOptions): Promise<UploadsBeginAnswer>;
+    get(id: string, options?: CallOptions): Promise<UploadsGetAnswer>;
+    parts(id: string, query?: UploadsPartsQuery, options?: CallOptions): Promise<UploadsPartsAnswer>;
+    complete(id: string, body?: UploadsCompleteBody, options?: CallOptions): Promise<UploadsCompleteAnswer>;
+    file(id: string, options?: CallOptions): Promise<FilesGetAnswer>;
+}
+/** The part size to ask for: 32 MiB, halved while that would leave fewer parts than are sent at once. */
+export declare function partSizeFor(fileBytes: number, atOnce: number): number;
+export declare function upload(calls: UploadCalls, fetcher: typeof fetch, source: Source, options?: UploadOptions): Promise<Item>;

package/dist/uploads.js

@@ -0,0 +1,259 @@
+// Uploading a file: begin, send the parts straight to storage, complete.
+//
+// The three steps the API defines (POST /v1/uploads, a PUT for each part to
+// the address given for it, POST /v1/uploads/:id/complete). No byte of the
+// file goes to the API. The rules are the website's (web/lib/uploads.ts):
+//
+//   - Each part goes with a Content-MD5, so storage refuses a part that
+//     changed on the way, and that part is sent again.
+//   - A part that fails for a passing reason (no answer, storage busy, its
+//     address run out) is sent again, a little later each time, six times at
+//     most. An address that is about to run out is replaced before it is used.
+//   - Storage itself says which parts it has. Carrying on with `resume` sends
+//     only the rest, from this process or another; so does a complete that
+//     finds a part missing.
+//   - A very large file can take storage a while to put together. Complete
+//     answers 202 until it has, and is asked again when it says.
+//
+// A part is read into memory whole to be hashed and sent, so the memory an
+// upload needs is about partBytes times the parts sent at once.
+import { base64, Md5 } from "./md5.js";
+import { FileFreakError, sleep } from "./transport.js";
+const MIB = 1024 * 1024;
+/** The API's bounds on a part. */
+const LEAST_PART = 8 * MIB;
+const MOST_PART = 64 * MIB;
+/** Parts sent at once, unless the caller says. Four of 32 MiB went at about 700 Mbps from a fast line (plan 04, B1.1). */
+const AT_ONCE = 4;
+/** How many times one part is tried before the upload stops. */
+const PART_ATTEMPTS = 6;
+/** An address this close to running out is not used; a fresh one is asked for instead. */
+const ADDRESS_MARGIN_MS = 60_000;
+/** How long to go on asking while storage puts a very large file together. */
+const ASSEMBLY_PATIENCE_MS = 30 * 60_000;
+/** An upload that stopped part way. What arrived stays in storage until the upload expires: carry on with `resume: uploadId`, or cancel it. */
+export class UploadStopped extends Error {
+    uploadId;
+    constructor(uploadId, cause) {
+        const why = (cause instanceof Error ? cause.message : String(cause)).replace(/[.\s]*$/, ".");
+        super(`The upload stopped. ${why} Carry on with resume: "${uploadId}", or cancel it.`, { cause });
+        this.uploadId = uploadId;
+        this.name = "UploadStopped";
+    }
+}
+/** The part size to ask for: 32 MiB, halved while that would leave fewer parts than are sent at once. */
+export function partSizeFor(fileBytes, atOnce) {
+    let size = 32 * MIB;
+    while (size > LEAST_PART && size > fileBytes / atOnce)
+        size /= 2;
+    return size;
+}
+function sizeOf(source) {
+    return source instanceof Blob ? source.size : source.byteLength;
+}
+async function bytesOf(source, from, to) {
+    if (source instanceof Blob)
+        return new Uint8Array(await source.slice(from, to).arrayBuffer());
+    if (source instanceof ArrayBuffer)
+        return new Uint8Array(source.slice(from, to));
+    return source.slice(from, to);
+}
+export async function upload(calls, fetcher, source, options = {}) {
+    const signal = options.signal;
+    const atOnce = Math.max(1, Math.min(16, options.concurrency ?? AT_ONCE));
+    const size = sizeOf(source);
+    const call = { signal };
+    let job;
+    if (options.resume) {
+        const state = await calls.get(options.resume, call);
+        if (state.upload.status === "completed" && state.upload.fileId)
+            return (await calls.file(state.upload.fileId, call)).item;
+        if (state.upload.status !== "pending") {
+            throw new FileFreakError(410, `upload_${state.upload.status}`, `That upload is ${state.upload.status}, so it cannot be carried on. Begin it again.`);
+        }
+        if (state.upload.sizeBytes !== size) {
+            throw new FileFreakError(400, "size_mismatch", `That upload is for ${state.upload.sizeBytes} bytes, and this source has ${size}. Carry it on with the same file it began with.`);
+        }
+        // Begun by a program that cut the file by its content, as ff does: its
+        // parts are blocks of different sizes, and this reads part n at n times
+        // partBytes, so every part would be sent from the wrong place and the
+        // upload could never complete.
+        if (state.blocks) {
+            throw new FileFreakError(400, "cut_by_content", "That upload was begun with its file cut by content, which this SDK does not do. Carry it on with the program that began it, or cancel it and begin again.");
+        }
+        job = { source, size, upload: state.upload, addresses: new Map(), fetching: null, done: new Set(state.uploadedParts), sentBytes: 0 };
+    }
+    else {
+        const name = options.name ?? (typeof File !== "undefined" && source instanceof File ? source.name : undefined);
+        if (!name)
+            throw new TypeError("Give the upload a name: { name: \"photo.jpg\" }.");
+        const type = options.contentType ?? (source instanceof Blob && source.type ? source.type : undefined);
+        const body = {
+            name,
+            sizeBytes: size,
+            parentId: options.parentId ?? null,
+            partBytes: Math.max(LEAST_PART, Math.min(MOST_PART, options.partBytes ?? partSizeFor(size, atOnce))),
+        };
+        if (options.onConflict)
+            body.onConflict = options.onConflict;
+        if (options.replacing)
+            body.replacing = options.replacing;
+        if (type)
+            body.contentType = type;
+        if (options.imageSizes)
+            body.imageSizes = options.imageSizes;
+        const begun = await calls.begin(body, { signal, ...(options.idempotencyKey ? { idempotencyKey: options.idempotencyKey } : {}) });
+        job = { source, size, upload: begun.upload, addresses: new Map(begun.parts.map((p) => [p.number, p])), fetching: null, done: new Set(), sentBytes: 0 };
+    }
+    options.onBegun?.(job.upload);
+    try {
+        tally(job, options);
+        await sendParts(calls, fetcher, job, atOnce, options);
+        return await finish(calls, fetcher, job, atOnce, options);
+    }
+    catch (e) {
+        if (signal?.aborted)
+            throw signal.reason;
+        throw new UploadStopped(job.upload.id, e);
+    }
+}
+function tally(job, options) {
+    let sent = 0;
+    for (const n of job.done)
+        sent += Math.min(job.size, (n + 1) * job.upload.partBytes) - n * job.upload.partBytes;
+    job.sentBytes = sent;
+    options.onProgress?.(sent, job.size);
+}
+async function sendParts(calls, fetcher, job, atOnce, options) {
+    const todo = [];
+    for (let n = 0; n < job.upload.partCount; n++)
+        if (!job.done.has(n))
+            todo.push(n);
+    let next = 0;
+    // The first part that cannot be sent stops the others: the file cannot be finished without it.
+    const stop = new AbortController();
+    const outer = options.signal;
+    const passOn = () => stop.abort(outer?.reason);
+    if (outer?.aborted)
+        passOn();
+    outer?.addEventListener("abort", passOn);
+    const worker = async () => {
+        for (;;) {
+            if (stop.signal.aborted)
+                throw stop.signal.reason;
+            const n = todo[next++];
+            if (n === undefined)
+                return;
+            await sendPart(calls, fetcher, job, n, stop.signal);
+            job.done.add(n);
+            tally(job, options);
+        }
+    };
+    try {
+        await Promise.all(Array.from({ length: Math.min(atOnce, todo.length) }, () => worker().catch((e) => {
+            stop.abort(e);
+            throw e;
+        })));
+    }
+    finally {
+        outer?.removeEventListener("abort", passOn);
+    }
+}
+async function sendPart(calls, fetcher, job, n, signal) {
+    const from = n * job.upload.partBytes;
+    let bytes;
+    try {
+        bytes = await bytesOf(job.source, from, Math.min(job.size, from + job.upload.partBytes));
+    }
+    catch (e) {
+        // Browsers and Node both refuse to read a file that changed after it was
+        // opened, which is what keeps one from going up half old and half new.
+        throw Object.assign(new FileFreakError(0, "unreadable", "The file could not be read. It may have been changed, moved or deleted since it was opened: open it again, unchanged, to carry on."), { cause: e });
+    }
+    const md5 = base64(new Md5().update(bytes).digest());
+    for (let attempt = 1;; attempt++) {
+        const part = await address(calls, job, n, signal);
+        let status = 0;
+        let damaged = false;
+        try {
+            const res = await fetcher(part.url, { method: part.method, headers: { ...part.headers, "Content-MD5": md5 }, body: bytes, signal });
+            if (res.ok) {
+                await res.body?.cancel().catch(() => undefined);
+                return;
+            }
+            status = res.status;
+            // Storage hashed what arrived, and it was not what was hashed here.
+            damaged = status === 400 && /Md5Mismatch/.test(await res.text().catch(() => ""));
+        }
+        catch (e) {
+            if (signal.aborted)
+                throw signal.reason;
+            // No answer: the connection dropped. Sent again below.
+            status = 0;
+            if (attempt >= PART_ATTEMPTS)
+                throw new FileFreakError(0, "network_error", `Storage could not be reached: ${e instanceof Error ? e.message : String(e)}.`);
+        }
+        // 403: the address ran out, or was for an upload that has since ended. A fresh address tells which.
+        if (status === 403)
+            job.addresses.delete(n);
+        const passing = damaged || status === 0 || status === 403 || status === 408 || status === 429 || status >= 500;
+        if (!passing)
+            throw new FileFreakError(status, "storage_refused", `Storage refused part ${n} of the file with ${status}.`);
+        if (attempt >= PART_ATTEMPTS) {
+            throw new FileFreakError(status, damaged ? "part_damaged" : "storage_unreachable", `Part ${n} could not be sent after ${PART_ATTEMPTS} tries.`);
+        }
+        // 1, 2, 4, 8, 16 seconds.
+        await sleep(Math.min(16_000, 1000 * 2 ** (attempt - 1)), signal);
+    }
+}
+async function address(calls, job, n, signal) {
+    for (;;) {
+        const known = job.addresses.get(n);
+        // Addresses last half an hour and a long upload outlives them. One that is
+        // about to run out is replaced before it is used, not after it has failed.
+        if (known && Date.parse(known.expiresAt) - Date.now() > ADDRESS_MARGIN_MS)
+            return known;
+        if (job.fetching) {
+            await job.fetching;
+            if (job.addresses.has(n) && Date.parse(job.addresses.get(n).expiresAt) - Date.now() > ADDRESS_MARGIN_MS)
+                continue;
+        }
+        const fetching = calls.parts(job.upload.id, { from: n, count: 100 }, { signal }).then(({ parts }) => {
+            for (const p of parts)
+                job.addresses.set(p.number, p);
+        });
+        job.fetching = fetching.finally(() => (job.fetching = null)).catch(() => undefined);
+        await fetching;
+        const part = job.addresses.get(n);
+        if (!part)
+            throw new FileFreakError(0, "no_address", `The API gave no address for part ${n}.`);
+        return part;
+    }
+}
+async function finish(calls, fetcher, job, atOnce, options) {
+    const started = Date.now();
+    const body = options.sha256 ? { sha256: options.sha256 } : {};
+    for (let round = 0;;) {
+        try {
+            const result = await calls.complete(job.upload.id, body, { signal: options.signal });
+            if ("item" in result)
+                return result.item;
+            // Storage is still putting a very large file together. Ask again.
+            if (Date.now() - started > ASSEMBLY_PATIENCE_MS) {
+                throw new FileFreakError(202, "still_assembling", "Storage is taking a long time to put this file together. Nothing is lost: complete it again in a few minutes.");
+            }
+            await sleep(Math.max(1, result.retryAfterSeconds) * 1000, options.signal);
+        }
+        catch (e) {
+            // Storage does not have every part after all. Send what is missing.
+            if (e instanceof FileFreakError && e.code === "upload_incomplete" && round++ < 2) {
+                const have = Array.isArray(e.body?.uploadedParts) ? e.body.uploadedParts.filter((n) => typeof n === "number") : [];
+                job.done = new Set(have);
+                tally(job, options);
+                await sendParts(calls, fetcher, job, atOnce, options);
+                continue;
+            }
+            throw e;
+        }
+    }
+}

package/package.json

@@ -1,6 +1,29 @@
 {
   "name": "@filefreak.io/sdk",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "description": "FileFreak's developer API for Node and the browser: every endpoint, uploads straight to storage in parts, downloads in ranges, and the changes to follow.",
+  "license": "Apache-2.0",
+  "homepage": "https://filefreak.io/developers/reference",
+  "type": "module",
+  "sideEffects": false,
+  "engines": {
+    "node": ">=20"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./node": {
+      "types": "./dist/node.d.ts",
+      "default": "./dist/node.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "publishConfig": {
+    "access": "public"
+  }
+}