@filefreak.io/sdk 0.0.0-stage → 0.1.0

package/dist/ranges.js

@@ -0,0 +1,303 @@
+// Fetching one file from storage as several ranges at once, each written
+// where it belongs.
+//
+// The website's engine for saving a large file (web/lib/rangedSave.ts),
+// without the page: the same measurements, the same rules. One request from
+// storage was measured at 360 to 420 Mbps on a line that carries about
+// 1 Gbps, and six ranges at once at 560 to 660 (plan 04, B1.5), so a file
+// large enough is fetched as up to six ranges, and a small one as one.
+//
+// One address, asked for once, because asking for another counts the file as
+// downloaded again. An address works for fifteen minutes and a request that
+// has started is not cut off when it passes, so each range is one long
+// request, all of them made at the start. When one finishes early it takes
+// the second half of the largest one left, as a new request. If that request
+// is not answered (the address has passed, say), the half goes back to the
+// range it was taken from, whose own request asked for it in the first place:
+// that range waits at the point it was cut until the new request has been
+// answered, so that it is still there to give back to. A range whose
+// connection breaks, or goes quiet, is asked for again from where it stopped.
+// After the address has passed that is refused, and the download stops with
+// an error saying so.
+import { FileFreakError } from "./transport.js";
+const MIB = 1024 * 1024;
+/** At most this many ranges at once. Six filled the line measured; more only adds requests. */
+export const RANGES_AT_ONCE = 6;
+/** No range is cut smaller than this: each is a request, and a request costs a round trip before its first byte. */
+export const LEAST_RANGE = 16 * MIB;
+/** Written to the sink this much at a time. Fewer writes, for this much memory for each range. */
+const WRITE_BYTES = 4 * MIB;
+/** A connection that breaks is asked for again this many times in a row, from where it stopped. */
+const TRIES = 5;
+/** A request with nothing arriving for this long is given up and asked for again. */
+const QUIET_MS = 30_000;
+/** A connection that broke or went quiet. The only failure worth asking again after. */
+class Broke extends Error {
+}
+/**
+ * Fetch `url` as ranges and write each where it belongs. Resolves when every
+ * byte has been written; rejects, with every request stopped, when one range
+ * cannot be finished.
+ */
+export async function fetchRanges(options) {
+    // The first range that fails stops the others: the file cannot be finished without it.
+    const stop = new AbortController();
+    const outer = options.signal;
+    const passOn = () => stop.abort(outer?.reason);
+    if (outer?.aborted)
+        passOn();
+    outer?.addEventListener("abort", passOn);
+    let written = 0;
+    const run = {
+        url: options.url,
+        sizeBytes: options.sizeBytes,
+        sink: options.sink,
+        signal: stop.signal,
+        fetch: options.fetch ?? globalThis.fetch.bind(globalThis),
+        sleep: options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))),
+        quietMs: options.quietMs ?? QUIET_MS,
+        wrote: (bytes) => {
+            written += bytes;
+            options.progress?.(written);
+        },
+    };
+    const least = options.leastRange ?? LEAST_RANGE;
+    const most = Math.max(1, Math.min(RANGES_AT_ONCE, options.ranges ?? RANGES_AT_ONCE));
+    const count = Math.max(1, Math.min(most, Math.floor(options.sizeBytes / least)));
+    const ranges = [];
+    for (let i = 0; i < count; i++) {
+        ranges.push({ at: Math.floor((i * options.sizeBytes) / count), end: Math.floor(((i + 1) * options.sizeBytes) / count), giving: null });
+    }
+    // One cut at a time, so that a range has at most one half out that could come back.
+    let cut = null;
+    let cutting = most > 1;
+    const worker = async (first) => {
+        await read(first, run);
+        for (;;) {
+            while (cut)
+                await cut;
+            if (!cutting || stop.signal.aborted)
+                return;
+            const from = ranges.filter((r) => r.end - r.at >= 2 * least).sort((a, b) => b.end - b.at - (a.end - a.at))[0];
+            if (!from)
+                return;
+            const before = from.end;
+            const half = { at: from.at + Math.floor((from.end - from.at) / 2), end: before, giving: null };
+            let settle = () => { };
+            cut = from.giving = new Promise((resolve) => (settle = resolve));
+            const settled = () => {
+                from.giving = null;
+                cut = null;
+                settle();
+            };
+            from.end = half.at;
+            ranges.push(half);
+            try {
+                await read(half, run, settled);
+            }
+            catch (e) {
+                if (!from.giving)
+                    throw e;
+                // Never answered, so nothing of it was written: the range it came
+                // from takes it back and carries on to where it was going. No more
+                // cuts, since whatever stopped this one would stop the next.
+                from.end = before;
+                ranges.splice(ranges.indexOf(half), 1);
+                cutting = false;
+                settled();
+                if (stop.signal.aborted)
+                    throw e;
+            }
+        }
+    };
+    try {
+        await Promise.all(ranges.slice().map((r) => worker(r).catch((e) => {
+            stop.abort(e);
+            throw e;
+        })));
+    }
+    finally {
+        outer?.removeEventListener("abort", passOn);
+    }
+}
+/**
+ * Read one range to its end, asking again from where it stopped when its
+ * connection breaks. With `answered`, the range is a half just taken from
+ * another: until storage has answered for it, any failure is thrown at once
+ * (so that the half can go back) rather than asked again, and `answered` is
+ * called when it has.
+ */
+async function read(range, run, answered) {
+    let tries = 0;
+    for (;;) {
+        // What is asked for is what the range holds now, which a cut that has
+        // not been answered could still change.
+        while (range.giving)
+            await range.giving;
+        if (range.at >= range.end)
+            return;
+        const startedAt = range.at;
+        const asking = new Asking(run);
+        try {
+            const res = await asking.fetch(range);
+            answered?.();
+            answered = undefined;
+            await drain(range, res, run, asking);
+        }
+        catch (e) {
+            if (run.signal.aborted || answered || !(e instanceof Broke))
+                throw e;
+            tries = await again(range.at > startedAt ? 0 : tries, run, e);
+            continue;
+        }
+        finally {
+            asking.done();
+        }
+        if (range.at >= range.end)
+            return;
+        tries = await again(range.at > startedAt ? 0 : tries, run, new Broke("the connection closed before the end of the range"));
+    }
+}
+/**
+ * One request, given up when nothing arrives on it for a while. Stopped with
+ * everything else when the download is.
+ */
+class Asking {
+    run;
+    controller = new AbortController();
+    timer;
+    stop = () => this.controller.abort(this.run.signal.reason);
+    constructor(run) {
+        this.run = run;
+        run.signal.addEventListener("abort", this.stop);
+    }
+    /** Something arrived, or is about to be waited for. */
+    listen() {
+        clearTimeout(this.timer);
+        this.timer = setTimeout(() => this.controller.abort(new Broke("nothing arrived for a while")), this.run.quietMs);
+    }
+    /** Nothing is expected for now: this side is not reading. */
+    deaf() {
+        clearTimeout(this.timer);
+    }
+    done() {
+        this.deaf();
+        this.run.signal.removeEventListener("abort", this.stop);
+    }
+    /** Ask for the rest of `range`, and check that storage answered with it. */
+    async fetch(range) {
+        this.listen();
+        let res;
+        try {
+            res = await this.run.fetch(this.run.url, { headers: { Range: `bytes=${range.at}-${range.end - 1}` }, signal: this.controller.signal });
+        }
+        catch (e) {
+            throw this.run.signal.aborted ? e : new Broke("the request failed", { cause: e });
+        }
+        const refuse = async (error) => {
+            await res.body?.cancel().catch(() => undefined);
+            throw error;
+        };
+        if (res.status === 403) {
+            return refuse(new FileFreakError(403, "address_expired", "The download stopped: a connection broke after the download's address had run out, so the rest could not be asked for. Download it again."));
+        }
+        if (res.status === 408 || res.status === 429 || res.status >= 500)
+            return refuse(new Broke(`storage answered ${res.status}`));
+        if (res.status !== 206)
+            return refuse(new FileFreakError(res.status, "storage_refused", `Storage answered ${res.status} when asked for part of the file.`));
+        // Content-Range is one of the headers storage lets a page read. Without
+        // it, a part written at the wrong place would go unnoticed.
+        const sent = /^bytes (\d+)-(\d+)\/(\d+)$/.exec(res.headers.get("Content-Range") ?? "");
+        if (!sent || !res.body)
+            return refuse(new FileFreakError(res.status, "storage_refused", "Storage did not say which part of the file it sent."));
+        if (Number(sent[3]) !== this.run.sizeBytes) {
+            return refuse(new FileFreakError(res.status, "file_changed", "The file in storage is not the size the API gave. It may have been replaced since; download it again."));
+        }
+        if (Number(sent[1]) !== range.at)
+            return refuse(new FileFreakError(res.status, "storage_refused", "Storage sent another part of the file than the one asked for."));
+        return res;
+    }
+    /** The next piece of the body, or null at its end. */
+    async next(reader) {
+        this.listen();
+        try {
+            const { done, value } = await reader.read();
+            return done ? null : value;
+        }
+        catch (e) {
+            throw this.run.signal.aborted ? e : new Broke("the connection broke", { cause: e });
+        }
+    }
+}
+/** Write what arrives on `res` into the range, until the range ends or the body does. */
+async function drain(range, res, run, asking) {
+    const reader = res.body.getReader();
+    const held = [];
+    let heldBytes = 0;
+    const flush = async () => {
+        if (heldBytes === 0)
+            return;
+        const data = join(held, heldBytes);
+        const position = range.at - heldBytes;
+        held.length = 0;
+        heldBytes = 0;
+        asking.deaf();
+        await run.sink.write(position, data);
+        run.wrote(data.byteLength);
+    };
+    try {
+        for (;;) {
+            let chunk = await asking.next(reader);
+            if (!chunk)
+                break;
+            while (chunk.byteLength > 0) {
+                if (range.at >= range.end) {
+                    await flush();
+                    if (range.giving) {
+                        // Cut here. This request goes on covering the rest, so wait,
+                        // reading nothing, to learn whether the rest comes back.
+                        asking.deaf();
+                        await range.giving;
+                        continue;
+                    }
+                    await reader.cancel().catch(() => undefined);
+                    return;
+                }
+                const take = chunk.subarray(0, Math.min(chunk.byteLength, range.end - range.at));
+                held.push(take);
+                heldBytes += take.byteLength;
+                range.at += take.byteLength;
+                chunk = chunk.subarray(take.byteLength);
+                if (heldBytes >= WRITE_BYTES)
+                    await flush();
+            }
+        }
+    }
+    finally {
+        // What arrived before a break is right, so it is written, and the range
+        // is asked for again from after it.
+        if (!run.signal.aborted)
+            await flush();
+    }
+}
+/** Wait before asking again, a little longer each time; give up after TRIES in a row with nothing arriving. */
+async function again(tries, run, why) {
+    if (tries + 1 >= TRIES) {
+        throw Object.assign(new FileFreakError(0, "storage_unreachable", "The download stopped: storage could not be reached after several tries. Download it again."), { cause: why });
+    }
+    await run.sleep(Math.min(8000, 500 * 2 ** tries));
+    if (run.signal.aborted)
+        throw run.signal.reason;
+    return tries + 1;
+}
+function join(parts, bytes) {
+    if (parts.length === 1)
+        return parts[0];
+    const out = new Uint8Array(bytes);
+    let at = 0;
+    for (const p of parts) {
+        out.set(p, at);
+        at += p.byteLength;
+    }
+    return out;
+}

package/dist/shards.d.ts

@@ -0,0 +1,59 @@
+export declare const SHARD_VERSION = 1;
+/** What a stored fragment adds to the bytes of the file it holds: the version byte, the nonce and the tag. */
+export declare const SHARD_OVERHEAD = 29;
+/**
+ * Why a fragment or a description was refused. A reader checks in this
+ * order, and says the first that fails: short, version, length,
+ * authentication. layout is a description of a file no fragment can match.
+ */
+export type ShardErrorCode = "short" | "version" | "length" | "authentication" | "layout";
+export declare class ShardError extends Error {
+    readonly code: ShardErrorCode;
+    constructor(code: ShardErrorCode, message: string);
+}
+/** How a file of `size` bytes is cut into fragments of `fragmentSize`, which the API chooses. */
+export declare class ShardLayout {
+    readonly size: number;
+    readonly fragmentSize: number;
+    readonly count: number;
+    constructor(size: number, fragmentSize: number);
+    /** Where in the file fragment `place` begins, and where it ends (exclusive). */
+    range(place: number): {
+        start: number;
+        end: number;
+    };
+    /** How many bytes of the file fragment `place` holds. */
+    length(place: number): number;
+}
+/**
+ * A content's key as WebCrypto holds it. Named through crypto.subtle and not
+ * as CryptoKey, which is the DOM's name for it and not a global in Node's
+ * types, which is all the API's typecheck reads this file with.
+ */
+export type ShardKey = Awaited<ReturnType<typeof crypto.subtle.importKey>>;
+/** Which fragment of which file: what a fragment's associated data binds it to. */
+export interface FragmentAt {
+    /** The content's own id, a UUID, which a copy of the file keeps. */
+    contentId: string;
+    layout: ShardLayout;
+    place: number;
+}
+/** A content's key, 32 bytes, made usable for its fragments and nothing else. */
+export declare function shardKey(raw: Uint8Array): Promise<ShardKey>;
+/** The bytes a fragment's tag also covers: the label, the content id, the place and the count. */
+export declare function associatedData(at: FragmentAt): Uint8Array<ArrayBuffer>;
+/** One fragment of a file, encrypted and ready to store, under a nonce of its own. */
+export declare function sealFragment(key: ShardKey, at: FragmentAt, plaintext: Uint8Array): Promise<Uint8Array>;
+/**
+ * sealFragment with the nonce chosen by the caller, which only the test
+ * files may do: two fragments sealed under one key with one nonce give away
+ * the difference between them, and the key's authentication with it. It is
+ * not exported from the package.
+ */
+export declare function sealWithNonce(key: ShardKey, at: FragmentAt, plaintext: Uint8Array, nonce: Uint8Array): Promise<Uint8Array>;
+/**
+ * A stored fragment's bytes of the file, once it is known to be this file's
+ * fragment at this place, unchanged. Nothing of a fragment that fails is
+ * returned: WebCrypto gives no plaintext until the tag has been checked.
+ */
+export declare function openFragment(key: ShardKey, at: FragmentAt, blob: Uint8Array): Promise<Uint8Array>;

package/dist/shards.js

@@ -0,0 +1,140 @@
+// The shard format, version 1: how a file under Shard Protection is kept, as
+// fragments each encrypted on its own. docs/reference/shard-format.md is the
+// definition, and docs/reference/shard-format-v1.json holds the test files
+// this is checked against (test/shards.test.ts), as the CLI's copy is: a
+// writer here whose bytes differ by one from the CLI's makes files the CLI
+// cannot read, and the reverse.
+//
+// WebCrypto, so the same code runs in a page, in a worker and in Node.
+export const SHARD_VERSION = 1;
+/** What a stored fragment adds to the bytes of the file it holds: the version byte, the nonce and the tag. */
+export const SHARD_OVERHEAD = 29;
+const NONCE_BYTES = 12;
+const HEADER_BYTES = 1 + NONCE_BYTES;
+const LABEL = new TextEncoder().encode("filefreak-shard-v1");
+/** The associated data numbers places and the count in 32 bits. */
+const MOST_FRAGMENTS = 0xffff_ffff;
+export class ShardError extends Error {
+    code;
+    constructor(code, message) {
+        super(message);
+        this.code = code;
+        this.name = "ShardError";
+    }
+}
+/** How a file of `size` bytes is cut into fragments of `fragmentSize`, which the API chooses. */
+export class ShardLayout {
+    size;
+    fragmentSize;
+    count;
+    constructor(size, fragmentSize) {
+        this.size = size;
+        this.fragmentSize = fragmentSize;
+        if (!Number.isSafeInteger(size) || size < 0)
+            throw new ShardError("layout", `A file's size is a whole number of bytes, not ${size}.`);
+        if (!Number.isSafeInteger(fragmentSize) || fragmentSize < 1) {
+            throw new ShardError("layout", `A fragment holds at least one byte of the file, not ${fragmentSize}.`);
+        }
+        // BigInt, so a size past 2^32 is divided exactly; an empty file is one
+        // empty fragment, so that even it has something to authenticate.
+        const count = size === 0 ? 1 : Number((BigInt(size) + BigInt(fragmentSize) - 1n) / BigInt(fragmentSize));
+        if (count > MOST_FRAGMENTS) {
+            throw new ShardError("layout", `A file of ${size} bytes in fragments of ${fragmentSize} is ${count} fragments, more than the format can number.`);
+        }
+        this.count = count;
+    }
+    /** Where in the file fragment `place` begins, and where it ends (exclusive). */
+    range(place) {
+        if (!Number.isInteger(place) || place < 0 || place >= this.count) {
+            throw new RangeError(`There is no fragment ${place}: this file has ${this.count}, numbered from 0.`);
+        }
+        const start = place * this.fragmentSize;
+        return { start, end: Math.min(start + this.fragmentSize, this.size) };
+    }
+    /** How many bytes of the file fragment `place` holds. */
+    length(place) {
+        const { start, end } = this.range(place);
+        return end - start;
+    }
+}
+/** A content's key, 32 bytes, made usable for its fragments and nothing else. */
+export function shardKey(raw) {
+    if (raw.length !== 32)
+        throw new RangeError(`A shard key is 32 bytes, not ${raw.length}.`);
+    return crypto.subtle.importKey("raw", new Uint8Array(raw), { name: "AES-GCM" }, false, ["encrypt", "decrypt"]);
+}
+/** The bytes a fragment's tag also covers: the label, the content id, the place and the count. */
+export function associatedData(at) {
+    const out = new Uint8Array(LABEL.length + 16 + 8);
+    out.set(LABEL, 0);
+    out.set(uuidBytes(at.contentId), LABEL.length);
+    const view = new DataView(out.buffer);
+    view.setUint32(LABEL.length + 16, at.place, false);
+    view.setUint32(LABEL.length + 20, at.layout.count, false);
+    return out;
+}
+/** One fragment of a file, encrypted and ready to store, under a nonce of its own. */
+export function sealFragment(key, at, plaintext) {
+    return sealWithNonce(key, at, plaintext, crypto.getRandomValues(new Uint8Array(NONCE_BYTES)));
+}
+/**
+ * sealFragment with the nonce chosen by the caller, which only the test
+ * files may do: two fragments sealed under one key with one nonce give away
+ * the difference between them, and the key's authentication with it. It is
+ * not exported from the package.
+ */
+export async function sealWithNonce(key, at, plaintext, nonce) {
+    const expected = at.layout.length(at.place);
+    if (plaintext.length !== expected) {
+        throw new ShardError("length", `Fragment ${at.place} holds ${expected} bytes of the file, not ${plaintext.length}.`);
+    }
+    if (nonce.length !== NONCE_BYTES)
+        throw new RangeError(`A nonce is ${NONCE_BYTES} bytes, not ${nonce.length}.`);
+    const iv = new Uint8Array(nonce);
+    const sealed = await crypto.subtle.encrypt({ name: "AES-GCM", iv, additionalData: associatedData(at), tagLength: 128 }, key, new Uint8Array(plaintext));
+    const out = new Uint8Array(HEADER_BYTES + sealed.byteLength);
+    out[0] = SHARD_VERSION;
+    out.set(iv, 1);
+    out.set(new Uint8Array(sealed), HEADER_BYTES);
+    return out;
+}
+/**
+ * A stored fragment's bytes of the file, once it is known to be this file's
+ * fragment at this place, unchanged. Nothing of a fragment that fails is
+ * returned: WebCrypto gives no plaintext until the tag has been checked.
+ */
+export async function openFragment(key, at, blob) {
+    const expected = at.layout.length(at.place);
+    if (blob.length < SHARD_OVERHEAD) {
+        throw new ShardError("short", `Fragment ${at.place} is ${blob.length} bytes, shorter than any fragment can be (${SHARD_OVERHEAD}).`);
+    }
+    if (blob[0] !== SHARD_VERSION) {
+        throw new ShardError("version", `Fragment ${at.place} is in version ${blob[0]} of the format, which this version of the SDK cannot read.`);
+    }
+    if (blob.length !== expected + SHARD_OVERHEAD) {
+        throw new ShardError("length", `Fragment ${at.place} is ${blob.length} bytes, and holding ${expected} bytes of the file it would be ${expected + SHARD_OVERHEAD}.`);
+    }
+    let plain;
+    try {
+        plain = await crypto.subtle.decrypt({ name: "AES-GCM", iv: new Uint8Array(blob.subarray(1, HEADER_BYTES)), additionalData: associatedData(at), tagLength: 128 }, key, new Uint8Array(blob.subarray(HEADER_BYTES)));
+    }
+    catch (e) {
+        // OperationError is WebCrypto's one answer to a tag that does not match.
+        // Anything else (a key made for something else) is not the fragment's fault.
+        if (e instanceof DOMException && e.name === "OperationError") {
+            throw new ShardError("authentication", `Fragment ${at.place} is not what was written as this file's fragment ${at.place}: it has been changed, or it belongs to another file or place.`);
+        }
+        throw e;
+    }
+    return new Uint8Array(plain);
+}
+function uuidBytes(id) {
+    if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id)) {
+        throw new RangeError(`A content id is a UUID, and ${JSON.stringify(id)} is not one.`);
+    }
+    const hex = id.replaceAll("-", "");
+    const out = new Uint8Array(16);
+    for (let i = 0; i < 16; i++)
+        out[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16);
+    return out;
+}

package/dist/transport.d.ts

@@ -0,0 +1,61 @@
+/** A refusal from the API, or a request that could not be answered at all. Branch on `code`, which does not change; `message` is for a person. */
+export declare class FileFreakError extends Error {
+    readonly status: number;
+    readonly code: string;
+    readonly body: Record<string, unknown> | null;
+    readonly retryAfterSeconds: number | null;
+    constructor(status: number, code: string, message: string, body?: Record<string, unknown> | null, retryAfterSeconds?: number | null);
+}
+/** A token as a string, or a function that fetches a fresh one from the developer's own server. */
+export type TokenSource = string | (() => Promise<string>);
+export interface ClientOptions {
+    /** An app's key, for a program on a server. Never put one in a web page: the API refuses a key sent from one. */
+    apiKey?: string;
+    /** For a web page: a token the developer's server made with POST /v1/tokens, or a function that asks the server for one. */
+    token?: TokenSource;
+    /** Where the API is. */
+    baseUrl?: string;
+    /** Another fetch, for a runtime without one or for tests. */
+    fetch?: typeof fetch;
+    /** How many times a request that failed for a passing reason is sent again. */
+    retries?: number;
+}
+export interface CallOptions {
+    signal?: AbortSignal;
+    /** For an endpoint that makes something: the caller's own key, so that the same request sent later, even from another process, makes nothing twice. */
+    idempotencyKey?: string;
+}
+export interface Request {
+    method: "GET" | "POST" | "PATCH" | "DELETE";
+    path: string;
+    query?: Record<string, unknown> | undefined;
+    body?: unknown;
+    /** The endpoint honours an Idempotency-Key. */
+    idempotent?: boolean;
+    /** The endpoint says it is safe to call again (finishing an upload). */
+    repeatable?: boolean;
+    options?: CallOptions;
+}
+export interface Response<T> {
+    status: number;
+    body: T;
+    headers: Headers;
+}
+export declare const DEFAULT_BASE_URL = "https://api.filefreak.io";
+/** Waits, or stops waiting when the signal is aborted. */
+export declare function sleep(ms: number, signal?: AbortSignal): Promise<void>;
+/** A wait that doubles with each try, with some spread so that many clients do not come back at once. */
+export declare function backoff(attempt: number): number;
+export declare class Transport {
+    readonly baseUrl: string;
+    private readonly fetcher;
+    private readonly retries;
+    private readonly apiKey;
+    private readonly tokenSource;
+    private token;
+    constructor(options: ClientOptions);
+    /** Another fetch, for reaching storage with the same retries and runtime. */
+    get fetch(): typeof fetch;
+    private credential;
+    send<T>(request: Request): Promise<Response<T>>;
+}