@filefreak.io/sdk 0.0.0-stage → 0.2.0

package/dist/ranges.d.ts

@@ -0,0 +1,34 @@
+/** At most this many ranges at once. Six filled the line measured; more only adds requests. */
+export declare const RANGES_AT_ONCE = 6;
+/** No range is cut smaller than this: each is a request, and a request costs a round trip before its first byte. */
+export declare const LEAST_RANGE: number;
+/** Where a download's bytes go. */
+export interface Sink {
+    /** Write `data` at `position` in the file. Called for several positions at once. */
+    write(position: number, data: Uint8Array<ArrayBuffer>): Promise<void>;
+    /** Called once every byte has been written. */
+    close?(): Promise<void>;
+    /** Called instead of close when the download fails or is stopped: a file with holes in it is worse than none. */
+    abort?(reason: unknown): Promise<void>;
+}
+export interface RangesOptions {
+    url: string;
+    sizeBytes: number;
+    sink: Sink;
+    signal?: AbortSignal | undefined;
+    /** How many bytes have been written, after each write. */
+    progress?: ((writtenBytes: number) => void) | undefined;
+    /** At most this many ranges at once. */
+    ranges?: number | undefined;
+    fetch?: typeof fetch;
+    sleep?: (ms: number) => Promise<void>;
+    quietMs?: number;
+    /** LEAST_RANGE, unless a test makes it smaller rather than make a file of gigabytes. */
+    leastRange?: number;
+}
+/**
+ * Fetch `url` as ranges and write each where it belongs. Resolves when every
+ * byte has been written; rejects, with every request stopped, when one range
+ * cannot be finished.
+ */
+export declare function fetchRanges(options: RangesOptions): Promise<void>;

package/dist/ranges.js

@@ -0,0 +1,303 @@
+// Fetching one file from storage as several ranges at once, each written
+// where it belongs.
+//
+// The website's engine for saving a large file (web/lib/rangedSave.ts),
+// without the page: the same measurements, the same rules. One request from
+// storage was measured at 360 to 420 Mbps on a line that carries about
+// 1 Gbps, and six ranges at once at 560 to 660 (plan 04, B1.5), so a file
+// large enough is fetched as up to six ranges, and a small one as one.
+//
+// One address, asked for once, because asking for another counts the file as
+// downloaded again. An address works for fifteen minutes and a request that
+// has started is not cut off when it passes, so each range is one long
+// request, all of them made at the start. When one finishes early it takes
+// the second half of the largest one left, as a new request. If that request
+// is not answered (the address has passed, say), the half goes back to the
+// range it was taken from, whose own request asked for it in the first place:
+// that range waits at the point it was cut until the new request has been
+// answered, so that it is still there to give back to. A range whose
+// connection breaks, or goes quiet, is asked for again from where it stopped.
+// After the address has passed that is refused, and the download stops with
+// an error saying so.
+import { FileFreakError } from "./transport.js";
+const MIB = 1024 * 1024;
+/** At most this many ranges at once. Six filled the line measured; more only adds requests. */
+export const RANGES_AT_ONCE = 6;
+/** No range is cut smaller than this: each is a request, and a request costs a round trip before its first byte. */
+export const LEAST_RANGE = 16 * MIB;
+/** Written to the sink this much at a time. Fewer writes, for this much memory for each range. */
+const WRITE_BYTES = 4 * MIB;
+/** A connection that breaks is asked for again this many times in a row, from where it stopped. */
+const TRIES = 5;
+/** A request with nothing arriving for this long is given up and asked for again. */
+const QUIET_MS = 30_000;
+/** A connection that broke or went quiet. The only failure worth asking again after. */
+class Broke extends Error {
+}
+/**
+ * Fetch `url` as ranges and write each where it belongs. Resolves when every
+ * byte has been written; rejects, with every request stopped, when one range
+ * cannot be finished.
+ */
+export async function fetchRanges(options) {
+    // The first range that fails stops the others: the file cannot be finished without it.
+    const stop = new AbortController();
+    const outer = options.signal;
+    const passOn = () => stop.abort(outer?.reason);
+    if (outer?.aborted)
+        passOn();
+    outer?.addEventListener("abort", passOn);
+    let written = 0;
+    const run = {
+        url: options.url,
+        sizeBytes: options.sizeBytes,
+        sink: options.sink,
+        signal: stop.signal,
+        fetch: options.fetch ?? globalThis.fetch.bind(globalThis),
+        sleep: options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))),
+        quietMs: options.quietMs ?? QUIET_MS,
+        wrote: (bytes) => {
+            written += bytes;
+            options.progress?.(written);
+        },
+    };
+    const least = options.leastRange ?? LEAST_RANGE;
+    const most = Math.max(1, Math.min(RANGES_AT_ONCE, options.ranges ?? RANGES_AT_ONCE));
+    const count = Math.max(1, Math.min(most, Math.floor(options.sizeBytes / least)));
+    const ranges = [];
+    for (let i = 0; i < count; i++) {
+        ranges.push({ at: Math.floor((i * options.sizeBytes) / count), end: Math.floor(((i + 1) * options.sizeBytes) / count), giving: null });
+    }
+    // One cut at a time, so that a range has at most one half out that could come back.
+    let cut = null;
+    let cutting = most > 1;
+    const worker = async (first) => {
+        await read(first, run);
+        for (;;) {
+            while (cut)
+                await cut;
+            if (!cutting || stop.signal.aborted)
+                return;
+            const from = ranges.filter((r) => r.end - r.at >= 2 * least).sort((a, b) => b.end - b.at - (a.end - a.at))[0];
+            if (!from)
+                return;
+            const before = from.end;
+            const half = { at: from.at + Math.floor((from.end - from.at) / 2), end: before, giving: null };
+            let settle = () => { };
+            cut = from.giving = new Promise((resolve) => (settle = resolve));
+            const settled = () => {
+                from.giving = null;
+                cut = null;
+                settle();
+            };
+            from.end = half.at;
+            ranges.push(half);
+            try {
+                await read(half, run, settled);
+            }
+            catch (e) {
+                if (!from.giving)
+                    throw e;
+                // Never answered, so nothing of it was written: the range it came
+                // from takes it back and carries on to where it was going. No more
+                // cuts, since whatever stopped this one would stop the next.
+                from.end = before;
+                ranges.splice(ranges.indexOf(half), 1);
+                cutting = false;
+                settled();
+                if (stop.signal.aborted)
+                    throw e;
+            }
+        }
+    };
+    try {
+        await Promise.all(ranges.slice().map((r) => worker(r).catch((e) => {
+            stop.abort(e);
+            throw e;
+        })));
+    }
+    finally {
+        outer?.removeEventListener("abort", passOn);
+    }
+}
+/**
+ * Read one range to its end, asking again from where it stopped when its
+ * connection breaks. With `answered`, the range is a half just taken from
+ * another: until storage has answered for it, any failure is thrown at once
+ * (so that the half can go back) rather than asked again, and `answered` is
+ * called when it has.
+ */
+async function read(range, run, answered) {
+    let tries = 0;
+    for (;;) {
+        // What is asked for is what the range holds now, which a cut that has
+        // not been answered could still change.
+        while (range.giving)
+            await range.giving;
+        if (range.at >= range.end)
+            return;
+        const startedAt = range.at;
+        const asking = new Asking(run);
+        try {
+            const res = await asking.fetch(range);
+            answered?.();
+            answered = undefined;
+            await drain(range, res, run, asking);
+        }
+        catch (e) {
+            if (run.signal.aborted || answered || !(e instanceof Broke))
+                throw e;
+            tries = await again(range.at > startedAt ? 0 : tries, run, e);
+            continue;
+        }
+        finally {
+            asking.done();
+        }
+        if (range.at >= range.end)
+            return;
+        tries = await again(range.at > startedAt ? 0 : tries, run, new Broke("the connection closed before the end of the range"));
+    }
+}
+/**
+ * One request, given up when nothing arrives on it for a while. Stopped with
+ * everything else when the download is.
+ */
+class Asking {
+    run;
+    controller = new AbortController();
+    timer;
+    stop = () => this.controller.abort(this.run.signal.reason);
+    constructor(run) {
+        this.run = run;
+        run.signal.addEventListener("abort", this.stop);
+    }
+    /** Something arrived, or is about to be waited for. */
+    listen() {
+        clearTimeout(this.timer);
+        this.timer = setTimeout(() => this.controller.abort(new Broke("nothing arrived for a while")), this.run.quietMs);
+    }
+    /** Nothing is expected for now: this side is not reading. */
+    deaf() {
+        clearTimeout(this.timer);
+    }
+    done() {
+        this.deaf();
+        this.run.signal.removeEventListener("abort", this.stop);
+    }
+    /** Ask for the rest of `range`, and check that storage answered with it. */
+    async fetch(range) {
+        this.listen();
+        let res;
+        try {
+            res = await this.run.fetch(this.run.url, { headers: { Range: `bytes=${range.at}-${range.end - 1}` }, signal: this.controller.signal });
+        }
+        catch (e) {
+            throw this.run.signal.aborted ? e : new Broke("the request failed", { cause: e });
+        }
+        const refuse = async (error) => {
+            await res.body?.cancel().catch(() => undefined);
+            throw error;
+        };
+        if (res.status === 403) {
+            return refuse(new FileFreakError(403, "address_expired", "The download stopped: a connection broke after the download's address had run out, so the rest could not be asked for. Download it again."));
+        }
+        if (res.status === 408 || res.status === 429 || res.status >= 500)
+            return refuse(new Broke(`storage answered ${res.status}`));
+        if (res.status !== 206)
+            return refuse(new FileFreakError(res.status, "storage_refused", `Storage answered ${res.status} when asked for part of the file.`));
+        // Content-Range is one of the headers storage lets a page read. Without
+        // it, a part written at the wrong place would go unnoticed.
+        const sent = /^bytes (\d+)-(\d+)\/(\d+)$/.exec(res.headers.get("Content-Range") ?? "");
+        if (!sent || !res.body)
+            return refuse(new FileFreakError(res.status, "storage_refused", "Storage did not say which part of the file it sent."));
+        if (Number(sent[3]) !== this.run.sizeBytes) {
+            return refuse(new FileFreakError(res.status, "file_changed", "The file in storage is not the size the API gave. It may have been replaced since; download it again."));
+        }
+        if (Number(sent[1]) !== range.at)
+            return refuse(new FileFreakError(res.status, "storage_refused", "Storage sent another part of the file than the one asked for."));
+        return res;
+    }
+    /** The next piece of the body, or null at its end. */
+    async next(reader) {
+        this.listen();
+        try {
+            const { done, value } = await reader.read();
+            return done ? null : value;
+        }
+        catch (e) {
+            throw this.run.signal.aborted ? e : new Broke("the connection broke", { cause: e });
+        }
+    }
+}
+/** Write what arrives on `res` into the range, until the range ends or the body does. */
+async function drain(range, res, run, asking) {
+    const reader = res.body.getReader();
+    const held = [];
+    let heldBytes = 0;
+    const flush = async () => {
+        if (heldBytes === 0)
+            return;
+        const data = join(held, heldBytes);
+        const position = range.at - heldBytes;
+        held.length = 0;
+        heldBytes = 0;
+        asking.deaf();
+        await run.sink.write(position, data);
+        run.wrote(data.byteLength);
+    };
+    try {
+        for (;;) {
+            let chunk = await asking.next(reader);
+            if (!chunk)
+                break;
+            while (chunk.byteLength > 0) {
+                if (range.at >= range.end) {
+                    await flush();
+                    if (range.giving) {
+                        // Cut here. This request goes on covering the rest, so wait,
+                        // reading nothing, to learn whether the rest comes back.
+                        asking.deaf();
+                        await range.giving;
+                        continue;
+                    }
+                    await reader.cancel().catch(() => undefined);
+                    return;
+                }
+                const take = chunk.subarray(0, Math.min(chunk.byteLength, range.end - range.at));
+                held.push(take);
+                heldBytes += take.byteLength;
+                range.at += take.byteLength;
+                chunk = chunk.subarray(take.byteLength);
+                if (heldBytes >= WRITE_BYTES)
+                    await flush();
+            }
+        }
+    }
+    finally {
+        // What arrived before a break is right, so it is written, and the range
+        // is asked for again from after it.
+        if (!run.signal.aborted)
+            await flush();
+    }
+}
+/** Wait before asking again, a little longer each time; give up after TRIES in a row with nothing arriving. */
+async function again(tries, run, why) {
+    if (tries + 1 >= TRIES) {
+        throw Object.assign(new FileFreakError(0, "storage_unreachable", "The download stopped: storage could not be reached after several tries. Download it again."), { cause: why });
+    }
+    await run.sleep(Math.min(8000, 500 * 2 ** tries));
+    if (run.signal.aborted)
+        throw run.signal.reason;
+    return tries + 1;
+}
+function join(parts, bytes) {
+    if (parts.length === 1)
+        return parts[0];
+    const out = new Uint8Array(bytes);
+    let at = 0;
+    for (const p of parts) {
+        out.set(p, at);
+        at += p.byteLength;
+    }
+    return out;
+}

package/dist/reader.d.ts

@@ -0,0 +1,76 @@
+import type { Sink } from "./ranges.js";
+/** The most bytes of fragments being fetched, or fetched and not yet written, at once in this program: five of the largest. */
+export declare const MOST_IN_FLIGHT_BYTES: number;
+/** Fragments fetched at once from one storage host. */
+export declare const PER_HOST = 6;
+/** Said when a fragment arrives and fails its check. */
+export declare const NOT_WHAT_WAS_STORED = "Part of this file is not what was stored. Nothing was saved.";
+/** Said when a fragment cannot be had from storage. */
+export declare const NOT_FETCHED = "Part of this file could not be fetched from storage. Nothing was saved; try again later.";
+/** A page of a protected file's map, as the reader uses it: files.shards's answer, or a link's. */
+export interface MapPage {
+    contentId: string;
+    sizeBytes: number;
+    fragmentBytes: number;
+    count: number;
+    /** The file's key, in base64. */
+    key: string;
+    fragments: ReadonlyArray<{
+        place: number;
+        url: string;
+    }>;
+    /** When this page's addresses stop working. */
+    expiresAt: string;
+}
+/** Asks for the page of a map that begins at a place. A function, so that a map is read the same way whoever gives it. */
+export type Pages = (from: number, signal: AbortSignal) => Promise<MapPage>;
+/**
+ * A number of units, taken and given back, first come first served: a
+ * budget of bytes, or turns at a host. A taker who asks for more than there
+ * ever is waits for all of it, and goes alone.
+ */
+export declare class Gate {
+    private readonly most;
+    private free;
+    private readonly queue;
+    constructor(most: number);
+    /** Wait for `units`. Answers the way to give them back, which may be called more than once. */
+    take(units: number, signal: AbortSignal): Promise<() => void>;
+    private giver;
+    private next;
+}
+export interface ReadOptions {
+    pages: Pages;
+    sink: Sink;
+    /**
+     * The file's size as it was listed. A map of another size is of a file
+     * replaced since, and is not read: a sink made for the size listed, as
+     * bytes() makes one, could not hold it.
+     */
+    sizeBytes?: number;
+    signal?: AbortSignal | undefined;
+    /** How many bytes of the file have been written, after each fragment. */
+    progress?: ((writtenBytes: number) => void) | undefined;
+    fetch?: typeof fetch;
+    sleep?: (ms: number, signal: AbortSignal) => Promise<void>;
+    now?: () => number;
+    /** Bytes of fragments in flight: the program's, unless a test gives a read its own. */
+    budget?: Gate;
+    /** Fragments fetched at once from one storage host. */
+    perHost?: number;
+    quietMs?: number;
+}
+/**
+ * Read the protected file whose map `pages` gives into `sink`. Resolves once
+ * every fragment has been written; the caller closes the sink, or aborts it
+ * when this rejects, as with fetchRanges. Rejects only once every fragment
+ * still in flight has stopped, so nothing is written to the sink after.
+ */
+export declare function readProtected(options: ReadOptions): Promise<void>;
+/**
+ * Fetch a file the way it is kept: through its map when it is protected,
+ * from its address when not. One converted since it was listed is refused
+ * the way the listing chose (409 not_protected, or shard_protected) before
+ * anything is written, and is fetched the other way, once.
+ */
+export declare function asItIsKept(isProtected: boolean, plain: () => Promise<void>, mapped: () => Promise<void>): Promise<void>;