@filefreak.io/sdk 0.1.0 → 0.2.0

package/dist/reader.js

@@ -0,0 +1,510 @@
+// Reading a file under Shard Protection (docs/reference/shard-format.md): its
+// map a page at a time, each fragment fetched whole from its storage lane,
+// checked and opened with the file's key (shards.ts), and written at its place
+// in the sink. The CLI's reader (cli/crates/transfer/src/read.rs), by the same
+// rules, with the same numbers and the same sentences, so a file reads the
+// same whichever program reads it.
+//
+// The bytes never pass through the API: a page of the map names an address in
+// a lane for each fragment, and they are fetched from there. A page needs
+// nothing from the one before, so one is asked for wherever the reader lacks
+// addresses: the first before anything is written, the next in the background
+// once a place close ahead has none, and a fresh one from a place whose
+// address has run out or is about to. Asking from place 0 is what counts the
+// file as downloaded, and place 0 is fetched first, so it is asked for again
+// only if fragment 0's own address runs out before it arrives.
+//
+// What is in flight is bounded twice. Each storage host is sent at most six
+// requests at once, and the whole program holds at most MOST_IN_FLIGHT_BYTES
+// of fragments being fetched or not yet written (the budget), however many
+// files it reads at once.
+//
+// A fragment whose connection breaks, goes quiet or meets a busy storage
+// account is asked for again, a little later each time, TRIES times. One that
+// arrives and fails its check is not: it would fail every time, and the read
+// stops with NOT_WHAT_WAS_STORED. The caller aborts the sink either way: a
+// file with holes in it, or with bytes that are not the file's, is worse than
+// none.
+//
+// The key is a WebCrypto key that cannot be exported, held inside one read
+// and let go of with it. No error here carries it.
+import { openFragment, ShardError, shardKey, ShardLayout, SHARD_OVERHEAD } from "./shards.js";
+import { backoff, FileFreakError, sleep } from "./transport.js";
+const MIB = 1024 * 1024;
+/** The most bytes of fragments being fetched, or fetched and not yet written, at once in this program: five of the largest. */
+export const MOST_IN_FLIGHT_BYTES = 320 * MIB;
+/** Fragments fetched at once from one storage host. */
+export const PER_HOST = 6;
+/** Once a place this close ahead of the next fragment started has no address, the page holding it is asked for. */
+const ASK_AHEAD = 200;
+/** A fragment that fails this many times in a row stops the read. */
+const TRIES = 5;
+/** An address this close to running out is not used: a fresh page is asked for instead. */
+const EXPIRY_MARGIN_MS = 60_000;
+/** A request with nothing arriving for this long is given up and asked for again. */
+const QUIET_MS = 30_000;
+/**
+ * At most this many fragments started and not yet written. The budget bounds
+ * their bytes, and this their number, which a map of a great many small
+ * fragments would otherwise not be.
+ */
+const MOST_STARTED = 64;
+/** Said when a fragment arrives and fails its check. */
+export const NOT_WHAT_WAS_STORED = "Part of this file is not what was stored. Nothing was saved.";
+/** Said when a fragment cannot be had from storage. */
+export const NOT_FETCHED = "Part of this file could not be fetched from storage. Nothing was saved; try again later.";
+/** Said when a later page of the map is of another content than the first. */
+const CHANGED = "This file changed while it was being read. Nothing was saved; run it again to fetch it as it is now.";
+const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+/**
+ * A number of units, taken and given back, first come first served: a
+ * budget of bytes, or turns at a host. A taker who asks for more than there
+ * ever is waits for all of it, and goes alone.
+ */
+export class Gate {
+    most;
+    free;
+    queue = [];
+    constructor(most) {
+        this.most = most;
+        this.most = Math.max(1, Math.floor(most));
+        this.free = this.most;
+    }
+    /** Wait for `units`. Answers the way to give them back, which may be called more than once. */
+    take(units, signal) {
+        const n = Math.min(Math.max(1, Math.ceil(units)), this.most);
+        if (signal.aborted)
+            return Promise.reject(signal.reason);
+        return new Promise((resolve, reject) => {
+            const entry = {
+                units: n,
+                go: () => {
+                    signal.removeEventListener("abort", left);
+                    resolve(this.giver(n));
+                },
+            };
+            const left = () => {
+                const at = this.queue.indexOf(entry);
+                if (at < 0)
+                    return;
+                this.queue.splice(at, 1);
+                reject(signal.reason);
+                // Whoever waited behind it may fit now.
+                this.next();
+            };
+            this.queue.push(entry);
+            signal.addEventListener("abort", left, { once: true });
+            this.next();
+        });
+    }
+    giver(n) {
+        let given = false;
+        return () => {
+            if (given)
+                return;
+            given = true;
+            this.free += n;
+            this.next();
+        };
+    }
+    next() {
+        while (this.queue.length > 0 && this.queue[0].units <= this.free) {
+            const entry = this.queue.shift();
+            this.free -= entry.units;
+            entry.go();
+        }
+    }
+}
+let processBudget = null;
+/** The budget every read in this program shares, unless it is given its own. */
+function shared() {
+    processBudget ??= new Gate(MOST_IN_FLIGHT_BYTES);
+    return processBudget;
+}
+function stopped(code, message, cause) {
+    const error = new FileFreakError(0, code, message);
+    return cause === undefined ? error : Object.assign(error, { cause });
+}
+function unreadable(what) {
+    return stopped("unreadable", `The API answered with a map whose ${what} could not be read.`);
+}
+const damaged = (cause) => stopped("fragment_damaged", NOT_WHAT_WAS_STORED, cause);
+const notFetched = (cause) => stopped("fragment_unavailable", NOT_FETCHED, cause);
+/**
+ * A page whose count, or one of whose places, the file's size and fragment
+ * size do not allow: a map that does not add up is not read from.
+ */
+function agrees(page, layout) {
+    if (page.count !== layout.count) {
+        throw stopped("layout", `This file's map does not add up: it says ${page.count} fragments, where ${layout.size} bytes in fragments of ${layout.fragmentSize} make ${layout.count}. Nothing was saved.`);
+    }
+    const wrong = page.fragments.find((f) => !Number.isInteger(f.place) || f.place < 0 || f.place >= layout.count);
+    if (wrong) {
+        throw stopped("layout", `This file's map names fragment ${wrong.place}, and the file has ${layout.count}, numbered from 0. Nothing was saved.`);
+    }
+}
+async function keyOf(text) {
+    let raw;
+    try {
+        raw = Uint8Array.from(atob(text), (c) => c.charCodeAt(0));
+    }
+    catch {
+        throw unreadable("key");
+    }
+    if (raw.length !== 32)
+        throw unreadable("key");
+    return await shardKey(raw);
+}
+/**
+ * Read the protected file whose map `pages` gives into `sink`. Resolves once
+ * every fragment has been written; the caller closes the sink, or aborts it
+ * when this rejects, as with fetchRanges. Rejects only once every fragment
+ * still in flight has stopped, so nothing is written to the sink after.
+ */
+export async function readProtected(options) {
+    const stop = new AbortController();
+    const outer = options.signal;
+    const passOn = () => stop.abort(outer?.reason);
+    if (outer?.aborted)
+        passOn();
+    outer?.addEventListener("abort", passOn);
+    try {
+        const first = await options.pages(0, stop.signal);
+        if (options.sizeBytes !== undefined && first.sizeBytes !== options.sizeBytes)
+            throw stopped("content_changed", CHANGED);
+        let layout;
+        try {
+            layout = new ShardLayout(first.sizeBytes, first.fragmentBytes);
+        }
+        catch (e) {
+            if (e instanceof ShardError)
+                throw stopped("layout", `${e.message} Nothing was saved.`);
+            throw e;
+        }
+        agrees(first, layout);
+        if (!UUID.test(first.contentId))
+            throw unreadable("content id");
+        const reading = new Reading(options, stop, layout, first.contentId, await keyOf(first.key));
+        reading.hold(first);
+        await reading.fill();
+    }
+    finally {
+        outer?.removeEventListener("abort", passOn);
+        // A page asked for ahead of need is not waited for.
+        if (!stop.signal.aborted)
+            stop.abort(new DOMException("The read is over.", "AbortError"));
+    }
+}
+/**
+ * Fetch a file the way it is kept: through its map when it is protected,
+ * from its address when not. One converted since it was listed is refused
+ * the way the listing chose (409 not_protected, or shard_protected) before
+ * anything is written, and is fetched the other way, once.
+ */
+export async function asItIsKept(isProtected, plain, mapped) {
+    const refused = isProtected ? "not_protected" : "shard_protected";
+    try {
+        await (isProtected ? mapped() : plain());
+    }
+    catch (e) {
+        if (!(e instanceof FileFreakError && e.code === refused))
+            throw e;
+        await (isProtected ? plain() : mapped());
+    }
+}
+/** One read: what every fragment of it shares. */
+class Reading {
+    options;
+    stop;
+    layout;
+    contentId;
+    key;
+    addresses = new Map();
+    /** Held while a page is asked for, so that fragments waiting on the same page ask once. */
+    asking = new Gate(1);
+    /** A page is being asked for ahead of need. */
+    looking = false;
+    hosts = new Map();
+    budget;
+    fetcher;
+    wait;
+    now;
+    quietMs;
+    written = 0;
+    constructor(options, stop, layout, contentId, key) {
+        this.options = options;
+        this.stop = stop;
+        this.layout = layout;
+        this.contentId = contentId;
+        this.key = key;
+        this.budget = options.budget ?? shared();
+        this.fetcher = options.fetch ?? fetch.bind(globalThis);
+        this.wait = options.sleep ?? sleep;
+        this.now = options.now ?? Date.now;
+        this.quietMs = options.quietMs ?? QUIET_MS;
+    }
+    get signal() {
+        return this.stop.signal;
+    }
+    /** Keep a page's addresses, once it is known to be a page of this file's map. */
+    hold(page) {
+        if (page.contentId !== this.contentId || page.sizeBytes !== this.layout.size || page.fragmentBytes !== this.layout.fragmentSize) {
+            throw stopped("content_changed", CHANGED);
+        }
+        agrees(page, this.layout);
+        const expiresAt = Date.parse(page.expiresAt);
+        if (!Number.isFinite(expiresAt))
+            throw unreadable("time its addresses run out");
+        for (const f of page.fragments)
+            this.addresses.set(f.place, { url: f.url, expiresAt });
+    }
+    /** The address for `place`, if there is one that will not run out in the next minute. */
+    usable(place) {
+        const a = this.addresses.get(place);
+        return a && a.expiresAt > this.now() + EXPIRY_MARGIN_MS ? a.url : null;
+    }
+    /** The first place from `from` on, and within ASK_AHEAD of it, with no address at all. */
+    missingAhead(from) {
+        const until = Math.min(from + ASK_AHEAD, this.layout.count);
+        for (let place = from; place < until; place++)
+            if (!this.addresses.has(place))
+                return place;
+        return null;
+    }
+    async addressFor(place) {
+        const ready = this.usable(place);
+        if (ready)
+            return ready;
+        const done = await this.asking.take(1, this.signal);
+        try {
+            // A page asked for while this one waited its turn may have brought it.
+            const now = this.usable(place);
+            if (now)
+                return now;
+            this.hold(await this.options.pages(place, this.signal));
+            // Used as given, however soon this computer's clock says it runs out:
+            // a page is as fresh as the API makes them, and a clock that disagrees
+            // with storage's is found out by storage refusing the address, not by
+            // asking for the page again for every fragment.
+            const given = this.addresses.get(place);
+            if (!given)
+                throw notFetched();
+            return given.url;
+        }
+        finally {
+            done();
+        }
+    }
+    /** Storage refused `url`, so it has run out, and so has every address given with it or before it. */
+    forget(url) {
+        let at = null;
+        for (const a of this.addresses.values()) {
+            if (a.url === url) {
+                at = a.expiresAt;
+                break;
+            }
+        }
+        if (at === null)
+            return;
+        for (const [place, a] of this.addresses)
+            if (a.expiresAt <= at)
+                this.addresses.delete(place);
+    }
+    /** A turn at the host `url` is on. Each storage lane is an account of its own, on a host of its own. */
+    turnAt(url) {
+        let host = "";
+        try {
+            host = new URL(url).host;
+        }
+        catch {
+            // An address that is not one is refused by fetch, and counted as a failure there.
+        }
+        let turns = this.hosts.get(host);
+        if (!turns) {
+            turns = new Gate(Math.max(1, this.options.perHost ?? PER_HOST));
+            this.hosts.set(host, turns);
+        }
+        return turns.take(1, this.signal);
+    }
+    /** One request for a stored fragment, held whole and at most `stored` bytes, so a longer one is caught as it arrives. */
+    async fetchOne(url, stored) {
+        const asking = new AbortController();
+        const stop = () => asking.abort(this.signal.reason);
+        this.signal.addEventListener("abort", stop);
+        let timer;
+        const listen = () => {
+            clearTimeout(timer);
+            timer = setTimeout(() => asking.abort(new DOMException("Nothing arrived for a while.", "TimeoutError")), this.quietMs);
+        };
+        try {
+            listen();
+            let res;
+            try {
+                res = await this.fetcher(url, { signal: asking.signal });
+            }
+            catch {
+                if (this.signal.aborted)
+                    throw this.signal.reason;
+                return "passing";
+            }
+            if (!res.ok || !res.body) {
+                await res.body?.cancel().catch(() => undefined);
+                if (res.status === 403)
+                    return "refused";
+                if (res.status === 408 || res.status === 429 || res.status >= 500)
+                    return "passing";
+                return "failed";
+            }
+            const blob = new Uint8Array(stored);
+            let at = 0;
+            const reader = res.body.getReader();
+            for (;;) {
+                listen();
+                let chunk;
+                try {
+                    chunk = await reader.read();
+                }
+                catch {
+                    if (this.signal.aborted)
+                        throw this.signal.reason;
+                    return "passing";
+                }
+                if (chunk.done)
+                    return blob.subarray(0, at);
+                if (at + chunk.value.byteLength > stored) {
+                    await reader.cancel().catch(() => undefined);
+                    return "too_long";
+                }
+                blob.set(chunk.value, at);
+                at += chunk.value.byteLength;
+            }
+        }
+        finally {
+            clearTimeout(timer);
+            this.signal.removeEventListener("abort", stop);
+        }
+    }
+    /** Fetch, check, open and write the fragment at `place`. */
+    async fragment(place) {
+        const { start } = this.layout.range(place);
+        const stored = this.layout.length(place) + SHARD_OVERHEAD;
+        let tries = 0;
+        let refreshed = false;
+        let blob;
+        for (;;) {
+            const url = await this.addressFor(place);
+            const done = await this.turnAt(url);
+            let fetched;
+            try {
+                fetched = await this.fetchOne(url, stored);
+            }
+            finally {
+                done();
+            }
+            if (fetched instanceof Uint8Array) {
+                blob = fetched;
+                break;
+            }
+            if (fetched === "refused" && !refreshed) {
+                // The address ran out: a fresh page from this place, once.
+                refreshed = true;
+                this.forget(url);
+                continue;
+            }
+            if (fetched === "passing") {
+                tries += 1;
+                if (tries >= TRIES)
+                    throw notFetched();
+                await this.wait(backoff(tries), this.signal);
+                continue;
+            }
+            if (fetched === "too_long")
+                throw damaged();
+            throw notFetched();
+        }
+        let plain;
+        try {
+            plain = await openFragment(this.key, { contentId: this.contentId, layout: this.layout, place }, blob);
+        }
+        catch (e) {
+            throw damaged(e);
+        }
+        if (this.signal.aborted)
+            throw this.signal.reason;
+        await this.options.sink.write(start, plain);
+        this.written += plain.byteLength;
+        this.options.progress?.(this.written);
+    }
+    /**
+     * Every fragment, started in place order as the budget allows, and waited
+     * for. The first that cannot be had stops the others: the file cannot be
+     * finished without it. It is what the read rejects with, once every other
+     * fragment has stopped.
+     */
+    async fill() {
+        const started = new Gate(MOST_STARTED);
+        const flying = new Set();
+        // The first reason the read stopped, which it rejects with.
+        const failures = [];
+        try {
+            for (let place = 0; place < this.layout.count; place++) {
+                const slot = await started.take(1, this.signal);
+                let held;
+                try {
+                    held = await this.budget.take(this.layout.length(place) + SHARD_OVERHEAD, this.signal);
+                }
+                catch (e) {
+                    slot();
+                    throw e;
+                }
+                this.lookAhead(place);
+                const task = this.fragment(place)
+                    .catch((e) => {
+                    failures.push(this.signal.aborted ? this.signal.reason : e);
+                    if (!this.signal.aborted)
+                        this.stop.abort(e);
+                })
+                    .finally(() => {
+                    held();
+                    slot();
+                    flying.delete(task);
+                });
+                flying.add(task);
+            }
+            await Promise.all([...flying]);
+        }
+        catch (e) {
+            // The signal stopped a wait above: the caller's, or a fragment's failure.
+            failures.push(e);
+        }
+        await Promise.allSettled([...flying]);
+        if (failures.length > 0)
+            throw failures[0];
+    }
+    /** Ask, in the background, for the page holding the first place close ahead of `from` with no address, unless one is being asked for already. */
+    lookAhead(from) {
+        const missing = this.missingAhead(from);
+        if (missing === null || this.looking)
+            return;
+        this.looking = true;
+        void (async () => {
+            try {
+                const done = await this.asking.take(1, this.signal);
+                try {
+                    if (!this.addresses.has(missing))
+                        this.hold(await this.options.pages(missing, this.signal));
+                }
+                finally {
+                    done();
+                }
+            }
+            catch {
+                // A page that cannot be had, or does not belong, is left to the
+                // fragment that needs it, which asks again and says why.
+            }
+            finally {
+                this.looking = false;
+            }
+        })();
+    }
+}

package/dist/shards.d.ts

@@ -56,4 +56,4 @@ export declare function sealWithNonce(key: ShardKey, at: FragmentAt, plaintext:
  * fragment at this place, unchanged. Nothing of a fragment that fails is
  * returned: WebCrypto gives no plaintext until the tag has been checked.
  */
-export declare function openFragment(key: ShardKey, at: FragmentAt, blob: Uint8Array): Promise<Uint8Array>;
+export declare function openFragment(key: ShardKey, at: FragmentAt, blob: Uint8Array): Promise<Uint8Array<ArrayBuffer>>;

package/dist/uploads.js

@@ -74,7 +74,7 @@ export async function upload(calls, fetcher, source, options = {}) {
         if (state.upload.sizeBytes !== size) {
             throw new FileFreakError(400, "size_mismatch", `That upload is for ${state.upload.sizeBytes} bytes, and this source has ${size}. Carry it on with the same file it began with.`);
         }
-        // Begun by a program that cut the file by its content, as ff does: its
+        // Begun by a program that cut the file by its content, as filefreak does: its
         // parts are blocks of different sizes, and this reads part n at n times
         // partBytes, so every part would be sent from the wrong place and the
         // upload could never complete.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@filefreak.io/sdk",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "FileFreak's developer API for Node and the browser: every endpoint, uploads straight to storage in parts, downloads in ranges, and the changes to follow.",
   "license": "Apache-2.0",
   "homepage": "https://filefreak.io/developers/reference",