@filefreak.io/sdk 0.1.0 → 0.3.0

package/dist/uploads.js

@@ -17,7 +17,21 @@
 //
 // A part is read into memory whole to be hashed and sent, so the memory an
 // upload needs is about partBytes times the parts sent at once.
+//
+// SEALED AS IT IS SENT (plan 04, F6 step 4, answer 46). Every begin says the
+// SDK can seal (protectable), as the website and the CLI do. Into a place
+// under Shard Protection the API answers with shards, and each part is then
+// a fragment: the file's bytes at that place, sealed with the file's own key
+// (sealFragment, a nonce of its own each time) before it is hashed and sent.
+// The key comes from uploads.get, whose answer nothing keeps, and is held
+// only while the upload runs. Fragments sent are confirmed (uploads.fragments)
+// as they go, at most one confirmation at a time: each is copied inside
+// storage to its own name and recorded, or is to be sealed again and sent.
+// Carrying on, every fragment not recorded is confirmed before it is sent
+// again, since storage may hold it already. A fragment is up to 64 MiB and
+// sealed in memory, so the memory a sealed upload needs is the same.
 import { base64, Md5 } from "./md5.js";
+import { sealFragment, ShardError, shardKey, ShardLayout } from "./shards.js";
 import { FileFreakError, sleep } from "./transport.js";
 const MIB = 1024 * 1024;
 /** The API's bounds on a part. */
@@ -31,6 +45,14 @@ const PART_ATTEMPTS = 6;
 const ADDRESS_MARGIN_MS = 60_000;
 /** How long to go on asking while storage puts a very large file together. */
 const ASSEMBLY_PATIENCE_MS = 30 * 60_000;
+/** Fragments sent are confirmed this often, or as soon as this many are waiting, and at most this many at once (the API's window). */
+const CONFIRM_MS = 2000;
+const CONFIRM_AT = 32;
+const CONFIRM_MOST = 100;
+/** How long confirming may answer only "still copying" before the upload stops and says so. */
+const CONFIRM_PATIENCE_MS = 5 * 60_000;
+/** How many times fragments storage did not keep are sealed again and sent before the upload stops. */
+const SEALED_ROUNDS = 4;
 /** An upload that stopped part way. What arrived stays in storage until the upload expires: carry on with `resume: uploadId`, or cancel it. */
 export class UploadStopped extends Error {
     uploadId;
@@ -41,6 +63,34 @@ export class UploadStopped extends Error {
         this.name = "UploadStopped";
     }
 }
+/**
+ * The key and the cut of a sealed upload, from what uploads.get says, with
+ * every fragment it has not recorded waiting to be confirmed: one sent
+ * before, by this process or another, may be in storage already.
+ */
+async function sealingOf(state, size) {
+    const shards = state.shards;
+    if (!shards)
+        throw new FileFreakError(0, "unreadable", "The API began a sealed upload and then gave no key for it.");
+    let layout;
+    let raw;
+    try {
+        layout = new ShardLayout(size, shards.fragmentBytes);
+        raw = Uint8Array.from(atob(shards.key), (c) => c.charCodeAt(0));
+    }
+    catch (e) {
+        throw Object.assign(new FileFreakError(0, "unreadable", "The API answered with a sealed upload whose cut or key could not be read."), { cause: e });
+    }
+    if (layout.count !== shards.count || layout.count !== state.upload.partCount || raw.length !== 32) {
+        throw new FileFreakError(0, "layout", `The API cut this file into ${shards.count} fragments, where ${size} bytes in fragments of ${shards.fragmentBytes} make ${layout.count}.`);
+    }
+    const recorded = new Set(state.uploadedParts);
+    const waiting = new Set();
+    for (let n = 0; n < layout.count; n++)
+        if (!recorded.has(n))
+            waiting.add(n);
+    return { key: await shardKey(raw), contentId: shards.contentId, layout, recorded, waiting };
+}
 /** The part size to ask for: 32 MiB, halved while that would leave fewer parts than are sent at once. */
 export function partSizeFor(fileBytes, atOnce) {
     let size = 32 * MIB;
@@ -74,14 +124,15 @@ export async function upload(calls, fetcher, source, options = {}) {
         if (state.upload.sizeBytes !== size) {
             throw new FileFreakError(400, "size_mismatch", `That upload is for ${state.upload.sizeBytes} bytes, and this source has ${size}. Carry it on with the same file it began with.`);
         }
-        // Begun by a program that cut the file by its content, as ff does: its
+        // Begun by a program that cut the file by its content, as filefreak does: its
         // parts are blocks of different sizes, and this reads part n at n times
         // partBytes, so every part would be sent from the wrong place and the
         // upload could never complete.
         if (state.blocks) {
             throw new FileFreakError(400, "cut_by_content", "That upload was begun with its file cut by content, which this SDK does not do. Carry it on with the program that began it, or cancel it and begin again.");
         }
-        job = { source, size, upload: state.upload, addresses: new Map(), fetching: null, done: new Set(state.uploadedParts), sentBytes: 0 };
+        const sealing = state.shards ? await sealingOf(state, size) : null;
+        job = { source, size, upload: state.upload, addresses: new Map(), fetching: null, done: new Set(state.uploadedParts), sentBytes: 0, sealing };
     }
     else {
         const name = options.name ?? (typeof File !== "undefined" && source instanceof File ? source.name : undefined);
@@ -93,6 +144,8 @@ export async function upload(calls, fetcher, source, options = {}) {
             sizeBytes: size,
             parentId: options.parentId ?? null,
             partBytes: Math.max(LEAST_PART, Math.min(MOST_PART, options.partBytes ?? partSizeFor(size, atOnce))),
+            // Into a protected place, the parts are fragments this sends sealed.
+            protectable: true,
         };
         if (options.onConflict)
             body.onConflict = options.onConflict;
@@ -103,12 +156,23 @@ export async function upload(calls, fetcher, source, options = {}) {
         if (options.imageSizes)
             body.imageSizes = options.imageSizes;
         const begun = await calls.begin(body, { signal, ...(options.idempotencyKey ? { idempotencyKey: options.idempotencyKey } : {}) });
-        job = { source, size, upload: begun.upload, addresses: new Map(begun.parts.map((p) => [p.number, p])), fetching: null, done: new Set(), sentBytes: 0 };
+        job = { source, size, upload: begun.upload, addresses: new Map(begun.parts.map((p) => [p.number, p])), fetching: null, done: new Set(), sentBytes: 0, sealing: null };
+        if (begun.shards) {
+            // Begin's answer is kept by the API under its Idempotency-Key, so the key is asked for here.
+            try {
+                job.sealing = await sealingOf(await calls.get(begun.upload.id, call), size);
+            }
+            catch (e) {
+                throw new UploadStopped(begun.upload.id, e);
+            }
+            // Nothing of this upload can have been sent yet.
+            job.sealing.waiting.clear();
+        }
     }
     options.onBegun?.(job.upload);
     try {
         tally(job, options);
-        await sendParts(calls, fetcher, job, atOnce, options);
+        await deliver(calls, fetcher, job, atOnce, options);
         return await finish(calls, fetcher, job, atOnce, options);
     }
     catch (e) {
@@ -124,6 +188,71 @@ function tally(job, options) {
     job.sentBytes = sent;
     options.onProgress?.(sent, job.size);
 }
+/** Every part storage does not have, sent: as they are for a plain upload, and confirmed until each is recorded for a sealed one. */
+async function deliver(calls, fetcher, job, atOnce, options) {
+    const sealing = job.sealing;
+    if (!sealing)
+        return await sendParts(calls, fetcher, job, atOnce, options);
+    for (let round = 0;; round++) {
+        // Whatever may be in storage already is asked about before it is sent again.
+        await confirmAll(calls, job, sealing, options);
+        if (sealing.recorded.size === job.upload.partCount)
+            return;
+        // Round 0 sends each fragment once; each later round sends again what storage did not keep.
+        if (round > SEALED_ROUNDS) {
+            const left = job.upload.partCount - sealing.recorded.size;
+            throw new FileFreakError(0, "fragments_refused", `Storage did not keep ${left} of this file's fragments after ${SEALED_ROUNDS + 1} tries.`);
+        }
+        let sending = true;
+        const sent = sendParts(calls, fetcher, job, atOnce, options).finally(() => (sending = false));
+        await Promise.all([sent, confirmWhile(calls, job, sealing, () => sending, options)]);
+    }
+}
+/** While fragments are being sent: those sent are confirmed every CONFIRM_MS, or as soon as CONFIRM_AT are waiting. */
+async function confirmWhile(calls, job, sealing, sending, options) {
+    let last = Date.now();
+    while (sending()) {
+        await sleep(100, options.signal);
+        if (!sending())
+            return;
+        if (sealing.waiting.size >= CONFIRM_AT || (sealing.waiting.size > 0 && Date.now() - last >= CONFIRM_MS)) {
+            await confirmSome(calls, job, sealing, options);
+            last = Date.now();
+        }
+    }
+}
+/** Every fragment waiting asked about until none is: each recorded, or to be sealed again and sent. */
+async function confirmAll(calls, job, sealing, options) {
+    let heard = Date.now();
+    while (sealing.waiting.size > 0) {
+        if (await confirmSome(calls, job, sealing, options)) {
+            heard = Date.now();
+            continue;
+        }
+        if (Date.now() - heard > CONFIRM_PATIENCE_MS) {
+            throw new FileFreakError(0, "still_copying", "Storage has not finished taking part of this file. Nothing is lost: carry it on with resume in a few minutes.");
+        }
+        // Only fragments storage is still copying, or another confirmation holds, are left.
+        await sleep(CONFIRM_MS, options.signal);
+    }
+}
+/** Ask about the first CONFIRM_MOST fragments waiting. True when one was recorded or is to be sent again: either way it stops waiting. */
+async function confirmSome(calls, job, sealing, options) {
+    const places = [...sealing.waiting].sort((a, b) => a - b).slice(0, CONFIRM_MOST);
+    const answer = await calls.fragments(job.upload.id, { places }, { signal: options.signal });
+    for (const n of answer.recorded) {
+        sealing.recorded.add(n);
+        sealing.waiting.delete(n);
+        job.done.add(n);
+    }
+    for (const n of answer.resend) {
+        sealing.waiting.delete(n);
+        // Sent again, sealed again: never the same nonce over other bytes.
+        job.done.delete(n);
+    }
+    tally(job, options);
+    return answer.recorded.length > 0 || answer.resend.length > 0;
+}
 async function sendParts(calls, fetcher, job, atOnce, options) {
     const todo = [];
     for (let n = 0; n < job.upload.partCount; n++)
@@ -146,6 +275,7 @@ async function sendParts(calls, fetcher, job, atOnce, options) {
                 return;
             await sendPart(calls, fetcher, job, n, stop.signal);
             job.done.add(n);
+            job.sealing?.waiting.add(n);
             tally(job, options);
         }
     };
@@ -160,16 +290,27 @@ async function sendParts(calls, fetcher, job, atOnce, options) {
     }
 }
 async function sendPart(calls, fetcher, job, n, signal) {
-    const from = n * job.upload.partBytes;
+    const sealing = job.sealing;
+    const { start: from, end: to } = sealing ? sealing.layout.range(n) : { start: n * job.upload.partBytes, end: Math.min(job.size, (n + 1) * job.upload.partBytes) };
     let bytes;
     try {
-        bytes = await bytesOf(job.source, from, Math.min(job.size, from + job.upload.partBytes));
+        bytes = await bytesOf(job.source, from, to);
     }
     catch (e) {
         // Browsers and Node both refuse to read a file that changed after it was
         // opened, which is what keeps one from going up half old and half new.
         throw Object.assign(new FileFreakError(0, "unreadable", "The file could not be read. It may have been changed, moved or deleted since it was opened: open it again, unchanged, to carry on."), { cause: e });
     }
+    if (sealing) {
+        try {
+            bytes = await sealFragment(sealing.key, { contentId: sealing.contentId, layout: sealing.layout, place: n }, bytes);
+        }
+        catch (e) {
+            if (e instanceof ShardError)
+                throw Object.assign(new FileFreakError(0, "layout", e.message), { cause: e });
+            throw e;
+        }
+    }
     const md5 = base64(new Md5().update(bytes).digest());
     for (let attempt = 1;; attempt++) {
         const part = await address(calls, job, n, signal);
@@ -249,8 +390,13 @@ async function finish(calls, fetcher, job, atOnce, options) {
             if (e instanceof FileFreakError && e.code === "upload_incomplete" && round++ < 2) {
                 const have = Array.isArray(e.body?.uploadedParts) ? e.body.uploadedParts.filter((n) => typeof n === "number") : [];
                 job.done = new Set(have);
+                if (job.sealing) {
+                    // The fragments recorded: any other may be in storage, and is confirmed first.
+                    job.sealing.recorded = new Set(have);
+                    job.sealing.waiting = new Set(Array.from({ length: job.upload.partCount }, (_, n) => n).filter((n) => !job.sealing.recorded.has(n)));
+                }
                 tally(job, options);
-                await sendParts(calls, fetcher, job, atOnce, options);
+                await deliver(calls, fetcher, job, atOnce, options);
                 continue;
             }
             throw e;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@filefreak.io/sdk",
-  "version": "0.1.0",
+  "version": "0.3.0",
   "description": "FileFreak's developer API for Node and the browser: every endpoint, uploads straight to storage in parts, downloads in ranges, and the changes to follow.",
   "license": "Apache-2.0",
   "homepage": "https://filefreak.io/developers/reference",