@filefreak.io/sdk 0.4.0 → 0.6.0

package/dist/uploads.js

@@ -16,7 +16,17 @@
 //     answers 202 until it has, and is asked again when it says.
 //
 // A part is read into memory whole to be hashed and sent, so the memory an
-// upload needs is about partBytes times the parts sent at once.
+// upload needs is about the size of a part times the parts sent at once.
+//
+// CUT BY CONTENT, as the CLI cuts a file (cut.ts). A file of 16 MiB or more
+// is read through once before it is begun, cut where its content says, and
+// begun with its blocks, each its size and SHA-256. Each part is then a
+// block, and a block the file it replaces has already, or one an earlier
+// part repeats, is not sent: the API lists those in reused. So a new version
+// of a large file sends only the blocks that changed. Each block is hashed
+// again as it is sent and refused if it is not the block that was declared,
+// since the file has changed. A block is 16 MiB at most, or 64 MiB in a file
+// past 32 GiB.
 //
 // SEALED AS IT IS SENT (plan 04, F6 step 4, answer 46). Every begin says the
 // SDK can seal (protectable), as the website and the CLI do. Into a place
@@ -30,9 +40,21 @@
 // Carrying on, every fragment not recorded is confirmed before it is sent
 // again, since storage may hold it already. A fragment is up to 64 MiB and
 // sealed in memory, so the memory a sealed upload needs is the same.
+//
+// Every begin also says the SDK writes version 2 of the shard format, in
+// which a protected file of more than 16 MiB is answered. Its fragments are
+// blocks: the file is cut again with a table made from its own key, each
+// block named by its id under the key (cutSealed), and the blocks declared
+// (uploads.blocks). Those the file's last version has under the same key are
+// kept and never sent, so a new version of a large protected file sends only
+// what changed, sealed here. Each block is checked against the id declared
+// for its place as it is sealed, and completing sends the outline's MAC. A
+// file not cut into two to ten thousand blocks, which version 2 does not
+// take, is begun again in version 1.
+import { cutPlain, cutSealed, hex, RULE } from "./cut.js";
 import { canDrawFrames, drawFrame, sendFrame, takesFrame } from "./frames.js";
 import { base64, Md5 } from "./md5.js";
-import { sealFragment, ShardError, shardKey, ShardLayout } from "./shards.js";
+import { BlockLayout, fileKeys, outlineMac, sealBlock, sealFragment, ShardError, shardKey, ShardLayout } from "./shards.js";
 import { FileFreakError, sleep } from "./transport.js";
 const MIB = 1024 * 1024;
 /** The API's bounds on a part. */
@@ -54,6 +76,8 @@ const CONFIRM_MOST = 100;
 const CONFIRM_PATIENCE_MS = 5 * 60_000;
 /** How many times fragments storage did not keep are sealed again and sent before the upload stops. */
 const SEALED_ROUNDS = 4;
+/** The versions of the shard format this writes. */
+const WRITES_FORMATS = [1, 2];
 /** An upload that stopped part way. What arrived stays in storage until the upload expires: carry on with `resume: uploadId`, or cancel it. */
 export class UploadStopped extends Error {
     uploadId;
@@ -64,33 +88,106 @@ export class UploadStopped extends Error {
         this.name = "UploadStopped";
     }
 }
-/**
- * The key and the cut of a sealed upload, from what uploads.get says, with
- * every fragment it has not recorded waiting to be confirmed: one sent
- * before, by this process or another, may be in storage already.
- */
-async function sealingOf(state, size) {
+/** A part that is not what was declared for it: the file has changed since it was cut. */
+function changed(part) {
+    return new FileFreakError(0, "source_changed", `Part ${part} of the file is not what it was when the upload began: the file has changed since.`);
+}
+/** A sealed upload's key, from what uploads.get says, and what it says of the file. */
+function keyOf(state) {
     const shards = state.shards;
     if (!shards)
         throw new FileFreakError(0, "unreadable", "The API began a sealed upload and then gave no key for it.");
-    let layout;
     let raw;
     try {
-        layout = new ShardLayout(size, shards.fragmentBytes);
         raw = Uint8Array.from(atob(shards.key), (c) => c.charCodeAt(0));
     }
     catch (e) {
-        throw Object.assign(new FileFreakError(0, "unreadable", "The API answered with a sealed upload whose cut or key could not be read."), { cause: e });
+        throw Object.assign(new FileFreakError(0, "unreadable", "The API answered with a sealed upload whose key could not be read."), { cause: e });
+    }
+    if (raw.length !== 32)
+        throw new FileFreakError(0, "unreadable", `The API answered with a key of ${raw.length} bytes, where a file's key is 32.`);
+    return { raw, shards };
+}
+/**
+ * The key and the cut of an upload sealed in version 1, from what
+ * uploads.get says, with every fragment it has not recorded waiting to be
+ * confirmed: one sent before, by this process or another, may be in storage
+ * already.
+ */
+async function sealingOf(state, size) {
+    const { raw, shards } = keyOf(state);
+    let layout;
+    try {
+        layout = new ShardLayout(size, shards.fragmentBytes);
+    }
+    catch (e) {
+        throw Object.assign(new FileFreakError(0, "unreadable", "The API answered with a sealed upload whose cut could not be read."), { cause: e });
     }
-    if (layout.count !== shards.count || layout.count !== state.upload.partCount || raw.length !== 32) {
+    if (layout.count !== shards.count || layout.count !== state.upload.partCount) {
         throw new FileFreakError(0, "layout", `The API cut this file into ${shards.count} fragments, where ${size} bytes in fragments of ${shards.fragmentBytes} make ${layout.count}.`);
     }
+    const key = await shardKey(raw);
+    const contentId = shards.contentId;
     const recorded = new Set(state.uploadedParts);
     const waiting = new Set();
     for (let n = 0; n < layout.count; n++)
         if (!recorded.has(n))
             waiting.add(n);
-    return { key: await shardKey(raw), contentId: shards.contentId, layout, recorded, waiting };
+    return {
+        layout,
+        seal: async (place, plain) => {
+            try {
+                return await sealFragment(key, { contentId, layout, place }, plain);
+            }
+            catch (e) {
+                if (e instanceof ShardError)
+                    throw Object.assign(new FileFreakError(0, "layout", e.message), { cause: e });
+                throw e;
+            }
+        },
+        outline: null,
+        recorded,
+        waiting,
+    };
+}
+/** What a file in version 2 is sealed with: each block checked, as it is sealed, against the id declared for its place. */
+async function blockSealing(keys, outline, recorded) {
+    let layout;
+    try {
+        layout = new BlockLayout(outline);
+    }
+    catch (e) {
+        if (e instanceof ShardError)
+            throw Object.assign(new FileFreakError(0, "layout", e.message), { cause: e });
+        throw e;
+    }
+    return {
+        layout,
+        seal: async (place, plain) => {
+            const { id, blob } = await sealBlock(keys, plain);
+            if (base64(id) !== base64(outline.blocks[place].id))
+                throw changed(place);
+            return blob;
+        },
+        outline: base64(await outlineMac(keys, outline)),
+        recorded,
+        waiting: new Set(),
+    };
+}
+/**
+ * Version 2: the file cut with its own key's table, and its blocks declared,
+ * with what to seal them with. Null for a file not cut into two to ten
+ * thousand blocks, which version 2 does not take.
+ */
+async function declare(calls, state, source, size, rule, signal) {
+    const { raw, shards } = keyOf(state);
+    const keys = await fileKeys(raw);
+    const blocks = await cutSealed(cuttable(source, size), raw, keys, rule, signal);
+    if (!blocks)
+        return null;
+    const declared = await calls.blocks(state.upload.id, { blocks: blocks.map((b) => ({ size: b.length, id: base64(b.id) })) }, { signal });
+    const sealing = await blockSealing(keys, { contentId: shards.contentId, size, blocks }, new Set(declared.reused));
+    return { upload: declared.upload, parts: declared.parts, sealing };
 }
 /** The part size to ask for: 32 MiB, halved while that would leave fewer parts than are sent at once. */
 export function partSizeFor(fileBytes, atOnce) {
@@ -109,44 +206,64 @@ async function bytesOf(source, from, to) {
         return new Uint8Array(source.slice(from, to));
     return source.slice(from, to);
 }
-export async function upload(calls, fetcher, source, options = {}) {
+/** Bytes of the file, or why they could not be read. */
+async function read(source, from, to) {
+    try {
+        return await bytesOf(source, from, to);
+    }
+    catch (e) {
+        // Browsers and Node both refuse to read a file that changed after it was
+        // opened, which is what keeps one from going up half old and half new.
+        throw Object.assign(new FileFreakError(0, "unreadable", "The file could not be read. It may have been changed, moved or deleted since it was opened: open it again, unchanged, to carry on."), { cause: e });
+    }
+}
+/** The file as the cutter reads it: where it is, when it is in memory, and a window at a time from a Blob. */
+function cuttable(source, size) {
+    if (source instanceof Blob)
+        return { size, read: (from, to) => read(source, from, to) };
+    return source instanceof ArrayBuffer ? new Uint8Array(source) : source;
+}
+/** Parts of one size, the last whatever remains. */
+function evenly(upload, size) {
+    return (n) => ({ start: n * upload.partBytes, end: Math.min(size, (n + 1) * upload.partBytes) });
+}
+/** Parts that are blocks of their own sizes. */
+function blocksOf(sizes) {
+    const starts = [0];
+    for (const size of sizes)
+        starts.push(starts[starts.length - 1] + size);
+    return (n) => ({ start: starts[n], end: starts[n + 1] });
+}
+/** A job for an upload sealed as it is sent: each part is a fragment, where its layout says. */
+function sealedJob(source, size, upload, sealing, addresses, done) {
+    return { source, size, upload, range: (n) => sealing.layout.range(n), hashes: null, addresses: new Map(addresses.map((p) => [p.number, p])), fetching: null, done: new Set(done), sentBytes: 0, sealing };
+}
+export async function upload(calls, fetcher, source, options = {}, rule = RULE) {
     const signal = options.signal;
     const atOnce = Math.max(1, Math.min(16, options.concurrency ?? AT_ONCE));
     const size = sizeOf(source);
-    const call = { signal };
     let job;
     if (options.resume) {
-        const state = await calls.get(options.resume, call);
-        if (state.upload.status === "completed" && state.upload.fileId)
-            return (await calls.file(state.upload.fileId, call)).item;
-        if (state.upload.status !== "pending") {
-            throw new FileFreakError(410, `upload_${state.upload.status}`, `That upload is ${state.upload.status}, so it cannot be carried on. Begin it again.`);
-        }
-        if (state.upload.sizeBytes !== size) {
-            throw new FileFreakError(400, "size_mismatch", `That upload is for ${state.upload.sizeBytes} bytes, and this source has ${size}. Carry it on with the same file it began with.`);
-        }
-        // Begun by a program that cut the file by its content, as filefreak does: its
-        // parts are blocks of different sizes, and this reads part n at n times
-        // partBytes, so every part would be sent from the wrong place and the
-        // upload could never complete.
-        if (state.blocks) {
-            throw new FileFreakError(400, "cut_by_content", "That upload was begun with its file cut by content, which this SDK does not do. Carry it on with the program that began it, or cancel it and begin again.");
-        }
-        const sealing = state.shards ? await sealingOf(state, size) : null;
-        job = { source, size, upload: state.upload, addresses: new Map(), fetching: null, done: new Set(state.uploadedParts), sentBytes: 0, sealing };
+        const carried = await carryOn(calls, source, size, options.resume, rule, signal);
+        if (!("upload" in carried))
+            return carried;
+        job = carried;
     }
     else {
         const name = options.name ?? (typeof File !== "undefined" && source instanceof File ? source.name : undefined);
         if (!name)
             throw new TypeError("Give the upload a name: { name: \"photo.jpg\" }.");
         const type = options.contentType ?? (source instanceof Blob && source.type ? source.type : undefined);
+        // Cut by its content first, as the CLI cuts a file, so that the blocks its next version still has are not sent again.
+        const blocks = await cutPlain(cuttable(source, size), rule, signal);
         const body = {
             name,
             sizeBytes: size,
             parentId: options.parentId ?? null,
-            partBytes: Math.max(LEAST_PART, Math.min(MOST_PART, options.partBytes ?? partSizeFor(size, atOnce))),
+            ...(blocks ? { blocks } : { partBytes: Math.max(LEAST_PART, Math.min(MOST_PART, options.partBytes ?? partSizeFor(size, atOnce))) }),
             // Into a protected place, the parts are fragments this sends sealed.
             protectable: true,
+            shardFormats: WRITES_FORMATS,
         };
         if (options.onConflict)
             body.onConflict = options.onConflict;
@@ -156,19 +273,7 @@ export async function upload(calls, fetcher, source, options = {}) {
             body.contentType = type;
         if (options.imageSizes)
             body.imageSizes = options.imageSizes;
-        const begun = await calls.begin(body, { signal, ...(options.idempotencyKey ? { idempotencyKey: options.idempotencyKey } : {}) });
-        job = { source, size, upload: begun.upload, addresses: new Map(begun.parts.map((p) => [p.number, p])), fetching: null, done: new Set(), sentBytes: 0, sealing: null };
-        if (begun.shards) {
-            // Begin's answer is kept by the API under its Idempotency-Key, so the key is asked for here.
-            try {
-                job.sealing = await sealingOf(await calls.get(begun.upload.id, call), size);
-            }
-            catch (e) {
-                throw new UploadStopped(begun.upload.id, e);
-            }
-            // Nothing of this upload can have been sent yet.
-            job.sealing.waiting.clear();
-        }
+        job = await begin(calls, source, size, body, blocks, options.idempotencyKey, rule, signal);
     }
     options.onBegun?.(job.upload);
     let item;
@@ -191,10 +296,93 @@ export async function upload(calls, fetcher, source, options = {}) {
     }
     return item;
 }
+/**
+ * Begin it. Sent as it is, its parts are its blocks when it was cut, and
+ * those the API has already are not sent. Sealed, its key is asked for, and
+ * in version 2 its blocks are cut and declared, or if it cannot be written
+ * in version 2 it is begun again in version 1.
+ */
+async function begin(calls, source, size, body, blocks, idempotencyKey, rule, signal) {
+    const begun = await calls.begin(body, { signal, ...(idempotencyKey ? { idempotencyKey } : {}) });
+    const addresses = new Map(begun.parts.map((p) => [p.number, p]));
+    if (!begun.shards) {
+        const range = blocks ? blocksOf(blocks.map((b) => b.size)) : evenly(begun.upload, size);
+        return { source, size, upload: begun.upload, range, hashes: blocks && blocks.map((b) => b.sha256), addresses, fetching: null, done: new Set(begun.reused), sentBytes: 0, sealing: null };
+    }
+    let job;
+    try {
+        // Begin's answer is kept by the API under its Idempotency-Key, so the key is asked for here.
+        const state = await calls.get(begun.upload.id, { signal });
+        if (state.shards?.format === 2) {
+            // Begun again in version 1 below, and answered in version 2 all the same: going on would begin again for ever.
+            if (!body.shardFormats?.includes(2))
+                throw new FileFreakError(0, "format_unsupported", "The API began this upload in version 2 of the shard format, which it was not offered.");
+            const declared = await declare(calls, state, source, size, rule, signal);
+            job = declared && sealedJob(source, size, declared.upload, declared.sealing, declared.parts, declared.sealing.recorded);
+            if (!job)
+                await calls.cancel(begun.upload.id, { signal });
+        }
+        else {
+            const sealing = await sealingOf(state, size);
+            // Nothing of this upload can have been sent yet.
+            sealing.waiting.clear();
+            job = sealedJob(source, size, begun.upload, sealing, begun.parts, []);
+        }
+    }
+    catch (e) {
+        throw new UploadStopped(begun.upload.id, e);
+    }
+    if (job)
+        return job;
+    // Not two to ten thousand blocks, which version 2 does not take. The API
+    // refuses one Idempotency-Key for two bodies, so this begin goes with the
+    // transport's own.
+    return await begin(calls, source, size, { ...body, shardFormats: [1] }, blocks, undefined, rule, signal);
+}
+/** An upload begun before, ready to send what storage does not have: or its file, if it was finished. */
+async function carryOn(calls, source, size, id, rule, signal) {
+    const state = await calls.get(id, { signal });
+    if (state.upload.status === "completed" && state.upload.fileId)
+        return (await calls.file(state.upload.fileId, { signal })).item;
+    if (state.upload.status !== "pending") {
+        throw new FileFreakError(410, `upload_${state.upload.status}`, `That upload is ${state.upload.status}, so it cannot be carried on. Begin it again.`);
+    }
+    if (state.upload.sizeBytes !== size) {
+        throw new FileFreakError(400, "size_mismatch", `That upload is for ${state.upload.sizeBytes} bytes, and this source has ${size}. Carry it on with the same file it began with.`);
+    }
+    if (state.shards?.format === 2) {
+        if (state.shards.blocks) {
+            // Declared before. What the file keeps from its last version, and what was
+            // recorded, are filled; anything else may be in storage, and is asked about first.
+            const blocks = state.shards.blocks.map((b) => ({ id: Uint8Array.from(atob(b.id), (c) => c.charCodeAt(0)), length: b.size }));
+            const sealing = await blockSealing(await fileKeys(keyOf(state).raw), { contentId: state.shards.contentId, size, blocks }, new Set(state.uploadedParts));
+            for (let n = 0; n < blocks.length; n++)
+                if (!sealing.recorded.has(n))
+                    sealing.waiting.add(n);
+            return sealedJob(source, size, state.upload, sealing, [], state.uploadedParts);
+        }
+        // Begun and never declared: nothing of it can have been sent.
+        const declared = await declare(calls, state, source, size, rule, signal);
+        if (declared)
+            return sealedJob(source, size, declared.upload, declared.sealing, declared.parts, declared.sealing.recorded);
+        await calls.cancel(id, { signal });
+        throw new FileFreakError(410, "upload_aborted", "That upload cannot be carried on: the file is not cut into 2 to 10,000 blocks, which version 2 of the shard format needs, so it has been cancelled. Begin it again, and it is written in version 1.");
+    }
+    if (state.shards)
+        return sealedJob(source, size, state.upload, await sealingOf(state, size), [], state.uploadedParts);
+    // Begun cut by content: each part is its block, checked against its SHA-256 as it is sent.
+    const range = state.blocks ? blocksOf(state.blocks.map((b) => b.size)) : evenly(state.upload, size);
+    const hashes = state.blocks && state.blocks.map((b) => b.sha256.toLowerCase());
+    return { source, size, upload: state.upload, range, hashes, addresses: new Map(), fetching: null, done: new Set([...state.uploadedParts, ...state.reused]), sentBytes: 0, sealing: null };
+}
 function tally(job, options) {
     let sent = 0;
-    for (const n of job.done)
-        sent += Math.min(job.size, (n + 1) * job.upload.partBytes) - n * job.upload.partBytes;
+    for (const n of job.done) {
+        if (n < 0 || n >= job.upload.partCount)
+            continue;
+        const { start, end } = job.range(n);
+        sent += end - start;
+    }
     job.sentBytes = sent;
     options.onProgress?.(sent, job.size);
 }
@@ -300,27 +488,12 @@ async function sendParts(calls, fetcher, job, atOnce, options) {
     }
 }
 async function sendPart(calls, fetcher, job, n, signal) {
-    const sealing = job.sealing;
-    const { start: from, end: to } = sealing ? sealing.layout.range(n) : { start: n * job.upload.partBytes, end: Math.min(job.size, (n + 1) * job.upload.partBytes) };
-    let bytes;
-    try {
-        bytes = await bytesOf(job.source, from, to);
-    }
-    catch (e) {
-        // Browsers and Node both refuse to read a file that changed after it was
-        // opened, which is what keeps one from going up half old and half new.
-        throw Object.assign(new FileFreakError(0, "unreadable", "The file could not be read. It may have been changed, moved or deleted since it was opened: open it again, unchanged, to carry on."), { cause: e });
-    }
-    if (sealing) {
-        try {
-            bytes = await sealFragment(sealing.key, { contentId: sealing.contentId, layout: sealing.layout, place: n }, bytes);
-        }
-        catch (e) {
-            if (e instanceof ShardError)
-                throw Object.assign(new FileFreakError(0, "layout", e.message), { cause: e });
-            throw e;
-        }
-    }
+    const { start: from, end: to } = job.range(n);
+    let bytes = await read(job.source, from, to);
+    if (job.hashes && hex(new Uint8Array(await crypto.subtle.digest("SHA-256", bytes))) !== job.hashes[n])
+        throw changed(n);
+    if (job.sealing)
+        bytes = await job.sealing.seal(n, bytes);
     const md5 = base64(new Md5().update(bytes).digest());
     for (let attempt = 1;; attempt++) {
         const part = await address(calls, job, n, signal);
@@ -383,7 +556,11 @@ async function address(calls, job, n, signal) {
 }
 async function finish(calls, fetcher, job, atOnce, options) {
     const started = Date.now();
-    const body = options.sha256 ? { sha256: options.sha256 } : {};
+    const body = {};
+    if (options.sha256)
+        body.sha256 = options.sha256;
+    if (job.sealing?.outline)
+        body.outline = job.sealing.outline;
     for (let round = 0;;) {
         try {
             const result = await calls.complete(job.upload.id, body, { signal: options.signal });
@@ -396,7 +573,8 @@ async function finish(calls, fetcher, job, atOnce, options) {
             await sleep(Math.max(1, result.retryAfterSeconds) * 1000, options.signal);
         }
         catch (e) {
-            // Storage does not have every part after all. Send what is missing.
+            // Storage does not have every part after all, or in version 2 a block the
+            // file kept went with the last version that had it. Send what is missing.
             if (e instanceof FileFreakError && e.code === "upload_incomplete" && round++ < 2) {
                 const have = Array.isArray(e.body?.uploadedParts) ? e.body.uploadedParts.filter((n) => typeof n === "number") : [];
                 job.done = new Set(have);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@filefreak.io/sdk",
-  "version": "0.4.0",
+  "version": "0.6.0",
   "description": "FileFreak's developer API for Node and the browser: every endpoint, uploads straight to storage in parts, downloads in ranges, and the changes to follow.",
   "license": "Apache-2.0",
   "homepage": "https://filefreak.io/developers/reference",