@filefreak.io/sdk 0.5.0 → 0.6.0

package/README.md

@@ -5,7 +5,8 @@ later and in browsers. No dependencies.
 
 Every endpoint is a function, typed from the same table the API is built
 from. Around them: uploads sent straight to storage in parts and carried on
-after an interruption, downloads fetched in ranges, listings and changes read
+after an interruption, a new version of a large file sending only what
+changed, downloads fetched in ranges, listings and changes read
 to the end, files under Shard Protection sealed as they are sent and read
 through their maps, what a share link holds saved with no key, and requests sent again only when that
 cannot make anything twice. The reference is at
@@ -43,6 +44,28 @@ an upload carried on asks which fragments storage has before it sends any.
 
 Keep keys on a server. A key sent from a web page is refused.
 
+## A new version of a large file
+
+A file of 16 MiB or more is read through before it is sent and cut where
+its content says, and each part is one of its blocks. A new version of
+it, sent with `onConflict: "replace"`, sends only the blocks the file does
+not have already: those it has are put together with the new ones inside
+storage. The SDK and the CLI cut a file alike, so the next version of a file
+either one sent keeps its blocks whichever sends it.
+
+```ts
+await ff.upload(await fromPath("disk.img"), { parentId: folder.id, onConflict: "replace" });
+```
+
+Into a folder under Shard Protection the same holds, with every block
+encrypted before it leaves. A protected file of more than 16 MiB is cut by
+its own key, so that nobody without the key can cut a copy of it the same
+way and compare, and its next version keeps the blocks the last one has
+when the key may read files; a key that may only write starts a new file
+key and sends every block. Each block is read again as it is sent and
+checked against what the upload began with, so a file changed while it is
+sent stops the upload (`source_changed`) rather than storing a mix of two.
+
 ## In a web page
 
 A page is given a token by your server instead. Your server makes it with its
@@ -190,7 +213,9 @@ await ff.files.delete(file.id, { ifRevision: file.revision });
 
 What reached storage is kept until the upload expires. Keep the upload's id
 from `onBegun`, and carry on with the same file: only the parts storage does
-not have are sent.
+not have are sent. A file cut by its content is carried on block by block,
+each checked against what the upload began with, so carrying on with another
+file is refused at its first block.
 
 ```ts
 import { UploadStopped } from "@filefreak.io/sdk";

package/dist/cut.d.ts

@@ -0,0 +1,96 @@
+import { type FileKeys, type OutlineBlock } from "./shards.js";
+/** How a file is cut. One rule for every file, so that a file cut once is cut the same way next time, which is what lets its blocks be found again. */
+export interface Rule {
+    /** A smaller plain file is not cut: it goes in one or two parts, and there is nothing to gain. */
+    cutFrom: number;
+    /** FastCDC's sizes. No piece but a file's last is shorter than least, and none is longer than most. */
+    least: number;
+    average: number;
+    most: number;
+    /** FastCDC's patterns at level 1: the stricter until a piece reaches the average, the looser after. Within the low 48 bits. */
+    strict: bigint;
+    loose: bigint;
+    /** Past this size pieces are put together in runs, about four to a run, so that a file of 150 GB is still within the API's 10,000 blocks. */
+    groupFrom: number;
+    /** A run ends once it is this long, so that one more piece, 16 MiB at most, leaves it within the API's 64 MiB. */
+    groupMost: number;
+    /** The most blocks the API takes. */
+    mostBlocks: number;
+    /** How much of a Blob is read at a time. */
+    readBytes: number;
+}
+export declare const RULE: Rule;
+/** FastCDC's 256 numbers: whole, for the hash at a cut, and their low 48 bits in halves of 24, for the hash as the bytes go by. */
+export declare class CutTable {
+    readonly values: BigUint64Array;
+    readonly low: Int32Array<ArrayBuffer>;
+    readonly high: Int32Array<ArrayBuffer>;
+    constructor(values: BigUint64Array);
+}
+/**
+ * The fastcdc crate's own table, which a plain file is cut with: for each
+ * value, the first eight bytes, big-endian, of the MD5 of 64 bytes of it
+ * (the crate's examples/table64.rs, after the C implementation it follows).
+ * Made here rather than copied in.
+ */
+export declare function publicTable(): CutTable;
+/** A protected file's own table: 2,048 bytes of HKDF-SHA256 from its key, with no salt and the info "filefreak-shard-v2 cut", as 256 big-endian numbers. */
+export declare function keyedTable(raw: Uint8Array): Promise<CutTable>;
+/** A piece of a file: how many bytes, and the hash it was cut at. */
+export interface Piece {
+    length: number;
+    hash: bigint;
+}
+/**
+ * The piece `data` begins with, where `data` runs to the end of the largest
+ * piece or of the file, whichever comes first. As the crate cuts: the hash
+ * starts `least` bytes in; places are tested two at a time, so the last byte
+ * of a piece cut short at an odd length is hashed and never tested; and a
+ * piece cut at a match ends before the byte that matched.
+ */
+export declare function piece(data: Uint8Array, table: CutTable, rule?: Rule): Piece;
+/**
+ * Whether a piece cut at `hash` ends a run, in a file past groupFrom: one in
+ * four, decided by the bytes where it was cut, so that a run ends at the
+ * same content wherever it moves to. Mixed first, because a hash at a cut
+ * matches a pattern and some of its bits are always zero.
+ */
+export declare function endsRun(hash: bigint): boolean;
+/** What is cut: its bytes, or a way to read them a window at a time. */
+export type Bytes = Uint8Array | {
+    size: number;
+    read(from: number, to: number): Promise<Uint8Array>;
+};
+/** A block of a file: how many bytes, and what names it. */
+export interface Named<T> {
+    length: number;
+    name: T;
+}
+/**
+ * A file cut into blocks with the table given: a piece to a block, or in a
+ * file past groupFrom, runs of pieces. `name` is given each block's bytes in
+ * order. Null as soon as there are more blocks than the API takes, without
+ * reading the rest. What is read is kept only while a piece or an open run
+ * still needs it.
+ */
+export declare function cut<T>(bytes: Bytes, table: CutTable, name: (block: Uint8Array) => Promise<T>, rule?: Rule, signal?: AbortSignal): Promise<Array<Named<T>> | null>;
+/** A plain block as beginning an upload takes it. */
+export interface PlainBlock {
+    size: number;
+    /** In lower-case hex. */
+    sha256: string;
+}
+/**
+ * A plain file cut for beginning its upload, each block its size and
+ * SHA-256. Null for a file too small to be worth cutting, or cut into more
+ * blocks than the API takes: either is sent in parts of one size.
+ */
+export declare function cutPlain(bytes: Bytes, rule?: Rule, signal?: AbortSignal): Promise<PlainBlock[] | null>;
+/**
+ * A protected file cut for version 2 of the shard format, with its own key's
+ * table, each block its id and length, as uploads.blocks takes them and the
+ * outline holds them. Null for a file not cut into two to ten thousand
+ * blocks, which version 2 does not take: it is written in version 1.
+ */
+export declare function cutSealed(bytes: Bytes, raw: Uint8Array, keys: FileKeys, rule?: Rule, signal?: AbortSignal): Promise<OutlineBlock[] | null>;
+export declare function hex(bytes: Uint8Array): string;

package/dist/cut.js

@@ -0,0 +1,220 @@
+// Cutting a file by its content, so that its next version sends only what
+// changed. The rule is the CLI's (cli/crates/transfer/src/blocks.rs), so
+// that of a file either one sent, the other's next version keeps the blocks.
+//
+// Where a piece ends is decided by the bytes there: FastCDC, as the fastcdc
+// crate runs it (version 5, its 2020 form, normalised at level 1), hashes the
+// bytes as it reads and cuts where the hash matches a pattern. A byte put in
+// near the start of a file moves the end of the piece it falls in and leaves
+// every later piece as it was, where parts of one size would all move. A
+// plain file is cut with the crate's own table of numbers and each block
+// named by its SHA-256, as beginning an upload takes them. A file under Shard
+// Protection, written in version 2 of the shard format, is cut with a table
+// made from the file's own key and each block named by its id
+// (docs/reference/shard-format.md, "Cutting a file"): with the public table,
+// whoever has the database could cut a file they hold and compare the
+// lengths. docs/reference/shard-format-v2.json holds both to the crate.
+//
+// The hash runs to 64 bits, more than a JavaScript number holds exactly. The
+// patterns look only at its low 48 bits, which nothing above them changes, so
+// only those are kept as the bytes go by, as two numbers of 24 bits. The
+// whole hash is wanted only at a cut, where it says whether a run ends in a
+// file past 32 GiB, and it depends only on the last 64 bytes hashed: it is
+// worked out again from those, as a BigInt.
+import { Md5 } from "./md5.js";
+import { blockId } from "./shards.js";
+const MIB = 1024 * 1024;
+const HALF = 0xff_ffff;
+export const RULE = {
+    cutFrom: 16 * MIB,
+    least: MIB,
+    average: 4 * MIB,
+    most: 16 * MIB,
+    strict: 0x0000d93777537000n,
+    loose: 0x0000d91767537000n,
+    groupFrom: 32 * 1024 * MIB,
+    groupMost: 48 * MIB,
+    mostBlocks: 10_000,
+    readBytes: 32 * MIB,
+};
+/** FastCDC's 256 numbers: whole, for the hash at a cut, and their low 48 bits in halves of 24, for the hash as the bytes go by. */
+export class CutTable {
+    values;
+    low = new Int32Array(256);
+    high = new Int32Array(256);
+    constructor(values) {
+        this.values = values;
+        if (values.length !== 256)
+            throw new RangeError(`A cutting table has 256 numbers, not ${values.length}.`);
+        for (let i = 0; i < 256; i++) {
+            this.low[i] = Number(values[i] & BigInt(HALF));
+            this.high[i] = Number((values[i] >> 24n) & BigInt(HALF));
+        }
+    }
+}
+let crateTable = null;
+/**
+ * The fastcdc crate's own table, which a plain file is cut with: for each
+ * value, the first eight bytes, big-endian, of the MD5 of 64 bytes of it
+ * (the crate's examples/table64.rs, after the C implementation it follows).
+ * Made here rather than copied in.
+ */
+export function publicTable() {
+    if (!crateTable) {
+        const values = new BigUint64Array(256);
+        for (let i = 0; i < 256; i++) {
+            const digest = new Md5().update(new Uint8Array(64).fill(i)).digest();
+            values[i] = new DataView(digest.buffer, digest.byteOffset, 8).getBigUint64(0, false);
+        }
+        crateTable = new CutTable(values);
+    }
+    return crateTable;
+}
+/** A protected file's own table: 2,048 bytes of HKDF-SHA256 from its key, with no salt and the info "filefreak-shard-v2 cut", as 256 big-endian numbers. */
+export async function keyedTable(raw) {
+    if (raw.length !== 32)
+        throw new RangeError(`A shard key is 32 bytes, not ${raw.length}.`);
+    const base = await crypto.subtle.importKey("raw", new Uint8Array(raw), "HKDF", false, ["deriveBits"]);
+    const bits = await crypto.subtle.deriveBits({ name: "HKDF", hash: "SHA-256", salt: new Uint8Array(0), info: new TextEncoder().encode("filefreak-shard-v2 cut") }, base, 2048 * 8);
+    const view = new DataView(bits);
+    const values = new BigUint64Array(256);
+    for (let i = 0; i < 256; i++)
+        values[i] = view.getBigUint64(i * 8, false);
+    return new CutTable(values);
+}
+/** A pattern as two halves of 24 bits, refused if it looks above the 48 bits the hash is kept to. */
+function halves(pattern) {
+    if (pattern >> 48n !== 0n)
+        throw new RangeError(`A cutting pattern looks only at the low 48 bits of the hash, and ${pattern.toString(16)} does not.`);
+    return [Number(pattern & BigInt(HALF)), Number(pattern >> 24n)];
+}
+/**
+ * The piece `data` begins with, where `data` runs to the end of the largest
+ * piece or of the file, whichever comes first. As the crate cuts: the hash
+ * starts `least` bytes in; places are tested two at a time, so the last byte
+ * of a piece cut short at an odd length is hashed and never tested; and a
+ * piece cut at a match ends before the byte that matched.
+ */
+export function piece(data, table, rule = RULE) {
+    const left = data.length;
+    if (left <= rule.least)
+        return { length: left, hash: 0n };
+    const end = Math.min(left, rule.most);
+    const strictTo = 2 * Math.floor(Math.min(rule.average, end) / 2);
+    const testedTo = 2 * Math.floor(end / 2);
+    const low = table.low;
+    const high = table.high;
+    let [maskLow, maskHigh] = halves(rule.strict);
+    const [looseLow, looseHigh] = halves(rule.loose);
+    let lo = 0;
+    let hi = 0;
+    for (let i = rule.least; i < testedTo; i++) {
+        if (i === strictTo) {
+            maskLow = looseLow;
+            maskHigh = looseHigh;
+        }
+        const b = data[i];
+        const sum = 2 * lo + low[b];
+        hi = (2 * hi + high[b] + (sum >>> 24)) & HALF;
+        lo = sum & HALF;
+        if ((lo & maskLow) === 0 && (hi & maskHigh) === 0)
+            return { length: i, hash: hashAt(data, i, table, rule, i % 2 === 0) };
+    }
+    return { length: end, hash: hashAt(data, end - 1, table, rule, false) };
+}
+/**
+ * The whole hash once byte `i` is hashed, from the bytes that still count in
+ * it: those from `least` on, and of them only the last 64, since each byte
+ * doubles it and 64 doublings leave nothing of an earlier one. Doubled again
+ * for a match at an even place, as the crate gives it.
+ */
+function hashAt(data, i, table, rule, doubled) {
+    let h = 0n;
+    for (let j = Math.max(rule.least, i - 63); j <= i; j++)
+        h = BigInt.asUintN(64, (h << 1n) + table.values[data[j]]);
+    return doubled ? BigInt.asUintN(64, h << 1n) : h;
+}
+/**
+ * Whether a piece cut at `hash` ends a run, in a file past groupFrom: one in
+ * four, decided by the bytes where it was cut, so that a run ends at the
+ * same content wherever it moves to. Mixed first, because a hash at a cut
+ * matches a pattern and some of its bits are always zero.
+ */
+export function endsRun(hash) {
+    return BigInt.asUintN(64, hash * 0x9e3779b97f4a7c15n) >> 62n === 0n;
+}
+/**
+ * A file cut into blocks with the table given: a piece to a block, or in a
+ * file past groupFrom, runs of pieces. `name` is given each block's bytes in
+ * order. Null as soon as there are more blocks than the API takes, without
+ * reading the rest. What is read is kept only while a piece or an open run
+ * still needs it.
+ */
+export async function cut(bytes, table, name, rule = RULE, signal) {
+    const reader = bytes instanceof Uint8Array ? null : bytes;
+    const size = sizeOf(bytes);
+    let buffer = bytes instanceof Uint8Array ? bytes : new Uint8Array(0);
+    let bufferAt = 0;
+    const grouping = size > rule.groupFrom;
+    const blocks = [];
+    let runAt = null;
+    for (let at = 0; at < size;) {
+        signal?.throwIfAborted();
+        const want = Math.min(size, at + rule.most);
+        if (reader && bufferAt + buffer.length < want) {
+            const keep = runAt ?? at;
+            const had = bufferAt + buffer.length;
+            const upTo = Math.min(size, Math.max(want, had + rule.readBytes));
+            const more = await reader.read(had, upTo);
+            if (more.length !== upTo - had)
+                throw new RangeError(`Reading bytes ${had} to ${upTo} gave ${more.length}: the file has changed since it was opened.`);
+            const joined = new Uint8Array(had - keep + more.length);
+            joined.set(buffer.subarray(keep - bufferAt));
+            joined.set(more, had - keep);
+            buffer = joined;
+            bufferAt = keep;
+        }
+        const { length, hash } = piece(buffer.subarray(at - bufferAt, want - bufferAt), table, rule);
+        const end = at + length;
+        runAt ??= at;
+        if (!grouping || end - runAt >= rule.groupMost || endsRun(hash) || end === size) {
+            blocks.push({ length: end - runAt, name: await name(buffer.subarray(runAt - bufferAt, end - bufferAt)) });
+            if (blocks.length > rule.mostBlocks)
+                return null;
+            runAt = null;
+        }
+        at = end;
+    }
+    return blocks;
+}
+/**
+ * A plain file cut for beginning its upload, each block its size and
+ * SHA-256. Null for a file too small to be worth cutting, or cut into more
+ * blocks than the API takes: either is sent in parts of one size.
+ */
+export async function cutPlain(bytes, rule = RULE, signal) {
+    if (sizeOf(bytes) < rule.cutFrom)
+        return null;
+    const blocks = await cut(bytes, publicTable(), async (block) => hex(new Uint8Array(await crypto.subtle.digest("SHA-256", own(block)))), rule, signal);
+    return blocks && blocks.map((b) => ({ size: b.length, sha256: b.name }));
+}
+/**
+ * A protected file cut for version 2 of the shard format, with its own key's
+ * table, each block its id and length, as uploads.blocks takes them and the
+ * outline holds them. Null for a file not cut into two to ten thousand
+ * blocks, which version 2 does not take: it is written in version 1.
+ */
+export async function cutSealed(bytes, raw, keys, rule = RULE, signal) {
+    const blocks = await cut(bytes, await keyedTable(raw), (block) => blockId(keys, block), rule, signal);
+    return blocks && blocks.length >= 2 ? blocks.map((b) => ({ id: b.name, length: b.length })) : null;
+}
+function sizeOf(bytes) {
+    return bytes instanceof Uint8Array ? bytes.length : bytes.size;
+}
+/** The bytes as WebCrypto's types take them, copied only when they are not in an ArrayBuffer of their own (a SharedArrayBuffer's). */
+function own(bytes) {
+    return bytes.buffer instanceof ArrayBuffer ? bytes : new Uint8Array(bytes);
+}
+export function hex(bytes) {
+    return Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("");
+}

package/dist/uploads.d.ts

@@ -1,4 +1,5 @@
-import type { FilesFrameAnswer, FilesGetAnswer, Item, Upload, UploadsBeginAnswer, UploadsBeginBody, UploadsCompleteAnswer, UploadsCompleteBody, UploadsFragmentsAnswer, UploadsFragmentsBody, UploadsGetAnswer, UploadsPartsAnswer, UploadsPartsQuery } from "./generated.js";
+import { type Rule } from "./cut.js";
+import type { FilesFrameAnswer, FilesGetAnswer, Item, Upload, UploadsBeginAnswer, UploadsBeginBody, UploadsBlocksAnswer, UploadsBlocksBody, UploadsCancelAnswer, UploadsCompleteAnswer, UploadsCompleteBody, UploadsFragmentsAnswer, UploadsFragmentsBody, UploadsGetAnswer, UploadsPartsAnswer, UploadsPartsQuery } from "./generated.js";
 import { type CallOptions } from "./transport.js";
 /** What can be uploaded. A File's name and type are used when none are given. */
 export type Source = Blob | ArrayBuffer | Uint8Array;
@@ -18,7 +19,7 @@ export interface UploadOptions {
     contentType?: string;
     /** Smaller copies to make of an image as soon as it is stored. */
     imageSizes?: number[];
-    /** Bytes in each part, from 8 MiB to 64 MiB. Left out, 32 MiB, or less for a small file. */
+    /** Bytes in each part, from 8 MiB to 64 MiB, for a file not cut by its content: one under 16 MiB, or one of more blocks than the API takes. Left out, 32 MiB, or less for a small file. */
     partBytes?: number;
     /** Parts sent at once. */
     concurrency?: number;
@@ -52,11 +53,13 @@ export interface UploadCalls {
     begin(body: UploadsBeginBody, options?: CallOptions): Promise<UploadsBeginAnswer>;
     get(id: string, options?: CallOptions): Promise<UploadsGetAnswer>;
     parts(id: string, query?: UploadsPartsQuery, options?: CallOptions): Promise<UploadsPartsAnswer>;
+    blocks(id: string, body: UploadsBlocksBody, options?: CallOptions): Promise<UploadsBlocksAnswer>;
     fragments(id: string, body: UploadsFragmentsBody, options?: CallOptions): Promise<UploadsFragmentsAnswer>;
     complete(id: string, body?: UploadsCompleteBody, options?: CallOptions): Promise<UploadsCompleteAnswer>;
+    cancel(id: string, options?: CallOptions): Promise<UploadsCancelAnswer>;
     file(id: string, options?: CallOptions): Promise<FilesGetAnswer>;
     frame(id: string, options?: CallOptions): Promise<FilesFrameAnswer>;
 }
 /** The part size to ask for: 32 MiB, halved while that would leave fewer parts than are sent at once. */
 export declare function partSizeFor(fileBytes: number, atOnce: number): number;
-export declare function upload(calls: UploadCalls, fetcher: typeof fetch, source: Source, options?: UploadOptions): Promise<Item>;
+export declare function upload(calls: UploadCalls, fetcher: typeof fetch, source: Source, options?: UploadOptions, rule?: Rule): Promise<Item>;

package/dist/uploads.js

@@ -16,7 +16,17 @@
 //     answers 202 until it has, and is asked again when it says.
 //
 // A part is read into memory whole to be hashed and sent, so the memory an
-// upload needs is about partBytes times the parts sent at once.
+// upload needs is about the size of a part times the parts sent at once.
+//
+// CUT BY CONTENT, as the CLI cuts a file (cut.ts). A file of 16 MiB or more
+// is read through once before it is begun, cut where its content says, and
+// begun with its blocks, each its size and SHA-256. Each part is then a
+// block, and a block the file it replaces has already, or one an earlier
+// part repeats, is not sent: the API lists those in reused. So a new version
+// of a large file sends only the blocks that changed. Each block is hashed
+// again as it is sent and refused if it is not the block that was declared,
+// since the file has changed. A block is 16 MiB at most, or 64 MiB in a file
+// past 32 GiB.
 //
 // SEALED AS IT IS SENT (plan 04, F6 step 4, answer 46). Every begin says the
 // SDK can seal (protectable), as the website and the CLI do. Into a place
@@ -30,9 +40,21 @@
 // Carrying on, every fragment not recorded is confirmed before it is sent
 // again, since storage may hold it already. A fragment is up to 64 MiB and
 // sealed in memory, so the memory a sealed upload needs is the same.
+//
+// Every begin also says the SDK writes version 2 of the shard format, in
+// which a protected file of more than 16 MiB is answered. Its fragments are
+// blocks: the file is cut again with a table made from its own key, each
+// block named by its id under the key (cutSealed), and the blocks declared
+// (uploads.blocks). Those the file's last version has under the same key are
+// kept and never sent, so a new version of a large protected file sends only
+// what changed, sealed here. Each block is checked against the id declared
+// for its place as it is sealed, and completing sends the outline's MAC. A
+// file not cut into two to ten thousand blocks, which version 2 does not
+// take, is begun again in version 1.
+import { cutPlain, cutSealed, hex, RULE } from "./cut.js";
 import { canDrawFrames, drawFrame, sendFrame, takesFrame } from "./frames.js";
 import { base64, Md5 } from "./md5.js";
-import { sealFragment, ShardError, shardKey, ShardLayout } from "./shards.js";
+import { BlockLayout, fileKeys, outlineMac, sealBlock, sealFragment, ShardError, shardKey, ShardLayout } from "./shards.js";
 import { FileFreakError, sleep } from "./transport.js";
 const MIB = 1024 * 1024;
 /** The API's bounds on a part. */
@@ -54,6 +76,8 @@ const CONFIRM_MOST = 100;
 const CONFIRM_PATIENCE_MS = 5 * 60_000;
 /** How many times fragments storage did not keep are sealed again and sent before the upload stops. */
 const SEALED_ROUNDS = 4;
+/** The versions of the shard format this writes. */
+const WRITES_FORMATS = [1, 2];
 /** An upload that stopped part way. What arrived stays in storage until the upload expires: carry on with `resume: uploadId`, or cancel it. */
 export class UploadStopped extends Error {
     uploadId;
@@ -64,33 +88,106 @@ export class UploadStopped extends Error {
         this.name = "UploadStopped";
     }
 }
-/**
- * The key and the cut of a sealed upload, from what uploads.get says, with
- * every fragment it has not recorded waiting to be confirmed: one sent
- * before, by this process or another, may be in storage already.
- */
-async function sealingOf(state, size) {
+/** A part that is not what was declared for it: the file has changed since it was cut. */
+function changed(part) {
+    return new FileFreakError(0, "source_changed", `Part ${part} of the file is not what it was when the upload began: the file has changed since.`);
+}
+/** A sealed upload's key, from what uploads.get says, and what it says of the file. */
+function keyOf(state) {
     const shards = state.shards;
     if (!shards)
         throw new FileFreakError(0, "unreadable", "The API began a sealed upload and then gave no key for it.");
-    let layout;
     let raw;
     try {
-        layout = new ShardLayout(size, shards.fragmentBytes);
         raw = Uint8Array.from(atob(shards.key), (c) => c.charCodeAt(0));
     }
     catch (e) {
-        throw Object.assign(new FileFreakError(0, "unreadable", "The API answered with a sealed upload whose cut or key could not be read."), { cause: e });
+        throw Object.assign(new FileFreakError(0, "unreadable", "The API answered with a sealed upload whose key could not be read."), { cause: e });
+    }
+    if (raw.length !== 32)
+        throw new FileFreakError(0, "unreadable", `The API answered with a key of ${raw.length} bytes, where a file's key is 32.`);
+    return { raw, shards };
+}
+/**
+ * The key and the cut of an upload sealed in version 1, from what
+ * uploads.get says, with every fragment it has not recorded waiting to be
+ * confirmed: one sent before, by this process or another, may be in storage
+ * already.
+ */
+async function sealingOf(state, size) {
+    const { raw, shards } = keyOf(state);
+    let layout;
+    try {
+        layout = new ShardLayout(size, shards.fragmentBytes);
+    }
+    catch (e) {
+        throw Object.assign(new FileFreakError(0, "unreadable", "The API answered with a sealed upload whose cut could not be read."), { cause: e });
     }
-    if (layout.count !== shards.count || layout.count !== state.upload.partCount || raw.length !== 32) {
+    if (layout.count !== shards.count || layout.count !== state.upload.partCount) {
         throw new FileFreakError(0, "layout", `The API cut this file into ${shards.count} fragments, where ${size} bytes in fragments of ${shards.fragmentBytes} make ${layout.count}.`);
     }
+    const key = await shardKey(raw);
+    const contentId = shards.contentId;
     const recorded = new Set(state.uploadedParts);
     const waiting = new Set();
     for (let n = 0; n < layout.count; n++)
         if (!recorded.has(n))
             waiting.add(n);
-    return { key: await shardKey(raw), contentId: shards.contentId, layout, recorded, waiting };
+    return {
+        layout,
+        seal: async (place, plain) => {
+            try {
+                return await sealFragment(key, { contentId, layout, place }, plain);
+            }
+            catch (e) {
+                if (e instanceof ShardError)
+                    throw Object.assign(new FileFreakError(0, "layout", e.message), { cause: e });
+                throw e;
+            }
+        },
+        outline: null,
+        recorded,
+        waiting,
+    };
+}
+/** What a file in version 2 is sealed with: each block checked, as it is sealed, against the id declared for its place. */
+async function blockSealing(keys, outline, recorded) {
+    let layout;
+    try {
+        layout = new BlockLayout(outline);
+    }
+    catch (e) {
+        if (e instanceof ShardError)
+            throw Object.assign(new FileFreakError(0, "layout", e.message), { cause: e });
+        throw e;
+    }
+    return {
+        layout,
+        seal: async (place, plain) => {
+            const { id, blob } = await sealBlock(keys, plain);
+            if (base64(id) !== base64(outline.blocks[place].id))
+                throw changed(place);
+            return blob;
+        },
+        outline: base64(await outlineMac(keys, outline)),
+        recorded,
+        waiting: new Set(),
+    };
+}
+/**
+ * Version 2: the file cut with its own key's table, and its blocks declared,
+ * with what to seal them with. Null for a file not cut into two to ten
+ * thousand blocks, which version 2 does not take.
+ */
+async function declare(calls, state, source, size, rule, signal) {
+    const { raw, shards } = keyOf(state);
+    const keys = await fileKeys(raw);
+    const blocks = await cutSealed(cuttable(source, size), raw, keys, rule, signal);
+    if (!blocks)
+        return null;
+    const declared = await calls.blocks(state.upload.id, { blocks: blocks.map((b) => ({ size: b.length, id: base64(b.id) })) }, { signal });
+    const sealing = await blockSealing(keys, { contentId: shards.contentId, size, blocks }, new Set(declared.reused));
+    return { upload: declared.upload, parts: declared.parts, sealing };
 }
 /** The part size to ask for: 32 MiB, halved while that would leave fewer parts than are sent at once. */
 export function partSizeFor(fileBytes, atOnce) {
@@ -109,44 +206,64 @@ async function bytesOf(source, from, to) {
         return new Uint8Array(source.slice(from, to));
     return source.slice(from, to);
 }
-export async function upload(calls, fetcher, source, options = {}) {
+/** Bytes of the file, or why they could not be read. */
+async function read(source, from, to) {
+    try {
+        return await bytesOf(source, from, to);
+    }
+    catch (e) {
+        // Browsers and Node both refuse to read a file that changed after it was
+        // opened, which is what keeps one from going up half old and half new.
+        throw Object.assign(new FileFreakError(0, "unreadable", "The file could not be read. It may have been changed, moved or deleted since it was opened: open it again, unchanged, to carry on."), { cause: e });
+    }
+}
+/** The file as the cutter reads it: where it is, when it is in memory, and a window at a time from a Blob. */
+function cuttable(source, size) {
+    if (source instanceof Blob)
+        return { size, read: (from, to) => read(source, from, to) };
+    return source instanceof ArrayBuffer ? new Uint8Array(source) : source;
+}
+/** Parts of one size, the last whatever remains. */
+function evenly(upload, size) {
+    return (n) => ({ start: n * upload.partBytes, end: Math.min(size, (n + 1) * upload.partBytes) });
+}
+/** Parts that are blocks of their own sizes. */
+function blocksOf(sizes) {
+    const starts = [0];
+    for (const size of sizes)
+        starts.push(starts[starts.length - 1] + size);
+    return (n) => ({ start: starts[n], end: starts[n + 1] });
+}
+/** A job for an upload sealed as it is sent: each part is a fragment, where its layout says. */
+function sealedJob(source, size, upload, sealing, addresses, done) {
+    return { source, size, upload, range: (n) => sealing.layout.range(n), hashes: null, addresses: new Map(addresses.map((p) => [p.number, p])), fetching: null, done: new Set(done), sentBytes: 0, sealing };
+}
+export async function upload(calls, fetcher, source, options = {}, rule = RULE) {
     const signal = options.signal;
     const atOnce = Math.max(1, Math.min(16, options.concurrency ?? AT_ONCE));
     const size = sizeOf(source);
-    const call = { signal };
     let job;
     if (options.resume) {
-        const state = await calls.get(options.resume, call);
-        if (state.upload.status === "completed" && state.upload.fileId)
-            return (await calls.file(state.upload.fileId, call)).item;
-        if (state.upload.status !== "pending") {
-            throw new FileFreakError(410, `upload_${state.upload.status}`, `That upload is ${state.upload.status}, so it cannot be carried on. Begin it again.`);
-        }
-        if (state.upload.sizeBytes !== size) {
-            throw new FileFreakError(400, "size_mismatch", `That upload is for ${state.upload.sizeBytes} bytes, and this source has ${size}. Carry it on with the same file it began with.`);
-        }
-        // Begun by a program that cut the file by its content, as filefreak does: its
-        // parts are blocks of different sizes, and this reads part n at n times
-        // partBytes, so every part would be sent from the wrong place and the
-        // upload could never complete.
-        if (state.blocks) {
-            throw new FileFreakError(400, "cut_by_content", "That upload was begun with its file cut by content, which this SDK does not do. Carry it on with the program that began it, or cancel it and begin again.");
-        }
-        const sealing = state.shards ? await sealingOf(state, size) : null;
-        job = { source, size, upload: state.upload, addresses: new Map(), fetching: null, done: new Set(state.uploadedParts), sentBytes: 0, sealing };
+        const carried = await carryOn(calls, source, size, options.resume, rule, signal);
+        if (!("upload" in carried))
+            return carried;
+        job = carried;
     }
     else {
         const name = options.name ?? (typeof File !== "undefined" && source instanceof File ? source.name : undefined);
         if (!name)
             throw new TypeError("Give the upload a name: { name: \"photo.jpg\" }.");
         const type = options.contentType ?? (source instanceof Blob && source.type ? source.type : undefined);
+        // Cut by its content first, as the CLI cuts a file, so that the blocks its next version still has are not sent again.
+        const blocks = await cutPlain(cuttable(source, size), rule, signal);
         const body = {
             name,
             sizeBytes: size,
             parentId: options.parentId ?? null,
-            partBytes: Math.max(LEAST_PART, Math.min(MOST_PART, options.partBytes ?? partSizeFor(size, atOnce))),
+            ...(blocks ? { blocks } : { partBytes: Math.max(LEAST_PART, Math.min(MOST_PART, options.partBytes ?? partSizeFor(size, atOnce))) }),
             // Into a protected place, the parts are fragments this sends sealed.
             protectable: true,
+            shardFormats: WRITES_FORMATS,
         };
         if (options.onConflict)
             body.onConflict = options.onConflict;
@@ -156,19 +273,7 @@ export async function upload(calls, fetcher, source, options = {}) {
             body.contentType = type;
         if (options.imageSizes)
             body.imageSizes = options.imageSizes;
-        const begun = await calls.begin(body, { signal, ...(options.idempotencyKey ? { idempotencyKey: options.idempotencyKey } : {}) });
-        job = { source, size, upload: begun.upload, addresses: new Map(begun.parts.map((p) => [p.number, p])), fetching: null, done: new Set(), sentBytes: 0, sealing: null };
-        if (begun.shards) {
-            // Begin's answer is kept by the API under its Idempotency-Key, so the key is asked for here.
-            try {
-                job.sealing = await sealingOf(await calls.get(begun.upload.id, call), size);
-            }
-            catch (e) {
-                throw new UploadStopped(begun.upload.id, e);
-            }
-            // Nothing of this upload can have been sent yet.
-            job.sealing.waiting.clear();
-        }
+        job = await begin(calls, source, size, body, blocks, options.idempotencyKey, rule, signal);
     }
     options.onBegun?.(job.upload);
     let item;
@@ -191,10 +296,93 @@ export async function upload(calls, fetcher, source, options = {}) {
     }
     return item;
 }
+/**
+ * Begin it. Sent as it is, its parts are its blocks when it was cut, and
+ * those the API has already are not sent. Sealed, its key is asked for, and
+ * in version 2 its blocks are cut and declared, or if it cannot be written
+ * in version 2 it is begun again in version 1.
+ */
+async function begin(calls, source, size, body, blocks, idempotencyKey, rule, signal) {
+    const begun = await calls.begin(body, { signal, ...(idempotencyKey ? { idempotencyKey } : {}) });
+    const addresses = new Map(begun.parts.map((p) => [p.number, p]));
+    if (!begun.shards) {
+        const range = blocks ? blocksOf(blocks.map((b) => b.size)) : evenly(begun.upload, size);
+        return { source, size, upload: begun.upload, range, hashes: blocks && blocks.map((b) => b.sha256), addresses, fetching: null, done: new Set(begun.reused), sentBytes: 0, sealing: null };
+    }
+    let job;
+    try {
+        // Begin's answer is kept by the API under its Idempotency-Key, so the key is asked for here.
+        const state = await calls.get(begun.upload.id, { signal });
+        if (state.shards?.format === 2) {
+            // Begun again in version 1 below, and answered in version 2 all the same: going on would begin again for ever.
+            if (!body.shardFormats?.includes(2))
+                throw new FileFreakError(0, "format_unsupported", "The API began this upload in version 2 of the shard format, which it was not offered.");
+            const declared = await declare(calls, state, source, size, rule, signal);
+            job = declared && sealedJob(source, size, declared.upload, declared.sealing, declared.parts, declared.sealing.recorded);
+            if (!job)
+                await calls.cancel(begun.upload.id, { signal });
+        }
+        else {
+            const sealing = await sealingOf(state, size);
+            // Nothing of this upload can have been sent yet.
+            sealing.waiting.clear();
+            job = sealedJob(source, size, begun.upload, sealing, begun.parts, []);
+        }
+    }
+    catch (e) {
+        throw new UploadStopped(begun.upload.id, e);
+    }
+    if (job)
+        return job;
+    // Not two to ten thousand blocks, which version 2 does not take. The API
+    // refuses one Idempotency-Key for two bodies, so this begin goes with the
+    // transport's own.
+    return await begin(calls, source, size, { ...body, shardFormats: [1] }, blocks, undefined, rule, signal);
+}
+/** An upload begun before, ready to send what storage does not have: or its file, if it was finished. */
+async function carryOn(calls, source, size, id, rule, signal) {
+    const state = await calls.get(id, { signal });
+    if (state.upload.status === "completed" && state.upload.fileId)
+        return (await calls.file(state.upload.fileId, { signal })).item;
+    if (state.upload.status !== "pending") {
+        throw new FileFreakError(410, `upload_${state.upload.status}`, `That upload is ${state.upload.status}, so it cannot be carried on. Begin it again.`);
+    }
+    if (state.upload.sizeBytes !== size) {
+        throw new FileFreakError(400, "size_mismatch", `That upload is for ${state.upload.sizeBytes} bytes, and this source has ${size}. Carry it on with the same file it began with.`);
+    }
+    if (state.shards?.format === 2) {
+        if (state.shards.blocks) {
+            // Declared before. What the file keeps from its last version, and what was
+            // recorded, are filled; anything else may be in storage, and is asked about first.
+            const blocks = state.shards.blocks.map((b) => ({ id: Uint8Array.from(atob(b.id), (c) => c.charCodeAt(0)), length: b.size }));
+            const sealing = await blockSealing(await fileKeys(keyOf(state).raw), { contentId: state.shards.contentId, size, blocks }, new Set(state.uploadedParts));
+            for (let n = 0; n < blocks.length; n++)
+                if (!sealing.recorded.has(n))
+                    sealing.waiting.add(n);
+            return sealedJob(source, size, state.upload, sealing, [], state.uploadedParts);
+        }
+        // Begun and never declared: nothing of it can have been sent.
+        const declared = await declare(calls, state, source, size, rule, signal);
+        if (declared)
+            return sealedJob(source, size, declared.upload, declared.sealing, declared.parts, declared.sealing.recorded);
+        await calls.cancel(id, { signal });
+        throw new FileFreakError(410, "upload_aborted", "That upload cannot be carried on: the file is not cut into 2 to 10,000 blocks, which version 2 of the shard format needs, so it has been cancelled. Begin it again, and it is written in version 1.");
+    }
+    if (state.shards)
+        return sealedJob(source, size, state.upload, await sealingOf(state, size), [], state.uploadedParts);
+    // Begun cut by content: each part is its block, checked against its SHA-256 as it is sent.
+    const range = state.blocks ? blocksOf(state.blocks.map((b) => b.size)) : evenly(state.upload, size);
+    const hashes = state.blocks && state.blocks.map((b) => b.sha256.toLowerCase());
+    return { source, size, upload: state.upload, range, hashes, addresses: new Map(), fetching: null, done: new Set([...state.uploadedParts, ...state.reused]), sentBytes: 0, sealing: null };
+}
 function tally(job, options) {
     let sent = 0;
-    for (const n of job.done)
-        sent += Math.min(job.size, (n + 1) * job.upload.partBytes) - n * job.upload.partBytes;
+    for (const n of job.done) {
+        if (n < 0 || n >= job.upload.partCount)
+            continue;
+        const { start, end } = job.range(n);
+        sent += end - start;
+    }
     job.sentBytes = sent;
     options.onProgress?.(sent, job.size);
 }
@@ -300,27 +488,12 @@ async function sendParts(calls, fetcher, job, atOnce, options) {
     }
 }
 async function sendPart(calls, fetcher, job, n, signal) {
-    const sealing = job.sealing;
-    const { start: from, end: to } = sealing ? sealing.layout.range(n) : { start: n * job.upload.partBytes, end: Math.min(job.size, (n + 1) * job.upload.partBytes) };
-    let bytes;
-    try {
-        bytes = await bytesOf(job.source, from, to);
-    }
-    catch (e) {
-        // Browsers and Node both refuse to read a file that changed after it was
-        // opened, which is what keeps one from going up half old and half new.
-        throw Object.assign(new FileFreakError(0, "unreadable", "The file could not be read. It may have been changed, moved or deleted since it was opened: open it again, unchanged, to carry on."), { cause: e });
-    }
-    if (sealing) {
-        try {
-            bytes = await sealFragment(sealing.key, { contentId: sealing.contentId, layout: sealing.layout, place: n }, bytes);
-        }
-        catch (e) {
-            if (e instanceof ShardError)
-                throw Object.assign(new FileFreakError(0, "layout", e.message), { cause: e });
-            throw e;
-        }
-    }
+    const { start: from, end: to } = job.range(n);
+    let bytes = await read(job.source, from, to);
+    if (job.hashes && hex(new Uint8Array(await crypto.subtle.digest("SHA-256", bytes))) !== job.hashes[n])
+        throw changed(n);
+    if (job.sealing)
+        bytes = await job.sealing.seal(n, bytes);
     const md5 = base64(new Md5().update(bytes).digest());
     for (let attempt = 1;; attempt++) {
         const part = await address(calls, job, n, signal);
@@ -383,7 +556,11 @@ async function address(calls, job, n, signal) {
 }
 async function finish(calls, fetcher, job, atOnce, options) {
     const started = Date.now();
-    const body = options.sha256 ? { sha256: options.sha256 } : {};
+    const body = {};
+    if (options.sha256)
+        body.sha256 = options.sha256;
+    if (job.sealing?.outline)
+        body.outline = job.sealing.outline;
     for (let round = 0;;) {
         try {
             const result = await calls.complete(job.upload.id, body, { signal: options.signal });
@@ -396,7 +573,8 @@ async function finish(calls, fetcher, job, atOnce, options) {
             await sleep(Math.max(1, result.retryAfterSeconds) * 1000, options.signal);
         }
         catch (e) {
-            // Storage does not have every part after all. Send what is missing.
+            // Storage does not have every part after all, or in version 2 a block the
+            // file kept went with the last version that had it. Send what is missing.
             if (e instanceof FileFreakError && e.code === "upload_incomplete" && round++ < 2) {
                 const have = Array.isArray(e.body?.uploadedParts) ? e.body.uploadedParts.filter((n) => typeof n === "number") : [];
                 job.done = new Set(have);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@filefreak.io/sdk",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "FileFreak's developer API for Node and the browser: every endpoint, uploads straight to storage in parts, downloads in ranges, and the changes to follow.",
   "license": "Apache-2.0",
   "homepage": "https://filefreak.io/developers/reference",